From b08ca49580c63abe79a650adbb0d14ca87a1cd24 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Tue, 16 Aug 2016 14:05:17 -0700 Subject: More controller work -- it builds! --- make-mac.mk | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'make-mac.mk') diff --git a/make-mac.mk b/make-mac.mk index e821c4cf..09e04eab 100644 --- a/make-mac.mk +++ b/make-mac.mk @@ -65,7 +65,7 @@ else STRIP=strip endif -CXXFLAGS=$(CFLAGS) -fno-rtti +CXXFLAGS=$(CFLAGS) -fno-rtti -mmacosx-version-min=10.7 -std=c++11 -stdlib=libc++ all: one @@ -78,7 +78,7 @@ one: $(OBJS) service/OneService.o one.o $(CODESIGN) -vvv zerotier-one cli: FORCE - $(CXX) -Os -mmacosx-version-min=10.7 -std=c++11 -stdlib=libc++ -o zerotier cli/zerotier.cpp osdep/OSUtils.cpp node/InetAddress.cpp node/Utils.cpp node/Salsa20.cpp node/Identity.cpp node/SHA512.cpp node/C25519.cpp -lcurl + $(CXX) $(CXXFLAGS) -o zerotier cli/zerotier.cpp osdep/OSUtils.cpp node/InetAddress.cpp node/Utils.cpp node/Salsa20.cpp node/Identity.cpp node/SHA512.cpp node/C25519.cpp -lcurl $(STRIP) zerotier selftest: $(OBJS) selftest.o -- cgit v1.2.3 From 402d99ea8d73d804294795cab12fceb6fe4234fe Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Tue, 16 Aug 2016 14:07:11 -0700 Subject: C++11 mode build fix. --- make-mac.mk | 2 +- service/OneService.cpp | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) (limited to 'make-mac.mk') diff --git a/make-mac.mk b/make-mac.mk index 09e04eab..63ffc3c8 100644 --- a/make-mac.mk +++ b/make-mac.mk @@ -65,7 +65,7 @@ else STRIP=strip endif -CXXFLAGS=$(CFLAGS) -fno-rtti -mmacosx-version-min=10.7 -std=c++11 -stdlib=libc++ +CXXFLAGS=$(CFLAGS) -mmacosx-version-min=10.7 -std=c++11 -stdlib=libc++ all: one diff --git a/service/OneService.cpp b/service/OneService.cpp index 0c9b0b8e..5c65dcc2 100644 --- a/service/OneService.cpp +++ b/service/OneService.cpp @@ -1119,13 +1119,13 @@ public: for(std::vector::iterator ip(n.managedIps.begin());ip!=n.managedIps.end();++ip) { if (std::find(newManagedIps.begin(),newManagedIps.end(),*ip) == newManagedIps.end()) { if (!n.tap->removeIp(*ip)) - fprintf(stderr,"ERROR: unable to remove ip address %s"ZT_EOL_S, ip->toString().c_str()); + fprintf(stderr,"ERROR: unable to remove ip address %s" ZT_EOL_S, ip->toString().c_str()); } } for(std::vector::iterator ip(newManagedIps.begin());ip!=newManagedIps.end();++ip) { if (std::find(n.managedIps.begin(),n.managedIps.end(),*ip) == n.managedIps.end()) { if (!n.tap->addIp(*ip)) - fprintf(stderr,"ERROR: unable to add ip address %s"ZT_EOL_S, ip->toString().c_str()); + fprintf(stderr,"ERROR: unable to add ip address %s" ZT_EOL_S, ip->toString().c_str()); } } @@ -1468,11 +1468,11 @@ public: #ifdef __WINDOWS__ FILE *tapFailLog = fopen((_homePath + ZT_PATH_SEPARATOR_S"port_error_log.txt").c_str(),"a"); if (tapFailLog) { - fprintf(tapFailLog,"%.16llx: %s"ZT_EOL_S,(unsigned long long)nwid,exc.what()); + fprintf(tapFailLog,"%.16llx: %s" ZT_EOL_S,(unsigned long long)nwid,exc.what()); fclose(tapFailLog); } #else - fprintf(stderr,"ERROR: unable to configure virtual network port: %s"ZT_EOL_S,exc.what()); + fprintf(stderr,"ERROR: unable to configure virtual network port: %s" ZT_EOL_S,exc.what()); #endif _nets.erase(nwid); return -999; @@ -1531,7 +1531,7 @@ public: case ZT_EVENT_TRACE: { if (metaData) { - ::fprintf(stderr,"%s"ZT_EOL_S,(const char *)metaData); + ::fprintf(stderr,"%s" ZT_EOL_S,(const char *)metaData); ::fflush(stderr); } } break; -- cgit v1.2.3 From a13f4d8353cf6a7f612333f00282b27cfe2ce9b3 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Wed, 17 Aug 2016 10:42:32 -0700 Subject: We now always build the controller in ZeroTier One, at least for desktop and server targets. Also means that ZeroTier One now requires C++11. (Still keeping C++11 out of the core in node/ though.) --- controller/EmbeddedNetworkController.cpp | 1462 ++++++++++++++++++++++++++++++ controller/EmbeddedNetworkController.hpp | 173 ++++ controller/SqliteNetworkController.cpp | 1462 ------------------------------ controller/SqliteNetworkController.hpp | 173 ---- make-linux.mk | 6 - make-mac.mk | 6 - objects.mk | 1 + selftest.cpp | 4 - service/ControlPlane.cpp | 20 +- service/ControlPlane.hpp | 10 +- service/OneService.cpp | 25 +- service/OneService.hpp | 12 - 12 files changed, 1646 insertions(+), 1708 deletions(-) create mode 100644 controller/EmbeddedNetworkController.cpp create mode 100644 controller/EmbeddedNetworkController.hpp delete mode 100644 controller/SqliteNetworkController.cpp delete mode 100644 controller/SqliteNetworkController.hpp (limited to 'make-mac.mk') diff --git a/controller/EmbeddedNetworkController.cpp b/controller/EmbeddedNetworkController.cpp new file mode 100644 index 00000000..479a65f5 --- /dev/null +++ b/controller/EmbeddedNetworkController.cpp @@ -0,0 +1,1462 @@ +/* + * ZeroTier One - Network Virtualization Everywhere + * Copyright (C) 2011-2015 ZeroTier, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +#include "../include/ZeroTierOne.h" +#include "../node/Constants.hpp" + +#include "EmbeddedNetworkController.hpp" + +#include "../node/Node.hpp" +#include "../node/Utils.hpp" +#include "../node/CertificateOfMembership.hpp" +#include "../node/NetworkConfig.hpp" +#include "../node/Dictionary.hpp" +#include "../node/InetAddress.hpp" +#include "../node/MAC.hpp" +#include "../node/Address.hpp" + +using json = nlohmann::json; + +// API version reported via JSON control plane +#define ZT_NETCONF_CONTROLLER_API_VERSION 3 + +// Number of requests to remember in member history +#define ZT_NETCONF_DB_MEMBER_HISTORY_LENGTH 8 + +// Min duration between requests for an address/nwid combo to prevent floods +#define ZT_NETCONF_MIN_REQUEST_PERIOD 1000 + +// Nodes are considered active if they've queried in less than this long +#define ZT_NETCONF_NODE_ACTIVE_THRESHOLD ((ZT_NETWORK_AUTOCONF_DELAY * 2) + 5000) + +namespace ZeroTier { + +static json _renderRule(ZT_VirtualNetworkRule &rule) +{ + char tmp[128]; + json r = json::object(); + r["not"] = ((rule.t & 0x80) != 0); + switch((rule.t) & 0x7f) { + case ZT_NETWORK_RULE_ACTION_DROP: + r["type"] = "ACTION_DROP"; + break; + case ZT_NETWORK_RULE_ACTION_ACCEPT: + r["type"] = "ACTION_ACCEPT"; + break; + case ZT_NETWORK_RULE_ACTION_TEE: + r["type"] = "ACTION_TEE"; + r["zt"] = Address(rule.v.zt).toString(); + break; + case ZT_NETWORK_RULE_ACTION_REDIRECT: + r["type"] = "ACTION_REDIRECT"; + r["zt"] = Address(rule.v.zt).toString(); + break; + case ZT_NETWORK_RULE_MATCH_SOURCE_ZEROTIER_ADDRESS: + r["type"] = "MATCH_SOURCE_ZEROTIER_ADDRESS"; + r["zt"] = Address(rule.v.zt).toString(); + break; + case ZT_NETWORK_RULE_MATCH_DEST_ZEROTIER_ADDRESS: + r["type"] = "MATCH_DEST_ZEROTIER_ADDRESS"; + r["zt"] = Address(rule.v.zt).toString(); + break; + case ZT_NETWORK_RULE_MATCH_VLAN_ID: + r["type"] = "MATCH_VLAN_ID"; + r["vlanId"] = (uint64_t)rule.v.vlanId; + break; + case ZT_NETWORK_RULE_MATCH_VLAN_PCP: + r["type"] = "MATCH_VLAN_PCP"; + r["vlanPcp"] = (uint64_t)rule.v.vlanPcp; + break; + case ZT_NETWORK_RULE_MATCH_VLAN_DEI: + r["type"] = "MATCH_VLAN_DEI"; + r["vlanDei"] = (uint64_t)rule.v.vlanDei; + break; + case ZT_NETWORK_RULE_MATCH_ETHERTYPE: + r["type"] = "MATCH_ETHERTYPE"; + r["etherType"] = (uint64_t)rule.v.etherType; + break; + case ZT_NETWORK_RULE_MATCH_MAC_SOURCE: + r["type"] = "MATCH_MAC_SOURCE"; + Utils::snprintf(tmp,sizeof(tmp),"%.2x:%.2x:%.2x:%.2x:%.2x:%.2x",(unsigned int)rule.v.mac[0],(unsigned int)rule.v.mac[1],(unsigned int)rule.v.mac[2],(unsigned int)rule.v.mac[3],(unsigned int)rule.v.mac[4],(unsigned int)rule.v.mac[5]); + r["mac"] = tmp; + break; + case ZT_NETWORK_RULE_MATCH_MAC_DEST: + r["type"] = "MATCH_MAC_DEST"; + Utils::snprintf(tmp,sizeof(tmp),"%.2x:%.2x:%.2x:%.2x:%.2x:%.2x",(unsigned int)rule.v.mac[0],(unsigned int)rule.v.mac[1],(unsigned int)rule.v.mac[2],(unsigned int)rule.v.mac[3],(unsigned int)rule.v.mac[4],(unsigned int)rule.v.mac[5]); + r["mac"] = tmp; + break; + case ZT_NETWORK_RULE_MATCH_IPV4_SOURCE: + r["type"] = "MATCH_IPV4_SOURCE"; + r["ip"] = InetAddress(&(rule.v.ipv4.ip),4,(unsigned int)rule.v.ipv4.mask).toString(); + break; + case ZT_NETWORK_RULE_MATCH_IPV4_DEST: + r["type"] = "MATCH_IPV4_DEST"; + r["ip"] = InetAddress(&(rule.v.ipv4.ip),4,(unsigned int)rule.v.ipv4.mask).toString(); + break; + case ZT_NETWORK_RULE_MATCH_IPV6_SOURCE: + r["type"] = "MATCH_IPV6_SOURCE"; + r["ip"] = InetAddress(rule.v.ipv6.ip,16,(unsigned int)rule.v.ipv6.mask).toString(); + break; + case ZT_NETWORK_RULE_MATCH_IPV6_DEST: + r["type"] = "MATCH_IPV6_DEST"; + r["ip"] = InetAddress(rule.v.ipv6.ip,16,(unsigned int)rule.v.ipv6.mask).toString(); + break; + case ZT_NETWORK_RULE_MATCH_IP_TOS: + r["type"] = "MATCH_IP_TOS"; + r["ipTos"] = (uint64_t)rule.v.ipTos; + break; + case ZT_NETWORK_RULE_MATCH_IP_PROTOCOL: + r["type"] = "MATCH_IP_PROTOCOL"; + r["ipProtocol"] = (uint64_t)rule.v.ipProtocol; + break; + case ZT_NETWORK_RULE_MATCH_IP_SOURCE_PORT_RANGE: + r["type"] = "MATCH_IP_SOURCE_PORT_RANGE"; + r["start"] = (uint64_t)rule.v.port[0]; + r["end"] = (uint64_t)rule.v.port[1]; + break; + case ZT_NETWORK_RULE_MATCH_IP_DEST_PORT_RANGE: + r["type"] = "MATCH_IP_DEST_PORT_RANGE"; + r["start"] = (uint64_t)rule.v.port[0]; + r["end"] = (uint64_t)rule.v.port[1]; + break; + case ZT_NETWORK_RULE_MATCH_CHARACTERISTICS: + r["type"] = "MATCH_CHARACTERISTICS"; + Utils::snprintf(tmp,sizeof(tmp),"%.16llx",rule.v.characteristics[0]); + r["mask"] = tmp; + Utils::snprintf(tmp,sizeof(tmp),"%.16llx",rule.v.characteristics[1]); + r["value"] = tmp; + break; + case ZT_NETWORK_RULE_MATCH_FRAME_SIZE_RANGE: + r["type"] = "MATCH_FRAME_SIZE_RANGE"; + r["start"] = (uint64_t)rule.v.frameSize[0]; + r["end"] = (uint64_t)rule.v.frameSize[1]; + break; + case ZT_NETWORK_RULE_MATCH_TAGS_SAMENESS: + r["type"] = "MATCH_TAGS_SAMENESS"; + r["id"] = (uint64_t)rule.v.tag.id; + r["value"] = (uint64_t)rule.v.tag.value; + break; + case ZT_NETWORK_RULE_MATCH_TAGS_BITWISE_AND: + r["type"] = "MATCH_TAGS_BITWISE_AND"; + r["id"] = (uint64_t)rule.v.tag.id; + r["value"] = (uint64_t)rule.v.tag.value; + break; + case ZT_NETWORK_RULE_MATCH_TAGS_BITWISE_OR: + r["type"] = "MATCH_TAGS_BITWISE_OR"; + r["id"] = (uint64_t)rule.v.tag.id; + r["value"] = (uint64_t)rule.v.tag.value; + break; + case ZT_NETWORK_RULE_MATCH_TAGS_BITWISE_XOR: + r["type"] = "MATCH_TAGS_BITWISE_XOR"; + r["id"] = (uint64_t)rule.v.tag.id; + r["value"] = (uint64_t)rule.v.tag.value; + break; + } + return r; +} + +static bool _parseRule(const json &r,ZT_VirtualNetworkRule &rule) +{ + if (r.is_object()) + return false; + std::string t = r["type"]; + memset(&rule,0,sizeof(ZT_VirtualNetworkRule)); + if (r.value("not",false)) + rule.t = 0x80; + else rule.t = 0x00; + if (t == "ACTION_DROP") { + rule.t |= ZT_NETWORK_RULE_ACTION_DROP; + return true; + } else if (t == "ACTION_ACCEPT") { + rule.t |= ZT_NETWORK_RULE_ACTION_ACCEPT; + return true; + } else if (t == "ACTION_TEE") { + rule.t |= ZT_NETWORK_RULE_ACTION_TEE; + rule.v.zt = Utils::hexStrToU64(r.value("zt","0").c_str()) & 0xffffffffffULL; + return true; + } else if (t == "ACTION_REDIRECT") { + rule.t |= ZT_NETWORK_RULE_ACTION_REDIRECT; + rule.v.zt = Utils::hexStrToU64(r.value("zt","0").c_str()) & 0xffffffffffULL; + return true; + } else if (t == "MATCH_SOURCE_ZEROTIER_ADDRESS") { + rule.t |= ZT_NETWORK_RULE_MATCH_SOURCE_ZEROTIER_ADDRESS; + rule.v.zt = Utils::hexStrToU64(r.value("zt","0").c_str()) & 0xffffffffffULL; + return true; + } else if (t == "MATCH_DEST_ZEROTIER_ADDRESS") { + rule.t |= ZT_NETWORK_RULE_MATCH_DEST_ZEROTIER_ADDRESS; + rule.v.zt = Utils::hexStrToU64(r.value("zt","0").c_str()) & 0xffffffffffULL; + return true; + } else if (t == "MATCH_VLAN_ID") { + rule.t |= ZT_NETWORK_RULE_MATCH_VLAN_ID; + rule.v.vlanId = (uint16_t)(r.value("vlanId",0ULL) & 0xffffULL); + return true; + } else if (t == "MATCH_VLAN_PCP") { + rule.t |= ZT_NETWORK_RULE_MATCH_VLAN_PCP; + rule.v.vlanPcp = (uint8_t)(r.value("vlanPcp",0ULL) & 0xffULL); + return true; + } else if (t == "MATCH_VLAN_DEI") { + rule.t |= ZT_NETWORK_RULE_MATCH_VLAN_DEI; + rule.v.vlanDei = (uint8_t)(r.value("vlanDei",0ULL) & 0xffULL); + return true; + } else if (t == "MATCH_ETHERTYPE") { + rule.t |= ZT_NETWORK_RULE_MATCH_ETHERTYPE; + rule.v.etherType = (uint16_t)(r.value("etherType",0ULL) & 0xffffULL); + return true; + } else if (t == "MATCH_MAC_SOURCE") { + rule.t |= ZT_NETWORK_RULE_MATCH_MAC_SOURCE; + const std::string mac(r.value("mac","0")); + Utils::unhex(mac.c_str(),(unsigned int)mac.length(),rule.v.mac,6); + return true; + } else if (t == "MATCH_MAC_DEST") { + rule.t |= ZT_NETWORK_RULE_MATCH_MAC_DEST; + const std::string mac(r.value("mac","0")); + Utils::unhex(mac.c_str(),(unsigned int)mac.length(),rule.v.mac,6); + return true; + } else if (t == "MATCH_IPV4_SOURCE") { + rule.t |= ZT_NETWORK_RULE_MATCH_IPV4_SOURCE; + InetAddress ip(r.value("ip","0.0.0.0")); + rule.v.ipv4.ip = reinterpret_cast(&ip)->sin_addr.s_addr; + rule.v.ipv4.mask = Utils::ntoh(reinterpret_cast(&ip)->sin_port) & 0xff; + if (rule.v.ipv4.mask > 32) rule.v.ipv4.mask = 32; + return true; + } else if (t == "MATCH_IPV4_DEST") { + rule.t |= ZT_NETWORK_RULE_MATCH_IPV4_DEST; + InetAddress ip(r.value("ip","0.0.0.0")); + rule.v.ipv4.ip = reinterpret_cast(&ip)->sin_addr.s_addr; + rule.v.ipv4.mask = Utils::ntoh(reinterpret_cast(&ip)->sin_port) & 0xff; + if (rule.v.ipv4.mask > 32) rule.v.ipv4.mask = 32; + return true; + } else if (t == "MATCH_IPV6_SOURCE") { + rule.t |= ZT_NETWORK_RULE_MATCH_IPV6_SOURCE; + InetAddress ip(r.value("ip","::0")); + memcpy(rule.v.ipv6.ip,reinterpret_cast(&ip)->sin6_addr.s6_addr,16); + rule.v.ipv6.mask = Utils::ntoh(reinterpret_cast(&ip)->sin6_port) & 0xff; + if (rule.v.ipv6.mask > 128) rule.v.ipv6.mask = 128; + return true; + } else if (t == "MATCH_IPV6_DEST") { + rule.t |= ZT_NETWORK_RULE_MATCH_IPV6_DEST; + InetAddress ip(r.value("ip","::0")); + memcpy(rule.v.ipv6.ip,reinterpret_cast(&ip)->sin6_addr.s6_addr,16); + rule.v.ipv6.mask = Utils::ntoh(reinterpret_cast(&ip)->sin6_port) & 0xff; + if (rule.v.ipv6.mask > 128) rule.v.ipv6.mask = 128; + return true; + } else if (t == "MATCH_IP_TOS") { + rule.t |= ZT_NETWORK_RULE_MATCH_IP_TOS; + rule.v.ipTos = (uint8_t)(r.value("ipTos",0ULL) & 0xffULL); + return true; + } else if (t == "MATCH_IP_PROTOCOL") { + rule.t |= ZT_NETWORK_RULE_MATCH_IP_PROTOCOL; + rule.v.ipProtocol = (uint8_t)(r.value("ipProtocol",0ULL) & 0xffULL); + return true; + } else if (t == "MATCH_IP_SOURCE_PORT_RANGE") { + rule.t |= ZT_NETWORK_RULE_MATCH_IP_SOURCE_PORT_RANGE; + rule.v.port[0] = (uint16_t)(r.value("start",0ULL) & 0xffffULL); + rule.v.port[1] = (uint16_t)(r.value("end",0ULL) & 0xffffULL); + return true; + } else if (t == "MATCH_IP_DEST_PORT_RANGE") { + rule.t |= ZT_NETWORK_RULE_MATCH_IP_DEST_PORT_RANGE; + rule.v.port[0] = (uint16_t)(r.value("start",0ULL) & 0xffffULL); + rule.v.port[1] = (uint16_t)(r.value("end",0ULL) & 0xffffULL); + return true; + } else if (t == "MATCH_CHARACTERISTICS") { + rule.t |= ZT_NETWORK_RULE_MATCH_CHARACTERISTICS; + if (r.count("mask")) { + auto v = r["mask"]; + if (v.is_number()) { + rule.v.characteristics[0] = v; + } else { + std::string tmp = v; + rule.v.characteristics[0] = Utils::hexStrToU64(tmp.c_str()); + } + } + if (r.count("value")) { + auto v = r["value"]; + if (v.is_number()) { + rule.v.characteristics[1] = v; + } else { + std::string tmp = v; + rule.v.characteristics[1] = Utils::hexStrToU64(tmp.c_str()); + } + } + return true; + } else if (t == "MATCH_FRAME_SIZE_RANGE") { + rule.t |= ZT_NETWORK_RULE_MATCH_FRAME_SIZE_RANGE; + rule.v.frameSize[0] = (uint16_t)(Utils::hexStrToU64(r.value("start","0").c_str()) & 0xffffULL); + rule.v.frameSize[1] = (uint16_t)(Utils::hexStrToU64(r.value("end","0").c_str()) & 0xffffULL); + return true; + } else if (t == "MATCH_TAGS_SAMENESS") { + rule.t |= ZT_NETWORK_RULE_MATCH_TAGS_SAMENESS; + rule.v.tag.id = (uint32_t)(Utils::hexStrToU64(r.value("id","0").c_str()) & 0xffffffffULL); + rule.v.tag.value = (uint32_t)(Utils::hexStrToU64(r.value("value","0").c_str()) & 0xffffffffULL); + return true; + } else if (t == "MATCH_TAGS_BITWISE_AND") { + rule.t |= ZT_NETWORK_RULE_MATCH_TAGS_BITWISE_AND; + rule.v.tag.id = (uint32_t)(Utils::hexStrToU64(r.value("id","0").c_str()) & 0xffffffffULL); + rule.v.tag.value = (uint32_t)(Utils::hexStrToU64(r.value("value","0").c_str()) & 0xffffffffULL); + return true; + } else if (t == "MATCH_TAGS_BITWISE_OR") { + rule.t |= ZT_NETWORK_RULE_MATCH_TAGS_BITWISE_OR; + rule.v.tag.id = (uint32_t)(Utils::hexStrToU64(r.value("id","0").c_str()) & 0xffffffffULL); + rule.v.tag.value = (uint32_t)(Utils::hexStrToU64(r.value("value","0").c_str()) & 0xffffffffULL); + return true; + } else if (t == "MATCH_TAGS_BITWISE_XOR") { + rule.t |= ZT_NETWORK_RULE_MATCH_TAGS_BITWISE_XOR; + rule.v.tag.id = (uint32_t)(Utils::hexStrToU64(r.value("id","0").c_str()) & 0xffffffffULL); + rule.v.tag.value = (uint32_t)(Utils::hexStrToU64(r.value("value","0").c_str()) & 0xffffffffULL); + return true; + } + return false; +} + +EmbeddedNetworkController::EmbeddedNetworkController(Node *node,const char *dbPath) : + _node(node), + _path(dbPath) +{ + OSUtils::mkdir(dbPath); + /* + if (sqlite3_open_v2(dbPath,&_db,SQLITE_OPEN_READWRITE|SQLITE_OPEN_CREATE,(const char *)0) != SQLITE_OK) + throw std::runtime_error("SqliteNetworkController cannot open database file"); + sqlite3_busy_timeout(_db,10000); + + sqlite3_exec(_db,"PRAGMA synchronous = OFF",0,0,0); + sqlite3_exec(_db,"PRAGMA journal_mode = MEMORY",0,0,0); + + sqlite3_stmt *s = (sqlite3_stmt *)0; + if ((sqlite3_prepare_v2(_db,"SELECT v FROM Config WHERE k = 'schemaVersion';",-1,&s,(const char **)0) == SQLITE_OK)&&(s)) { + int schemaVersion = -1234; + if (sqlite3_step(s) == SQLITE_ROW) { + schemaVersion = sqlite3_column_int(s,0); + } + + sqlite3_finalize(s); + + if (schemaVersion == -1234) { + sqlite3_close(_db); + throw std::runtime_error("SqliteNetworkController schemaVersion not found in Config table (init failure?)"); + } + + if (schemaVersion < 2) { + // Create NodeHistory table to upgrade from version 1 to version 2 + if (sqlite3_exec(_db, + "CREATE TABLE NodeHistory (\n" + " nodeId char(10) NOT NULL REFERENCES Node(id) ON DELETE CASCADE,\n" + " networkId char(16) NOT NULL REFERENCES Network(id) ON DELETE CASCADE,\n" + " networkVisitCounter INTEGER NOT NULL DEFAULT(0),\n" + " networkRequestAuthorized INTEGER NOT NULL DEFAULT(0),\n" + " requestTime INTEGER NOT NULL DEFAULT(0),\n" + " clientMajorVersion INTEGER NOT NULL DEFAULT(0),\n" + " clientMinorVersion INTEGER NOT NULL DEFAULT(0),\n" + " clientRevision INTEGER NOT NULL DEFAULT(0),\n" + " networkRequestMetaData VARCHAR(1024),\n" + " fromAddress VARCHAR(128)\n" + ");\n" + "CREATE INDEX NodeHistory_nodeId ON NodeHistory (nodeId);\n" + "CREATE INDEX NodeHistory_networkId ON NodeHistory (networkId);\n" + "CREATE INDEX NodeHistory_requestTime ON NodeHistory (requestTime);\n" + "UPDATE \"Config\" SET \"v\" = 2 WHERE \"k\" = 'schemaVersion';\n" + ,0,0,0) != SQLITE_OK) { + char err[1024]; + Utils::snprintf(err,sizeof(err),"SqliteNetworkController cannot upgrade the database to version 2: %s",sqlite3_errmsg(_db)); + sqlite3_close(_db); + throw std::runtime_error(err); + } else { + schemaVersion = 2; + } + } + + if (schemaVersion < 3) { + // Create Route table to upgrade from version 2 to version 3 and migrate old + // data. Also delete obsolete Gateway table that was never actually used, and + // migrate Network flags to a bitwise flags field instead of ASCII cruft. + if (sqlite3_exec(_db, + "DROP TABLE Gateway;\n" + "CREATE TABLE Route (\n" + " networkId char(16) NOT NULL REFERENCES Network(id) ON DELETE CASCADE,\n" + " target blob(16) NOT NULL,\n" + " via blob(16),\n" + " targetNetmaskBits integer NOT NULL,\n" + " ipVersion integer NOT NULL,\n" + " flags integer NOT NULL,\n" + " metric integer NOT NULL\n" + ");\n" + "CREATE INDEX Route_networkId ON Route (networkId);\n" + "INSERT INTO Route SELECT DISTINCT networkId,\"ip\" AS \"target\",NULL AS \"via\",ipNetmaskBits AS targetNetmaskBits,ipVersion,0 AS \"flags\",0 AS \"metric\" FROM IpAssignment WHERE nodeId IS NULL AND \"type\" = 1;\n" + "ALTER TABLE Network ADD COLUMN \"flags\" integer NOT NULL DEFAULT(0);\n" + "UPDATE Network SET \"flags\" = (\"flags\" | 1) WHERE v4AssignMode = 'zt';\n" + "UPDATE Network SET \"flags\" = (\"flags\" | 2) WHERE v6AssignMode = 'rfc4193';\n" + "UPDATE Network SET \"flags\" = (\"flags\" | 4) WHERE v6AssignMode = '6plane';\n" + "ALTER TABLE Member ADD COLUMN \"flags\" integer NOT NULL DEFAULT(0);\n" + "DELETE FROM IpAssignment WHERE nodeId IS NULL AND \"type\" = 1;\n" + "UPDATE \"Config\" SET \"v\" = 3 WHERE \"k\" = 'schemaVersion';\n" + ,0,0,0) != SQLITE_OK) { + char err[1024]; + Utils::snprintf(err,sizeof(err),"SqliteNetworkController cannot upgrade the database to version 3: %s",sqlite3_errmsg(_db)); + sqlite3_close(_db); + throw std::runtime_error(err); + } else { + schemaVersion = 3; + } + } + + if (schemaVersion < 4) { + // Turns out this was overkill and a huge performance drag. Will be revisiting this + // more later but for now a brief snapshot of recent history stored in Member is fine. + // Also prepare for implementation of proof of work requests. + if (sqlite3_exec(_db, + "DROP TABLE NodeHistory;\n" + "ALTER TABLE Member ADD COLUMN lastRequestTime integer NOT NULL DEFAULT(0);\n" + "ALTER TABLE Member ADD COLUMN lastPowDifficulty integer NOT NULL DEFAULT(0);\n" + "ALTER TABLE Member ADD COLUMN lastPowTime integer NOT NULL DEFAULT(0);\n" + "ALTER TABLE Member ADD COLUMN recentHistory blob;\n" + "CREATE INDEX Member_networkId_lastRequestTime ON Member(networkId, lastRequestTime);\n" + "UPDATE \"Config\" SET \"v\" = 4 WHERE \"k\" = 'schemaVersion';\n" + ,0,0,0) != SQLITE_OK) { + char err[1024]; + Utils::snprintf(err,sizeof(err),"SqliteNetworkController cannot upgrade the database to version 3: %s",sqlite3_errmsg(_db)); + sqlite3_close(_db); + throw std::runtime_error(err); + } else { + schemaVersion = 4; + } + } + + if (schemaVersion < 5) { + // Upgrade old rough draft Rule table to new release format + if (sqlite3_exec(_db, + "DROP TABLE Relay;\n" + "DROP INDEX Rule_networkId_ruleNo;\n" + "ALTER TABLE \"Rule\" RENAME TO RuleOld;\n" + "CREATE TABLE Rule (\n" + " networkId char(16) NOT NULL REFERENCES Network(id) ON DELETE CASCADE,\n" + " capId integer,\n" + " ruleNo integer NOT NULL,\n" + " ruleType integer NOT NULL DEFAULT(0),\n" + " \"addr\" blob(16),\n" + " \"int1\" integer,\n" + " \"int2\" integer,\n" + " \"int3\" integer,\n" + " \"int4\" integer\n" + ");\n" + "INSERT INTO \"Rule\" SELECT networkId,(ruleNo*2) AS ruleNo,37 AS \"ruleType\",etherType AS \"int1\" FROM RuleOld WHERE RuleOld.etherType IS NOT NULL AND RuleOld.etherType > 0;\n" + "INSERT INTO \"Rule\" SELECT networkId,((ruleNo*2)+1) AS ruleNo,1 AS \"ruleType\" FROM RuleOld;\n" + "DROP TABLE RuleOld;\n" + "CREATE INDEX Rule_networkId_capId ON Rule (networkId,capId);\n" + "CREATE TABLE MemberTC (\n" + " networkId char(16) NOT NULL REFERENCES Network(id) ON DELETE CASCADE,\n" + " nodeId char(10) NOT NULL REFERENCES Node(id) ON DELETE CASCADE,\n" + " tagId integer,\n" + " tagValue integer,\n" + " capId integer,\n" + " capMaxCustodyChainLength integer NOT NULL DEFAULT(1)\n" + ");\n" + "CREATE INDEX MemberTC_networkId_nodeId ON MemberTC (networkId,nodeId);\n" + "UPDATE \"Config\" SET \"v\" = 5 WHERE \"k\" = 'schemaVersion';\n" + ,0,0,0) != SQLITE_OK) { + char err[1024]; + Utils::snprintf(err,sizeof(err),"SqliteNetworkController cannot upgrade the database to version 3: %s",sqlite3_errmsg(_db)); + sqlite3_close(_db); + throw std::runtime_error(err); + } else { + schemaVersion = 5; + } + } + + if (schemaVersion != ZT_NETCONF_SQLITE_SCHEMA_VERSION) { + sqlite3_close(_db); + throw std::runtime_error("SqliteNetworkController database schema version mismatch"); + } + } else { + // Prepare statement will fail if Config table doesn't exist, which means our DB + // needs to be initialized. + if (sqlite3_exec(_db,ZT_NETCONF_SCHEMA_SQL"INSERT INTO Config (k,v) VALUES ('schemaVersion',"ZT_NETCONF_SQLITE_SCHEMA_VERSION_STR");",0,0,0) != SQLITE_OK) { + char err[1024]; + Utils::snprintf(err,sizeof(err),"SqliteNetworkController cannot initialize database and/or insert schemaVersion into Config table: %s",sqlite3_errmsg(_db)); + sqlite3_close(_db); + throw std::runtime_error(err); + } + } + + if ( + + (sqlite3_prepare_v2(_db,"SELECT name,private,enableBroadcast,allowPassiveBridging,\"flags\",multicastLimit,creationTime,revision,memberRevisionCounter,(SELECT COUNT(1) FROM Member WHERE Member.networkId = Network.id AND Member.authorized > 0) FROM Network WHERE id = ?",-1,&_sGetNetworkById,(const char **)0) != SQLITE_OK) + ||(sqlite3_prepare_v2(_db,"SELECT revision FROM Network WHERE id = ?",-1,&_sGetNetworkRevision,(const char **)0) != SQLITE_OK) + ||(sqlite3_prepare_v2(_db,"UPDATE Network SET revision = ? WHERE id = ?",-1,&_sSetNetworkRevision,(const char **)0) != SQLITE_OK) + ||(sqlite3_prepare_v2(_db,"INSERT INTO Network (id,name,creationTime,revision) VALUES (?,?,?,1)",-1,&_sCreateNetwork,(const char **)0) != SQLITE_OK) + ||(sqlite3_prepare_v2(_db,"DELETE FROM Network WHERE id = ?",-1,&_sDeleteNetwork,(const char **)0) != SQLITE_OK) + ||(sqlite3_prepare_v2(_db,"SELECT id FROM Network ORDER BY id ASC",-1,&_sListNetworks,(const char **)0) != SQLITE_OK) + ||(sqlite3_prepare_v2(_db,"UPDATE Network SET memberRevisionCounter = (memberRevisionCounter + 1) WHERE id = ?",-1,&_sIncrementMemberRevisionCounter,(const char **)0) != SQLITE_OK) + + ||(sqlite3_prepare_v2(_db,"SELECT identity FROM Node WHERE id = ?",-1,&_sGetNodeIdentity,(const char **)0) != SQLITE_OK) + ||(sqlite3_prepare_v2(_db,"INSERT OR REPLACE INTO Node (id,identity) VALUES (?,?)",-1,&_sCreateOrReplaceNode,(const char **)0) != SQLITE_OK) + + ||(sqlite3_prepare_v2(_db,"INSERT INTO Rule (networkId,ruleNo,nodeId,ztSource,ztDest,vlanId,vlanPcp,vlanDei,) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)",-1,&_sCreateRule,(const char **)0) != SQLITE_OK) + ||(sqlite3_prepare_v2(_db,"SELECT ruleNo,nodeId,sourcePort,destPort,vlanId,vlanPcp,etherType,macSource,macDest,ipSource,ipDest,ipTos,ipProtocol,ipSourcePort,ipDestPort,\"flags\",invFlags,\"action\" FROM Rule WHERE networkId = ? ORDER BY ruleNo ASC",-1,&_sListRules,(const char **)0) != SQLITE_OK) + ||(sqlite3_prepare_v2(_db,"DELETE FROM Rule WHERE networkId = ?",-1,&_sDeleteRulesForNetwork,(const char **)0) != SQLITE_OK) + + ||(sqlite3_prepare_v2(_db,"SELECT ipRangeStart,ipRangeEnd FROM IpAssignmentPool WHERE networkId = ? AND ipVersion = ?",-1,&_sGetIpAssignmentPools,(const char **)0) != SQLITE_OK) + ||(sqlite3_prepare_v2(_db,"SELECT ipRangeStart,ipRangeEnd,ipVersion FROM IpAssignmentPool WHERE networkId = ? ORDER BY ipRangeStart ASC",-1,&_sGetIpAssignmentPools2,(const char **)0) != SQLITE_OK) + ||(sqlite3_prepare_v2(_db,"INSERT INTO IpAssignmentPool (networkId,ipRangeStart,ipRangeEnd,ipVersion) VALUES (?,?,?,?)",-1,&_sCreateIpAssignmentPool,(const char **)0) != SQLITE_OK) + ||(sqlite3_prepare_v2(_db,"DELETE FROM IpAssignmentPool WHERE networkId = ?",-1,&_sDeleteIpAssignmentPoolsForNetwork,(const char **)0) != SQLITE_OK) + + ||(sqlite3_prepare_v2(_db,"SELECT ip,ipNetmaskBits,ipVersion FROM IpAssignment WHERE networkId = ? AND nodeId = ? AND \"type\" = 0 ORDER BY ip ASC",-1,&_sGetIpAssignmentsForNode,(const char **)0) != SQLITE_OK) + ||(sqlite3_prepare_v2(_db,"SELECT 1 FROM IpAssignment WHERE networkId = ? AND ip = ? AND ipVersion = ? AND \"type\" = ?",-1,&_sCheckIfIpIsAllocated,(const char **)0) != SQLITE_OK) + ||(sqlite3_prepare_v2(_db,"INSERT INTO IpAssignment (networkId,nodeId,\"type\",ip,ipNetmaskBits,ipVersion) VALUES (?,?,?,?,?,?)",-1,&_sAllocateIp,(const char **)0) != SQLITE_OK) + ||(sqlite3_prepare_v2(_db,"DELETE FROM IpAssignment WHERE networkId = ? AND nodeId = ? AND \"type\" = ?",-1,&_sDeleteIpAllocations,(const char **)0) != SQLITE_OK) + + ||(sqlite3_prepare_v2(_db,"SELECT rowid,authorized,activeBridge,memberRevision,\"flags\",lastRequestTime,recentHistory FROM Member WHERE networkId = ? AND nodeId = ?",-1,&_sGetMember,(const char **)0) != SQLITE_OK) + ||(sqlite3_prepare_v2(_db,"SELECT m.authorized,m.activeBridge,m.memberRevision,n.identity,m.flags,m.lastRequestTime,m.recentHistory FROM Member AS m LEFT OUTER JOIN Node AS n ON n.id = m.nodeId WHERE m.networkId = ? AND m.nodeId = ?",-1,&_sGetMember2,(const char **)0) != SQLITE_OK) + ||(sqlite3_prepare_v2(_db,"INSERT INTO Member (networkId,nodeId,authorized,activeBridge,memberRevision) VALUES (?,?,?,0,(SELECT memberRevisionCounter FROM Network WHERE id = ?))",-1,&_sCreateMember,(const char **)0) != SQLITE_OK) + ||(sqlite3_prepare_v2(_db,"SELECT nodeId FROM Member WHERE networkId = ? AND activeBridge > 0 AND authorized > 0",-1,&_sGetActiveBridges,(const char **)0) != SQLITE_OK) + ||(sqlite3_prepare_v2(_db,"SELECT m.nodeId,m.memberRevision FROM Member AS m WHERE m.networkId = ? ORDER BY m.nodeId ASC",-1,&_sListNetworkMembers,(const char **)0) != SQLITE_OK) + ||(sqlite3_prepare_v2(_db,"UPDATE Member SET authorized = ?,memberRevision = (SELECT memberRevisionCounter FROM Network WHERE id = ?) WHERE rowid = ?",-1,&_sUpdateMemberAuthorized,(const char **)0) != SQLITE_OK) + ||(sqlite3_prepare_v2(_db,"UPDATE Member SET activeBridge = ?,memberRevision = (SELECT memberRevisionCounter FROM Network WHERE id = ?) WHERE rowid = ?",-1,&_sUpdateMemberActiveBridge,(const char **)0) != SQLITE_OK) + ||(sqlite3_prepare_v2(_db,"UPDATE Member SET \"lastRequestTime\" = ?, \"recentHistory\" = ? WHERE rowid = ?",-1,&_sUpdateMemberHistory,(const char **)0) != SQLITE_OK) + ||(sqlite3_prepare_v2(_db,"DELETE FROM Member WHERE networkId = ? AND nodeId = ?",-1,&_sDeleteMember,(const char **)0) != SQLITE_OK) + ||(sqlite3_prepare_v2(_db,"DELETE FROM Member WHERE networkId = ?",-1,&_sDeleteAllNetworkMembers,(const char **)0) != SQLITE_OK) + ||(sqlite3_prepare_v2(_db,"SELECT nodeId,recentHistory FROM Member WHERE networkId = ? AND lastRequestTime >= ?",-1,&_sGetActiveNodesOnNetwork,(const char **)0) != SQLITE_OK) + + ||(sqlite3_prepare_v2(_db,"INSERT INTO Route (networkId,target,via,targetNetmaskBits,ipVersion,flags,metric) VALUES (?,?,?,?,?,?,?)",-1,&_sCreateRoute,(const char **)0) != SQLITE_OK) + ||(sqlite3_prepare_v2(_db,"SELECT DISTINCT target,via,targetNetmaskBits,ipVersion,flags,metric FROM \"Route\" WHERE networkId = ? ORDER BY ipVersion,target,via",-1,&_sGetRoutes,(const char **)0) != SQLITE_OK) + ||(sqlite3_prepare_v2(_db,"DELETE FROM \"Route\" WHERE networkId = ?",-1,&_sDeleteRoutes,(const char **)0) != SQLITE_OK) + + ||(sqlite3_prepare_v2(_db,"SELECT \"v\" FROM \"Config\" WHERE \"k\" = ?",-1,&_sGetConfig,(const char **)0) != SQLITE_OK) + ||(sqlite3_prepare_v2(_db,"INSERT OR REPLACE INTO \"Config\" (\"k\",\"v\") VALUES (?,?)",-1,&_sSetConfig,(const char **)0) != SQLITE_OK) + + ) { + std::string err(std::string("SqliteNetworkController unable to initialize one or more prepared statements: ") + sqlite3_errmsg(_db)); + sqlite3_close(_db); + throw std::runtime_error(err); + } + + sqlite3_reset(_sGetConfig); + sqlite3_bind_text(_sGetConfig,1,"instanceId",10,SQLITE_STATIC); + if (sqlite3_step(_sGetConfig) != SQLITE_ROW) { + unsigned char sr[32]; + Utils::getSecureRandom(sr,32); + for(unsigned int i=0;i<32;++i) + _instanceId.push_back("0123456789abcdef"[(unsigned int)sr[i] & 0xf]); + + sqlite3_reset(_sSetConfig); + sqlite3_bind_text(_sSetConfig,1,"instanceId",10,SQLITE_STATIC); + sqlite3_bind_text(_sSetConfig,2,_instanceId.c_str(),-1,SQLITE_STATIC); + if (sqlite3_step(_sSetConfig) != SQLITE_DONE) + throw std::runtime_error("SqliteNetworkController unable to read or initialize instanceId"); + } else { + const char *iid = reinterpret_cast(sqlite3_column_text(_sGetConfig,0)); + if (!iid) + throw std::runtime_error("SqliteNetworkController unable to read instanceId (it's NULL)"); + _instanceId = iid; + } + +#ifdef ZT_NETCONF_SQLITE_TRACE + sqlite3_trace(_db,sqliteTraceFunc,(void *)0); +#endif + + _backupThread = Thread::start(this); + */ +} + +EmbeddedNetworkController::~EmbeddedNetworkController() +{ +} + +NetworkController::ResultCode EmbeddedNetworkController::doNetworkConfigRequest(const InetAddress &fromAddr,const Identity &signingId,const Identity &identity,uint64_t nwid,const Dictionary &metaData,NetworkConfig &nc) +{ + if (((!signingId)||(!signingId.hasPrivate()))||(signingId.address().toInt() != (nwid >> 24))) { + return NetworkController::NETCONF_QUERY_INTERNAL_SERVER_ERROR; + } + + const uint64_t now = OSUtils::now(); + + // Check rate limit circuit breaker to prevent flooding + { + Mutex::Lock _l(_lastRequestTime_m); + uint64_t &lrt = _lastRequestTime[std::pair(identity.address().toInt(),nwid)]; + if ((now - lrt) <= ZT_NETCONF_MIN_REQUEST_PERIOD) + return NetworkController::NETCONF_QUERY_IGNORE; + lrt = now; + } + + json network(_readJson(_networkJP(nwid,false))); + if (!network.size()) + return NetworkController::NETCONF_QUERY_OBJECT_NOT_FOUND; + const std::string memberJP(_memberJP(nwid,identity.address(),false)); + json member(_readJson(memberJP)); + + { + std::string haveIdStr = member.value("identity",""); + if (haveIdStr.length() > 0) { + try { + if (Identity(haveIdStr.c_str()) != identity) + return NetworkController::NETCONF_QUERY_ACCESS_DENIED; + } catch ( ... ) { + return NetworkController::NETCONF_QUERY_ACCESS_DENIED; + } + } else { + member["identity"] = identity.toString(false); + } + } + + // Make sure these are always present no matter what, and increment member revision since we will always at least log something + member["id"] = identity.address().toString(); + member["address"] = member["id"]; + member["nwid"] = network["id"]; + member["memberRevision"] = member.value("memberRevision",0ULL) + 1; + + // Update member log + { + json rlEntry = json::object(); + rlEntry["ts"] = now; + rlEntry["authorized"] = member["authorized"]; + rlEntry["clientMajorVersion"] = metaData.getUI(ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_NODE_MAJOR_VERSION,0); + rlEntry["clientMinorVersion"] = metaData.getUI(ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_NODE_MINOR_VERSION,0); + rlEntry["clientRevision"] = metaData.getUI(ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_NODE_REVISION,0); + rlEntry["clientProtocolVersion"] = metaData.getUI(ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_PROTOCOL_VERSION,0); + if (fromAddr) + rlEntry["fromAddr"] = fromAddr.toString(); + json recentLog = json::array(); + recentLog.push_back(rlEntry); + auto oldLog = member["recentLog"]; + if (oldLog.is_array()) { + for(unsigned long i=0;i= ZT_NETCONF_DB_MEMBER_HISTORY_LENGTH) + break; + } + } + member["recentLog"] = recentLog; + } + + // Stop if network is private and member is not authorized + if ( (network.value("private",true)) && (!member.value("authorized",false)) ) { + _writeJson(memberJP,member); + return NetworkController::NETCONF_QUERY_ACCESS_DENIED; + } + // Else compose and send network config + + nc.networkId = nwid; + nc.type = network.value("private",true) ? ZT_NETWORK_TYPE_PRIVATE : ZT_NETWORK_TYPE_PUBLIC; + nc.timestamp = now; + nc.revision = network.value("revision",0ULL); + nc.issuedTo = identity.address(); + if (network.value("enableBroadcast",true)) nc.flags |= ZT_NETWORKCONFIG_FLAG_ENABLE_BROADCAST; + if (network.value("allowPassiveBridging",false)) nc.flags |= ZT_NETWORKCONFIG_FLAG_ALLOW_PASSIVE_BRIDGING; + Utils::scopy(nc.name,sizeof(nc.name),network.value("name","").c_str()); + nc.multicastLimit = (unsigned int)network.value("multicastLimit",32ULL); + + bool amActiveBridge = false; + { + json ab = network["activeBridges"]; + if (ab.is_array()) { + for(unsigned long i=0;i= ZT_MAX_NETWORK_RULES) + break; + auto rule = rules[i]; + if (_parseRule(rule,nc.rules[nc.ruleCount])) + ++nc.ruleCount; + } + } + + if (routes.is_array()) { + for(unsigned long i=0;i= ZT_MAX_NETWORK_ROUTES) + break; + auto route = routes[i]; + InetAddress t(route.value("target","")); + InetAddress v(route.value("via","")); + if ((t)&&(v)&&(t.ss_family == v.ss_family)) { + ZT_VirtualNetworkRoute *r = &(nc.routes[nc.routeCount]); + *(reinterpret_cast(&(r->target))) = t; + *(reinterpret_cast(&(r->via))) = v; + ++nc.routeCount; + } + } + } + + bool haveManagedIpv4AutoAssignment = false; + bool haveManagedIpv6AutoAssignment = false; // "special" NDP-emulated address types do not count + json ipAssignments = member["ipAssignments"]; + if (ipAssignments.is_array()) { + for(unsigned long i=0;i(&(nc.routes[rk].target))->containsAddress(ip)) ) + routedNetmaskBits = reinterpret_cast(&(nc.routes[rk].target))->netmaskBits(); + } + + if (routedNetmaskBits > 0) { + if (nc.staticIpCount < ZT_MAX_ZT_ASSIGNED_ADDRESSES) { + ip.setPort(routedNetmaskBits); + nc.staticIps[nc.staticIpCount++] = ip; + } + if (ip.ss_family == AF_INET) + haveManagedIpv4AutoAssignment = true; + else if (ip.ss_family == AF_INET6) + haveManagedIpv6AutoAssignment = true; + } + } + } else { + ipAssignments = json::array(); + } + + std::set allocatedIps; + bool allocatedIpsLoaded = false; + + if ( (ipAssignmentPools.is_array()) && ((v6AssignMode.is_object())&&(v6AssignMode.value("zt",false))) && (!haveManagedIpv6AutoAssignment) && (!amActiveBridge) ) { + if (!allocatedIpsLoaded) allocatedIps = _getAlreadyAllocatedIps(nwid); + for(unsigned long p=0;((p s[1])&&((e[1] - s[1]) >= 0xffffffffffULL)) { + // First see if we can just cram a ZeroTier ID into the higher 64 bits. If so do that. + xx[0] = Utils::hton(x[0]); + xx[1] = Utils::hton(x[1] + identity.address().toInt()); + } else { + // Otherwise pick random addresses -- this technically doesn't explore the whole range if the lower 64 bit range is >= 1 but that won't matter since that would be huge anyway + Utils::getSecureRandom((void *)xx,16); + if ((e[0] > s[0])) + xx[0] %= (e[0] - s[0]); + else xx[0] = 0; + if ((e[1] > s[1])) + xx[1] %= (e[1] - s[1]); + else xx[1] = 0; + xx[0] = Utils::hton(x[0] + xx[0]); + xx[1] = Utils::hton(x[1] + xx[1]); + } + + InetAddress ip6((const void *)xx,16,0); + + // Check if this IP is within a local-to-Ethernet routed network + int routedNetmaskBits = 0; + for(unsigned int rk=0;rk(&(nc.routes[rk].target))->containsAddress(ip6)) ) + routedNetmaskBits = reinterpret_cast(&(nc.routes[rk].target))->netmaskBits(); + } + + // If it's routed, then try to claim and assign it and if successful end loop + if ((routedNetmaskBits > 0)&&(!allocatedIps.count(ip6))) { + ipAssignments.push_back(ip6.toIpString()); + member["ipAssignments"] = ipAssignments; + ip6.setPort((unsigned int)routedNetmaskBits); + if (nc.staticIpCount < ZT_MAX_ZT_ASSIGNED_ADDRESSES) + nc.staticIps[nc.staticIpCount++] = ip6; + haveManagedIpv4AutoAssignment = true; + break; + } + } + } + } + } + } + + if ( (ipAssignmentPools.is_array()) && ((v4AssignMode.is_object())&&(v4AssignMode.value("zt",false))) && (!haveManagedIpv4AutoAssignment) && (!amActiveBridge) ) { + if (!allocatedIpsLoaded) allocatedIps = _getAlreadyAllocatedIps(nwid); + for(unsigned long p=0;((p(&ipRangeStart)->sin_addr.s_addr)); + uint32_t ipRangeEnd = Utils::ntoh((uint32_t)(reinterpret_cast(&ipRangeEnd)->sin_addr.s_addr)); + if ((ipRangeEnd <= ipRangeStart)||(ipRangeStart == 0)) + continue; + uint32_t ipRangeLen = ipRangeEnd - ipRangeStart; + + // Start with the LSB of the member's address + uint32_t ipTrialCounter = (uint32_t)(identity.address().toInt() & 0xffffffff); + + for(uint32_t k=ipRangeStart,trialCount=0;(k<=ipRangeEnd)&&(trialCount < 1000);++k,++trialCount) { + uint32_t ip = (ipRangeLen > 0) ? (ipRangeStart + (ipTrialCounter % ipRangeLen)) : ipRangeStart; + ++ipTrialCounter; + if ((ip & 0x000000ff) == 0x000000ff) + continue; // don't allow addresses that end in .255 + + // Check if this IP is within a local-to-Ethernet routed network + int routedNetmaskBits = 0; + for(unsigned int rk=0;rk(&(nc.routes[rk].target))->sin_addr.s_addr)); + int targetBits = Utils::ntoh((uint16_t)(reinterpret_cast(&(nc.routes[rk].target))->sin_port)); + if ((ip & (0xffffffff << (32 - targetBits))) == targetIp) { + routedNetmaskBits = targetBits; + break; + } + } + } + + InetAddress ip4(Utils::hton(ip),0); + + // If it's routed, then try to claim and assign it and if successful end loop + if ((routedNetmaskBits > 0)&&(!allocatedIps.count(ip4))) { + ipAssignments.push_back(ip4.toIpString()); + member["ipAssignments"] = ipAssignments; + if (nc.staticIpCount < ZT_MAX_ZT_ASSIGNED_ADDRESSES) { + struct sockaddr_in *const v4ip = reinterpret_cast(&(nc.staticIps[nc.staticIpCount++])); + v4ip->sin_family = AF_INET; + v4ip->sin_port = Utils::hton((uint16_t)routedNetmaskBits); + v4ip->sin_addr.s_addr = Utils::hton(ip); + } + haveManagedIpv4AutoAssignment = true; + break; + } + } + } + } + } + } + + if (network.value("private",true)) { + CertificateOfMembership com(now,ZT_NETWORK_COM_DEFAULT_REVISION_MAX_DELTA,nwid,identity.address()); + if (com.sign(signingId)) { + nc.com = com; + } else { + return NETCONF_QUERY_INTERNAL_SERVER_ERROR; + } + } + + _writeJson(memberJP,member); + return NetworkController::NETCONF_QUERY_OK; +} + +unsigned int EmbeddedNetworkController::handleControlPlaneHttpGET( + const std::vector &path, + const std::map &urlArgs, + const std::map &headers, + const std::string &body, + std::string &responseBody, + std::string &responseContentType) +{ + if ((path.size() > 0)&&(path[0] == "network")) { + + if ((path.size() >= 2)&&(path[1].length() == 16)) { + const uint64_t nwid = Utils::hexStrToU64(path[1].c_str()); + char nwids[24]; + Utils::snprintf(nwids,sizeof(nwids),"%.16llx",(unsigned long long)nwid); + + json network(_readJson(_networkJP(nwid,false))); + if (!network.size()) + return 404; + + if (path.size() >= 3) { + + if (path[2] == "member") { + + if (path.size() >= 4) { + const uint64_t address = Utils::hexStrToU64(path[3].c_str()); + + json member(_readJson(_memberJP(nwid,Address(address),false))); + if (!member.size()) + return 404; + + char addrs[24]; + Utils::snprintf(addrs,sizeof(addrs),"%.10llx",address); + + member["clock"] = OSUtils::now(); + responseBody = member.dump(2); + responseContentType = "application/json"; + + return 200; + } else { + + responseBody = "{"; + std::vector members(OSUtils::listSubdirectories((_networkBP(nwid,false) + ZT_PATH_SEPARATOR_S + "member").c_str())); + for(std::vector::iterator i(members.begin());i!=members.end();++i) { + if (i->length() == ZT_ADDRESS_LENGTH_HEX) { + json member(_readJson(_memberJP(nwid,Address(Utils::hexStrToU64(i->c_str())),false))); + if (member.size()) { + responseBody.append((responseBody.length() == 1) ? "\"" : ",\""); + responseBody.append(*i); + responseBody.append("\":"); + const std::string rc = member.value("memberRevision","0"); + responseBody.append(rc); + } + } + } + responseBody.push_back('}'); + responseContentType = "application/json"; + + return 200; + } + + } else if ((path[2] == "active")&&(path.size() == 3)) { + + responseBody = "{"; + std::vector members(OSUtils::listSubdirectories((_networkBP(nwid,false) + ZT_PATH_SEPARATOR_S + "member").c_str())); + const uint64_t threshold = OSUtils::now() - ZT_NETCONF_NODE_ACTIVE_THRESHOLD; + for(std::vector::iterator i(members.begin());i!=members.end();++i) { + if (i->length() == ZT_ADDRESS_LENGTH_HEX) { + json member(_readJson(_memberJP(nwid,Address(Utils::hexStrToU64(i->c_str())),false))); + if (member.size()) { + auto recentLog = member["recentLog"]; + if ((recentLog.is_array())&&(recentLog.size() > 0)) { + auto mostRecentLog = recentLog[0]; + if ((mostRecentLog.is_object())&&((uint64_t)mostRecentLog.value("ts",0ULL) >= threshold)) { + responseBody.append((responseBody.length() == 1) ? "\"" : ",\""); + responseBody.append(*i); + responseBody.append("\":"); + responseBody.append(mostRecentLog.dump()); + } + } + } + } + } + responseBody.push_back('}'); + responseContentType = "application/json"; + return 200; + + } else if ((path[2] == "test")&&(path.size() >= 4)) { + + Mutex::Lock _l(_circuitTests_m); + std::map< uint64_t,_CircuitTestEntry >::iterator cte(_circuitTests.find(Utils::hexStrToU64(path[3].c_str()))); + if ((cte != _circuitTests.end())&&(cte->second.test)) { + + responseBody = "["; + responseBody.append(cte->second.jsonResults); + responseBody.push_back(']'); + responseContentType = "application/json"; + + return 200; + + } // else 404 + + } // else 404 + + } else { + + nlohmann::json o(network); + o["clock"] = OSUtils::now(); + responseBody = o.dump(2); + responseContentType = "application/json"; + return 200; + + } + } else if (path.size() == 1) { + + responseBody = "["; + std::vector networks(OSUtils::listSubdirectories((_path + ZT_PATH_SEPARATOR_S + "network").c_str())); + for(auto i(networks.begin());i!=networks.end();++i) { + if (i->length() == 16) { + responseBody.append((responseBody.length() == 1) ? "\"" : ",\""); + responseBody.append(*i); + responseBody.append("\""); + } + } + responseBody.push_back(']'); + responseContentType = "application/json"; + return 200; + + } // else 404 + + } else { + + char tmp[4096]; + Utils::snprintf(tmp,sizeof(tmp),"{\n\t\"controller\": true,\n\t\"apiVersion\": %d,\n\t\"clock\": %llu\n}\n",ZT_NETCONF_CONTROLLER_API_VERSION,(unsigned long long)OSUtils::now()); + responseBody = tmp; + responseContentType = "application/json"; + return 200; + + } + + return 404; +} + +unsigned int EmbeddedNetworkController::handleControlPlaneHttpPOST( + const std::vector &path, + const std::map &urlArgs, + const std::map &headers, + const std::string &body, + std::string &responseBody, + std::string &responseContentType) +{ + if (path.empty()) + return 404; + + json b; + try { + b = json::parse(body); + if (!b.is_object()) + return 400; + } catch ( ... ) { + return 400; + } + + if (path[0] == "network") { + + if ((path.size() >= 2)&&(path[1].length() == 16)) { + uint64_t nwid = Utils::hexStrToU64(path[1].c_str()); + char nwids[24]; + Utils::snprintf(nwids,sizeof(nwids),"%.16llx",(unsigned long long)nwid); + + if (path.size() >= 3) { + json network(_readJson(_networkJP(nwid,false))); + if (!network.size()) + return 404; + + if ((path.size() == 4)&&(path[2] == "member")&&(path[3].length() == 10)) { + uint64_t address = Utils::hexStrToU64(path[3].c_str()); + char addrs[24]; + Utils::snprintf(addrs,sizeof(addrs),"%.10llx",(unsigned long long)address); + + json member(_readJson(_memberJP(nwid,Address(address),true))); + + try { + if (b.count("authorized")) member["authorized"] = b.value("authorized",false); + if ((b.count("identity"))&&(!member.count("identity"))) member["identity"] = b.value("identity",""); // allow identity to be populated only if not already known + + if (b.count("ipAssignments")) { + auto ipa = b["ipAssignments"]; + if (ipa.is_array()) { + json mipa(json::array()); + for(unsigned long i=0;itestId),sizeof(test->testId)); + test->credentialNetworkId = nwid; + test->ptr = (void *)this; + json hops = b["hops"]; + if (hops.is_array()) { + for(unsigned long i=0;ihops[test->hopCount].addresses[test->hops[test->hopCount].breadth++] = Utils::hexStrToU64(s.c_str()) & 0xffffffffffULL; + } + } else if (hops2.is_string()) { + std::string s = hops2; + test->hops[test->hopCount].addresses[test->hops[test->hopCount].breadth++] = Utils::hexStrToU64(s.c_str()) & 0xffffffffffULL; + } + } + } + test->reportAtEveryHop = (b.value("reportAtEveryHop",true) ? 1 : 0); + + if (!test->hopCount) { + ::free((void *)test); + return 400; + } + + test->timestamp = OSUtils::now(); + + _CircuitTestEntry &te = _circuitTests[test->testId]; + te.test = test; + te.jsonResults = ""; + + _node->circuitTestBegin(test,&(EmbeddedNetworkController::_circuitTestCallback)); + + char json[1024]; + Utils::snprintf(json,sizeof(json),"{\"testId\":\"%.16llx\"}",test->testId); + responseBody = json; + responseContentType = "application/json"; + return 200; + + } // else 404 + + } else { + // POST to network ID + + // Magic ID ending with ______ picks a random unused network ID + if (path[1].substr(10) == "______") { + nwid = 0; + uint64_t nwidPrefix = (Utils::hexStrToU64(path[1].substr(0,10).c_str()) << 24) & 0xffffffffff000000ULL; + uint64_t nwidPostfix = 0; + for(unsigned long k=0;k<100000;++k) { // sanity limit on trials + Utils::getSecureRandom(&nwidPostfix,sizeof(nwidPostfix)); + uint64_t tryNwid = nwidPrefix | (nwidPostfix & 0xffffffULL); + if ((tryNwid & 0xffffffULL) == 0ULL) tryNwid |= 1ULL; + Utils::snprintf(nwids,sizeof(nwids),"%.16llx",(unsigned long long)tryNwid); + if (!OSUtils::fileExists(_networkJP(tryNwid,false).c_str())) { + nwid = tryNwid; + break; + } + } + if (!nwid) + return 503; + } + + json network(_readJson(_networkJP(nwid,true))); + + try { + if (b.count("name")) network["name"] = b.value("name",""); + if (b.count("private")) network["private"] = b.value("private",true); + if (b.count("enableBroadcast")) network["enableBroadcast"] = b.value("enableBroadcast",false); + if (b.count("allowPassiveBridging")) network["allowPassiveBridging"] = b.value("allowPassiveBridging",false); + if (b.count("multicastLimit")) network["multicastLimit"] = b.value("multicastLimit",32ULL); + + if (b.count("activeBridges")) { + auto ab = b["activeBridges"]; + if (ab.is_array()) { + json ab2 = json::array(); + for(unsigned long i=0;i v6m(Utils::split(b.value("v6AssignMode","").c_str(),",","","")); + std::sort(v6m.begin(),v6m.end()); + v6m.erase(std::unique(v6m.begin(),v6m.end()),v6m.end()); + for(std::vector::iterator i(v6m.begin());i!=v6m.end();++i) { + if (*i == "rfc4193") + nv6m["rfc4193"] = true; + else if (*i == "zt") + nv6m["zt"] = true; + else if (*i == "6plane") + nv6m["6plane"] = true; + } + } else if (b["v6AssignMode"].is_object()) { + auto v6m = b["v6AssignMode"]; + if (v6m.count("rfc4193")) nv6m["rfc4193"] = v6m.value("rfc4193",false); + if (v6m.count("zt")) nv6m["rfc4193"] = v6m.value("zt",false); + if (v6m.count("6plane")) nv6m["rfc4193"] = v6m.value("6plane",false); + } + if (!nv6m.count("rfc4193")) nv6m["rfc4193"] = false; + if (!nv6m.count("zt")) nv6m["zt"] = false; + if (!nv6m.count("6plane")) nv6m["6plane"] = false; + } + + if (b.count("routes")) { + auto rts = b["routes"]; + if (rts.is_array()) { + for(unsigned long i=0;i &path, + const std::map &urlArgs, + const std::map &headers, + const std::string &body, + std::string &responseBody, + std::string &responseContentType) +{ + if (path.empty()) + return 404; + + if (path[0] == "network") { + if ((path.size() >= 2)&&(path[1].length() == 16)) { + const uint64_t nwid = Utils::hexStrToU64(path[1].c_str()); + + json network(_readJson(_networkJP(nwid,false))); + if (!network.size()) + return 404; + + if (path.size() >= 3) { + if ((path.size() == 4)&&(path[2] == "member")&&(path[3].length() == 10)) { + const uint64_t address = Utils::hexStrToU64(path[3].c_str()); + + json member(_readJson(_memberJP(nwid,Address(address),false))); + if (!member.size()) + return 404; + + OSUtils::rmDashRf(_memberBP(nwid,Address(address),false).c_str()); + + responseBody = member.dump(2); + responseContentType = "application/json"; + return 200; + } + } else { + OSUtils::rmDashRf(_networkBP(nwid,false).c_str()); + responseBody = network.dump(2); + responseContentType = "application/json"; + return 200; + } + } // else 404 + + } // else 404 + + return 404; +} + +void EmbeddedNetworkController::_circuitTestCallback(ZT_Node *node,ZT_CircuitTest *test,const ZT_CircuitTestReport *report) +{ + char tmp[65535]; + EmbeddedNetworkController *const self = reinterpret_cast(test->ptr); + + if (!test) + return; + if (!report) + return; + + Mutex::Lock _l(self->_circuitTests_m); + std::map< uint64_t,_CircuitTestEntry >::iterator cte(self->_circuitTests.find(test->testId)); + + if (cte == self->_circuitTests.end()) { // sanity check: a circuit test we didn't launch? + self->_node->circuitTestEnd(test); + ::free((void *)test); + return; + } + + Utils::snprintf(tmp,sizeof(tmp), + "%s{\n" + "\t\"timestamp\": %llu," ZT_EOL_S + "\t\"testId\": \"%.16llx\"," ZT_EOL_S + "\t\"upstream\": \"%.10llx\"," ZT_EOL_S + "\t\"current\": \"%.10llx\"," ZT_EOL_S + "\t\"receivedTimestamp\": %llu," ZT_EOL_S + "\t\"remoteTimestamp\": %llu," ZT_EOL_S + "\t\"sourcePacketId\": \"%.16llx\"," ZT_EOL_S + "\t\"flags\": %llu," ZT_EOL_S + "\t\"sourcePacketHopCount\": %u," ZT_EOL_S + "\t\"errorCode\": %u," ZT_EOL_S + "\t\"vendor\": %d," ZT_EOL_S + "\t\"protocolVersion\": %u," ZT_EOL_S + "\t\"majorVersion\": %u," ZT_EOL_S + "\t\"minorVersion\": %u," ZT_EOL_S + "\t\"revision\": %u," ZT_EOL_S + "\t\"platform\": %d," ZT_EOL_S + "\t\"architecture\": %d," ZT_EOL_S + "\t\"receivedOnLocalAddress\": \"%s\"," ZT_EOL_S + "\t\"receivedFromRemoteAddress\": \"%s\"" ZT_EOL_S + "}", + ((cte->second.jsonResults.length() > 0) ? ",\n" : ""), + (unsigned long long)report->timestamp, + (unsigned long long)test->testId, + (unsigned long long)report->upstream, + (unsigned long long)report->current, + (unsigned long long)OSUtils::now(), + (unsigned long long)report->remoteTimestamp, + (unsigned long long)report->sourcePacketId, + (unsigned long long)report->flags, + report->sourcePacketHopCount, + report->errorCode, + (int)report->vendor, + report->protocolVersion, + report->majorVersion, + report->minorVersion, + report->revision, + (int)report->platform, + (int)report->architecture, + reinterpret_cast(&(report->receivedOnLocalAddress))->toString().c_str(), + reinterpret_cast(&(report->receivedFromRemoteAddress))->toString().c_str()); + + cte->second.jsonResults.append(tmp); +} + +} // namespace ZeroTier diff --git a/controller/EmbeddedNetworkController.hpp b/controller/EmbeddedNetworkController.hpp new file mode 100644 index 00000000..f6e6b098 --- /dev/null +++ b/controller/EmbeddedNetworkController.hpp @@ -0,0 +1,173 @@ +/* + * ZeroTier One - Network Virtualization Everywhere + * Copyright (C) 2011-2015 ZeroTier, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +#ifndef ZT_SQLITENETWORKCONTROLLER_HPP +#define ZT_SQLITENETWORKCONTROLLER_HPP + +#include + +#include +#include +#include +#include + +#include "../node/Constants.hpp" + +#include "../node/NetworkController.hpp" +#include "../node/Mutex.hpp" +#include "../node/Utils.hpp" +#include "../node/InetAddress.hpp" + +#include "../osdep/OSUtils.hpp" + +#include "../ext/json/json.hpp" + +namespace ZeroTier { + +class Node; + +class EmbeddedNetworkController : public NetworkController +{ +public: + EmbeddedNetworkController(Node *node,const char *dbPath); + virtual ~EmbeddedNetworkController(); + + virtual NetworkController::ResultCode doNetworkConfigRequest( + const InetAddress &fromAddr, + const Identity &signingId, + const Identity &identity, + uint64_t nwid, + const Dictionary &metaData, + NetworkConfig &nc); + + unsigned int handleControlPlaneHttpGET( + const std::vector &path, + const std::map &urlArgs, + const std::map &headers, + const std::string &body, + std::string &responseBody, + std::string &responseContentType); + unsigned int handleControlPlaneHttpPOST( + const std::vector &path, + const std::map &urlArgs, + const std::map &headers, + const std::string &body, + std::string &responseBody, + std::string &responseContentType); + unsigned int handleControlPlaneHttpDELETE( + const std::vector &path, + const std::map &urlArgs, + const std::map &headers, + const std::string &body, + std::string &responseBody, + std::string &responseContentType); + +private: + static void _circuitTestCallback(ZT_Node *node,ZT_CircuitTest *test,const ZT_CircuitTestReport *report); + + inline nlohmann::json _readJson(const std::string &path) + { + std::string buf; + if (OSUtils::readFile(path.c_str(),buf)) { + try { + return nlohmann::json::parse(buf); + } catch ( ... ) {} + } + return nlohmann::json::object(); + } + + inline bool _writeJson(const std::string &path,const nlohmann::json &obj) + { + std::string buf(obj.dump(2)); + return OSUtils::writeFile(path.c_str(),buf); + } + + inline std::string _networkBP(const uint64_t nwid,bool create) + { + char tmp[64]; + Utils::snprintf(tmp,sizeof(tmp),"%.16llx",nwid); + std::string p(_path + ZT_PATH_SEPARATOR_S + "network"); + if (create) OSUtils::mkdir(p.c_str()); + p.push_back(ZT_PATH_SEPARATOR); + p.append(tmp); + if (create) OSUtils::mkdir(p.c_str()); + return p; + } + inline std::string _networkJP(const uint64_t nwid,bool create) + { + return (_networkBP(nwid,create) + ZT_PATH_SEPARATOR + "config.json"); + } + inline std::string _memberBP(const uint64_t nwid,const Address &member,bool create) + { + std::string p(_networkBP(nwid,create)); + p.push_back(ZT_PATH_SEPARATOR); + p.append("member"); + if (create) OSUtils::mkdir(p.c_str()); + p.push_back(ZT_PATH_SEPARATOR); + p.append(member.toString()); + if (create) OSUtils::mkdir(p.c_str()); + return p; + } + inline std::string _memberJP(const uint64_t nwid,const Address &member,bool create) + { + return (_memberBP(nwid,member,create) + ZT_PATH_SEPARATOR + "config.json"); + } + + inline std::set _getAlreadyAllocatedIps(const uint64_t nwid) + { + std::set ips; + std::string bp(_networkBP(nwid,false) + ZT_PATH_SEPARATOR_S + "member"); + std::vector members(OSUtils::listSubdirectories(bp.c_str())); + for(std::vector::iterator m(members.begin());m!=members.end();++m) { + if (m->length() == ZT_ADDRESS_LENGTH_HEX) { + nlohmann::json mj = _readJson(bp + ZT_PATH_SEPARATOR_S + *m + ZT_PATH_SEPARATOR_S + "config.json"); + auto ipAssignments = mj["ipAssignments"]; + if (ipAssignments.is_array()) { + for(unsigned long i=0;i _circuitTests; + Mutex _circuitTests_m; + + // Last request time by address, for rate limitation + std::map< std::pair,uint64_t > _lastRequestTime; + Mutex _lastRequestTime_m; +}; + +} // namespace ZeroTier + +#endif diff --git a/controller/SqliteNetworkController.cpp b/controller/SqliteNetworkController.cpp deleted file mode 100644 index 863f93f3..00000000 --- a/controller/SqliteNetworkController.cpp +++ /dev/null @@ -1,1462 +0,0 @@ -/* - * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2015 ZeroTier, Inc. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - */ - -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include - -#include "../include/ZeroTierOne.h" -#include "../node/Constants.hpp" - -#include "SqliteNetworkController.hpp" - -#include "../node/Node.hpp" -#include "../node/Utils.hpp" -#include "../node/CertificateOfMembership.hpp" -#include "../node/NetworkConfig.hpp" -#include "../node/Dictionary.hpp" -#include "../node/InetAddress.hpp" -#include "../node/MAC.hpp" -#include "../node/Address.hpp" - -using json = nlohmann::json; - -// API version reported via JSON control plane -#define ZT_NETCONF_CONTROLLER_API_VERSION 3 - -// Number of requests to remember in member history -#define ZT_NETCONF_DB_MEMBER_HISTORY_LENGTH 8 - -// Min duration between requests for an address/nwid combo to prevent floods -#define ZT_NETCONF_MIN_REQUEST_PERIOD 1000 - -// Nodes are considered active if they've queried in less than this long -#define ZT_NETCONF_NODE_ACTIVE_THRESHOLD ((ZT_NETWORK_AUTOCONF_DELAY * 2) + 5000) - -namespace ZeroTier { - -static json _renderRule(ZT_VirtualNetworkRule &rule) -{ - char tmp[128]; - json r = json::object(); - r["not"] = ((rule.t & 0x80) != 0); - switch((rule.t) & 0x7f) { - case ZT_NETWORK_RULE_ACTION_DROP: - r["type"] = "ACTION_DROP"; - break; - case ZT_NETWORK_RULE_ACTION_ACCEPT: - r["type"] = "ACTION_ACCEPT"; - break; - case ZT_NETWORK_RULE_ACTION_TEE: - r["type"] = "ACTION_TEE"; - r["zt"] = Address(rule.v.zt).toString(); - break; - case ZT_NETWORK_RULE_ACTION_REDIRECT: - r["type"] = "ACTION_REDIRECT"; - r["zt"] = Address(rule.v.zt).toString(); - break; - case ZT_NETWORK_RULE_MATCH_SOURCE_ZEROTIER_ADDRESS: - r["type"] = "MATCH_SOURCE_ZEROTIER_ADDRESS"; - r["zt"] = Address(rule.v.zt).toString(); - break; - case ZT_NETWORK_RULE_MATCH_DEST_ZEROTIER_ADDRESS: - r["type"] = "MATCH_DEST_ZEROTIER_ADDRESS"; - r["zt"] = Address(rule.v.zt).toString(); - break; - case ZT_NETWORK_RULE_MATCH_VLAN_ID: - r["type"] = "MATCH_VLAN_ID"; - r["vlanId"] = (uint64_t)rule.v.vlanId; - break; - case ZT_NETWORK_RULE_MATCH_VLAN_PCP: - r["type"] = "MATCH_VLAN_PCP"; - r["vlanPcp"] = (uint64_t)rule.v.vlanPcp; - break; - case ZT_NETWORK_RULE_MATCH_VLAN_DEI: - r["type"] = "MATCH_VLAN_DEI"; - r["vlanDei"] = (uint64_t)rule.v.vlanDei; - break; - case ZT_NETWORK_RULE_MATCH_ETHERTYPE: - r["type"] = "MATCH_ETHERTYPE"; - r["etherType"] = (uint64_t)rule.v.etherType; - break; - case ZT_NETWORK_RULE_MATCH_MAC_SOURCE: - r["type"] = "MATCH_MAC_SOURCE"; - Utils::snprintf(tmp,sizeof(tmp),"%.2x:%.2x:%.2x:%.2x:%.2x:%.2x",(unsigned int)rule.v.mac[0],(unsigned int)rule.v.mac[1],(unsigned int)rule.v.mac[2],(unsigned int)rule.v.mac[3],(unsigned int)rule.v.mac[4],(unsigned int)rule.v.mac[5]); - r["mac"] = tmp; - break; - case ZT_NETWORK_RULE_MATCH_MAC_DEST: - r["type"] = "MATCH_MAC_DEST"; - Utils::snprintf(tmp,sizeof(tmp),"%.2x:%.2x:%.2x:%.2x:%.2x:%.2x",(unsigned int)rule.v.mac[0],(unsigned int)rule.v.mac[1],(unsigned int)rule.v.mac[2],(unsigned int)rule.v.mac[3],(unsigned int)rule.v.mac[4],(unsigned int)rule.v.mac[5]); - r["mac"] = tmp; - break; - case ZT_NETWORK_RULE_MATCH_IPV4_SOURCE: - r["type"] = "MATCH_IPV4_SOURCE"; - r["ip"] = InetAddress(&(rule.v.ipv4.ip),4,(unsigned int)rule.v.ipv4.mask).toString(); - break; - case ZT_NETWORK_RULE_MATCH_IPV4_DEST: - r["type"] = "MATCH_IPV4_DEST"; - r["ip"] = InetAddress(&(rule.v.ipv4.ip),4,(unsigned int)rule.v.ipv4.mask).toString(); - break; - case ZT_NETWORK_RULE_MATCH_IPV6_SOURCE: - r["type"] = "MATCH_IPV6_SOURCE"; - r["ip"] = InetAddress(rule.v.ipv6.ip,16,(unsigned int)rule.v.ipv6.mask).toString(); - break; - case ZT_NETWORK_RULE_MATCH_IPV6_DEST: - r["type"] = "MATCH_IPV6_DEST"; - r["ip"] = InetAddress(rule.v.ipv6.ip,16,(unsigned int)rule.v.ipv6.mask).toString(); - break; - case ZT_NETWORK_RULE_MATCH_IP_TOS: - r["type"] = "MATCH_IP_TOS"; - r["ipTos"] = (uint64_t)rule.v.ipTos; - break; - case ZT_NETWORK_RULE_MATCH_IP_PROTOCOL: - r["type"] = "MATCH_IP_PROTOCOL"; - r["ipProtocol"] = (uint64_t)rule.v.ipProtocol; - break; - case ZT_NETWORK_RULE_MATCH_IP_SOURCE_PORT_RANGE: - r["type"] = "MATCH_IP_SOURCE_PORT_RANGE"; - r["start"] = (uint64_t)rule.v.port[0]; - r["end"] = (uint64_t)rule.v.port[1]; - break; - case ZT_NETWORK_RULE_MATCH_IP_DEST_PORT_RANGE: - r["type"] = "MATCH_IP_DEST_PORT_RANGE"; - r["start"] = (uint64_t)rule.v.port[0]; - r["end"] = (uint64_t)rule.v.port[1]; - break; - case ZT_NETWORK_RULE_MATCH_CHARACTERISTICS: - r["type"] = "MATCH_CHARACTERISTICS"; - Utils::snprintf(tmp,sizeof(tmp),"%.16llx",rule.v.characteristics[0]); - r["mask"] = tmp; - Utils::snprintf(tmp,sizeof(tmp),"%.16llx",rule.v.characteristics[1]); - r["value"] = tmp; - break; - case ZT_NETWORK_RULE_MATCH_FRAME_SIZE_RANGE: - r["type"] = "MATCH_FRAME_SIZE_RANGE"; - r["start"] = (uint64_t)rule.v.frameSize[0]; - r["end"] = (uint64_t)rule.v.frameSize[1]; - break; - case ZT_NETWORK_RULE_MATCH_TAGS_SAMENESS: - r["type"] = "MATCH_TAGS_SAMENESS"; - r["id"] = (uint64_t)rule.v.tag.id; - r["value"] = (uint64_t)rule.v.tag.value; - break; - case ZT_NETWORK_RULE_MATCH_TAGS_BITWISE_AND: - r["type"] = "MATCH_TAGS_BITWISE_AND"; - r["id"] = (uint64_t)rule.v.tag.id; - r["value"] = (uint64_t)rule.v.tag.value; - break; - case ZT_NETWORK_RULE_MATCH_TAGS_BITWISE_OR: - r["type"] = "MATCH_TAGS_BITWISE_OR"; - r["id"] = (uint64_t)rule.v.tag.id; - r["value"] = (uint64_t)rule.v.tag.value; - break; - case ZT_NETWORK_RULE_MATCH_TAGS_BITWISE_XOR: - r["type"] = "MATCH_TAGS_BITWISE_XOR"; - r["id"] = (uint64_t)rule.v.tag.id; - r["value"] = (uint64_t)rule.v.tag.value; - break; - } - return r; -} - -static bool _parseRule(const json &r,ZT_VirtualNetworkRule &rule) -{ - if (r.is_object()) - return false; - std::string t = r["type"]; - memset(&rule,0,sizeof(ZT_VirtualNetworkRule)); - if (r.value("not",false)) - rule.t = 0x80; - else rule.t = 0x00; - if (t == "ACTION_DROP") { - rule.t |= ZT_NETWORK_RULE_ACTION_DROP; - return true; - } else if (t == "ACTION_ACCEPT") { - rule.t |= ZT_NETWORK_RULE_ACTION_ACCEPT; - return true; - } else if (t == "ACTION_TEE") { - rule.t |= ZT_NETWORK_RULE_ACTION_TEE; - rule.v.zt = Utils::hexStrToU64(r.value("zt","0").c_str()) & 0xffffffffffULL; - return true; - } else if (t == "ACTION_REDIRECT") { - rule.t |= ZT_NETWORK_RULE_ACTION_REDIRECT; - rule.v.zt = Utils::hexStrToU64(r.value("zt","0").c_str()) & 0xffffffffffULL; - return true; - } else if (t == "MATCH_SOURCE_ZEROTIER_ADDRESS") { - rule.t |= ZT_NETWORK_RULE_MATCH_SOURCE_ZEROTIER_ADDRESS; - rule.v.zt = Utils::hexStrToU64(r.value("zt","0").c_str()) & 0xffffffffffULL; - return true; - } else if (t == "MATCH_DEST_ZEROTIER_ADDRESS") { - rule.t |= ZT_NETWORK_RULE_MATCH_DEST_ZEROTIER_ADDRESS; - rule.v.zt = Utils::hexStrToU64(r.value("zt","0").c_str()) & 0xffffffffffULL; - return true; - } else if (t == "MATCH_VLAN_ID") { - rule.t |= ZT_NETWORK_RULE_MATCH_VLAN_ID; - rule.v.vlanId = (uint16_t)(r.value("vlanId",0ULL) & 0xffffULL); - return true; - } else if (t == "MATCH_VLAN_PCP") { - rule.t |= ZT_NETWORK_RULE_MATCH_VLAN_PCP; - rule.v.vlanPcp = (uint8_t)(r.value("vlanPcp",0ULL) & 0xffULL); - return true; - } else if (t == "MATCH_VLAN_DEI") { - rule.t |= ZT_NETWORK_RULE_MATCH_VLAN_DEI; - rule.v.vlanDei = (uint8_t)(r.value("vlanDei",0ULL) & 0xffULL); - return true; - } else if (t == "MATCH_ETHERTYPE") { - rule.t |= ZT_NETWORK_RULE_MATCH_ETHERTYPE; - rule.v.etherType = (uint16_t)(r.value("etherType",0ULL) & 0xffffULL); - return true; - } else if (t == "MATCH_MAC_SOURCE") { - rule.t |= ZT_NETWORK_RULE_MATCH_MAC_SOURCE; - const std::string mac(r.value("mac","0")); - Utils::unhex(mac.c_str(),(unsigned int)mac.length(),rule.v.mac,6); - return true; - } else if (t == "MATCH_MAC_DEST") { - rule.t |= ZT_NETWORK_RULE_MATCH_MAC_DEST; - const std::string mac(r.value("mac","0")); - Utils::unhex(mac.c_str(),(unsigned int)mac.length(),rule.v.mac,6); - return true; - } else if (t == "MATCH_IPV4_SOURCE") { - rule.t |= ZT_NETWORK_RULE_MATCH_IPV4_SOURCE; - InetAddress ip(r.value("ip","0.0.0.0")); - rule.v.ipv4.ip = reinterpret_cast(&ip)->sin_addr.s_addr; - rule.v.ipv4.mask = Utils::ntoh(reinterpret_cast(&ip)->sin_port) & 0xff; - if (rule.v.ipv4.mask > 32) rule.v.ipv4.mask = 32; - return true; - } else if (t == "MATCH_IPV4_DEST") { - rule.t |= ZT_NETWORK_RULE_MATCH_IPV4_DEST; - InetAddress ip(r.value("ip","0.0.0.0")); - rule.v.ipv4.ip = reinterpret_cast(&ip)->sin_addr.s_addr; - rule.v.ipv4.mask = Utils::ntoh(reinterpret_cast(&ip)->sin_port) & 0xff; - if (rule.v.ipv4.mask > 32) rule.v.ipv4.mask = 32; - return true; - } else if (t == "MATCH_IPV6_SOURCE") { - rule.t |= ZT_NETWORK_RULE_MATCH_IPV6_SOURCE; - InetAddress ip(r.value("ip","::0")); - memcpy(rule.v.ipv6.ip,reinterpret_cast(&ip)->sin6_addr.s6_addr,16); - rule.v.ipv6.mask = Utils::ntoh(reinterpret_cast(&ip)->sin6_port) & 0xff; - if (rule.v.ipv6.mask > 128) rule.v.ipv6.mask = 128; - return true; - } else if (t == "MATCH_IPV6_DEST") { - rule.t |= ZT_NETWORK_RULE_MATCH_IPV6_DEST; - InetAddress ip(r.value("ip","::0")); - memcpy(rule.v.ipv6.ip,reinterpret_cast(&ip)->sin6_addr.s6_addr,16); - rule.v.ipv6.mask = Utils::ntoh(reinterpret_cast(&ip)->sin6_port) & 0xff; - if (rule.v.ipv6.mask > 128) rule.v.ipv6.mask = 128; - return true; - } else if (t == "MATCH_IP_TOS") { - rule.t |= ZT_NETWORK_RULE_MATCH_IP_TOS; - rule.v.ipTos = (uint8_t)(r.value("ipTos",0ULL) & 0xffULL); - return true; - } else if (t == "MATCH_IP_PROTOCOL") { - rule.t |= ZT_NETWORK_RULE_MATCH_IP_PROTOCOL; - rule.v.ipProtocol = (uint8_t)(r.value("ipProtocol",0ULL) & 0xffULL); - return true; - } else if (t == "MATCH_IP_SOURCE_PORT_RANGE") { - rule.t |= ZT_NETWORK_RULE_MATCH_IP_SOURCE_PORT_RANGE; - rule.v.port[0] = (uint16_t)(r.value("start",0ULL) & 0xffffULL); - rule.v.port[1] = (uint16_t)(r.value("end",0ULL) & 0xffffULL); - return true; - } else if (t == "MATCH_IP_DEST_PORT_RANGE") { - rule.t |= ZT_NETWORK_RULE_MATCH_IP_DEST_PORT_RANGE; - rule.v.port[0] = (uint16_t)(r.value("start",0ULL) & 0xffffULL); - rule.v.port[1] = (uint16_t)(r.value("end",0ULL) & 0xffffULL); - return true; - } else if (t == "MATCH_CHARACTERISTICS") { - rule.t |= ZT_NETWORK_RULE_MATCH_CHARACTERISTICS; - if (r.count("mask")) { - auto v = r["mask"]; - if (v.is_number()) { - rule.v.characteristics[0] = v; - } else { - std::string tmp = v; - rule.v.characteristics[0] = Utils::hexStrToU64(tmp.c_str()); - } - } - if (r.count("value")) { - auto v = r["value"]; - if (v.is_number()) { - rule.v.characteristics[1] = v; - } else { - std::string tmp = v; - rule.v.characteristics[1] = Utils::hexStrToU64(tmp.c_str()); - } - } - return true; - } else if (t == "MATCH_FRAME_SIZE_RANGE") { - rule.t |= ZT_NETWORK_RULE_MATCH_FRAME_SIZE_RANGE; - rule.v.frameSize[0] = (uint16_t)(Utils::hexStrToU64(r.value("start","0").c_str()) & 0xffffULL); - rule.v.frameSize[1] = (uint16_t)(Utils::hexStrToU64(r.value("end","0").c_str()) & 0xffffULL); - return true; - } else if (t == "MATCH_TAGS_SAMENESS") { - rule.t |= ZT_NETWORK_RULE_MATCH_TAGS_SAMENESS; - rule.v.tag.id = (uint32_t)(Utils::hexStrToU64(r.value("id","0").c_str()) & 0xffffffffULL); - rule.v.tag.value = (uint32_t)(Utils::hexStrToU64(r.value("value","0").c_str()) & 0xffffffffULL); - return true; - } else if (t == "MATCH_TAGS_BITWISE_AND") { - rule.t |= ZT_NETWORK_RULE_MATCH_TAGS_BITWISE_AND; - rule.v.tag.id = (uint32_t)(Utils::hexStrToU64(r.value("id","0").c_str()) & 0xffffffffULL); - rule.v.tag.value = (uint32_t)(Utils::hexStrToU64(r.value("value","0").c_str()) & 0xffffffffULL); - return true; - } else if (t == "MATCH_TAGS_BITWISE_OR") { - rule.t |= ZT_NETWORK_RULE_MATCH_TAGS_BITWISE_OR; - rule.v.tag.id = (uint32_t)(Utils::hexStrToU64(r.value("id","0").c_str()) & 0xffffffffULL); - rule.v.tag.value = (uint32_t)(Utils::hexStrToU64(r.value("value","0").c_str()) & 0xffffffffULL); - return true; - } else if (t == "MATCH_TAGS_BITWISE_XOR") { - rule.t |= ZT_NETWORK_RULE_MATCH_TAGS_BITWISE_XOR; - rule.v.tag.id = (uint32_t)(Utils::hexStrToU64(r.value("id","0").c_str()) & 0xffffffffULL); - rule.v.tag.value = (uint32_t)(Utils::hexStrToU64(r.value("value","0").c_str()) & 0xffffffffULL); - return true; - } - return false; -} - -SqliteNetworkController::SqliteNetworkController(Node *node,const char *dbPath) : - _node(node), - _path(dbPath) -{ - OSUtils::mkdir(dbPath); - /* - if (sqlite3_open_v2(dbPath,&_db,SQLITE_OPEN_READWRITE|SQLITE_OPEN_CREATE,(const char *)0) != SQLITE_OK) - throw std::runtime_error("SqliteNetworkController cannot open database file"); - sqlite3_busy_timeout(_db,10000); - - sqlite3_exec(_db,"PRAGMA synchronous = OFF",0,0,0); - sqlite3_exec(_db,"PRAGMA journal_mode = MEMORY",0,0,0); - - sqlite3_stmt *s = (sqlite3_stmt *)0; - if ((sqlite3_prepare_v2(_db,"SELECT v FROM Config WHERE k = 'schemaVersion';",-1,&s,(const char **)0) == SQLITE_OK)&&(s)) { - int schemaVersion = -1234; - if (sqlite3_step(s) == SQLITE_ROW) { - schemaVersion = sqlite3_column_int(s,0); - } - - sqlite3_finalize(s); - - if (schemaVersion == -1234) { - sqlite3_close(_db); - throw std::runtime_error("SqliteNetworkController schemaVersion not found in Config table (init failure?)"); - } - - if (schemaVersion < 2) { - // Create NodeHistory table to upgrade from version 1 to version 2 - if (sqlite3_exec(_db, - "CREATE TABLE NodeHistory (\n" - " nodeId char(10) NOT NULL REFERENCES Node(id) ON DELETE CASCADE,\n" - " networkId char(16) NOT NULL REFERENCES Network(id) ON DELETE CASCADE,\n" - " networkVisitCounter INTEGER NOT NULL DEFAULT(0),\n" - " networkRequestAuthorized INTEGER NOT NULL DEFAULT(0),\n" - " requestTime INTEGER NOT NULL DEFAULT(0),\n" - " clientMajorVersion INTEGER NOT NULL DEFAULT(0),\n" - " clientMinorVersion INTEGER NOT NULL DEFAULT(0),\n" - " clientRevision INTEGER NOT NULL DEFAULT(0),\n" - " networkRequestMetaData VARCHAR(1024),\n" - " fromAddress VARCHAR(128)\n" - ");\n" - "CREATE INDEX NodeHistory_nodeId ON NodeHistory (nodeId);\n" - "CREATE INDEX NodeHistory_networkId ON NodeHistory (networkId);\n" - "CREATE INDEX NodeHistory_requestTime ON NodeHistory (requestTime);\n" - "UPDATE \"Config\" SET \"v\" = 2 WHERE \"k\" = 'schemaVersion';\n" - ,0,0,0) != SQLITE_OK) { - char err[1024]; - Utils::snprintf(err,sizeof(err),"SqliteNetworkController cannot upgrade the database to version 2: %s",sqlite3_errmsg(_db)); - sqlite3_close(_db); - throw std::runtime_error(err); - } else { - schemaVersion = 2; - } - } - - if (schemaVersion < 3) { - // Create Route table to upgrade from version 2 to version 3 and migrate old - // data. Also delete obsolete Gateway table that was never actually used, and - // migrate Network flags to a bitwise flags field instead of ASCII cruft. - if (sqlite3_exec(_db, - "DROP TABLE Gateway;\n" - "CREATE TABLE Route (\n" - " networkId char(16) NOT NULL REFERENCES Network(id) ON DELETE CASCADE,\n" - " target blob(16) NOT NULL,\n" - " via blob(16),\n" - " targetNetmaskBits integer NOT NULL,\n" - " ipVersion integer NOT NULL,\n" - " flags integer NOT NULL,\n" - " metric integer NOT NULL\n" - ");\n" - "CREATE INDEX Route_networkId ON Route (networkId);\n" - "INSERT INTO Route SELECT DISTINCT networkId,\"ip\" AS \"target\",NULL AS \"via\",ipNetmaskBits AS targetNetmaskBits,ipVersion,0 AS \"flags\",0 AS \"metric\" FROM IpAssignment WHERE nodeId IS NULL AND \"type\" = 1;\n" - "ALTER TABLE Network ADD COLUMN \"flags\" integer NOT NULL DEFAULT(0);\n" - "UPDATE Network SET \"flags\" = (\"flags\" | 1) WHERE v4AssignMode = 'zt';\n" - "UPDATE Network SET \"flags\" = (\"flags\" | 2) WHERE v6AssignMode = 'rfc4193';\n" - "UPDATE Network SET \"flags\" = (\"flags\" | 4) WHERE v6AssignMode = '6plane';\n" - "ALTER TABLE Member ADD COLUMN \"flags\" integer NOT NULL DEFAULT(0);\n" - "DELETE FROM IpAssignment WHERE nodeId IS NULL AND \"type\" = 1;\n" - "UPDATE \"Config\" SET \"v\" = 3 WHERE \"k\" = 'schemaVersion';\n" - ,0,0,0) != SQLITE_OK) { - char err[1024]; - Utils::snprintf(err,sizeof(err),"SqliteNetworkController cannot upgrade the database to version 3: %s",sqlite3_errmsg(_db)); - sqlite3_close(_db); - throw std::runtime_error(err); - } else { - schemaVersion = 3; - } - } - - if (schemaVersion < 4) { - // Turns out this was overkill and a huge performance drag. Will be revisiting this - // more later but for now a brief snapshot of recent history stored in Member is fine. - // Also prepare for implementation of proof of work requests. - if (sqlite3_exec(_db, - "DROP TABLE NodeHistory;\n" - "ALTER TABLE Member ADD COLUMN lastRequestTime integer NOT NULL DEFAULT(0);\n" - "ALTER TABLE Member ADD COLUMN lastPowDifficulty integer NOT NULL DEFAULT(0);\n" - "ALTER TABLE Member ADD COLUMN lastPowTime integer NOT NULL DEFAULT(0);\n" - "ALTER TABLE Member ADD COLUMN recentHistory blob;\n" - "CREATE INDEX Member_networkId_lastRequestTime ON Member(networkId, lastRequestTime);\n" - "UPDATE \"Config\" SET \"v\" = 4 WHERE \"k\" = 'schemaVersion';\n" - ,0,0,0) != SQLITE_OK) { - char err[1024]; - Utils::snprintf(err,sizeof(err),"SqliteNetworkController cannot upgrade the database to version 3: %s",sqlite3_errmsg(_db)); - sqlite3_close(_db); - throw std::runtime_error(err); - } else { - schemaVersion = 4; - } - } - - if (schemaVersion < 5) { - // Upgrade old rough draft Rule table to new release format - if (sqlite3_exec(_db, - "DROP TABLE Relay;\n" - "DROP INDEX Rule_networkId_ruleNo;\n" - "ALTER TABLE \"Rule\" RENAME TO RuleOld;\n" - "CREATE TABLE Rule (\n" - " networkId char(16) NOT NULL REFERENCES Network(id) ON DELETE CASCADE,\n" - " capId integer,\n" - " ruleNo integer NOT NULL,\n" - " ruleType integer NOT NULL DEFAULT(0),\n" - " \"addr\" blob(16),\n" - " \"int1\" integer,\n" - " \"int2\" integer,\n" - " \"int3\" integer,\n" - " \"int4\" integer\n" - ");\n" - "INSERT INTO \"Rule\" SELECT networkId,(ruleNo*2) AS ruleNo,37 AS \"ruleType\",etherType AS \"int1\" FROM RuleOld WHERE RuleOld.etherType IS NOT NULL AND RuleOld.etherType > 0;\n" - "INSERT INTO \"Rule\" SELECT networkId,((ruleNo*2)+1) AS ruleNo,1 AS \"ruleType\" FROM RuleOld;\n" - "DROP TABLE RuleOld;\n" - "CREATE INDEX Rule_networkId_capId ON Rule (networkId,capId);\n" - "CREATE TABLE MemberTC (\n" - " networkId char(16) NOT NULL REFERENCES Network(id) ON DELETE CASCADE,\n" - " nodeId char(10) NOT NULL REFERENCES Node(id) ON DELETE CASCADE,\n" - " tagId integer,\n" - " tagValue integer,\n" - " capId integer,\n" - " capMaxCustodyChainLength integer NOT NULL DEFAULT(1)\n" - ");\n" - "CREATE INDEX MemberTC_networkId_nodeId ON MemberTC (networkId,nodeId);\n" - "UPDATE \"Config\" SET \"v\" = 5 WHERE \"k\" = 'schemaVersion';\n" - ,0,0,0) != SQLITE_OK) { - char err[1024]; - Utils::snprintf(err,sizeof(err),"SqliteNetworkController cannot upgrade the database to version 3: %s",sqlite3_errmsg(_db)); - sqlite3_close(_db); - throw std::runtime_error(err); - } else { - schemaVersion = 5; - } - } - - if (schemaVersion != ZT_NETCONF_SQLITE_SCHEMA_VERSION) { - sqlite3_close(_db); - throw std::runtime_error("SqliteNetworkController database schema version mismatch"); - } - } else { - // Prepare statement will fail if Config table doesn't exist, which means our DB - // needs to be initialized. - if (sqlite3_exec(_db,ZT_NETCONF_SCHEMA_SQL"INSERT INTO Config (k,v) VALUES ('schemaVersion',"ZT_NETCONF_SQLITE_SCHEMA_VERSION_STR");",0,0,0) != SQLITE_OK) { - char err[1024]; - Utils::snprintf(err,sizeof(err),"SqliteNetworkController cannot initialize database and/or insert schemaVersion into Config table: %s",sqlite3_errmsg(_db)); - sqlite3_close(_db); - throw std::runtime_error(err); - } - } - - if ( - - (sqlite3_prepare_v2(_db,"SELECT name,private,enableBroadcast,allowPassiveBridging,\"flags\",multicastLimit,creationTime,revision,memberRevisionCounter,(SELECT COUNT(1) FROM Member WHERE Member.networkId = Network.id AND Member.authorized > 0) FROM Network WHERE id = ?",-1,&_sGetNetworkById,(const char **)0) != SQLITE_OK) - ||(sqlite3_prepare_v2(_db,"SELECT revision FROM Network WHERE id = ?",-1,&_sGetNetworkRevision,(const char **)0) != SQLITE_OK) - ||(sqlite3_prepare_v2(_db,"UPDATE Network SET revision = ? WHERE id = ?",-1,&_sSetNetworkRevision,(const char **)0) != SQLITE_OK) - ||(sqlite3_prepare_v2(_db,"INSERT INTO Network (id,name,creationTime,revision) VALUES (?,?,?,1)",-1,&_sCreateNetwork,(const char **)0) != SQLITE_OK) - ||(sqlite3_prepare_v2(_db,"DELETE FROM Network WHERE id = ?",-1,&_sDeleteNetwork,(const char **)0) != SQLITE_OK) - ||(sqlite3_prepare_v2(_db,"SELECT id FROM Network ORDER BY id ASC",-1,&_sListNetworks,(const char **)0) != SQLITE_OK) - ||(sqlite3_prepare_v2(_db,"UPDATE Network SET memberRevisionCounter = (memberRevisionCounter + 1) WHERE id = ?",-1,&_sIncrementMemberRevisionCounter,(const char **)0) != SQLITE_OK) - - ||(sqlite3_prepare_v2(_db,"SELECT identity FROM Node WHERE id = ?",-1,&_sGetNodeIdentity,(const char **)0) != SQLITE_OK) - ||(sqlite3_prepare_v2(_db,"INSERT OR REPLACE INTO Node (id,identity) VALUES (?,?)",-1,&_sCreateOrReplaceNode,(const char **)0) != SQLITE_OK) - - ||(sqlite3_prepare_v2(_db,"INSERT INTO Rule (networkId,ruleNo,nodeId,ztSource,ztDest,vlanId,vlanPcp,vlanDei,) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)",-1,&_sCreateRule,(const char **)0) != SQLITE_OK) - ||(sqlite3_prepare_v2(_db,"SELECT ruleNo,nodeId,sourcePort,destPort,vlanId,vlanPcp,etherType,macSource,macDest,ipSource,ipDest,ipTos,ipProtocol,ipSourcePort,ipDestPort,\"flags\",invFlags,\"action\" FROM Rule WHERE networkId = ? ORDER BY ruleNo ASC",-1,&_sListRules,(const char **)0) != SQLITE_OK) - ||(sqlite3_prepare_v2(_db,"DELETE FROM Rule WHERE networkId = ?",-1,&_sDeleteRulesForNetwork,(const char **)0) != SQLITE_OK) - - ||(sqlite3_prepare_v2(_db,"SELECT ipRangeStart,ipRangeEnd FROM IpAssignmentPool WHERE networkId = ? AND ipVersion = ?",-1,&_sGetIpAssignmentPools,(const char **)0) != SQLITE_OK) - ||(sqlite3_prepare_v2(_db,"SELECT ipRangeStart,ipRangeEnd,ipVersion FROM IpAssignmentPool WHERE networkId = ? ORDER BY ipRangeStart ASC",-1,&_sGetIpAssignmentPools2,(const char **)0) != SQLITE_OK) - ||(sqlite3_prepare_v2(_db,"INSERT INTO IpAssignmentPool (networkId,ipRangeStart,ipRangeEnd,ipVersion) VALUES (?,?,?,?)",-1,&_sCreateIpAssignmentPool,(const char **)0) != SQLITE_OK) - ||(sqlite3_prepare_v2(_db,"DELETE FROM IpAssignmentPool WHERE networkId = ?",-1,&_sDeleteIpAssignmentPoolsForNetwork,(const char **)0) != SQLITE_OK) - - ||(sqlite3_prepare_v2(_db,"SELECT ip,ipNetmaskBits,ipVersion FROM IpAssignment WHERE networkId = ? AND nodeId = ? AND \"type\" = 0 ORDER BY ip ASC",-1,&_sGetIpAssignmentsForNode,(const char **)0) != SQLITE_OK) - ||(sqlite3_prepare_v2(_db,"SELECT 1 FROM IpAssignment WHERE networkId = ? AND ip = ? AND ipVersion = ? AND \"type\" = ?",-1,&_sCheckIfIpIsAllocated,(const char **)0) != SQLITE_OK) - ||(sqlite3_prepare_v2(_db,"INSERT INTO IpAssignment (networkId,nodeId,\"type\",ip,ipNetmaskBits,ipVersion) VALUES (?,?,?,?,?,?)",-1,&_sAllocateIp,(const char **)0) != SQLITE_OK) - ||(sqlite3_prepare_v2(_db,"DELETE FROM IpAssignment WHERE networkId = ? AND nodeId = ? AND \"type\" = ?",-1,&_sDeleteIpAllocations,(const char **)0) != SQLITE_OK) - - ||(sqlite3_prepare_v2(_db,"SELECT rowid,authorized,activeBridge,memberRevision,\"flags\",lastRequestTime,recentHistory FROM Member WHERE networkId = ? AND nodeId = ?",-1,&_sGetMember,(const char **)0) != SQLITE_OK) - ||(sqlite3_prepare_v2(_db,"SELECT m.authorized,m.activeBridge,m.memberRevision,n.identity,m.flags,m.lastRequestTime,m.recentHistory FROM Member AS m LEFT OUTER JOIN Node AS n ON n.id = m.nodeId WHERE m.networkId = ? AND m.nodeId = ?",-1,&_sGetMember2,(const char **)0) != SQLITE_OK) - ||(sqlite3_prepare_v2(_db,"INSERT INTO Member (networkId,nodeId,authorized,activeBridge,memberRevision) VALUES (?,?,?,0,(SELECT memberRevisionCounter FROM Network WHERE id = ?))",-1,&_sCreateMember,(const char **)0) != SQLITE_OK) - ||(sqlite3_prepare_v2(_db,"SELECT nodeId FROM Member WHERE networkId = ? AND activeBridge > 0 AND authorized > 0",-1,&_sGetActiveBridges,(const char **)0) != SQLITE_OK) - ||(sqlite3_prepare_v2(_db,"SELECT m.nodeId,m.memberRevision FROM Member AS m WHERE m.networkId = ? ORDER BY m.nodeId ASC",-1,&_sListNetworkMembers,(const char **)0) != SQLITE_OK) - ||(sqlite3_prepare_v2(_db,"UPDATE Member SET authorized = ?,memberRevision = (SELECT memberRevisionCounter FROM Network WHERE id = ?) WHERE rowid = ?",-1,&_sUpdateMemberAuthorized,(const char **)0) != SQLITE_OK) - ||(sqlite3_prepare_v2(_db,"UPDATE Member SET activeBridge = ?,memberRevision = (SELECT memberRevisionCounter FROM Network WHERE id = ?) WHERE rowid = ?",-1,&_sUpdateMemberActiveBridge,(const char **)0) != SQLITE_OK) - ||(sqlite3_prepare_v2(_db,"UPDATE Member SET \"lastRequestTime\" = ?, \"recentHistory\" = ? WHERE rowid = ?",-1,&_sUpdateMemberHistory,(const char **)0) != SQLITE_OK) - ||(sqlite3_prepare_v2(_db,"DELETE FROM Member WHERE networkId = ? AND nodeId = ?",-1,&_sDeleteMember,(const char **)0) != SQLITE_OK) - ||(sqlite3_prepare_v2(_db,"DELETE FROM Member WHERE networkId = ?",-1,&_sDeleteAllNetworkMembers,(const char **)0) != SQLITE_OK) - ||(sqlite3_prepare_v2(_db,"SELECT nodeId,recentHistory FROM Member WHERE networkId = ? AND lastRequestTime >= ?",-1,&_sGetActiveNodesOnNetwork,(const char **)0) != SQLITE_OK) - - ||(sqlite3_prepare_v2(_db,"INSERT INTO Route (networkId,target,via,targetNetmaskBits,ipVersion,flags,metric) VALUES (?,?,?,?,?,?,?)",-1,&_sCreateRoute,(const char **)0) != SQLITE_OK) - ||(sqlite3_prepare_v2(_db,"SELECT DISTINCT target,via,targetNetmaskBits,ipVersion,flags,metric FROM \"Route\" WHERE networkId = ? ORDER BY ipVersion,target,via",-1,&_sGetRoutes,(const char **)0) != SQLITE_OK) - ||(sqlite3_prepare_v2(_db,"DELETE FROM \"Route\" WHERE networkId = ?",-1,&_sDeleteRoutes,(const char **)0) != SQLITE_OK) - - ||(sqlite3_prepare_v2(_db,"SELECT \"v\" FROM \"Config\" WHERE \"k\" = ?",-1,&_sGetConfig,(const char **)0) != SQLITE_OK) - ||(sqlite3_prepare_v2(_db,"INSERT OR REPLACE INTO \"Config\" (\"k\",\"v\") VALUES (?,?)",-1,&_sSetConfig,(const char **)0) != SQLITE_OK) - - ) { - std::string err(std::string("SqliteNetworkController unable to initialize one or more prepared statements: ") + sqlite3_errmsg(_db)); - sqlite3_close(_db); - throw std::runtime_error(err); - } - - sqlite3_reset(_sGetConfig); - sqlite3_bind_text(_sGetConfig,1,"instanceId",10,SQLITE_STATIC); - if (sqlite3_step(_sGetConfig) != SQLITE_ROW) { - unsigned char sr[32]; - Utils::getSecureRandom(sr,32); - for(unsigned int i=0;i<32;++i) - _instanceId.push_back("0123456789abcdef"[(unsigned int)sr[i] & 0xf]); - - sqlite3_reset(_sSetConfig); - sqlite3_bind_text(_sSetConfig,1,"instanceId",10,SQLITE_STATIC); - sqlite3_bind_text(_sSetConfig,2,_instanceId.c_str(),-1,SQLITE_STATIC); - if (sqlite3_step(_sSetConfig) != SQLITE_DONE) - throw std::runtime_error("SqliteNetworkController unable to read or initialize instanceId"); - } else { - const char *iid = reinterpret_cast(sqlite3_column_text(_sGetConfig,0)); - if (!iid) - throw std::runtime_error("SqliteNetworkController unable to read instanceId (it's NULL)"); - _instanceId = iid; - } - -#ifdef ZT_NETCONF_SQLITE_TRACE - sqlite3_trace(_db,sqliteTraceFunc,(void *)0); -#endif - - _backupThread = Thread::start(this); - */ -} - -SqliteNetworkController::~SqliteNetworkController() -{ -} - -NetworkController::ResultCode SqliteNetworkController::doNetworkConfigRequest(const InetAddress &fromAddr,const Identity &signingId,const Identity &identity,uint64_t nwid,const Dictionary &metaData,NetworkConfig &nc) -{ - if (((!signingId)||(!signingId.hasPrivate()))||(signingId.address().toInt() != (nwid >> 24))) { - return NetworkController::NETCONF_QUERY_INTERNAL_SERVER_ERROR; - } - - const uint64_t now = OSUtils::now(); - - // Check rate limit circuit breaker to prevent flooding - { - Mutex::Lock _l(_lastRequestTime_m); - uint64_t &lrt = _lastRequestTime[std::pair(identity.address().toInt(),nwid)]; - if ((now - lrt) <= ZT_NETCONF_MIN_REQUEST_PERIOD) - return NetworkController::NETCONF_QUERY_IGNORE; - lrt = now; - } - - json network(_readJson(_networkJP(nwid,false))); - if (!network.size()) - return NetworkController::NETCONF_QUERY_OBJECT_NOT_FOUND; - const std::string memberJP(_memberJP(nwid,identity.address(),false)); - json member(_readJson(memberJP)); - - { - std::string haveIdStr = member.value("identity",""); - if (haveIdStr.length() > 0) { - try { - if (Identity(haveIdStr.c_str()) != identity) - return NetworkController::NETCONF_QUERY_ACCESS_DENIED; - } catch ( ... ) { - return NetworkController::NETCONF_QUERY_ACCESS_DENIED; - } - } else { - member["identity"] = identity.toString(false); - } - } - - // Make sure these are always present no matter what, and increment member revision since we will always at least log something - member["id"] = identity.address().toString(); - member["address"] = member["id"]; - member["nwid"] = network["id"]; - member["memberRevision"] = member.value("memberRevision",0ULL) + 1; - - // Update member log - { - json rlEntry = json::object(); - rlEntry["ts"] = now; - rlEntry["authorized"] = member["authorized"]; - rlEntry["clientMajorVersion"] = metaData.getUI(ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_NODE_MAJOR_VERSION,0); - rlEntry["clientMinorVersion"] = metaData.getUI(ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_NODE_MINOR_VERSION,0); - rlEntry["clientRevision"] = metaData.getUI(ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_NODE_REVISION,0); - rlEntry["clientProtocolVersion"] = metaData.getUI(ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_PROTOCOL_VERSION,0); - if (fromAddr) - rlEntry["fromAddr"] = fromAddr.toString(); - json recentLog = json::array(); - recentLog.push_back(rlEntry); - auto oldLog = member["recentLog"]; - if (oldLog.is_array()) { - for(unsigned long i=0;i= ZT_NETCONF_DB_MEMBER_HISTORY_LENGTH) - break; - } - } - member["recentLog"] = recentLog; - } - - // Stop if network is private and member is not authorized - if ( (network.value("private",true)) && (!member.value("authorized",false)) ) { - _writeJson(memberJP,member); - return NetworkController::NETCONF_QUERY_ACCESS_DENIED; - } - // Else compose and send network config - - nc.networkId = nwid; - nc.type = network.value("private",true) ? ZT_NETWORK_TYPE_PRIVATE : ZT_NETWORK_TYPE_PUBLIC; - nc.timestamp = now; - nc.revision = network.value("revision",0ULL); - nc.issuedTo = identity.address(); - if (network.value("enableBroadcast",true)) nc.flags |= ZT_NETWORKCONFIG_FLAG_ENABLE_BROADCAST; - if (network.value("allowPassiveBridging",false)) nc.flags |= ZT_NETWORKCONFIG_FLAG_ALLOW_PASSIVE_BRIDGING; - Utils::scopy(nc.name,sizeof(nc.name),network.value("name","").c_str()); - nc.multicastLimit = (unsigned int)network.value("multicastLimit",32ULL); - - bool amActiveBridge = false; - { - json ab = network["activeBridges"]; - if (ab.is_array()) { - for(unsigned long i=0;i= ZT_MAX_NETWORK_RULES) - break; - auto rule = rules[i]; - if (_parseRule(rule,nc.rules[nc.ruleCount])) - ++nc.ruleCount; - } - } - - if (routes.is_array()) { - for(unsigned long i=0;i= ZT_MAX_NETWORK_ROUTES) - break; - auto route = routes[i]; - InetAddress t(route.value("target","")); - InetAddress v(route.value("via","")); - if ((t)&&(v)&&(t.ss_family == v.ss_family)) { - ZT_VirtualNetworkRoute *r = &(nc.routes[nc.routeCount]); - *(reinterpret_cast(&(r->target))) = t; - *(reinterpret_cast(&(r->via))) = v; - ++nc.routeCount; - } - } - } - - bool haveManagedIpv4AutoAssignment = false; - bool haveManagedIpv6AutoAssignment = false; // "special" NDP-emulated address types do not count - json ipAssignments = member["ipAssignments"]; - if (ipAssignments.is_array()) { - for(unsigned long i=0;i(&(nc.routes[rk].target))->containsAddress(ip)) ) - routedNetmaskBits = reinterpret_cast(&(nc.routes[rk].target))->netmaskBits(); - } - - if (routedNetmaskBits > 0) { - if (nc.staticIpCount < ZT_MAX_ZT_ASSIGNED_ADDRESSES) { - ip.setPort(routedNetmaskBits); - nc.staticIps[nc.staticIpCount++] = ip; - } - if (ip.ss_family == AF_INET) - haveManagedIpv4AutoAssignment = true; - else if (ip.ss_family == AF_INET6) - haveManagedIpv6AutoAssignment = true; - } - } - } else { - ipAssignments = json::array(); - } - - std::set allocatedIps; - bool allocatedIpsLoaded = false; - - if ( (ipAssignmentPools.is_array()) && ((v6AssignMode.is_object())&&(v6AssignMode.value("zt",false))) && (!haveManagedIpv6AutoAssignment) && (!amActiveBridge) ) { - if (!allocatedIpsLoaded) allocatedIps = _getAlreadyAllocatedIps(nwid); - for(unsigned long p=0;((p s[1])&&((e[1] - s[1]) >= 0xffffffffffULL)) { - // First see if we can just cram a ZeroTier ID into the higher 64 bits. If so do that. - xx[0] = Utils::hton(x[0]); - xx[1] = Utils::hton(x[1] + identity.address().toInt()); - } else { - // Otherwise pick random addresses -- this technically doesn't explore the whole range if the lower 64 bit range is >= 1 but that won't matter since that would be huge anyway - Utils::getSecureRandom((void *)xx,16); - if ((e[0] > s[0])) - xx[0] %= (e[0] - s[0]); - else xx[0] = 0; - if ((e[1] > s[1])) - xx[1] %= (e[1] - s[1]); - else xx[1] = 0; - xx[0] = Utils::hton(x[0] + xx[0]); - xx[1] = Utils::hton(x[1] + xx[1]); - } - - InetAddress ip6((const void *)xx,16,0); - - // Check if this IP is within a local-to-Ethernet routed network - int routedNetmaskBits = 0; - for(unsigned int rk=0;rk(&(nc.routes[rk].target))->containsAddress(ip6)) ) - routedNetmaskBits = reinterpret_cast(&(nc.routes[rk].target))->netmaskBits(); - } - - // If it's routed, then try to claim and assign it and if successful end loop - if ((routedNetmaskBits > 0)&&(!allocatedIps.count(ip6))) { - ipAssignments.push_back(ip6.toIpString()); - member["ipAssignments"] = ipAssignments; - ip6.setPort((unsigned int)routedNetmaskBits); - if (nc.staticIpCount < ZT_MAX_ZT_ASSIGNED_ADDRESSES) - nc.staticIps[nc.staticIpCount++] = ip6; - haveManagedIpv4AutoAssignment = true; - break; - } - } - } - } - } - } - - if ( (ipAssignmentPools.is_array()) && ((v4AssignMode.is_object())&&(v4AssignMode.value("zt",false))) && (!haveManagedIpv4AutoAssignment) && (!amActiveBridge) ) { - if (!allocatedIpsLoaded) allocatedIps = _getAlreadyAllocatedIps(nwid); - for(unsigned long p=0;((p(&ipRangeStart)->sin_addr.s_addr)); - uint32_t ipRangeEnd = Utils::ntoh((uint32_t)(reinterpret_cast(&ipRangeEnd)->sin_addr.s_addr)); - if ((ipRangeEnd <= ipRangeStart)||(ipRangeStart == 0)) - continue; - uint32_t ipRangeLen = ipRangeEnd - ipRangeStart; - - // Start with the LSB of the member's address - uint32_t ipTrialCounter = (uint32_t)(identity.address().toInt() & 0xffffffff); - - for(uint32_t k=ipRangeStart,trialCount=0;(k<=ipRangeEnd)&&(trialCount < 1000);++k,++trialCount) { - uint32_t ip = (ipRangeLen > 0) ? (ipRangeStart + (ipTrialCounter % ipRangeLen)) : ipRangeStart; - ++ipTrialCounter; - if ((ip & 0x000000ff) == 0x000000ff) - continue; // don't allow addresses that end in .255 - - // Check if this IP is within a local-to-Ethernet routed network - int routedNetmaskBits = 0; - for(unsigned int rk=0;rk(&(nc.routes[rk].target))->sin_addr.s_addr)); - int targetBits = Utils::ntoh((uint16_t)(reinterpret_cast(&(nc.routes[rk].target))->sin_port)); - if ((ip & (0xffffffff << (32 - targetBits))) == targetIp) { - routedNetmaskBits = targetBits; - break; - } - } - } - - InetAddress ip4(Utils::hton(ip),0); - - // If it's routed, then try to claim and assign it and if successful end loop - if ((routedNetmaskBits > 0)&&(!allocatedIps.count(ip4))) { - ipAssignments.push_back(ip4.toIpString()); - member["ipAssignments"] = ipAssignments; - if (nc.staticIpCount < ZT_MAX_ZT_ASSIGNED_ADDRESSES) { - struct sockaddr_in *const v4ip = reinterpret_cast(&(nc.staticIps[nc.staticIpCount++])); - v4ip->sin_family = AF_INET; - v4ip->sin_port = Utils::hton((uint16_t)routedNetmaskBits); - v4ip->sin_addr.s_addr = Utils::hton(ip); - } - haveManagedIpv4AutoAssignment = true; - break; - } - } - } - } - } - } - - if (network.value("private",true)) { - CertificateOfMembership com(now,ZT_NETWORK_COM_DEFAULT_REVISION_MAX_DELTA,nwid,identity.address()); - if (com.sign(signingId)) { - nc.com = com; - } else { - return NETCONF_QUERY_INTERNAL_SERVER_ERROR; - } - } - - _writeJson(memberJP,member); - return NetworkController::NETCONF_QUERY_OK; -} - -unsigned int SqliteNetworkController::handleControlPlaneHttpGET( - const std::vector &path, - const std::map &urlArgs, - const std::map &headers, - const std::string &body, - std::string &responseBody, - std::string &responseContentType) -{ - if ((path.size() > 0)&&(path[0] == "network")) { - - if ((path.size() >= 2)&&(path[1].length() == 16)) { - const uint64_t nwid = Utils::hexStrToU64(path[1].c_str()); - char nwids[24]; - Utils::snprintf(nwids,sizeof(nwids),"%.16llx",(unsigned long long)nwid); - - json network(_readJson(_networkJP(nwid,false))); - if (!network.size()) - return 404; - - if (path.size() >= 3) { - - if (path[2] == "member") { - - if (path.size() >= 4) { - const uint64_t address = Utils::hexStrToU64(path[3].c_str()); - - json member(_readJson(_memberJP(nwid,Address(address),false))); - if (!member.size()) - return 404; - - char addrs[24]; - Utils::snprintf(addrs,sizeof(addrs),"%.10llx",address); - - member["clock"] = OSUtils::now(); - responseBody = member.dump(2); - responseContentType = "application/json"; - - return 200; - } else { - - responseBody = "{"; - std::vector members(OSUtils::listSubdirectories((_networkBP(nwid,false) + ZT_PATH_SEPARATOR_S + "member").c_str())); - for(std::vector::iterator i(members.begin());i!=members.end();++i) { - if (i->length() == ZT_ADDRESS_LENGTH_HEX) { - json member(_readJson(_memberJP(nwid,Address(Utils::hexStrToU64(i->c_str())),false))); - if (member.size()) { - responseBody.append((responseBody.length() == 1) ? "\"" : ",\""); - responseBody.append(*i); - responseBody.append("\":"); - const std::string rc = member.value("memberRevision","0"); - responseBody.append(rc); - } - } - } - responseBody.push_back('}'); - responseContentType = "application/json"; - - return 200; - } - - } else if ((path[2] == "active")&&(path.size() == 3)) { - - responseBody = "{"; - std::vector members(OSUtils::listSubdirectories((_networkBP(nwid,false) + ZT_PATH_SEPARATOR_S + "member").c_str())); - const uint64_t threshold = OSUtils::now() - ZT_NETCONF_NODE_ACTIVE_THRESHOLD; - for(std::vector::iterator i(members.begin());i!=members.end();++i) { - if (i->length() == ZT_ADDRESS_LENGTH_HEX) { - json member(_readJson(_memberJP(nwid,Address(Utils::hexStrToU64(i->c_str())),false))); - if (member.size()) { - auto recentLog = member["recentLog"]; - if ((recentLog.is_array())&&(recentLog.size() > 0)) { - auto mostRecentLog = recentLog[0]; - if ((mostRecentLog.is_object())&&((uint64_t)mostRecentLog.value("ts",0ULL) >= threshold)) { - responseBody.append((responseBody.length() == 1) ? "\"" : ",\""); - responseBody.append(*i); - responseBody.append("\":"); - responseBody.append(mostRecentLog.dump()); - } - } - } - } - } - responseBody.push_back('}'); - responseContentType = "application/json"; - return 200; - - } else if ((path[2] == "test")&&(path.size() >= 4)) { - - Mutex::Lock _l(_circuitTests_m); - std::map< uint64_t,_CircuitTestEntry >::iterator cte(_circuitTests.find(Utils::hexStrToU64(path[3].c_str()))); - if ((cte != _circuitTests.end())&&(cte->second.test)) { - - responseBody = "["; - responseBody.append(cte->second.jsonResults); - responseBody.push_back(']'); - responseContentType = "application/json"; - - return 200; - - } // else 404 - - } // else 404 - - } else { - - nlohmann::json o(network); - o["clock"] = OSUtils::now(); - responseBody = o.dump(2); - responseContentType = "application/json"; - return 200; - - } - } else if (path.size() == 1) { - - responseBody = "["; - std::vector networks(OSUtils::listSubdirectories((_path + ZT_PATH_SEPARATOR_S + "network").c_str())); - for(auto i(networks.begin());i!=networks.end();++i) { - if (i->length() == 16) { - responseBody.append((responseBody.length() == 1) ? "\"" : ",\""); - responseBody.append(*i); - responseBody.append("\""); - } - } - responseBody.push_back(']'); - responseContentType = "application/json"; - return 200; - - } // else 404 - - } else { - - char tmp[4096]; - Utils::snprintf(tmp,sizeof(tmp),"{\n\t\"controller\": true,\n\t\"apiVersion\": %d,\n\t\"clock\": %llu\n}\n",ZT_NETCONF_CONTROLLER_API_VERSION,(unsigned long long)OSUtils::now()); - responseBody = tmp; - responseContentType = "application/json"; - return 200; - - } - - return 404; -} - -unsigned int SqliteNetworkController::handleControlPlaneHttpPOST( - const std::vector &path, - const std::map &urlArgs, - const std::map &headers, - const std::string &body, - std::string &responseBody, - std::string &responseContentType) -{ - if (path.empty()) - return 404; - - json b; - try { - b = json::parse(body); - if (!b.is_object()) - return 400; - } catch ( ... ) { - return 400; - } - - if (path[0] == "network") { - - if ((path.size() >= 2)&&(path[1].length() == 16)) { - uint64_t nwid = Utils::hexStrToU64(path[1].c_str()); - char nwids[24]; - Utils::snprintf(nwids,sizeof(nwids),"%.16llx",(unsigned long long)nwid); - - if (path.size() >= 3) { - json network(_readJson(_networkJP(nwid,false))); - if (!network.size()) - return 404; - - if ((path.size() == 4)&&(path[2] == "member")&&(path[3].length() == 10)) { - uint64_t address = Utils::hexStrToU64(path[3].c_str()); - char addrs[24]; - Utils::snprintf(addrs,sizeof(addrs),"%.10llx",(unsigned long long)address); - - json member(_readJson(_memberJP(nwid,Address(address),true))); - - try { - if (b.count("authorized")) member["authorized"] = b.value("authorized",false); - if ((b.count("identity"))&&(!member.count("identity"))) member["identity"] = b.value("identity",""); // allow identity to be populated only if not already known - - if (b.count("ipAssignments")) { - auto ipa = b["ipAssignments"]; - if (ipa.is_array()) { - json mipa(json::array()); - for(unsigned long i=0;itestId),sizeof(test->testId)); - test->credentialNetworkId = nwid; - test->ptr = (void *)this; - json hops = b["hops"]; - if (hops.is_array()) { - for(unsigned long i=0;ihops[test->hopCount].addresses[test->hops[test->hopCount].breadth++] = Utils::hexStrToU64(s.c_str()) & 0xffffffffffULL; - } - } else if (hops2.is_string()) { - std::string s = hops2; - test->hops[test->hopCount].addresses[test->hops[test->hopCount].breadth++] = Utils::hexStrToU64(s.c_str()) & 0xffffffffffULL; - } - } - } - test->reportAtEveryHop = (b.value("reportAtEveryHop",true) ? 1 : 0); - - if (!test->hopCount) { - ::free((void *)test); - return 400; - } - - test->timestamp = OSUtils::now(); - - _CircuitTestEntry &te = _circuitTests[test->testId]; - te.test = test; - te.jsonResults = ""; - - _node->circuitTestBegin(test,&(SqliteNetworkController::_circuitTestCallback)); - - char json[1024]; - Utils::snprintf(json,sizeof(json),"{\"testId\":\"%.16llx\"}",test->testId); - responseBody = json; - responseContentType = "application/json"; - return 200; - - } // else 404 - - } else { - // POST to network ID - - // Magic ID ending with ______ picks a random unused network ID - if (path[1].substr(10) == "______") { - nwid = 0; - uint64_t nwidPrefix = (Utils::hexStrToU64(path[1].substr(0,10).c_str()) << 24) & 0xffffffffff000000ULL; - uint64_t nwidPostfix = 0; - for(unsigned long k=0;k<100000;++k) { // sanity limit on trials - Utils::getSecureRandom(&nwidPostfix,sizeof(nwidPostfix)); - uint64_t tryNwid = nwidPrefix | (nwidPostfix & 0xffffffULL); - if ((tryNwid & 0xffffffULL) == 0ULL) tryNwid |= 1ULL; - Utils::snprintf(nwids,sizeof(nwids),"%.16llx",(unsigned long long)tryNwid); - if (!OSUtils::fileExists(_networkJP(tryNwid,false).c_str())) { - nwid = tryNwid; - break; - } - } - if (!nwid) - return 503; - } - - json network(_readJson(_networkJP(nwid,true))); - - try { - if (b.count("name")) network["name"] = b.value("name",""); - if (b.count("private")) network["private"] = b.value("private",true); - if (b.count("enableBroadcast")) network["enableBroadcast"] = b.value("enableBroadcast",false); - if (b.count("allowPassiveBridging")) network["allowPassiveBridging"] = b.value("allowPassiveBridging",false); - if (b.count("multicastLimit")) network["multicastLimit"] = b.value("multicastLimit",32ULL); - - if (b.count("activeBridges")) { - auto ab = b["activeBridges"]; - if (ab.is_array()) { - json ab2 = json::array(); - for(unsigned long i=0;i v6m(Utils::split(b.value("v6AssignMode","").c_str(),",","","")); - std::sort(v6m.begin(),v6m.end()); - v6m.erase(std::unique(v6m.begin(),v6m.end()),v6m.end()); - for(std::vector::iterator i(v6m.begin());i!=v6m.end();++i) { - if (*i == "rfc4193") - nv6m["rfc4193"] = true; - else if (*i == "zt") - nv6m["zt"] = true; - else if (*i == "6plane") - nv6m["6plane"] = true; - } - } else if (b["v6AssignMode"].is_object()) { - auto v6m = b["v6AssignMode"]; - if (v6m.count("rfc4193")) nv6m["rfc4193"] = v6m.value("rfc4193",false); - if (v6m.count("zt")) nv6m["rfc4193"] = v6m.value("zt",false); - if (v6m.count("6plane")) nv6m["rfc4193"] = v6m.value("6plane",false); - } - if (!nv6m.count("rfc4193")) nv6m["rfc4193"] = false; - if (!nv6m.count("zt")) nv6m["zt"] = false; - if (!nv6m.count("6plane")) nv6m["6plane"] = false; - } - - if (b.count("routes")) { - auto rts = b["routes"]; - if (rts.is_array()) { - for(unsigned long i=0;i &path, - const std::map &urlArgs, - const std::map &headers, - const std::string &body, - std::string &responseBody, - std::string &responseContentType) -{ - if (path.empty()) - return 404; - - if (path[0] == "network") { - if ((path.size() >= 2)&&(path[1].length() == 16)) { - const uint64_t nwid = Utils::hexStrToU64(path[1].c_str()); - - json network(_readJson(_networkJP(nwid,false))); - if (!network.size()) - return 404; - - if (path.size() >= 3) { - if ((path.size() == 4)&&(path[2] == "member")&&(path[3].length() == 10)) { - const uint64_t address = Utils::hexStrToU64(path[3].c_str()); - - json member(_readJson(_memberJP(nwid,Address(address),false))); - if (!member.size()) - return 404; - - OSUtils::rmDashRf(_memberBP(nwid,Address(address),false).c_str()); - - responseBody = member.dump(2); - responseContentType = "application/json"; - return 200; - } - } else { - OSUtils::rmDashRf(_networkBP(nwid,false).c_str()); - responseBody = network.dump(2); - responseContentType = "application/json"; - return 200; - } - } // else 404 - - } // else 404 - - return 404; -} - -void SqliteNetworkController::_circuitTestCallback(ZT_Node *node,ZT_CircuitTest *test,const ZT_CircuitTestReport *report) -{ - char tmp[65535]; - SqliteNetworkController *const self = reinterpret_cast(test->ptr); - - if (!test) - return; - if (!report) - return; - - Mutex::Lock _l(self->_circuitTests_m); - std::map< uint64_t,_CircuitTestEntry >::iterator cte(self->_circuitTests.find(test->testId)); - - if (cte == self->_circuitTests.end()) { // sanity check: a circuit test we didn't launch? - self->_node->circuitTestEnd(test); - ::free((void *)test); - return; - } - - Utils::snprintf(tmp,sizeof(tmp), - "%s{\n" - "\t\"timestamp\": %llu," ZT_EOL_S - "\t\"testId\": \"%.16llx\"," ZT_EOL_S - "\t\"upstream\": \"%.10llx\"," ZT_EOL_S - "\t\"current\": \"%.10llx\"," ZT_EOL_S - "\t\"receivedTimestamp\": %llu," ZT_EOL_S - "\t\"remoteTimestamp\": %llu," ZT_EOL_S - "\t\"sourcePacketId\": \"%.16llx\"," ZT_EOL_S - "\t\"flags\": %llu," ZT_EOL_S - "\t\"sourcePacketHopCount\": %u," ZT_EOL_S - "\t\"errorCode\": %u," ZT_EOL_S - "\t\"vendor\": %d," ZT_EOL_S - "\t\"protocolVersion\": %u," ZT_EOL_S - "\t\"majorVersion\": %u," ZT_EOL_S - "\t\"minorVersion\": %u," ZT_EOL_S - "\t\"revision\": %u," ZT_EOL_S - "\t\"platform\": %d," ZT_EOL_S - "\t\"architecture\": %d," ZT_EOL_S - "\t\"receivedOnLocalAddress\": \"%s\"," ZT_EOL_S - "\t\"receivedFromRemoteAddress\": \"%s\"" ZT_EOL_S - "}", - ((cte->second.jsonResults.length() > 0) ? ",\n" : ""), - (unsigned long long)report->timestamp, - (unsigned long long)test->testId, - (unsigned long long)report->upstream, - (unsigned long long)report->current, - (unsigned long long)OSUtils::now(), - (unsigned long long)report->remoteTimestamp, - (unsigned long long)report->sourcePacketId, - (unsigned long long)report->flags, - report->sourcePacketHopCount, - report->errorCode, - (int)report->vendor, - report->protocolVersion, - report->majorVersion, - report->minorVersion, - report->revision, - (int)report->platform, - (int)report->architecture, - reinterpret_cast(&(report->receivedOnLocalAddress))->toString().c_str(), - reinterpret_cast(&(report->receivedFromRemoteAddress))->toString().c_str()); - - cte->second.jsonResults.append(tmp); -} - -} // namespace ZeroTier diff --git a/controller/SqliteNetworkController.hpp b/controller/SqliteNetworkController.hpp deleted file mode 100644 index 288ea23a..00000000 --- a/controller/SqliteNetworkController.hpp +++ /dev/null @@ -1,173 +0,0 @@ -/* - * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2015 ZeroTier, Inc. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - */ - -#ifndef ZT_SQLITENETWORKCONTROLLER_HPP -#define ZT_SQLITENETWORKCONTROLLER_HPP - -#include - -#include -#include -#include -#include - -#include "../node/Constants.hpp" - -#include "../node/NetworkController.hpp" -#include "../node/Mutex.hpp" -#include "../node/Utils.hpp" -#include "../node/InetAddress.hpp" - -#include "../osdep/OSUtils.hpp" - -#include "../ext/json/json.hpp" - -namespace ZeroTier { - -class Node; - -class SqliteNetworkController : public NetworkController -{ -public: - SqliteNetworkController(Node *node,const char *dbPath); - virtual ~SqliteNetworkController(); - - virtual NetworkController::ResultCode doNetworkConfigRequest( - const InetAddress &fromAddr, - const Identity &signingId, - const Identity &identity, - uint64_t nwid, - const Dictionary &metaData, - NetworkConfig &nc); - - unsigned int handleControlPlaneHttpGET( - const std::vector &path, - const std::map &urlArgs, - const std::map &headers, - const std::string &body, - std::string &responseBody, - std::string &responseContentType); - unsigned int handleControlPlaneHttpPOST( - const std::vector &path, - const std::map &urlArgs, - const std::map &headers, - const std::string &body, - std::string &responseBody, - std::string &responseContentType); - unsigned int handleControlPlaneHttpDELETE( - const std::vector &path, - const std::map &urlArgs, - const std::map &headers, - const std::string &body, - std::string &responseBody, - std::string &responseContentType); - -private: - static void _circuitTestCallback(ZT_Node *node,ZT_CircuitTest *test,const ZT_CircuitTestReport *report); - - inline nlohmann::json _readJson(const std::string &path) - { - std::string buf; - if (OSUtils::readFile(path.c_str(),buf)) { - try { - return nlohmann::json::parse(buf); - } catch ( ... ) {} - } - return nlohmann::json::object(); - } - - inline bool _writeJson(const std::string &path,const nlohmann::json &obj) - { - std::string buf(obj.dump(2)); - return OSUtils::writeFile(path.c_str(),buf); - } - - inline std::string _networkBP(const uint64_t nwid,bool create) - { - char tmp[64]; - Utils::snprintf(tmp,sizeof(tmp),"%.16llx",nwid); - std::string p(_path + ZT_PATH_SEPARATOR_S + "network"); - if (create) OSUtils::mkdir(p.c_str()); - p.push_back(ZT_PATH_SEPARATOR); - p.append(tmp); - if (create) OSUtils::mkdir(p.c_str()); - return p; - } - inline std::string _networkJP(const uint64_t nwid,bool create) - { - return (_networkBP(nwid,create) + ZT_PATH_SEPARATOR + "config.json"); - } - inline std::string _memberBP(const uint64_t nwid,const Address &member,bool create) - { - std::string p(_networkBP(nwid,create)); - p.push_back(ZT_PATH_SEPARATOR); - p.append("member"); - if (create) OSUtils::mkdir(p.c_str()); - p.push_back(ZT_PATH_SEPARATOR); - p.append(member.toString()); - if (create) OSUtils::mkdir(p.c_str()); - return p; - } - inline std::string _memberJP(const uint64_t nwid,const Address &member,bool create) - { - return (_memberBP(nwid,member,create) + ZT_PATH_SEPARATOR + "config.json"); - } - - inline std::set _getAlreadyAllocatedIps(const uint64_t nwid) - { - std::set ips; - std::string bp(_networkBP(nwid,false) + ZT_PATH_SEPARATOR_S + "member"); - std::vector members(OSUtils::listSubdirectories(bp.c_str())); - for(std::vector::iterator m(members.begin());m!=members.end();++m) { - if (m->length() == ZT_ADDRESS_LENGTH_HEX) { - nlohmann::json mj = _readJson(bp + ZT_PATH_SEPARATOR_S + *m + ZT_PATH_SEPARATOR_S + "config.json"); - auto ipAssignments = mj["ipAssignments"]; - if (ipAssignments.is_array()) { - for(unsigned long i=0;i _circuitTests; - Mutex _circuitTests_m; - - // Last request time by address, for rate limitation - std::map< std::pair,uint64_t > _lastRequestTime; - Mutex _lastRequestTime_m; -}; - -} // namespace ZeroTier - -#endif diff --git a/make-linux.mk b/make-linux.mk index acc22a63..d0d206a1 100644 --- a/make-linux.mk +++ b/make-linux.mk @@ -82,12 +82,6 @@ ifeq ($(ZT_USE_MINIUPNPC),1) endif endif -ifeq ($(ZT_ENABLE_NETWORK_CONTROLLER),1) - DEFS+=-DZT_ENABLE_NETWORK_CONTROLLER - LDLIBS+=-L/usr/local/lib -lsqlite3 - OBJS+=controller/SqliteNetworkController.o -endif - ifeq ($(ZT_ENABLE_CLUSTER),1) DEFS+=-DZT_ENABLE_CLUSTER endif diff --git a/make-mac.mk b/make-mac.mk index 63ffc3c8..f00f8d6b 100644 --- a/make-mac.mk +++ b/make-mac.mk @@ -45,12 +45,6 @@ ifeq ($(ZT_USE_MINIUPNPC),1) OBJS+=ext/libnatpmp/natpmp.o ext/libnatpmp/getgateway.o ext/miniupnpc/connecthostport.o ext/miniupnpc/igd_desc_parse.o ext/miniupnpc/minisoap.o ext/miniupnpc/minissdpc.o ext/miniupnpc/miniupnpc.o ext/miniupnpc/miniwget.o ext/miniupnpc/minixml.o ext/miniupnpc/portlistingparse.o ext/miniupnpc/receivedata.o ext/miniupnpc/upnpcommands.o ext/miniupnpc/upnpdev.o ext/miniupnpc/upnperrors.o ext/miniupnpc/upnpreplyparse.o osdep/PortMapper.o endif -ifeq ($(ZT_ENABLE_NETWORK_CONTROLLER),1) - DEFS+=-DZT_ENABLE_NETWORK_CONTROLLER - LIBS+=-L/usr/local/lib -lsqlite3 - OBJS+=controller/SqliteNetworkController.o -endif - # Debug mode -- dump trace output, build binary with -g ifeq ($(ZT_DEBUG),1) DEFS+=-DZT_TRACE diff --git a/objects.mk b/objects.mk index 11b03c81..f92a907e 100644 --- a/objects.mk +++ b/objects.mk @@ -1,4 +1,5 @@ OBJS=\ + controller/EmbeddedNetworkController.o \ node/C25519.o \ node/Capability.o \ node/CertificateOfMembership.o \ diff --git a/selftest.cpp b/selftest.cpp index f4232851..ab05c9ef 100644 --- a/selftest.cpp +++ b/selftest.cpp @@ -53,10 +53,6 @@ #include "osdep/PortMapper.hpp" #include "osdep/Thread.hpp" -#ifdef ZT_ENABLE_NETWORK_CONTROLLER -#include "controller/SqliteNetworkController.hpp" -#endif // ZT_ENABLE_NETWORK_CONTROLLER - #ifdef __WINDOWS__ #include #endif diff --git a/service/ControlPlane.cpp b/service/ControlPlane.cpp index a10697a9..7aa757a9 100644 --- a/service/ControlPlane.cpp +++ b/service/ControlPlane.cpp @@ -34,9 +34,7 @@ #include "../ext/json-parser/json.h" #endif -#ifdef ZT_ENABLE_NETWORK_CONTROLLER -#include "../controller/SqliteNetworkController.hpp" -#endif +#include "../controller/EmbeddedNetworkController.hpp" #include "../node/InetAddress.hpp" #include "../node/Node.hpp" @@ -254,9 +252,7 @@ static void _jsonAppend(unsigned int depth,std::string &buf,const ZT_Peer *peer) ControlPlane::ControlPlane(OneService *svc,Node *n,const char *uiStaticPath) : _svc(svc), _node(n), -#ifdef ZT_ENABLE_NETWORK_CONTROLLER - _controller((SqliteNetworkController *)0), -#endif + _controller((EmbeddedNetworkController *)0), _uiStaticPath((uiStaticPath) ? uiStaticPath : "") { } @@ -499,13 +495,9 @@ unsigned int ControlPlane::handleRequest( responseContentType = "text/plain"; scode = 200; } else { -#ifdef ZT_ENABLE_NETWORK_CONTROLLER if (_controller) scode = _controller->handleControlPlaneHttpGET(std::vector(ps.begin()+1,ps.end()),urlArgs,headers,body,responseBody,responseContentType); else scode = 404; -#else - scode = 404; -#endif } } else scode = 401; // isAuth == false @@ -559,13 +551,9 @@ unsigned int ControlPlane::handleRequest( } else scode = 500; } } else { -#ifdef ZT_ENABLE_NETWORK_CONTROLLER if (_controller) scode = _controller->handleControlPlaneHttpPOST(std::vector(ps.begin()+1,ps.end()),urlArgs,headers,body,responseBody,responseContentType); else scode = 404; -#else - scode = 404; -#endif } } else scode = 401; // isAuth == false @@ -594,13 +582,9 @@ unsigned int ControlPlane::handleRequest( _node->freeQueryResult((void *)nws); } else scode = 500; } else { -#ifdef ZT_ENABLE_NETWORK_CONTROLLER if (_controller) scode = _controller->handleControlPlaneHttpDELETE(std::vector(ps.begin()+1,ps.end()),urlArgs,headers,body,responseBody,responseContentType); else scode = 404; -#else - scode = 404; -#endif } } else { diff --git a/service/ControlPlane.hpp b/service/ControlPlane.hpp index 08a9d6e0..ec9b94d7 100644 --- a/service/ControlPlane.hpp +++ b/service/ControlPlane.hpp @@ -31,7 +31,7 @@ namespace ZeroTier { class OneService; class Node; -class SqliteNetworkController; +class EmbeddedNetworkController; struct InetAddress; /** @@ -43,18 +43,16 @@ public: ControlPlane(OneService *svc,Node *n,const char *uiStaticPath); ~ControlPlane(); -#ifdef ZT_ENABLE_NETWORK_CONTROLLER /** * Set controller, which will be available under /controller * * @param c Network controller instance */ - inline void setController(SqliteNetworkController *c) + inline void setController(EmbeddedNetworkController *c) { Mutex::Lock _l(_lock); _controller = c; } -#endif /** * Add an authentication token for API access @@ -89,9 +87,7 @@ public: private: OneService *const _svc; Node *const _node; -#ifdef ZT_ENABLE_NETWORK_CONTROLLER - SqliteNetworkController *_controller; -#endif + EmbeddedNetworkController *_controller; std::string _uiStaticPath; std::set _authTokens; Mutex _lock; diff --git a/service/OneService.cpp b/service/OneService.cpp index 5c65dcc2..74628168 100644 --- a/service/OneService.cpp +++ b/service/OneService.cpp @@ -69,11 +69,7 @@ */ //#define ZT_BREAK_UDP -#ifdef ZT_ENABLE_NETWORK_CONTROLLER -#include "../controller/SqliteNetworkController.hpp" -#else -class SqliteNetworkController; -#endif // ZT_ENABLE_NETWORK_CONTROLLER +#include "../controller/EmbeddedNetworkController.hpp" #ifdef __WINDOWS__ #include @@ -129,7 +125,7 @@ namespace ZeroTier { typedef BSDEthernetTap EthernetTap; } #define ZT_TAP_CHECK_MULTICAST_INTERVAL 5000 // Path under ZT1 home for controller database if controller is enabled -#define ZT_CONTROLLER_DB_PATH "controller.db" +#define ZT_CONTROLLER_DB_PATH "controller.d" // TCP fallback relay host -- geo-distributed using Amazon Route53 geo-aware DNS #define ZT_TCP_FALLBACK_RELAY "tcp-fallback.zerotier.com" @@ -487,9 +483,7 @@ public: const std::string _homePath; BackgroundResolver _tcpFallbackResolver; -#ifdef ZT_ENABLE_NETWORK_CONTROLLER - SqliteNetworkController *_controller; -#endif + EmbeddedNetworkController *_controller; Phy _phy; Node *_node; @@ -579,9 +573,7 @@ public: OneServiceImpl(const char *hp,unsigned int port) : _homePath((hp) ? hp : ".") ,_tcpFallbackResolver(ZT_TCP_FALLBACK_RELAY) -#ifdef ZT_ENABLE_NETWORK_CONTROLLER - ,_controller((SqliteNetworkController *)0) -#endif + ,_controller((EmbeddedNetworkController *)0) ,_phy(this,false,true) ,_node((Node *)0) ,_controlPlane((ControlPlane *)0) @@ -673,9 +665,7 @@ public: #ifdef ZT_USE_MINIUPNPC delete _portMapper; #endif -#ifdef ZT_ENABLE_NETWORK_CONTROLLER delete _controller; -#endif #ifdef ZT_ENABLE_CLUSTER delete _clusterDefinition; #endif @@ -794,10 +784,8 @@ public: } } -#ifdef ZT_ENABLE_NETWORK_CONTROLLER - _controller = new SqliteNetworkController(_node,(_homePath + ZT_PATH_SEPARATOR_S + ZT_CONTROLLER_DB_PATH).c_str(),(_homePath + ZT_PATH_SEPARATOR_S + "circuitTestResults.d").c_str()); + _controller = new EmbeddedNetworkController(_node,(_homePath + ZT_PATH_SEPARATOR_S + ZT_CONTROLLER_DB_PATH).c_str()); _node->setNetconfMaster((void *)_controller); -#endif #ifdef ZT_ENABLE_CLUSTER if (OSUtils::fileExists((_homePath + ZT_PATH_SEPARATOR_S + "cluster").c_str())) { @@ -850,10 +838,7 @@ public: _controlPlane = new ControlPlane(this,_node,(_homePath + ZT_PATH_SEPARATOR_S + "ui").c_str()); _controlPlane->addAuthToken(authToken.c_str()); - -#ifdef ZT_ENABLE_NETWORK_CONTROLLER _controlPlane->setController(_controller); -#endif { // Remember networks from previous session std::vector networksDotD(OSUtils::listDirectory((_homePath + ZT_PATH_SEPARATOR_S + "networks.d").c_str())); diff --git a/service/OneService.hpp b/service/OneService.hpp index cead381d..72ff7d84 100644 --- a/service/OneService.hpp +++ b/service/OneService.hpp @@ -25,18 +25,6 @@ namespace ZeroTier { /** * Local service for ZeroTier One as system VPN/NFV provider - * - * If built with ZT_ENABLE_NETWORK_CONTROLLER defined, this includes and - * runs controller/SqliteNetworkController with a database called - * controller.db in the specified home directory. - * - * If built with ZT_AUTO_UPDATE, an official ZeroTier update URL is - * periodically checked and updates are automatically downloaded, verified - * against a built-in list of update signing keys, and installed. This is - * only supported for certain platforms. - * - * If built with ZT_ENABLE_CLUSTER, a 'cluster' file is checked and if - * present is read to determine the identity of other cluster members. */ class OneService { -- cgit v1.2.3 From 8e3463d47a8e7565784f349f359ebe7f4a4d0e57 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Wed, 24 Aug 2016 13:37:57 -0700 Subject: Add length limit to TEE and REDIRECT, and completely factor out old C json-parser to eliminate a dependency. --- AUTHORS.md | 8 +- controller/EmbeddedNetworkController.cpp | 18 +- ext/json-parser/AUTHORS | 20 - ext/json-parser/LICENSE | 26 - ext/json-parser/README.md | 97 --- ext/json-parser/json.c | 1012 ------------------------------ ext/json-parser/json.h | 283 --------- include/ZeroTierOne.h | 14 + make-freebsd.mk | 2 +- make-linux.mk | 6 - make-mac.mk | 2 +- node/Capability.hpp | 9 + node/Network.cpp | 6 +- node/Packet.hpp | 1 + one.cpp | 288 +++------ service/ControlPlane.cpp | 33 +- 16 files changed, 158 insertions(+), 1667 deletions(-) delete mode 100644 ext/json-parser/AUTHORS delete mode 100644 ext/json-parser/LICENSE delete mode 100644 ext/json-parser/README.md delete mode 100644 ext/json-parser/json.c delete mode 100644 ext/json-parser/json.h (limited to 'make-mac.mk') diff --git a/AUTHORS.md b/AUTHORS.md index aa9e9111..90a64ef6 100644 --- a/AUTHORS.md +++ b/AUTHORS.md @@ -39,11 +39,11 @@ These are included in ext/ for platforms that do not have them available in comm * Home page: https://github.com/joyent/http-parser/ * License grant: MIT/Expat - * json-parser by James McLaughlin + * C++11 json (nlohmann/json) by Niels Lohmann - * Files: ext/json-parser/* - * Home page: https://github.com/udp/json-parser/ - * License grant: BSD attribution + * Files: ext/json/* + * Home page: https://github.com/nlohmann/json + * License grant: MIT * TunTapOSX by Mattias Nissler diff --git a/controller/EmbeddedNetworkController.cpp b/controller/EmbeddedNetworkController.cpp index 4e8d1aad..0a4a17f8 100644 --- a/controller/EmbeddedNetworkController.cpp +++ b/controller/EmbeddedNetworkController.cpp @@ -121,11 +121,15 @@ static json _renderRule(ZT_VirtualNetworkRule &rule) break; case ZT_NETWORK_RULE_ACTION_TEE: r["type"] = "ACTION_TEE"; - r["zt"] = Address(rule.v.zt).toString(); + r["address"] = Address(rule.v.fwd.address).toString(); + r["flags"] = (uint64_t)rule.v.fwd.flags; + r["length"] = (uint64_t)rule.v.fwd.length; break; case ZT_NETWORK_RULE_ACTION_REDIRECT: r["type"] = "ACTION_REDIRECT"; - r["zt"] = Address(rule.v.zt).toString(); + r["address"] = Address(rule.v.fwd.address).toString(); + r["flags"] = (uint64_t)rule.v.fwd.flags; + r["length"] = (uint64_t)rule.v.fwd.length; break; case ZT_NETWORK_RULE_MATCH_SOURCE_ZEROTIER_ADDRESS: r["type"] = "MATCH_SOURCE_ZEROTIER_ADDRESS"; @@ -235,7 +239,7 @@ static bool _parseRule(const json &r,ZT_VirtualNetworkRule &rule) { if (r.is_object()) return false; - std::string t = r["type"]; + const std::string t(_jS(r["type"],"")); memset(&rule,0,sizeof(ZT_VirtualNetworkRule)); if (_jB(r["not"],false)) rule.t = 0x80; @@ -248,11 +252,15 @@ static bool _parseRule(const json &r,ZT_VirtualNetworkRule &rule) return true; } else if (t == "ACTION_TEE") { rule.t |= ZT_NETWORK_RULE_ACTION_TEE; - rule.v.zt = Utils::hexStrToU64(_jS(r["zt"],"0").c_str()) & 0xffffffffffULL; + rule.v.fwd.address = Utils::hexStrToU64(_jS(r["address"],"0").c_str()) & 0xffffffffffULL; + rule.v.fwd.flags = (uint32_t)(_jI(r["flags"],0ULL) & 0xffffffffULL); + rule.v.fwd.length = (uint16_t)(_jI(r["length"],0ULL) & 0xffffULL); return true; } else if (t == "ACTION_REDIRECT") { rule.t |= ZT_NETWORK_RULE_ACTION_REDIRECT; - rule.v.zt = Utils::hexStrToU64(_jS(r["zt"],"0").c_str()) & 0xffffffffffULL; + rule.v.fwd.address = Utils::hexStrToU64(_jS(r["zt"],"0").c_str()) & 0xffffffffffULL; + rule.v.fwd.flags = (uint32_t)(_jI(r["flags"],0ULL) & 0xffffffffULL); + rule.v.fwd.length = (uint16_t)(_jI(r["length"],0ULL) & 0xffffULL); return true; } else if (t == "MATCH_SOURCE_ZEROTIER_ADDRESS") { rule.t |= ZT_NETWORK_RULE_MATCH_SOURCE_ZEROTIER_ADDRESS; diff --git a/ext/json-parser/AUTHORS b/ext/json-parser/AUTHORS deleted file mode 100644 index 6a5c799f..00000000 --- a/ext/json-parser/AUTHORS +++ /dev/null @@ -1,20 +0,0 @@ -All contributors arranged by first commit: - -James McLaughlin -Alex Gartrell -Peter Scott -Mathias Kaerlev -Emiel Mols -Czarek Tomczak -Nicholas Braden -Ivan Kozub -Árpád Goretity -Igor Gnatenko -Haïkel Guémar -Tobias Waldekranz -Patrick Donnelly -Wilmer van der Gaast -Jin Wei -François Cartegnie -Matthijs Boelstra - diff --git a/ext/json-parser/LICENSE b/ext/json-parser/LICENSE deleted file mode 100644 index 1aee375e..00000000 --- a/ext/json-parser/LICENSE +++ /dev/null @@ -1,26 +0,0 @@ - - Copyright (C) 2012, 2013 James McLaughlin et al. All rights reserved. - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions - are met: - - 1. Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - - 2. Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - - THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND - ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE - FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS - OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT - LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY - OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF - SUCH DAMAGE. - diff --git a/ext/json-parser/README.md b/ext/json-parser/README.md deleted file mode 100644 index e0b70b6d..00000000 --- a/ext/json-parser/README.md +++ /dev/null @@ -1,97 +0,0 @@ -Very low footprint JSON parser written in portable ANSI C. - -* BSD licensed with no dependencies (i.e. just drop the C file into your project) -* Never recurses or allocates more memory than it needs -* Very simple API with operator sugar for C++ - -[![Build Status](https://secure.travis-ci.org/udp/json-parser.png)](http://travis-ci.org/udp/json-parser) - -_Want to serialize? Check out [json-builder](https://github.com/udp/json-builder)!_ - -Installing ----------- - -There is now a makefile which will produce a libjsonparser static and dynamic library. However, this -is _not_ required to build json-parser, and the source files (`json.c` and `json.h`) should be happy -in any build system you already have in place. - - -API ---- - - json_value * json_parse (const json_char * json, - size_t length); - - json_value * json_parse_ex (json_settings * settings, - const json_char * json, - size_t length, - char * error); - - void json_value_free (json_value *); - -The `type` field of `json_value` is one of: - -* `json_object` (see `u.object.length`, `u.object.values[x].name`, `u.object.values[x].value`) -* `json_array` (see `u.array.length`, `u.array.values`) -* `json_integer` (see `u.integer`) -* `json_double` (see `u.dbl`) -* `json_string` (see `u.string.ptr`, `u.string.length`) -* `json_boolean` (see `u.boolean`) -* `json_null` - - -Compile-Time Options --------------------- - - -DJSON_TRACK_SOURCE - -Stores the source location (line and column number) inside each `json_value`. - -This is useful for application-level error reporting. - - -Runtime Options ---------------- - - settings |= json_enable_comments; - -Enables C-style `// line` and `/* block */` comments. - - size_t value_extra - -The amount of space (if any) to allocate at the end of each `json_value`, in -order to give the application space to add metadata. - - void * (* mem_alloc) (size_t, int zero, void * user_data); - void (* mem_free) (void *, void * user_data); - -Custom allocator routines. If NULL, the default `malloc` and `free` will be used. - -The `user_data` pointer will be forwarded from `json_settings` to allow application -context to be passed. - - -Changes in version 1.1.0 ------------------------- - -* UTF-8 byte order marks are now skipped if present - -* Allows cross-compilation by honoring --host if given (@wkz) - -* Maximum size for error buffer is now exposed in header (@LB--) - -* GCC warning for `static` after `const` fixed (@batrick) - -* Optional support for C-style line and block comments added (@Jin-W-FS) - -* `name_length` field added to object values - -* It is now possible to retrieve the source line/column number of a parsed `json_value` when `JSON_TRACK_SOURCE` is enabled - -* The application may now extend `json_value` using the `value_extra` setting - -* Un-ambiguate pow call in the case of C++ overloaded pow (@fcartegnie) - -* Fix null pointer de-reference when a non-existing array is closed and no root value is present - - diff --git a/ext/json-parser/json.c b/ext/json-parser/json.c deleted file mode 100644 index 166cdcb6..00000000 --- a/ext/json-parser/json.c +++ /dev/null @@ -1,1012 +0,0 @@ -/* vim: set et ts=3 sw=3 sts=3 ft=c: - * - * Copyright (C) 2012, 2013, 2014 James McLaughlin et al. All rights reserved. - * https://github.com/udp/json-parser - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS - * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT - * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY - * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF - * SUCH DAMAGE. - */ - -#include "json.h" - -#ifdef _MSC_VER - #ifndef _CRT_SECURE_NO_WARNINGS - #define _CRT_SECURE_NO_WARNINGS - #endif - #pragma warning(disable:4996) -#endif - -const struct _json_value json_value_none; - -#include -#include -#include -#include - -typedef unsigned int json_uchar; - -static unsigned char hex_value (json_char c) -{ - if (isdigit(c)) - return c - '0'; - - switch (c) { - case 'a': case 'A': return 0x0A; - case 'b': case 'B': return 0x0B; - case 'c': case 'C': return 0x0C; - case 'd': case 'D': return 0x0D; - case 'e': case 'E': return 0x0E; - case 'f': case 'F': return 0x0F; - default: return 0xFF; - } -} - -typedef struct -{ - unsigned long used_memory; - - unsigned int uint_max; - unsigned long ulong_max; - - json_settings settings; - int first_pass; - - const json_char * ptr; - unsigned int cur_line, cur_col; - -} json_state; - -static void * default_alloc (size_t size, int zero, void * user_data) -{ - return zero ? calloc (1, size) : malloc (size); -} - -static void default_free (void * ptr, void * user_data) -{ - free (ptr); -} - -static void * json_alloc (json_state * state, unsigned long size, int zero) -{ - if ((state->ulong_max - state->used_memory) < size) - return 0; - - if (state->settings.max_memory - && (state->used_memory += size) > state->settings.max_memory) - { - return 0; - } - - return state->settings.mem_alloc (size, zero, state->settings.user_data); -} - -static int new_value (json_state * state, - json_value ** top, json_value ** root, json_value ** alloc, - json_type type) -{ - json_value * value; - int values_size; - - if (!state->first_pass) - { - value = *top = *alloc; - *alloc = (*alloc)->_reserved.next_alloc; - - if (!*root) - *root = value; - - switch (value->type) - { - case json_array: - - if (value->u.array.length == 0) - break; - - if (! (value->u.array.values = (json_value **) json_alloc - (state, value->u.array.length * sizeof (json_value *), 0)) ) - { - return 0; - } - - value->u.array.length = 0; - break; - - case json_object: - - if (value->u.object.length == 0) - break; - - values_size = sizeof (*value->u.object.values) * value->u.object.length; - - if (! (value->u.object.values = (json_object_entry *) json_alloc - (state, values_size + ((unsigned long) value->u.object.values), 0)) ) - { - return 0; - } - - value->_reserved.object_mem = (*(char **) &value->u.object.values) + values_size; - - value->u.object.length = 0; - break; - - case json_string: - - if (! (value->u.string.ptr = (json_char *) json_alloc - (state, (value->u.string.length + 1) * sizeof (json_char), 0)) ) - { - return 0; - } - - value->u.string.length = 0; - break; - - default: - break; - }; - - return 1; - } - - if (! (value = (json_value *) json_alloc - (state, sizeof (json_value) + state->settings.value_extra, 1))) - { - return 0; - } - - if (!*root) - *root = value; - - value->type = type; - value->parent = *top; - - #ifdef JSON_TRACK_SOURCE - value->line = state->cur_line; - value->col = state->cur_col; - #endif - - if (*alloc) - (*alloc)->_reserved.next_alloc = value; - - *alloc = *top = value; - - return 1; -} - -#define whitespace \ - case '\n': ++ state.cur_line; state.cur_col = 0; \ - case ' ': case '\t': case '\r' - -#define string_add(b) \ - do { if (!state.first_pass) string [string_length] = b; ++ string_length; } while (0); - -#define line_and_col \ - state.cur_line, state.cur_col - -static const long - flag_next = 1 << 0, - flag_reproc = 1 << 1, - flag_need_comma = 1 << 2, - flag_seek_value = 1 << 3, - flag_escaped = 1 << 4, - flag_string = 1 << 5, - flag_need_colon = 1 << 6, - flag_done = 1 << 7, - flag_num_negative = 1 << 8, - flag_num_zero = 1 << 9, - flag_num_e = 1 << 10, - flag_num_e_got_sign = 1 << 11, - flag_num_e_negative = 1 << 12, - flag_line_comment = 1 << 13, - flag_block_comment = 1 << 14; - -json_value * json_parse_ex (json_settings * settings, - const json_char * json, - size_t length, - char * error_buf) -{ - json_char error [json_error_max]; - const json_char * end; - json_value * top, * root, * alloc = 0; - json_state state = { 0 }; - long flags; - long num_digits = 0, num_e = 0; - json_int_t num_fraction = 0; - - /* Skip UTF-8 BOM - */ - if (length >= 3 && ((unsigned char) json [0]) == 0xEF - && ((unsigned char) json [1]) == 0xBB - && ((unsigned char) json [2]) == 0xBF) - { - json += 3; - length -= 3; - } - - error[0] = '\0'; - end = (json + length); - - memcpy (&state.settings, settings, sizeof (json_settings)); - - if (!state.settings.mem_alloc) - state.settings.mem_alloc = default_alloc; - - if (!state.settings.mem_free) - state.settings.mem_free = default_free; - - memset (&state.uint_max, 0xFF, sizeof (state.uint_max)); - memset (&state.ulong_max, 0xFF, sizeof (state.ulong_max)); - - state.uint_max -= 8; /* limit of how much can be added before next check */ - state.ulong_max -= 8; - - for (state.first_pass = 1; state.first_pass >= 0; -- state.first_pass) - { - json_uchar uchar; - unsigned char uc_b1, uc_b2, uc_b3, uc_b4; - json_char * string = 0; - unsigned int string_length = 0; - - top = root = 0; - flags = flag_seek_value; - - state.cur_line = 1; - - for (state.ptr = json ;; ++ state.ptr) - { - json_char b = (state.ptr == end ? 0 : *state.ptr); - - if (flags & flag_string) - { - if (!b) - { sprintf (error, "Unexpected EOF in string (at %d:%d)", line_and_col); - goto e_failed; - } - - if (string_length > state.uint_max) - goto e_overflow; - - if (flags & flag_escaped) - { - flags &= ~ flag_escaped; - - switch (b) - { - case 'b': string_add ('\b'); break; - case 'f': string_add ('\f'); break; - case 'n': string_add ('\n'); break; - case 'r': string_add ('\r'); break; - case 't': string_add ('\t'); break; - case 'u': - - if (end - state.ptr < 4 || - (uc_b1 = hex_value (*++ state.ptr)) == 0xFF || - (uc_b2 = hex_value (*++ state.ptr)) == 0xFF || - (uc_b3 = hex_value (*++ state.ptr)) == 0xFF || - (uc_b4 = hex_value (*++ state.ptr)) == 0xFF) - { - sprintf (error, "Invalid character value `%c` (at %d:%d)", b, line_and_col); - goto e_failed; - } - - uc_b1 = (uc_b1 << 4) | uc_b2; - uc_b2 = (uc_b3 << 4) | uc_b4; - uchar = (uc_b1 << 8) | uc_b2; - - if ((uchar & 0xF800) == 0xD800) { - json_uchar uchar2; - - if (end - state.ptr < 6 || (*++ state.ptr) != '\\' || (*++ state.ptr) != 'u' || - (uc_b1 = hex_value (*++ state.ptr)) == 0xFF || - (uc_b2 = hex_value (*++ state.ptr)) == 0xFF || - (uc_b3 = hex_value (*++ state.ptr)) == 0xFF || - (uc_b4 = hex_value (*++ state.ptr)) == 0xFF) - { - sprintf (error, "Invalid character value `%c` (at %d:%d)", b, line_and_col); - goto e_failed; - } - - uc_b1 = (uc_b1 << 4) | uc_b2; - uc_b2 = (uc_b3 << 4) | uc_b4; - uchar2 = (uc_b1 << 8) | uc_b2; - - uchar = 0x010000 | ((uchar & 0x3FF) << 10) | (uchar2 & 0x3FF); - } - - if (sizeof (json_char) >= sizeof (json_uchar) || (uchar <= 0x7F)) - { - string_add ((json_char) uchar); - break; - } - - if (uchar <= 0x7FF) - { - if (state.first_pass) - string_length += 2; - else - { string [string_length ++] = 0xC0 | (uchar >> 6); - string [string_length ++] = 0x80 | (uchar & 0x3F); - } - - break; - } - - if (uchar <= 0xFFFF) { - if (state.first_pass) - string_length += 3; - else - { string [string_length ++] = 0xE0 | (uchar >> 12); - string [string_length ++] = 0x80 | ((uchar >> 6) & 0x3F); - string [string_length ++] = 0x80 | (uchar & 0x3F); - } - - break; - } - - if (state.first_pass) - string_length += 4; - else - { string [string_length ++] = 0xF0 | (uchar >> 18); - string [string_length ++] = 0x80 | ((uchar >> 12) & 0x3F); - string [string_length ++] = 0x80 | ((uchar >> 6) & 0x3F); - string [string_length ++] = 0x80 | (uchar & 0x3F); - } - - break; - - default: - string_add (b); - }; - - continue; - } - - if (b == '\\') - { - flags |= flag_escaped; - continue; - } - - if (b == '"') - { - if (!state.first_pass) - string [string_length] = 0; - - flags &= ~ flag_string; - string = 0; - - switch (top->type) - { - case json_string: - - top->u.string.length = string_length; - flags |= flag_next; - - break; - - case json_object: - - if (state.first_pass) - (*(json_char **) &top->u.object.values) += string_length + 1; - else - { - top->u.object.values [top->u.object.length].name - = (json_char *) top->_reserved.object_mem; - - top->u.object.values [top->u.object.length].name_length - = string_length; - - (*(json_char **) &top->_reserved.object_mem) += string_length + 1; - } - - flags |= flag_seek_value | flag_need_colon; - continue; - - default: - break; - }; - } - else - { - string_add (b); - continue; - } - } - - if (state.settings.settings & json_enable_comments) - { - if (flags & (flag_line_comment | flag_block_comment)) - { - if (flags & flag_line_comment) - { - if (b == '\r' || b == '\n' || !b) - { - flags &= ~ flag_line_comment; - -- state.ptr; /* so null can be reproc'd */ - } - - continue; - } - - if (flags & flag_block_comment) - { - if (!b) - { sprintf (error, "%d:%d: Unexpected EOF in block comment", line_and_col); - goto e_failed; - } - - if (b == '*' && state.ptr < (end - 1) && state.ptr [1] == '/') - { - flags &= ~ flag_block_comment; - ++ state.ptr; /* skip closing sequence */ - } - - continue; - } - } - else if (b == '/') - { - if (! (flags & (flag_seek_value | flag_done)) && top->type != json_object) - { sprintf (error, "%d:%d: Comment not allowed here", line_and_col); - goto e_failed; - } - - if (++ state.ptr == end) - { sprintf (error, "%d:%d: EOF unexpected", line_and_col); - goto e_failed; - } - - switch (b = *state.ptr) - { - case '/': - flags |= flag_line_comment; - continue; - - case '*': - flags |= flag_block_comment; - continue; - - default: - sprintf (error, "%d:%d: Unexpected `%c` in comment opening sequence", line_and_col, b); - goto e_failed; - }; - } - } - - if (flags & flag_done) - { - if (!b) - break; - - switch (b) - { - whitespace: - continue; - - default: - - sprintf (error, "%d:%d: Trailing garbage: `%c`", - state.cur_line, state.cur_col, b); - - goto e_failed; - }; - } - - if (flags & flag_seek_value) - { - switch (b) - { - whitespace: - continue; - - case ']': - - if (top && top->type == json_array) - flags = (flags & ~ (flag_need_comma | flag_seek_value)) | flag_next; - else - { sprintf (error, "%d:%d: Unexpected ]", line_and_col); - goto e_failed; - } - - break; - - default: - - if (flags & flag_need_comma) - { - if (b == ',') - { flags &= ~ flag_need_comma; - continue; - } - else - { - sprintf (error, "%d:%d: Expected , before %c", - state.cur_line, state.cur_col, b); - - goto e_failed; - } - } - - if (flags & flag_need_colon) - { - if (b == ':') - { flags &= ~ flag_need_colon; - continue; - } - else - { - sprintf (error, "%d:%d: Expected : before %c", - state.cur_line, state.cur_col, b); - - goto e_failed; - } - } - - flags &= ~ flag_seek_value; - - switch (b) - { - case '{': - - if (!new_value (&state, &top, &root, &alloc, json_object)) - goto e_alloc_failure; - - continue; - - case '[': - - if (!new_value (&state, &top, &root, &alloc, json_array)) - goto e_alloc_failure; - - flags |= flag_seek_value; - continue; - - case '"': - - if (!new_value (&state, &top, &root, &alloc, json_string)) - goto e_alloc_failure; - - flags |= flag_string; - - string = top->u.string.ptr; - string_length = 0; - - continue; - - case 't': - - if ((end - state.ptr) < 3 || *(++ state.ptr) != 'r' || - *(++ state.ptr) != 'u' || *(++ state.ptr) != 'e') - { - goto e_unknown_value; - } - - if (!new_value (&state, &top, &root, &alloc, json_boolean)) - goto e_alloc_failure; - - top->u.boolean = 1; - - flags |= flag_next; - break; - - case 'f': - - if ((end - state.ptr) < 4 || *(++ state.ptr) != 'a' || - *(++ state.ptr) != 'l' || *(++ state.ptr) != 's' || - *(++ state.ptr) != 'e') - { - goto e_unknown_value; - } - - if (!new_value (&state, &top, &root, &alloc, json_boolean)) - goto e_alloc_failure; - - flags |= flag_next; - break; - - case 'n': - - if ((end - state.ptr) < 3 || *(++ state.ptr) != 'u' || - *(++ state.ptr) != 'l' || *(++ state.ptr) != 'l') - { - goto e_unknown_value; - } - - if (!new_value (&state, &top, &root, &alloc, json_null)) - goto e_alloc_failure; - - flags |= flag_next; - break; - - default: - - if (isdigit (b) || b == '-') - { - if (!new_value (&state, &top, &root, &alloc, json_integer)) - goto e_alloc_failure; - - if (!state.first_pass) - { - while (isdigit (b) || b == '+' || b == '-' - || b == 'e' || b == 'E' || b == '.') - { - if ( (++ state.ptr) == end) - { - b = 0; - break; - } - - b = *state.ptr; - } - - flags |= flag_next | flag_reproc; - break; - } - - flags &= ~ (flag_num_negative | flag_num_e | - flag_num_e_got_sign | flag_num_e_negative | - flag_num_zero); - - num_digits = 0; - num_fraction = 0; - num_e = 0; - - if (b != '-') - { - flags |= flag_reproc; - break; - } - - flags |= flag_num_negative; - continue; - } - else - { sprintf (error, "%d:%d: Unexpected %c when seeking value", line_and_col, b); - goto e_failed; - } - }; - }; - } - else - { - switch (top->type) - { - case json_object: - - switch (b) - { - whitespace: - continue; - - case '"': - - if (flags & flag_need_comma) - { sprintf (error, "%d:%d: Expected , before \"", line_and_col); - goto e_failed; - } - - flags |= flag_string; - - string = (json_char *) top->_reserved.object_mem; - string_length = 0; - - break; - - case '}': - - flags = (flags & ~ flag_need_comma) | flag_next; - break; - - case ',': - - if (flags & flag_need_comma) - { - flags &= ~ flag_need_comma; - break; - } - - default: - sprintf (error, "%d:%d: Unexpected `%c` in object", line_and_col, b); - goto e_failed; - }; - - break; - - case json_integer: - case json_double: - - if (isdigit (b)) - { - ++ num_digits; - - if (top->type == json_integer || flags & flag_num_e) - { - if (! (flags & flag_num_e)) - { - if (flags & flag_num_zero) - { sprintf (error, "%d:%d: Unexpected `0` before `%c`", line_and_col, b); - goto e_failed; - } - - if (num_digits == 1 && b == '0') - flags |= flag_num_zero; - } - else - { - flags |= flag_num_e_got_sign; - num_e = (num_e * 10) + (b - '0'); - continue; - } - - top->u.integer = (top->u.integer * 10) + (b - '0'); - continue; - } - - num_fraction = (num_fraction * 10) + (b - '0'); - continue; - } - - if (b == '+' || b == '-') - { - if ( (flags & flag_num_e) && !(flags & flag_num_e_got_sign)) - { - flags |= flag_num_e_got_sign; - - if (b == '-') - flags |= flag_num_e_negative; - - continue; - } - } - else if (b == '.' && top->type == json_integer) - { - if (!num_digits) - { sprintf (error, "%d:%d: Expected digit before `.`", line_and_col); - goto e_failed; - } - - top->type = json_double; - top->u.dbl = (double) top->u.integer; - - num_digits = 0; - continue; - } - - if (! (flags & flag_num_e)) - { - if (top->type == json_double) - { - if (!num_digits) - { sprintf (error, "%d:%d: Expected digit after `.`", line_and_col); - goto e_failed; - } - - top->u.dbl += ((double) num_fraction) / (pow (10.0, (double) num_digits)); - } - - if (b == 'e' || b == 'E') - { - flags |= flag_num_e; - - if (top->type == json_integer) - { - top->type = json_double; - top->u.dbl = (double) top->u.integer; - } - - num_digits = 0; - flags &= ~ flag_num_zero; - - continue; - } - } - else - { - if (!num_digits) - { sprintf (error, "%d:%d: Expected digit after `e`", line_and_col); - goto e_failed; - } - - top->u.dbl *= pow (10.0, (double) - (flags & flag_num_e_negative ? - num_e : num_e)); - } - - if (flags & flag_num_negative) - { - if (top->type == json_integer) - top->u.integer = - top->u.integer; - else - top->u.dbl = - top->u.dbl; - } - - flags |= flag_next | flag_reproc; - break; - - default: - break; - }; - } - - if (flags & flag_reproc) - { - flags &= ~ flag_reproc; - -- state.ptr; - } - - if (flags & flag_next) - { - flags = (flags & ~ flag_next) | flag_need_comma; - - if (!top->parent) - { - /* root value done */ - - flags |= flag_done; - continue; - } - - if (top->parent->type == json_array) - flags |= flag_seek_value; - - if (!state.first_pass) - { - json_value * parent = top->parent; - - switch (parent->type) - { - case json_object: - - parent->u.object.values - [parent->u.object.length].value = top; - - break; - - case json_array: - - parent->u.array.values - [parent->u.array.length] = top; - - break; - - default: - break; - }; - } - - if ( (++ top->parent->u.array.length) > state.uint_max) - goto e_overflow; - - top = top->parent; - - continue; - } - } - - alloc = root; - } - - return root; - -e_unknown_value: - - sprintf (error, "%d:%d: Unknown value", line_and_col); - goto e_failed; - -e_alloc_failure: - - strcpy (error, "Memory allocation failure"); - goto e_failed; - -e_overflow: - - sprintf (error, "%d:%d: Too long (caught overflow)", line_and_col); - goto e_failed; - -e_failed: - - if (error_buf) - { - if (*error) - strcpy (error_buf, error); - else - strcpy (error_buf, "Unknown error"); - } - - if (state.first_pass) - alloc = root; - - while (alloc) - { - top = alloc->_reserved.next_alloc; - state.settings.mem_free (alloc, state.settings.user_data); - alloc = top; - } - - if (!state.first_pass) - json_value_free_ex (&state.settings, root); - - return 0; -} - -json_value * json_parse (const json_char * json, size_t length) -{ - json_settings settings = { 0 }; - return json_parse_ex (&settings, json, length, 0); -} - -void json_value_free_ex (json_settings * settings, json_value * value) -{ - json_value * cur_value; - - if (!value) - return; - - value->parent = 0; - - while (value) - { - switch (value->type) - { - case json_array: - - if (!value->u.array.length) - { - settings->mem_free (value->u.array.values, settings->user_data); - break; - } - - value = value->u.array.values [-- value->u.array.length]; - continue; - - case json_object: - - if (!value->u.object.length) - { - settings->mem_free (value->u.object.values, settings->user_data); - break; - } - - value = value->u.object.values [-- value->u.object.length].value; - continue; - - case json_string: - - settings->mem_free (value->u.string.ptr, settings->user_data); - break; - - default: - break; - }; - - cur_value = value; - value = value->parent; - settings->mem_free (cur_value, settings->user_data); - } -} - -void json_value_free (json_value * value) -{ - json_settings settings = { 0 }; - settings.mem_free = default_free; - json_value_free_ex (&settings, value); -} - diff --git a/ext/json-parser/json.h b/ext/json-parser/json.h deleted file mode 100644 index f6549ec4..00000000 --- a/ext/json-parser/json.h +++ /dev/null @@ -1,283 +0,0 @@ - -/* vim: set et ts=3 sw=3 sts=3 ft=c: - * - * Copyright (C) 2012, 2013, 2014 James McLaughlin et al. All rights reserved. - * https://github.com/udp/json-parser - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS - * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT - * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY - * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF - * SUCH DAMAGE. - */ - -#ifndef _JSON_H -#define _JSON_H - -#ifndef json_char - #define json_char char -#endif - -#ifndef json_int_t - #ifndef _MSC_VER - #include - #define json_int_t int64_t - #else - #define json_int_t __int64 - #endif -#endif - -#include - -#ifdef __cplusplus - - #include - - extern "C" - { - -#endif - -typedef struct -{ - unsigned long max_memory; - int settings; - - /* Custom allocator support (leave null to use malloc/free) - */ - - void * (* mem_alloc) (size_t, int zero, void * user_data); - void (* mem_free) (void *, void * user_data); - - void * user_data; /* will be passed to mem_alloc and mem_free */ - - size_t value_extra; /* how much extra space to allocate for values? */ - -} json_settings; - -#define json_enable_comments 0x01 - -typedef enum -{ - json_none, - json_object, - json_array, - json_integer, - json_double, - json_string, - json_boolean, - json_null - -} json_type; - -extern const struct _json_value json_value_none; - -typedef struct _json_object_entry -{ - json_char * name; - unsigned int name_length; - - struct _json_value * value; - -} json_object_entry; - -typedef struct _json_value -{ - struct _json_value * parent; - - json_type type; - - union - { - int boolean; - json_int_t integer; - double dbl; - - struct - { - unsigned int length; - json_char * ptr; /* null terminated */ - - } string; - - struct - { - unsigned int length; - - json_object_entry * values; - - #if defined(__cplusplus) && __cplusplus >= 201103L - decltype(values) begin () const - { return values; - } - decltype(values) end () const - { return values + length; - } - #endif - - } object; - - struct - { - unsigned int length; - struct _json_value ** values; - - #if defined(__cplusplus) && __cplusplus >= 201103L - decltype(values) begin () const - { return values; - } - decltype(values) end () const - { return values + length; - } - #endif - - } array; - - } u; - - union - { - struct _json_value * next_alloc; - void * object_mem; - - } _reserved; - - #ifdef JSON_TRACK_SOURCE - - /* Location of the value in the source JSON - */ - unsigned int line, col; - - #endif - - - /* Some C++ operator sugar */ - - #ifdef __cplusplus - - public: - - inline _json_value () - { memset (this, 0, sizeof (_json_value)); - } - - inline const struct _json_value &operator [] (int index) const - { - if (type != json_array || index < 0 - || ((unsigned int) index) >= u.array.length) - { - return json_value_none; - } - - return *u.array.values [index]; - } - - inline const struct _json_value &operator [] (const char * index) const - { - if (type != json_object) - return json_value_none; - - for (unsigned int i = 0; i < u.object.length; ++ i) - if (!strcmp (u.object.values [i].name, index)) - return *u.object.values [i].value; - - return json_value_none; - } - - inline operator const char * () const - { - switch (type) - { - case json_string: - return u.string.ptr; - - default: - return ""; - }; - } - - inline operator json_int_t () const - { - switch (type) - { - case json_integer: - return u.integer; - - case json_double: - return (json_int_t) u.dbl; - - default: - return 0; - }; - } - - inline operator bool () const - { - if (type != json_boolean) - return false; - - return u.boolean != 0; - } - - inline operator double () const - { - switch (type) - { - case json_integer: - return (double) u.integer; - - case json_double: - return u.dbl; - - default: - return 0; - }; - } - - #endif - -} json_value; - -json_value * json_parse (const json_char * json, - size_t length); - -#define json_error_max 128 -json_value * json_parse_ex (json_settings * settings, - const json_char * json, - size_t length, - char * error); - -void json_value_free (json_value *); - - -/* Not usually necessary, unless you used a custom mem_alloc and now want to - * use a custom mem_free. - */ -void json_value_free_ex (json_settings * settings, - json_value *); - - -#ifdef __cplusplus - } /* extern "C" */ -#endif - -#endif - - diff --git a/include/ZeroTierOne.h b/include/ZeroTierOne.h index aa7ecc2c..5864e346 100644 --- a/include/ZeroTierOne.h +++ b/include/ZeroTierOne.h @@ -468,6 +468,11 @@ enum ZT_VirtualNetworkType ZT_NETWORK_TYPE_PUBLIC = 1 }; +/* + - TEE : should use a field to indicate how many bytes of each packet max are TEE'd + - Controller : web hooks for auth, optional required re-auth? or auth for a period of time? auto-expiring auth? +*/ + /** * The type of a virtual network rules table entry * @@ -721,6 +726,15 @@ typedef struct uint32_t id; uint32_t value; } tag; + + /** + * Destinations for TEE and REDIRECT + */ + struct { + uint64_t address; + uint32_t flags; + uint16_t length; + } fwd; } v; } ZT_VirtualNetworkRule; diff --git a/make-freebsd.mk b/make-freebsd.mk index e7bd9fd2..cb9a2e6d 100644 --- a/make-freebsd.mk +++ b/make-freebsd.mk @@ -6,7 +6,7 @@ DEFS= LIBS= include objects.mk -OBJS+=osdep/BSDEthernetTap.o ext/lz4/lz4.o ext/json-parser/json.o ext/http-parser/http_parser.o +OBJS+=osdep/BSDEthernetTap.o ext/lz4/lz4.o ext/http-parser/http_parser.o # "make official" is a shortcut for this ifeq ($(ZT_OFFICIAL_RELEASE),1) diff --git a/make-linux.mk b/make-linux.mk index fe9ecad8..1fc27bde 100644 --- a/make-linux.mk +++ b/make-linux.mk @@ -51,12 +51,6 @@ else LDLIBS+=-lhttp_parser DEFS+=-DZT_USE_SYSTEM_HTTP_PARSER endif -ifeq ($(wildcard /usr/include/json-parser/json.h),) - OBJS+=ext/json-parser/json.o -else - LDLIBS+=-ljsonparser - DEFS+=-DZT_USE_SYSTEM_JSON_PARSER -endif ifeq ($(ZT_USE_MINIUPNPC),1) OBJS+=osdep/PortMapper.o diff --git a/make-mac.mk b/make-mac.mk index f00f8d6b..ee90ae4c 100644 --- a/make-mac.mk +++ b/make-mac.mk @@ -11,7 +11,7 @@ LIBS= ARCH_FLAGS=-arch x86_64 include objects.mk -OBJS+=osdep/OSXEthernetTap.o ext/lz4/lz4.o ext/json-parser/json.o ext/http-parser/http_parser.o +OBJS+=osdep/OSXEthernetTap.o ext/lz4/lz4.o ext/http-parser/http_parser.o # Disable codesign since open source users will not have ZeroTier's certs CODESIGN=echo diff --git a/node/Capability.hpp b/node/Capability.hpp index b0620891..0b352725 100644 --- a/node/Capability.hpp +++ b/node/Capability.hpp @@ -181,6 +181,11 @@ public: break; case ZT_NETWORK_RULE_ACTION_TEE: case ZT_NETWORK_RULE_ACTION_REDIRECT: + b.append((uint8_t)14); + b.append((uint64_t)rules[i].v.fwd.address); + b.append((uint32_t)rules[i].v.fwd.flags); + b.append((uint16_t)rules[i].v.fwd.length); + break; case ZT_NETWORK_RULE_MATCH_SOURCE_ZEROTIER_ADDRESS: case ZT_NETWORK_RULE_MATCH_DEST_ZEROTIER_ADDRESS: b.append((uint8_t)5); @@ -266,6 +271,10 @@ public: break; case ZT_NETWORK_RULE_ACTION_TEE: case ZT_NETWORK_RULE_ACTION_REDIRECT: + rules[ruleCount].v.fwd.address = b.template at(p); + rules[ruleCount].v.fwd.flags = b.template at(p + 8); + rules[ruleCount].v.fwd.length = b.template at(p + 12); + break; case ZT_NETWORK_RULE_MATCH_SOURCE_ZEROTIER_ADDRESS: case ZT_NETWORK_RULE_MATCH_DEST_ZEROTIER_ADDRESS: rules[ruleCount].v.zt = Address(b.field(p,ZT_ADDRESS_LENGTH),ZT_ADDRESS_LENGTH).toInt(); diff --git a/node/Network.cpp b/node/Network.cpp index 1319df4e..e12dd027 100644 --- a/node/Network.cpp +++ b/node/Network.cpp @@ -154,13 +154,13 @@ static int _doZtFilter( break; case ZT_NETWORK_RULE_ACTION_TEE: case ZT_NETWORK_RULE_ACTION_REDIRECT: { - Packet outp(Address(rules[rn].v.zt),RR->identity.address(),Packet::VERB_EXT_FRAME); + Packet outp(Address(rules[rn].v.fwd.address),RR->identity.address(),Packet::VERB_EXT_FRAME); outp.append(nconf.networkId); - outp.append((uint8_t)((rt == ZT_NETWORK_RULE_ACTION_REDIRECT) ? 0x04 : 0x02)); + outp.append((uint8_t)( ((rt == ZT_NETWORK_RULE_ACTION_REDIRECT) ? 0x04 : 0x02) | (inbound ? 0x08 : 0x00) )); macDest.appendTo(outp); macSource.appendTo(outp); outp.append((uint16_t)etherType); - outp.append(frameData,frameLen); + outp.append(frameData,(rules[rn].v.fwd.length != 0) ? ((frameLen < (unsigned int)rules[rn].v.fwd.length) ? frameLen : (unsigned int)rules[rn].v.fwd.length) : frameLen); outp.compress(); RR->sw->send(outp,true); diff --git a/node/Packet.hpp b/node/Packet.hpp index 0a5d3fec..570bace9 100644 --- a/node/Packet.hpp +++ b/node/Packet.hpp @@ -657,6 +657,7 @@ public: * 0x01 - Certificate of network membership attached (DEPRECATED) * 0x02 - Packet is a TEE'd packet * 0x04 - Packet is a REDIRECT'ed packet + * 0x08 - TEE/REDIRECT'ed packet is on inbound side of connection * * An extended frame carries full MAC addressing, making them a * superset of VERB_FRAME. They're used for bridging or when we diff --git a/one.cpp b/one.cpp index 9f7a0a29..f3442933 100644 --- a/one.cpp +++ b/one.cpp @@ -48,16 +48,12 @@ #include #include +#include +#include #include "version.h" #include "include/ZeroTierOne.h" -#ifdef ZT_USE_SYSTEM_JSON_PARSER -#include -#else -#include "ext/json-parser/json.h" -#endif - #include "node/Identity.hpp" #include "node/CertificateOfMembership.hpp" #include "node/Utils.hpp" @@ -68,6 +64,8 @@ #include "service/OneService.hpp" +#include "ext/json/json.hpp" + #define ZT_PID_PATH "zerotier-one.pid" using namespace ZeroTier; @@ -283,221 +281,135 @@ static int cli(int argc,char **argv) return 1; } } else if ((command == "info")||(command == "status")) { - unsigned int scode = Http::GET( - 1024 * 1024 * 16, - 60000, - (const struct sockaddr *)&addr, - "/status", - requestHeaders, - responseHeaders, - responseBody); + const unsigned int scode = Http::GET(1024 * 1024 * 16,60000,(const struct sockaddr *)&addr,"/status",requestHeaders,responseHeaders,responseBody); + + nlohmann::json j; + try { + j = nlohmann::json::parse(responseBody); + } catch (std::exception &exc) { + printf("%u %s invalid JSON response (%s)" ZT_EOL_S,scode,command.c_str(),exc.what()); + return 1; + } catch ( ... ) { + printf("%u %s invalid JSON response (unknown exception)" ZT_EOL_S,scode,command.c_str()); + return 1; + } + if (scode == 200) { + std::ostringstream out; if (json) { - printf("%s",cliFixJsonCRs(responseBody).c_str()); - return 0; + out << j.dump(2) << ZT_EOL_S; } else { - json_value *j = json_parse(responseBody.c_str(),responseBody.length()); - bool good = false; - if (j) { - if (j->type == json_object) { - const char *address = (const char *)0; - bool online = false; - const char *version = (const char *)0; - for(unsigned int k=0;ku.object.length;++k) { - if ((!strcmp(j->u.object.values[k].name,"address"))&&(j->u.object.values[k].value->type == json_string)) - address = j->u.object.values[k].value->u.string.ptr; - else if ((!strcmp(j->u.object.values[k].name,"version"))&&(j->u.object.values[k].value->type == json_string)) - version = j->u.object.values[k].value->u.string.ptr; - else if ((!strcmp(j->u.object.values[k].name,"online"))&&(j->u.object.values[k].value->type == json_boolean)) - online = (j->u.object.values[k].value->u.boolean != 0); - } - if ((address)&&(version)) { - printf("200 info %s %s %s" ZT_EOL_S,address,(online ? "ONLINE" : "OFFLINE"),version); - good = true; - } - } - json_value_free(j); - } - if (good) { - return 0; - } else { - printf("%u %s invalid JSON response" ZT_EOL_S,scode,command.c_str()); - return 1; - } + if (j.is_object()) + out << "200 info " << j["address"].get() << " " << j["version"].get() << " " << ((j["tcpFallbackActive"]) ? "TUNNELED" : ((j["online"]) ? "ONLINE" : "OFFLINE")) << ZT_EOL_S; } + printf("%s",out.str().c_str()); + return 0; } else { printf("%u %s %s" ZT_EOL_S,scode,command.c_str(),responseBody.c_str()); return 1; } } else if (command == "listpeers") { - unsigned int scode = Http::GET( - 1024 * 1024 * 16, - 60000, - (const struct sockaddr *)&addr, - "/peer", - requestHeaders, - responseHeaders, - responseBody); + const unsigned int scode = Http::GET(1024 * 1024 * 16,60000,(const struct sockaddr *)&addr,"/peer",requestHeaders,responseHeaders,responseBody); + + nlohmann::json j; + try { + j = nlohmann::json::parse(responseBody); + } catch (std::exception &exc) { + printf("%u %s invalid JSON response (%s)" ZT_EOL_S,scode,command.c_str(),exc.what()); + return 1; + } catch ( ... ) { + printf("%u %s invalid JSON response (unknown exception)" ZT_EOL_S,scode,command.c_str()); + return 1; + } + if (scode == 200) { + std::ostringstream out; if (json) { - printf("%s",cliFixJsonCRs(responseBody).c_str()); - return 0; + out << j.dump(2) << ZT_EOL_S; } else { - printf("200 listpeers " ZT_EOL_S); - json_value *j = json_parse(responseBody.c_str(),responseBody.length()); - if (j) { - if (j->type == json_array) { - for(unsigned int p=0;pu.array.length;++p) { - json_value *jp = j->u.array.values[p]; - if (jp->type == json_object) { - const char *address = (const char *)0; - std::string paths; - int64_t latency = 0; - int64_t versionMajor = -1,versionMinor = -1,versionRev = -1; - const char *role = (const char *)0; - for(unsigned int k=0;ku.object.length;++k) { - if ((!strcmp(jp->u.object.values[k].name,"address"))&&(jp->u.object.values[k].value->type == json_string)) - address = jp->u.object.values[k].value->u.string.ptr; - else if ((!strcmp(jp->u.object.values[k].name,"versionMajor"))&&(jp->u.object.values[k].value->type == json_integer)) - versionMajor = jp->u.object.values[k].value->u.integer; - else if ((!strcmp(jp->u.object.values[k].name,"versionMinor"))&&(jp->u.object.values[k].value->type == json_integer)) - versionMinor = jp->u.object.values[k].value->u.integer; - else if ((!strcmp(jp->u.object.values[k].name,"versionRev"))&&(jp->u.object.values[k].value->type == json_integer)) - versionRev = jp->u.object.values[k].value->u.integer; - else if ((!strcmp(jp->u.object.values[k].name,"role"))&&(jp->u.object.values[k].value->type == json_string)) - role = jp->u.object.values[k].value->u.string.ptr; - else if ((!strcmp(jp->u.object.values[k].name,"latency"))&&(jp->u.object.values[k].value->type == json_integer)) - latency = jp->u.object.values[k].value->u.integer; - else if ((!strcmp(jp->u.object.values[k].name,"paths"))&&(jp->u.object.values[k].value->type == json_array)) { - for(unsigned int pp=0;ppu.object.values[k].value->u.array.length;++pp) { - json_value *jpath = jp->u.object.values[k].value->u.array.values[pp]; - if (jpath->type == json_object) { - const char *paddr = (const char *)0; - int64_t lastSend = 0; - int64_t lastReceive = 0; - bool preferred = false; - bool active = false; - for(unsigned int kk=0;kku.object.length;++kk) { - if ((!strcmp(jpath->u.object.values[kk].name,"address"))&&(jpath->u.object.values[kk].value->type == json_string)) - paddr = jpath->u.object.values[kk].value->u.string.ptr; - else if ((!strcmp(jpath->u.object.values[kk].name,"lastSend"))&&(jpath->u.object.values[kk].value->type == json_integer)) - lastSend = jpath->u.object.values[kk].value->u.integer; - else if ((!strcmp(jpath->u.object.values[kk].name,"lastReceive"))&&(jpath->u.object.values[kk].value->type == json_integer)) - lastReceive = jpath->u.object.values[kk].value->u.integer; - else if ((!strcmp(jpath->u.object.values[kk].name,"preferred"))&&(jpath->u.object.values[kk].value->type == json_boolean)) - preferred = (jpath->u.object.values[kk].value->u.boolean != 0); - else if ((!strcmp(jpath->u.object.values[kk].name,"active"))&&(jpath->u.object.values[kk].value->type == json_boolean)) - active = (jpath->u.object.values[kk].value->u.boolean != 0); - } - if ((paddr)&&(active)) { - int64_t now = (int64_t)OSUtils::now(); - if (lastSend > 0) - lastSend = now - lastSend; - if (lastReceive > 0) - lastReceive = now - lastReceive; - char pathtmp[256]; - Utils::snprintf(pathtmp,sizeof(pathtmp),"%s;%lld;%lld;%s", - paddr, - lastSend, - lastReceive, - (preferred ? "preferred" : "active")); - if (paths.length()) - paths.push_back(','); - paths.append(pathtmp); - } - } - } - } - } - if ((address)&&(role)) { - char verstr[64]; - if ((versionMajor >= 0)&&(versionMinor >= 0)&&(versionRev >= 0)) - Utils::snprintf(verstr,sizeof(verstr),"%lld.%lld.%lld",versionMajor,versionMinor,versionRev); - else { - verstr[0] = '-'; - verstr[1] = (char)0; - } - printf("200 listpeers %s %s %lld %s %s" ZT_EOL_S,address,(paths.length()) ? paths.c_str() : "-",(long long)latency,verstr,role); + out << "200 listpeers " << ZT_EOL_S; + if (j.is_array()) { + for(unsigned long k=0;k= 0) { + Utils::snprintf(ver,sizeof(ver),"%lld.%lld.%lld",vmaj,vmin,vrev); + } else { + ver[0] = '-'; + ver[1] = (char)0; + } + out << "200 listpeers " << p["address"].get() << " " << bestPath << " " << p["latency"] << " " << ver << " " << p["role"].get() << ZT_EOL_S; } - json_value_free(j); } - return 0; } + printf("%s",out.str().c_str()); + return 0; } else { printf("%u %s %s" ZT_EOL_S,scode,command.c_str(),responseBody.c_str()); return 1; } } else if (command == "listnetworks") { - unsigned int scode = Http::GET( - 1024 * 1024 * 16, - 60000, - (const struct sockaddr *)&addr, - "/network", - requestHeaders, - responseHeaders, - responseBody); + const unsigned int scode = Http::GET(1024 * 1024 * 16,60000,(const struct sockaddr *)&addr,"/network",requestHeaders,responseHeaders,responseBody); + + nlohmann::json j; + try { + j = nlohmann::json::parse(responseBody); + } catch (std::exception &exc) { + printf("%u %s invalid JSON response (%s)" ZT_EOL_S,scode,command.c_str(),exc.what()); + return 1; + } catch ( ... ) { + printf("%u %s invalid JSON response (unknown exception)" ZT_EOL_S,scode,command.c_str()); + return 1; + } + if (scode == 200) { + std::ostringstream out; if (json) { - printf("%s",cliFixJsonCRs(responseBody).c_str()); - return 0; + out << j.dump(2) << ZT_EOL_S; } else { - printf("200 listnetworks " ZT_EOL_S); - json_value *j = json_parse(responseBody.c_str(),responseBody.length()); - if (j) { - if (j->type == json_array) { - for(unsigned int p=0;pu.array.length;++p) { - json_value *jn = j->u.array.values[p]; - if (jn->type == json_object) { - const char *nwid = (const char *)0; - const char *name = ""; - const char *mac = (const char *)0; - const char *status = (const char *)0; - const char *type = (const char *)0; - const char *portDeviceName = ""; - std::string ips; - for(unsigned int k=0;ku.object.length;++k) { - if ((!strcmp(jn->u.object.values[k].name,"nwid"))&&(jn->u.object.values[k].value->type == json_string)) - nwid = jn->u.object.values[k].value->u.string.ptr; - else if ((!strcmp(jn->u.object.values[k].name,"name"))&&(jn->u.object.values[k].value->type == json_string)) - name = jn->u.object.values[k].value->u.string.ptr; - else if ((!strcmp(jn->u.object.values[k].name,"mac"))&&(jn->u.object.values[k].value->type == json_string)) - mac = jn->u.object.values[k].value->u.string.ptr; - else if ((!strcmp(jn->u.object.values[k].name,"status"))&&(jn->u.object.values[k].value->type == json_string)) - status = jn->u.object.values[k].value->u.string.ptr; - else if ((!strcmp(jn->u.object.values[k].name,"type"))&&(jn->u.object.values[k].value->type == json_string)) - type = jn->u.object.values[k].value->u.string.ptr; - else if ((!strcmp(jn->u.object.values[k].name,"portDeviceName"))&&(jn->u.object.values[k].value->type == json_string)) - portDeviceName = jn->u.object.values[k].value->u.string.ptr; - else if ((!strcmp(jn->u.object.values[k].name,"assignedAddresses"))&&(jn->u.object.values[k].value->type == json_array)) { - for(unsigned int a=0;au.object.values[k].value->u.array.length;++a) { - json_value *aa = jn->u.object.values[k].value->u.array.values[a]; - if (aa->type == json_string) { - if (ips.length()) - ips.push_back(','); - ips.append(aa->u.string.ptr); - } - } + out << "200 listnetworks " << ZT_EOL_S; + if (j.is_array()) { + for(unsigned long i=0;i 0) aa.push_back(','); + aa.append(addr); } } - if ((nwid)&&(mac)&&(status)&&(type)) { - printf("200 listnetworks %s %s %s %s %s %s %s" ZT_EOL_S, - nwid, - (((name)&&(name[0])) ? name : "-"), - mac, - status, - type, - (((portDeviceName)&&(portDeviceName[0])) ? portDeviceName : "-"), - ((ips.length() > 0) ? ips.c_str() : "-")); - } } + if (aa.length() == 0) aa = "-"; + out << "200 listnetworks " << n["nwid"].get() << " " << n["name"].get() << " " << n["mac"].get() << " " << n["status"].get() << " " << n["type"].get() << " " << n["portDeviceName"].get() << " " << aa << ZT_EOL_S; } } - json_value_free(j); } } + printf("%s",out.str().c_str()); + return 0; } else { printf("%u %s %s" ZT_EOL_S,scode,command.c_str(),responseBody.c_str()); return 1; diff --git a/service/ControlPlane.cpp b/service/ControlPlane.cpp index 7aa757a9..5ed6b8b7 100644 --- a/service/ControlPlane.cpp +++ b/service/ControlPlane.cpp @@ -28,11 +28,7 @@ #include "../ext/http-parser/http_parser.h" #endif -#ifdef ZT_USE_SYSTEM_JSON_PARSER -#include -#else -#include "../ext/json-parser/json.h" -#endif +#include "../ext/json/json.hpp" #include "../controller/EmbeddedNetworkController.hpp" @@ -519,23 +515,18 @@ unsigned int ControlPlane::handleRequest( OneService::NetworkSettings localSettings; _svc->getNetworkSettings(nws->networks[i].nwid,localSettings); - json_value *j = json_parse(body.c_str(),body.length()); - if (j) { - if (j->type == json_object) { - for(unsigned int k=0;ku.object.length;++k) { - if (!strcmp(j->u.object.values[k].name,"allowManaged")) { - if (j->u.object.values[k].value->type == json_boolean) - localSettings.allowManaged = (j->u.object.values[k].value->u.boolean != 0); - } else if (!strcmp(j->u.object.values[k].name,"allowGlobal")) { - if (j->u.object.values[k].value->type == json_boolean) - localSettings.allowGlobal = (j->u.object.values[k].value->u.boolean != 0); - } else if (!strcmp(j->u.object.values[k].name,"allowDefault")) { - if (j->u.object.values[k].value->type == json_boolean) - localSettings.allowDefault = (j->u.object.values[k].value->u.boolean != 0); - } - } + try { + nlohmann::json j(nlohmann::json::parse(body)); + if (j.is_object()) { + auto allowManaged = j["allowManaged"]; + if (allowManaged.is_boolean()) localSettings.allowManaged = (bool)allowManaged; + auto allowGlobal = j["allowGlobal"]; + if (allowGlobal.is_boolean()) localSettings.allowGlobal = (bool)allowGlobal; + auto allowDefault = j["allowDefault"]; + if (allowDefault.is_boolean()) localSettings.allowDefault = (bool)allowDefault; } - json_value_free(j); + } catch ( ... ) { + // discard invalid JSON } _svc->setNetworkSettings(nws->networks[i].nwid,localSettings); -- cgit v1.2.3 From c175a5fcd702ebeb64213fbd80bcda0b37cb043e Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Fri, 23 Dec 2016 15:11:44 -0800 Subject: Change org names, add makefile rule for Mac to build new UI. --- make-mac.mk | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) (limited to 'make-mac.mk') diff --git a/make-mac.mk b/make-mac.mk index ee90ae4c..875f01e5 100644 --- a/make-mac.mk +++ b/make-mac.mk @@ -28,8 +28,8 @@ ifeq ($(ZT_OFFICIAL_RELEASE),1) ZT_USE_MINIUPNPC=1 CODESIGN=codesign PRODUCTSIGN=productsign - CODESIGN_APP_CERT="Developer ID Application: ZeroTier Networks LLC (8ZD9JUCZ4V)" - CODESIGN_INSTALLER_CERT="Developer ID Installer: ZeroTier Networks LLC (8ZD9JUCZ4V)" + CODESIGN_APP_CERT="Developer ID Application: ZeroTier, Inc (8ZD9JUCZ4V)" + CODESIGN_INSTALLER_CERT="Developer ID Installer: ZeroTier, Inc (8ZD9JUCZ4V)" endif ifeq ($(ZT_ENABLE_CLUSTER),1) @@ -61,7 +61,7 @@ endif CXXFLAGS=$(CFLAGS) -mmacosx-version-min=10.7 -std=c++11 -stdlib=libc++ -all: one +all: one macui one: $(OBJS) service/OneService.o one.o $(CXX) $(CXXFLAGS) -o zerotier-one $(OBJS) service/OneService.o one.o $(LIBS) @@ -71,6 +71,11 @@ one: $(OBJS) service/OneService.o one.o $(CODESIGN) -f -s $(CODESIGN_APP_CERT) zerotier-one $(CODESIGN) -vvv zerotier-one +macui: FORCE + cd macui && xcodebuild -target "ZeroTier One" -configuration Release + $(CODESIGN) -f -s $(CODESIGN_APP_CERT) "macui/build/Release/ZeroTier One.app" + $(CODESIGN) -vvv "macui/build/Release/ZeroTier One.app" + cli: FORCE $(CXX) $(CXXFLAGS) -o zerotier cli/zerotier.cpp osdep/OSUtils.cpp node/InetAddress.cpp node/Utils.cpp node/Salsa20.cpp node/Identity.cpp node/SHA512.cpp node/C25519.cpp -lcurl $(STRIP) zerotier @@ -93,10 +98,9 @@ official: FORCE make ZT_OFFICIAL_RELEASE=1 mac-dist-pkg clean: - rm -rf *.dSYM build-* *.pkg *.dmg *.o node/*.o controller/*.o service/*.o osdep/*.o ext/http-parser/*.o ext/lz4/*.o ext/json-parser/*.o $(OBJS) zerotier-one zerotier-idtool zerotier-selftest zerotier-cli zerotier ZeroTierOneInstaller-* mkworld doc/node_modules + rm -rf *.dSYM build-* *.pkg *.dmg *.o node/*.o controller/*.o service/*.o osdep/*.o ext/http-parser/*.o ext/lz4/*.o ext/json-parser/*.o $(OBJS) zerotier-one zerotier-idtool zerotier-selftest zerotier-cli zerotier ZeroTierOneInstaller-* mkworld doc/node_modules macui/build distclean: clean - rm -rf doc/node_modules # For those building from source -- installs signed binary tap driver in system ZT home install-mac-tap: FORCE -- cgit v1.2.3 From 24017659df4ebad3531c722374ac2f40dc9e2928 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Fri, 23 Dec 2016 15:25:04 -0800 Subject: Update installer, etc., for Mac. --- ext/installfiles/mac/ZeroTier One.pkgproj | 121 +----------------------------- make-mac.mk | 6 +- 2 files changed, 7 insertions(+), 120 deletions(-) (limited to 'make-mac.mk') diff --git a/ext/installfiles/mac/ZeroTier One.pkgproj b/ext/installfiles/mac/ZeroTier One.pkgproj index d9730527..4215fd22 100755 --- a/ext/installfiles/mac/ZeroTier One.pkgproj +++ b/ext/installfiles/mac/ZeroTier One.pkgproj @@ -37,7 +37,7 @@ GID 80 PATH - mac-ui-macgap1-wrapper/bin/ZeroTier One.app + ../../../macui/build/Release/ZeroTier One.app PATH_TYPE 1 PERMISSIONS @@ -121,119 +121,6 @@ UID 0 - - CHILDREN - - - CHILDREN - - GID - 0 - PATH - ui/index.html - PATH_TYPE - 1 - PERMISSIONS - 420 - TYPE - 3 - UID - 0 - - - CHILDREN - - GID - 0 - PATH - ui/main.js - PATH_TYPE - 1 - PERMISSIONS - 420 - TYPE - 3 - UID - 0 - - - CHILDREN - - GID - 0 - PATH - ui/react.min.js - PATH_TYPE - 1 - PERMISSIONS - 420 - TYPE - 3 - UID - 0 - - - CHILDREN - - GID - 0 - PATH - ui/simpleajax.min.js - PATH_TYPE - 1 - PERMISSIONS - 420 - TYPE - 3 - UID - 0 - - - CHILDREN - - GID - 0 - PATH - ui/zerotier.css - PATH_TYPE - 1 - PERMISSIONS - 420 - TYPE - 3 - UID - 0 - - - CHILDREN - - GID - 0 - PATH - ui/ztui.min.js - PATH_TYPE - 1 - PERMISSIONS - 420 - TYPE - 3 - UID - 0 - - - GID - 0 - PATH - ui - PATH_TYPE - 0 - PERMISSIONS - 493 - TYPE - 2 - UID - 0 - CHILDREN @@ -759,7 +646,7 @@ OVERWRITE_PERMISSIONS VERSION - 1.1.14 + 1.1.17 PROJECT_COMMENTS @@ -773,7 +660,7 @@ ZW50LVN0eWxlLVR5cGUiIGNvbnRlbnQ9InRleHQvY3NzIj4KPHRp dGxlPjwvdGl0bGU+CjxtZXRhIG5hbWU9IkdlbmVyYXRvciIgY29u dGVudD0iQ29jb2EgSFRNTCBXcml0ZXIiPgo8bWV0YSBuYW1lPSJD - b2NvYVZlcnNpb24iIGNvbnRlbnQ9IjE0MDQuNDciPgo8c3R5bGUg + b2NvYVZlcnNpb24iIGNvbnRlbnQ9IjE1MDQuNzYiPgo8c3R5bGUg dHlwZT0idGV4dC9jc3MiPgpwLnAxIHttYXJnaW46IDAuMHB4IDAu MHB4IDAuMHB4IDAuMHB4OyBsaW5lLWhlaWdodDogMTQuMHB4OyBm b250OiAxMi4wcHggSGVsdmV0aWNhOyBjb2xvcjogIzAwMDAwMDsg @@ -782,7 +669,7 @@ b2R5Pgo8cCBjbGFzcz0icDEiPjxzcGFuIGNsYXNzPSJzMSI+WmVy b1RpZXIgT25lIC0gTmV0d29yayBWaXJ0dWFsaXphdGlvbiBFdmVy eXdoZXJlPC9zcGFuPjwvcD4KPHAgY2xhc3M9InAxIj48c3BhbiBj - bGFzcz0iczEiPihjKTIwMTEtMjAxNiBaZXJvVGllciwgSW5jLjwv + bGFzcz0iczEiPihjKTIwMTEtMjAxNyBaZXJvVGllciwgSW5jLjwv c3Bhbj48L3A+CjxwIGNsYXNzPSJwMSI+PHNwYW4gY2xhc3M9InMx Ij5jb250YWN0QHplcm90aWVyLmNvbTwvc3Bhbj48L3A+CjxwIGNs YXNzPSJwMSI+PHNwYW4gY2xhc3M9InMxIj48YnI+Cjwvc3Bhbj48 diff --git a/make-mac.mk b/make-mac.mk index 875f01e5..0fbd3817 100644 --- a/make-mac.mk +++ b/make-mac.mk @@ -69,12 +69,12 @@ one: $(OBJS) service/OneService.o one.o ln -sf zerotier-one zerotier-idtool ln -sf zerotier-one zerotier-cli $(CODESIGN) -f -s $(CODESIGN_APP_CERT) zerotier-one - $(CODESIGN) -vvv zerotier-one + #$(CODESIGN) -vvv zerotier-one macui: FORCE cd macui && xcodebuild -target "ZeroTier One" -configuration Release $(CODESIGN) -f -s $(CODESIGN_APP_CERT) "macui/build/Release/ZeroTier One.app" - $(CODESIGN) -vvv "macui/build/Release/ZeroTier One.app" + #$(CODESIGN) -vvv "macui/build/Release/ZeroTier One.app" cli: FORCE $(CXX) $(CXXFLAGS) -o zerotier cli/zerotier.cpp osdep/OSUtils.cpp node/InetAddress.cpp node/Utils.cpp node/Salsa20.cpp node/Identity.cpp node/SHA512.cpp node/C25519.cpp -lcurl @@ -94,7 +94,7 @@ mac-dist-pkg: FORCE # For ZeroTier, Inc. to build official signed packages official: FORCE make clean - make ZT_OFFICIAL_RELEASE=1 -j 4 one + make ZT_OFFICIAL_RELEASE=1 -j 4 one macui make ZT_OFFICIAL_RELEASE=1 mac-dist-pkg clean: -- cgit v1.2.3 From 6fb49f68fc2ac3539732794f553f635610c92ef2 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Tue, 10 Jan 2017 12:20:29 -0800 Subject: Comment out and disable old auto-update code. --- make-mac.mk | 6 +----- service/OneService.cpp | 38 ++++---------------------------------- service/OneService.hpp | 5 ----- 3 files changed, 5 insertions(+), 44 deletions(-) (limited to 'make-mac.mk') diff --git a/make-mac.mk b/make-mac.mk index 0fbd3817..79f50ca1 100644 --- a/make-mac.mk +++ b/make-mac.mk @@ -24,7 +24,7 @@ ZT_USE_MINIUPNPC?=1 # For internal use only -- signs everything with ZeroTier's developer cert ifeq ($(ZT_OFFICIAL_RELEASE),1) - DEFS+=-DZT_OFFICIAL_RELEASE -DZT_AUTO_UPDATE + DEFS+=-DZT_OFFICIAL_RELEASE ZT_USE_MINIUPNPC=1 CODESIGN=codesign PRODUCTSIGN=productsign @@ -36,10 +36,6 @@ ifeq ($(ZT_ENABLE_CLUSTER),1) DEFS+=-DZT_ENABLE_CLUSTER endif -ifeq ($(ZT_AUTO_UPDATE),1) - DEFS+=-DZT_AUTO_UPDATE -endif - ifeq ($(ZT_USE_MINIUPNPC),1) DEFS+=-DMACOSX -DZT_USE_MINIUPNPC -DMINIUPNP_STATICLIB -D_DARWIN_C_SOURCE -DMINIUPNPC_SET_SOCKET_TIMEOUT -DMINIUPNPC_GET_SRC_ADDR -D_BSD_SOURCE -D_DEFAULT_SOURCE -DOS_STRING=\"Darwin/15.0.0\" -DMINIUPNPC_VERSION_STRING=\"2.0\" -DUPNP_VERSION_STRING=\"UPnP/1.1\" -DENABLE_STRNATPMPERR OBJS+=ext/libnatpmp/natpmp.o ext/libnatpmp/getgateway.o ext/miniupnpc/connecthostport.o ext/miniupnpc/igd_desc_parse.o ext/miniupnpc/minisoap.o ext/miniupnpc/minissdpc.o ext/miniupnpc/miniupnpc.o ext/miniupnpc/miniwget.o ext/miniupnpc/minixml.o ext/miniupnpc/portlistingparse.o ext/miniupnpc/receivedata.o ext/miniupnpc/upnpcommands.o ext/miniupnpc/upnpdev.o ext/miniupnpc/upnperrors.o ext/miniupnpc/upnpreplyparse.o osdep/PortMapper.o diff --git a/service/OneService.cpp b/service/OneService.cpp index 96b3a960..cd267982 100644 --- a/service/OneService.cpp +++ b/service/OneService.cpp @@ -194,6 +194,8 @@ static std::string _jS(const json &jv,const char *dfl) return std::string((dfl) ? dfl : ""); } +#if 0 + #ifdef ZT_AUTO_UPDATE #define ZT_AUTO_UPDATE_MAX_HTTP_RESPONSE_SIZE (1024 * 1024 * 64) #define ZT_AUTO_UPDATE_CHECK_PERIOD 21600000 @@ -403,6 +405,8 @@ public: static BackgroundSoftwareUpdateChecker backgroundSoftwareUpdateChecker; #endif // ZT_AUTO_UPDATE +#endif + static std::string _trimString(const std::string &s) { unsigned long end = (unsigned long)s.length(); @@ -947,9 +951,6 @@ public: uint64_t lastTcpFallbackResolve = 0; uint64_t lastBindRefresh = 0; uint64_t lastLocalInterfaceAddressCheck = (OSUtils::now() - ZT_LOCAL_INTERFACE_CHECK_INTERVAL) + 15000; // do this in 15s to give portmapper time to configure and other things time to settle -#ifdef ZT_AUTO_UPDATE - uint64_t lastSoftwareUpdateCheck = 0; -#endif // ZT_AUTO_UPDATE for(;;) { _run_m.lock(); if (!_run) { @@ -994,13 +995,6 @@ public: dl = _nextBackgroundTaskDeadline; } -#ifdef ZT_AUTO_UPDATE - if ((now - lastSoftwareUpdateCheck) >= ZT_AUTO_UPDATE_CHECK_PERIOD) { - lastSoftwareUpdateCheck = now; - Thread::start(&backgroundSoftwareUpdateChecker); - } -#endif // ZT_AUTO_UPDATE - if ((now - lastTcpFallbackResolve) >= ZT_TCP_FALLBACK_RERESOLVE_DELAY) { lastTcpFallbackResolve = now; _tcpFallbackResolver.resolveNow(); @@ -2245,30 +2239,6 @@ std::string OneService::platformDefaultHomePath() return OSUtils::platformDefaultHomePath(); } -std::string OneService::autoUpdateUrl() -{ -#ifdef ZT_AUTO_UPDATE - -/* -#if defined(__LINUX__) && ( defined(__i386__) || defined(__x86_64) || defined(__x86_64__) || defined(__amd64) || defined(__i386) ) - if (sizeof(void *) == 8) - return "http://download.zerotier.com/ZeroTierOneInstaller-linux-x64-LATEST.nfo"; - else return "http://download.zerotier.com/ZeroTierOneInstaller-linux-x86-LATEST.nfo"; -#endif -*/ - -#if defined(__APPLE__) && ( defined(__i386__) || defined(__x86_64) || defined(__x86_64__) || defined(__amd64) || defined(__i386) ) - return "http://download.zerotier.com/update/mac_intel/"; -#endif - -#ifdef __WINDOWS__ - return "http://download.zerotier.com/update/win_intel/"; -#endif - -#endif // ZT_AUTO_UPDATE - return std::string(); -} - OneService *OneService::newInstance(const char *hp,unsigned int port) { return new OneServiceImpl(hp,port); } OneService::~OneService() {} diff --git a/service/OneService.hpp b/service/OneService.hpp index aebc051b..7aa3b361 100644 --- a/service/OneService.hpp +++ b/service/OneService.hpp @@ -81,11 +81,6 @@ public: */ static std::string platformDefaultHomePath(); - /** - * @return Auto-update URL or empty string if auto-updates unsupported or not enabled - */ - static std::string autoUpdateUrl(); - /** * Create a new instance of the service * -- cgit v1.2.3 From 565842b5ec69135153ba1632d7d16a0c426946e1 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Wed, 11 Jan 2017 14:37:31 -0800 Subject: Software updater work. --- make-linux.mk | 26 +++++ make-mac.mk | 3 + objects.mk | 3 +- osdep/OSUtils.cpp | 18 ++++ osdep/OSUtils.hpp | 1 + service/SoftwareUpdater.cpp | 235 ++++++++++++++++++++++++++++++++++++++++++++ service/SoftwareUpdater.hpp | 171 ++++++++++++++++++++++++++++++++ 7 files changed, 456 insertions(+), 1 deletion(-) create mode 100644 service/SoftwareUpdater.cpp create mode 100644 service/SoftwareUpdater.hpp (limited to 'make-mac.mk') diff --git a/make-linux.mk b/make-linux.mk index ed8ec44b..90644e3d 100644 --- a/make-linux.mk +++ b/make-linux.mk @@ -113,6 +113,32 @@ endif #LDFLAGS= #STRIP=echo +# Determine system build architecture from compiler target +CC_MACH=$(shell $(CC) -dumpmachine | cut -d '-' -f 1) +ZT_ARCHITECTURE=0 +ifeq ($(CC_MACH),x86_64) + ZT_ARCHITECTURE=2 +endif +ifeq ($(CC_MACH),amd64) + ZT_ARCHITECTURE=2 +endif +ifeq ($(CC_MACH),i386) + ZT_ARCHITECTURE=1 +endif +ifeq ($(CC_MACH),i686) + ZT_ARCHITECTURE=1 +endif +ifeq ($(CC_MACH),arm) + ZT_ARCHITECTURE=3 +endif +ifeq ($(CC_MACH),arm64) + ZT_ARCHITECTURE=4 +endif +ifeq ($(CC_MACH),aarch64) + ZT_ARCHITECTURE=4 +endif +DEFS+=-DZT_BUILD_PLATFORM=1 -DZT_BUILD_ARCHITECTURE=$(ZT_ARCHITECTURE) + all: one one: $(OBJS) service/OneService.o one.o osdep/LinuxEthernetTap.o osdep/LinuxDropPrivileges.o diff --git a/make-mac.mk b/make-mac.mk index 79f50ca1..e9de6f49 100644 --- a/make-mac.mk +++ b/make-mac.mk @@ -57,6 +57,9 @@ endif CXXFLAGS=$(CFLAGS) -mmacosx-version-min=10.7 -std=c++11 -stdlib=libc++ +# 3 == MacOS, 2 == X64 +DEFS+=-DZT_BUILD_PLATFORM=3 -DZT_BUILD_ARCHITECTURE=2 + all: one macui one: $(OBJS) service/OneService.o one.o diff --git a/objects.mk b/objects.mk index 078a92a7..38d6bef3 100644 --- a/objects.mk +++ b/objects.mk @@ -31,4 +31,5 @@ OBJS=\ osdep/Http.o \ osdep/OSUtils.o \ service/ClusterGeoIpService.o \ - service/ControlPlane.o + service/ControlPlane.o \ + service/SoftwareUpdater.o diff --git a/osdep/OSUtils.cpp b/osdep/OSUtils.cpp index a6757bcc..08d67bbe 100644 --- a/osdep/OSUtils.cpp +++ b/osdep/OSUtils.cpp @@ -23,6 +23,7 @@ #include #include "../node/Constants.hpp" +#include "../node/Utils.hpp" #ifdef __UNIX_LIKE__ #include @@ -453,6 +454,23 @@ std::string OSUtils::jsonString(const nlohmann::json &jv,const char *dfl) return std::string((dfl) ? dfl : ""); } +std::string OSUtils::jsonBinFromHex(const nlohmann::json &jv) +{ + std::string s(jsonString(jv,"")); + if (s.length() > 0) { + char *buf = new char[(s.length() / 2) + 1]; + try { + unsigned int l = Utils::unhex(s,buf,(unsigned int)s.length()); + std::string b(buf,l); + delete [] buf; + return b; + } catch ( ... ) { + delete [] buf; + } + } + return std::string(); +} + // Used to convert HTTP header names to ASCII lower case const unsigned char OSUtils::TOLOWER_TABLE[256] = { 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, ' ', '!', '"', '#', '$', '%', '&', 0x27, '(', ')', '*', '+', ',', '-', '.', '/', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', ':', ';', '<', '=', '>', '?', '@', 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z', '{', '|', '}', '~', '_', '`', 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z', '{', '|', '}', '~', 0x7f, 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f, 0xa0, 0xa1, 0xa2, 0xa3, 0xa4, 0xa5, 0xa6, 0xa7, 0xa8, 0xa9, 0xaa, 0xab, 0xac, 0xad, 0xae, 0xaf, 0xb0, 0xb1, 0xb2, 0xb3, 0xb4, 0xb5, 0xb6, 0xb7, 0xb8, 0xb9, 0xba, 0xbb, 0xbc, 0xbd, 0xbe, 0xbf, 0xc0, 0xc1, 0xc2, 0xc3, 0xc4, 0xc5, 0xc6, 0xc7, 0xc8, 0xc9, 0xca, 0xcb, 0xcc, 0xcd, 0xce, 0xcf, 0xd0, 0xd1, 0xd2, 0xd3, 0xd4, 0xd5, 0xd6, 0xd7, 0xd8, 0xd9, 0xda, 0xdb, 0xdc, 0xdd, 0xde, 0xdf, 0xe0, 0xe1, 0xe2, 0xe3, 0xe4, 0xe5, 0xe6, 0xe7, 0xe8, 0xe9, 0xea, 0xeb, 0xec, 0xed, 0xee, 0xef, 0xf0, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0xf7, 0xf8, 0xf9, 0xfa, 0xfb, 0xfc, 0xfd, 0xfe, 0xff }; diff --git a/osdep/OSUtils.hpp b/osdep/OSUtils.hpp index 7aa83f17..2fe01979 100644 --- a/osdep/OSUtils.hpp +++ b/osdep/OSUtils.hpp @@ -274,6 +274,7 @@ public: static uint64_t jsonInt(const nlohmann::json &jv,const uint64_t dfl); static bool jsonBool(const nlohmann::json &jv,const bool dfl); static std::string jsonString(const nlohmann::json &jv,const char *dfl); + static std::string jsonBinFromHex(const nlohmann::json &jv); private: static const unsigned char TOLOWER_TABLE[256]; diff --git a/service/SoftwareUpdater.cpp b/service/SoftwareUpdater.cpp new file mode 100644 index 00000000..3c77a55a --- /dev/null +++ b/service/SoftwareUpdater.cpp @@ -0,0 +1,235 @@ +/* + * ZeroTier One - Network Virtualization Everywhere + * Copyright (C) 2011-2016 ZeroTier, Inc. https://www.zerotier.com/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +#include +#include +#include +#include + +#include "SoftwareUpdater.hpp" + +#include "../version.h" + +#include "../node/Constants.hpp" +#include "../node/Utils.hpp" +#include "../node/SHA512.hpp" +#include "../node/Buffer.hpp" +#include "../node/Node.hpp" +#include "../osdep/OSUtils.hpp" + +namespace ZeroTier { + +SoftwareUpdater::SoftwareUpdater(Node &node,const char *homePath,bool updateDistributor) : + _node(node), + _lastCheckTime(OSUtils::now()), // check in the future in case we just started, in which case we're probably offline + _homePath(homePath) +{ + // Load all updates we are distributing if we are an update distributor and have an update-dist.d folder + if (updateDistributor) { + std::string udd(_homePath + ZT_PATH_SEPARATOR_S + "update-dist.d"); + std::vector ud(OSUtils::listDirectory(udd.c_str())); + for(std::vector::iterator u(ud.begin());u!=ud.end();++u) { + // Each update has a companion .json file describing it. Other files are ignored. + if ((u->length() > 5)&&(u->substr(u->length() - 5,5) == ".json")) { + std::string buf; + if (OSUtils::readFile((udd + ZT_PATH_SEPARATOR_S + *u).c_str(),buf)) { + try { + _D d; + d.meta = OSUtils::jsonParse(buf); + std::string metaHash = OSUtils::jsonBinFromHex(d.meta[ZT_SOFTWARE_UPDATE_JSON_UPDATE_HASH]); + if ((metaHash.length() == ZT_SHA512_DIGEST_LEN)&&(OSUtils::readFile((udd + ZT_PATH_SEPARATOR_S + u->substr(0,u->length() - 5)).c_str(),d.bin))) { + uint8_t sha512[ZT_SHA512_DIGEST_LEN]; + SHA512::hash(sha512,d.bin.data(),(unsigned int)d.bin.length()); + if (!memcmp(sha512,metaHash.data(),ZT_SHA512_DIGEST_LEN)) { // double check that hash in JSON is correct + _dist[Array(sha512)] = d; + } + } + } catch ( ... ) {} // ignore bad meta JSON, etc. + } + } + } + } +} + +SoftwareUpdater::~SoftwareUpdater() +{ +} + +void SoftwareUpdater::handleSoftwareUpdateUserMessage(uint64_t origin,const void *data,unsigned int len) +{ + if (!len) return; + try { + const MessageVerb v = (MessageVerb)reinterpret_cast(data)[0]; + switch(v) { + case VERB_GET_LATEST: + case VERB_LATEST: { + nlohmann::json req = OSUtils::jsonParse(std::string(reinterpret_cast(data) + 1,len - 1)); + if (req.is_object()) { + const unsigned int rvMaj = (unsigned int)OSUtils::jsonInt(req[ZT_SOFTWARE_UPDATE_JSON_VERSION_MAJOR],0); + const unsigned int rvMin = (unsigned int)OSUtils::jsonInt(req[ZT_SOFTWARE_UPDATE_JSON_VERSION_MINOR],0); + const unsigned int rvRev = (unsigned int)OSUtils::jsonInt(req[ZT_SOFTWARE_UPDATE_JSON_VERSION_REVISION],0); + if (v == VERB_GET_LATEST) { + + if (_dist.size() > 0) { + const nlohmann::json *latest = (const nlohmann::json *)0; + const std::string rSigner = OSUtils::jsonString(req[ZT_SOFTWARE_UPDATE_JSON_EXPECT_SIGNED_BY],""); + for(std::map< Array,_D >::const_iterator d(_dist.begin());d!=_dist.end();++d) { + if (OSUtils::jsonString(d->second.meta[ZT_SOFTWARE_UPDATE_JSON_UPDATE_SIGNED_BY],"") == rSigner) { + const unsigned int dvMaj = (unsigned int)OSUtils::jsonInt(d->second.meta[ZT_SOFTWARE_UPDATE_JSON_VERSION_MAJOR],0); + const unsigned int dvMin = (unsigned int)OSUtils::jsonInt(d->second.meta[ZT_SOFTWARE_UPDATE_JSON_VERSION_MINOR],0); + const unsigned int dvRev = (unsigned int)OSUtils::jsonInt(d->second.meta[ZT_SOFTWARE_UPDATE_JSON_VERSION_REVISION],0); + if (Utils::compareVersion(dvMaj,dvMin,dvRev,rvMaj,rvMin,rvRev) > 0) + latest = &(d->second.meta); + } + } + if (latest) { + std::string lj; + lj.push_back((char)VERB_LATEST); + lj.append(OSUtils::jsonDump(*latest)); + _node.sendUserMessage(origin,ZT_SOFTWARE_UPDATE_USER_MESSAGE_TYPE,lj.data(),(unsigned int)lj.length()); + } + } // else no reply, since we have nothing to distribute + + } else { // VERB_LATEST + + if ((origin == ZT_SOFTWARE_UPDATE_SERVICE)&& + (Utils::compareVersion(rvMaj,rvMin,rvRev,ZEROTIER_ONE_VERSION_MAJOR,ZEROTIER_ONE_VERSION_MINOR,ZEROTIER_ONE_VERSION_REVISION) > 0)&& + (OSUtils::jsonString(req[ZT_SOFTWARE_UPDATE_JSON_UPDATE_SIGNED_BY],"") == ZT_SOFTWARE_UPDATE_SIGNING_AUTHORITY)) { + const unsigned long len = (unsigned long)OSUtils::jsonInt(req[ZT_SOFTWARE_UPDATE_JSON_UPDATE_SIZE],0); + const std::string hash = OSUtils::jsonBinFromHex(req[ZT_SOFTWARE_UPDATE_JSON_UPDATE_HASH]); + if ((len <= ZT_SOFTWARE_UPDATE_MAX_SIZE)&&(hash.length() >= 16)) { + if (_latestMeta != req) { + _latestMeta = req; + _latestBin = ""; + memcpy(_latestBinHashPrefix.data,hash.data(),16); + _latestBinLength = len; + _latestBinValid = false; + } + + Buffer<128> gd; + gd.append((uint8_t)VERB_GET_DATA); + gd.append(_latestBinHashPrefix.data,16); + gd.append((uint32_t)_latestBin.length()); + _node.sendUserMessage(ZT_SOFTWARE_UPDATE_SERVICE,ZT_SOFTWARE_UPDATE_USER_MESSAGE_TYPE,gd.data(),gd.size()); + } + } + + } + } + } break; + + case VERB_GET_DATA: + if ((len >= 21)&&(_dist.size() > 0)) { + std::map< Array,_D >::iterator d(_dist.find(Array(reinterpret_cast(data) + 1))); + if (d != _dist.end()) { + unsigned long idx = (unsigned long)*(reinterpret_cast(data) + 17) << 24; + idx |= (unsigned long)*(reinterpret_cast(data) + 18) << 16; + idx |= (unsigned long)*(reinterpret_cast(data) + 19) << 8; + idx |= (unsigned long)*(reinterpret_cast(data) + 20); + if (idx < d->second.bin.length()) { + Buffer buf; + buf.append((uint8_t)VERB_DATA); + buf.append(reinterpret_cast(data) + 1,16); + buf.append((uint32_t)idx); + buf.append(d->second.bin.data() + idx,std::max((unsigned long)ZT_SOFTWARE_UPDATE_CHUNK_SIZE,(unsigned long)(d->second.bin.length() - idx))); + _node.sendUserMessage(origin,ZT_SOFTWARE_UPDATE_USER_MESSAGE_TYPE,buf.data(),buf.size()); + } + } + } + break; + + case VERB_DATA: + if ((len >= 21)&&(!memcmp(_latestBinHashPrefix.data,reinterpret_cast(data) + 1,16))) { + unsigned long idx = (unsigned long)*(reinterpret_cast(data) + 17) << 24; + idx |= (unsigned long)*(reinterpret_cast(data) + 18) << 16; + idx |= (unsigned long)*(reinterpret_cast(data) + 19) << 8; + idx |= (unsigned long)*(reinterpret_cast(data) + 20); + if (idx == _latestBin.length()) + _latestBin.append(reinterpret_cast(data) + 21,len - 21); + + if (_latestBin.length() < _latestBinLength) { + Buffer<128> gd; + gd.append((uint8_t)VERB_GET_DATA); + gd.append(_latestBinHashPrefix.data,16); + gd.append((uint32_t)_latestBin.length()); + _node.sendUserMessage(ZT_SOFTWARE_UPDATE_SERVICE,ZT_SOFTWARE_UPDATE_USER_MESSAGE_TYPE,gd.data(),gd.size()); + } + } + break; + + default: + break; + } + } catch ( ... ) { + // Discard bad messages + } +} + +nlohmann::json SoftwareUpdater::check() +{ + if (_latestBinLength > 0) { + if (_latestBin.length() >= _latestBinLength) { + if (_latestBinValid) { + return _latestMeta; + } else { + // This is the important security verification part! + + try { + // (1) Check the hash itself to make sure the image is basically okay + uint8_t sha512[ZT_SHA512_DIGEST_LEN]; + SHA512::hash(sha512,_latestBin.data(),(unsigned int)_latestBin.length()); + if (Utils::hex(sha512,ZT_SHA512_DIGEST_LEN) == OSUtils::jsonString(_latestMeta[ZT_SOFTWARE_UPDATE_JSON_UPDATE_HASH],"")) { + // (2) Check signature by signing authority + std::string sig(OSUtils::jsonBinFromHex(_latestMeta[ZT_SOFTWARE_UPDATE_JSON_UPDATE_SIGNATURE])); + if (Identity(ZT_SOFTWARE_UPDATE_SIGNING_AUTHORITY).verify(_latestBin.data(),(unsigned int)_latestBin.length(),sig.data(),(unsigned int)sig.length())) { + // If we passed both of these, the update is good! + _latestBinValid = true; + return _latestMeta; + } + } + } catch ( ... ) {} // any exception equals verification failure + + // If we get here, checks failed. + _latestMeta = nlohmann::json(); + _latestBin = ""; + _latestBinLength = 0; + _latestBinValid = false; + } + } else { + Buffer<128> gd; + gd.append((uint8_t)VERB_GET_DATA); + gd.append(_latestBinHashPrefix.data,16); + gd.append((uint32_t)_latestBin.length()); + _node.sendUserMessage(ZT_SOFTWARE_UPDATE_SERVICE,ZT_SOFTWARE_UPDATE_USER_MESSAGE_TYPE,gd.data(),gd.size()); + } + } + + const uint64_t now = OSUtils::now(); + if ((now - _lastCheckTime) >= ZT_SOFTWARE_UPDATE_CHECK_PERIOD) { + } + + return nlohmann::json(); +} + +void SoftwareUpdater::apply() +{ + if ((_latestBin.length() == _latestBinLength)&&(_latestBinLength > 0)&&(_latestBinValid)) { + } +} + +} // namespace ZeroTier diff --git a/service/SoftwareUpdater.hpp b/service/SoftwareUpdater.hpp new file mode 100644 index 00000000..5fb465f8 --- /dev/null +++ b/service/SoftwareUpdater.hpp @@ -0,0 +1,171 @@ +/* + * ZeroTier One - Network Virtualization Everywhere + * Copyright (C) 2011-2016 ZeroTier, Inc. https://www.zerotier.com/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +#ifndef ZT_SOFTWAREUPDATER_HPP +#define ZT_SOFTWAREUPDATER_HPP + +#include + +#include +#include +#include + +#include "../include/ZeroTierOne.h" +#include "../node/Identity.hpp" +#include "../node/Array.hpp" + +#include "../ext/json/json.hpp" + +/** + * VERB_USER_MESSAGE type ID for software update messages + */ +#define ZT_SOFTWARE_UPDATE_USER_MESSAGE_TYPE 1000 + +/** + * ZeroTier address of node that provides software updates + */ +#define ZT_SOFTWARE_UPDATE_SERVICE 0xc1243d3869ULL + +/** + * ZeroTier identity that must be used to sign software updates + */ +#define ZT_SOFTWARE_UPDATE_SIGNING_AUTHORITY "" + +/** + * Chunk size for in-band downloads (can be changed, designed to always fit in one UDP packet easily) + */ +#define ZT_SOFTWARE_UPDATE_CHUNK_SIZE 1380 + +/** + * Sanity limit for the size of an update binary image + */ +#define ZT_SOFTWARE_UPDATE_MAX_SIZE (1024 * 1024 * 256) + +/** + * How often (ms) do we check? + */ +#define ZT_SOFTWARE_UPDATE_CHECK_PERIOD (60 * 60 * 1000) + +#define ZT_SOFTWARE_UPDATE_JSON_VERSION_MAJOR "versionMajor" +#define ZT_SOFTWARE_UPDATE_JSON_VERSION_MINOR "versionMinor" +#define ZT_SOFTWARE_UPDATE_JSON_VERSION_REVISION "versionRev" +#define ZT_SOFTWARE_UPDATE_JSON_EXPECT_SIGNED_BY "expectedSigner" +#define ZT_SOFTWARE_UPDATE_JSON_PLATFORM "platform" +#define ZT_SOFTWARE_UPDATE_JSON_ARCHITECTURE "arch" +#define ZT_SOFTWARE_UPDATE_JSON_WORD_SIZE "wordSize" +#define ZT_SOFTWARE_UPDATE_JSON_VENDOR "vendor" +#define ZT_SOFTWARE_UPDATE_JSON_CHANNEL "channel" +#define ZT_SOFTWARE_UPDATE_JSON_UPDATE_SIGNED_BY "updateSigner" +#define ZT_SOFTWARE_UPDATE_JSON_UPDATE_SIGNATURE "updateSig" +#define ZT_SOFTWARE_UPDATE_JSON_UPDATE_HASH "updateHash" +#define ZT_SOFTWARE_UPDATE_JSON_UPDATE_SIZE "updateSize" +#define ZT_SOFTWARE_UPDATE_JSON_EXEC_ARGS "updateExecArgs" + +namespace ZeroTier { + +class Node; + +/** + * This class handles retrieving and executing updates, or serving them + */ +class SoftwareUpdater +{ +public: + /** + * Each message begins with an 8-bit message verb + */ + enum MessageVerb + { + /** + * Payload: JSON containing current system platform, version, etc. + */ + VERB_GET_LATEST = 1, + + /** + * Payload: JSON describing latest update for this target. (No response is sent if there is none.) + */ + VERB_LATEST = 2, + + /** + * Payload: + * <[16] first 128 bits of hash of data object> + * <[4] 32-bit index of chunk to get> + */ + VERB_GET_DATA = 3, + + /** + * Payload: + * <[16] first 128 bits of hash of data object> + * <[4] 32-bit index of chunk> + * <[...] chunk data> + */ + VERB_DATA = 4 + }; + + SoftwareUpdater(Node &node,const char *homePath,bool updateDistributor); + ~SoftwareUpdater(); + + /** + * Handle a software update user message + * + * @param origin ZeroTier address of message origin + * @param data Message payload + * @param len Length of message + */ + void handleSoftwareUpdateUserMessage(uint64_t origin,const void *data,unsigned int len); + + /** + * Check for updates and do other update-related housekeeping + * + * It should be called about every 10 seconds. + * + * @return Null JSON object or update information if there is an update downloaded and ready + */ + nlohmann::json check(); + + /** + * Apply any ready update now + * + * Depending on the platform this function may never return and may forcibly + * exit the process. It does nothing if no update is ready. + */ + void apply(); + +private: + Node &_node; + uint64_t _lastCheckTime; + std::string _homePath; + + // Offered software updates if we are an update host (we have update-dist.d and update hosting is enabled) + struct _D + { + nlohmann::json meta; + std::string bin; + }; + std::map< Array,_D > _dist; // key is first 16 bytes of hash + + nlohmann::json _latestMeta; + std::string _latestBin; + Array _latestBinHashPrefix; + unsigned long _latestBinLength; + bool _latestBinValid; +}; + +} // namespace ZeroTier + +#endif -- cgit v1.2.3 From c6b0b076055f0b9b579c21bbbe83171a554a2600 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Thu, 12 Jan 2017 13:18:11 -0800 Subject: More software update work, and settings in local.conf. --- make-freebsd.mk | 40 +++++++++++++++--------- make-linux.mk | 64 +++++++++++---------------------------- make-mac.mk | 41 ++++++++++--------------- service/OneService.cpp | 19 ++++++++++-- service/SoftwareUpdater.cpp | 74 ++++++++++++++++++++++++--------------------- service/SoftwareUpdater.hpp | 33 ++++++++++++++++---- 6 files changed, 142 insertions(+), 129 deletions(-) (limited to 'make-mac.mk') diff --git a/make-freebsd.mk b/make-freebsd.mk index cb9a2e6d..7de85c8c 100644 --- a/make-freebsd.mk +++ b/make-freebsd.mk @@ -1,6 +1,5 @@ CC=cc CXX=c++ - INCLUDES= DEFS= LIBS= @@ -8,11 +7,6 @@ LIBS= include objects.mk OBJS+=osdep/BSDEthernetTap.o ext/lz4/lz4.o ext/http-parser/http_parser.o -# "make official" is a shortcut for this -ifeq ($(ZT_OFFICIAL_RELEASE),1) - DEFS+=-DZT_OFFICIAL_RELEASE -endif - # Build with ZT_ENABLE_CLUSTER=1 to build with cluster support ifeq ($(ZT_ENABLE_CLUSTER),1) DEFS+=-DZT_ENABLE_CLUSTER @@ -34,6 +28,32 @@ else STRIP=strip --strip-all endif +# Determine system build architecture from compiler target +CC_MACH=$(shell $(CC) -dumpmachine | cut -d '-' -f 1) +ZT_ARCHITECTURE=0 +ifeq ($(CC_MACH),x86_64) + ZT_ARCHITECTURE=2 +endif +ifeq ($(CC_MACH),amd64) + ZT_ARCHITECTURE=2 +endif +ifeq ($(CC_MACH),i386) + ZT_ARCHITECTURE=1 +endif +ifeq ($(CC_MACH),i686) + ZT_ARCHITECTURE=1 +endif +ifeq ($(CC_MACH),arm) + ZT_ARCHITECTURE=3 +endif +ifeq ($(CC_MACH),arm64) + ZT_ARCHITECTURE=4 +endif +ifeq ($(CC_MACH),aarch64) + ZT_ARCHITECTURE=4 +endif +DEFS+=-DZT_BUILD_PLATFORM=7 -DZT_BUILD_ARCHITECTURE=$(ZT_ARCHITECTURE) -DZT_SOFTWARE_UPDATE_DEFAULT="disable" + CXXFLAGS+=$(CFLAGS) -fno-rtti all: one @@ -48,18 +68,10 @@ selftest: $(OBJS) selftest.o $(CXX) $(CXXFLAGS) $(LDFLAGS) -o zerotier-selftest selftest.o $(OBJS) $(LIBS) $(STRIP) zerotier-selftest -# No installer on FreeBSD yet -#installer: one FORCE -# ./buildinstaller.sh - clean: rm -rf *.o node/*.o controller/*.o osdep/*.o service/*.o ext/http-parser/*.o ext/lz4/*.o ext/json-parser/*.o build-* zerotier-one zerotier-idtool zerotier-selftest zerotier-cli ZeroTierOneInstaller-* debug: FORCE make -j 4 ZT_DEBUG=1 -#official: FORCE -# make -j 4 ZT_OFFICIAL_RELEASE=1 -# ./buildinstaller.sh - FORCE: diff --git a/make-linux.mk b/make-linux.mk index 90644e3d..cb91ab88 100644 --- a/make-linux.mk +++ b/make-linux.mk @@ -1,24 +1,3 @@ -# -# Makefile for ZeroTier One on Linux -# -# This is confirmed to work on distributions newer than CentOS 6 (the -# one used for reference builds) and on 32 and 64 bit x86 and ARM -# machines. It should also work on other 'normal' machines and recent -# distributions. Editing might be required for tiny devices or weird -# distros. -# -# Targets -# one: zerotier-one and symlinks (cli and idtool) -# manpages: builds manpages, requires 'ronn' or nodeJS (will use either) -# all: builds 'one' and 'manpages' -# selftest: zerotier-selftest -# debug: builds 'one' and 'selftest' with tracing and debug flags -# clean: removes all built files, objects, other trash -# distclean: removes a few other things that might be present -# debian: build DEB packages; deb dev tools must be present -# redhat: build RPM packages; rpm dev tools must be present -# - # Automagically pick clang or gcc, with preference for clang # This is only done if we have not overridden these with an environment or CLI variable ifeq ($(origin CC),default) @@ -28,8 +7,6 @@ ifeq ($(origin CXX),default) CXX=$(shell if [ -e /usr/bin/clang++ ]; then echo clang++; else echo g++; fi) endif -#UNAME_M=$(shell $(CC) -dumpmachine | cut -d '-' -f 1) - INCLUDES?= DEFS?=-D_FORTIFY_SOURCE=2 LDLIBS?= @@ -52,28 +29,23 @@ else DEFS+=-DZT_USE_SYSTEM_HTTP_PARSER endif -ifeq ($(ZT_USE_MINIUPNPC),1) - OBJS+=osdep/PortMapper.o - - DEFS+=-DZT_USE_MINIUPNPC - - # Auto-detect libminiupnpc at least v2.0 - MINIUPNPC_IS_NEW_ENOUGH=$(shell grep -sqr '.*define.*MINIUPNPC_VERSION.*"2.."' /usr/include/miniupnpc/miniupnpc.h && echo 1) - ifeq ($(MINIUPNPC_IS_NEW_ENOUGH),1) - DEFS+=-DZT_USE_SYSTEM_MINIUPNPC - LDLIBS+=-lminiupnpc - else - DEFS+=-DMINIUPNP_STATICLIB -DMINIUPNPC_SET_SOCKET_TIMEOUT -DMINIUPNPC_GET_SRC_ADDR -D_BSD_SOURCE -D_DEFAULT_SOURCE -D_XOPEN_SOURCE=600 -DOS_STRING=\"Linux\" -DMINIUPNPC_VERSION_STRING=\"2.0\" -DUPNP_VERSION_STRING=\"UPnP/1.1\" -DENABLE_STRNATPMPERR - OBJS+=ext/miniupnpc/connecthostport.o ext/miniupnpc/igd_desc_parse.o ext/miniupnpc/minisoap.o ext/miniupnpc/minissdpc.o ext/miniupnpc/miniupnpc.o ext/miniupnpc/miniwget.o ext/miniupnpc/minixml.o ext/miniupnpc/portlistingparse.o ext/miniupnpc/receivedata.o ext/miniupnpc/upnpcommands.o ext/miniupnpc/upnpdev.o ext/miniupnpc/upnperrors.o ext/miniupnpc/upnpreplyparse.o - endif - - # Auto-detect libnatpmp - ifeq ($(wildcard /usr/include/natpmp.h),) - OBJS+=ext/libnatpmp/natpmp.o ext/libnatpmp/getgateway.o - else - LDLIBS+=-lnatpmp - DEFS+=-DZT_USE_SYSTEM_NATPMP - endif +# Auto-detect miniupnpc and nat-pmp as well and use system libs if present, +# otherwise build into binary as done on Mac and Windows. +OBJS+=osdep/PortMapper.o +DEFS+=-DZT_USE_MINIUPNPC +MINIUPNPC_IS_NEW_ENOUGH=$(shell grep -sqr '.*define.*MINIUPNPC_VERSION.*"2.."' /usr/include/miniupnpc/miniupnpc.h && echo 1) +ifeq ($(MINIUPNPC_IS_NEW_ENOUGH),1) + DEFS+=-DZT_USE_SYSTEM_MINIUPNPC + LDLIBS+=-lminiupnpc +else + DEFS+=-DMINIUPNP_STATICLIB -DMINIUPNPC_SET_SOCKET_TIMEOUT -DMINIUPNPC_GET_SRC_ADDR -D_BSD_SOURCE -D_DEFAULT_SOURCE -D_XOPEN_SOURCE=600 -DOS_STRING=\"Linux\" -DMINIUPNPC_VERSION_STRING=\"2.0\" -DUPNP_VERSION_STRING=\"UPnP/1.1\" -DENABLE_STRNATPMPERR + OBJS+=ext/miniupnpc/connecthostport.o ext/miniupnpc/igd_desc_parse.o ext/miniupnpc/minisoap.o ext/miniupnpc/minissdpc.o ext/miniupnpc/miniupnpc.o ext/miniupnpc/miniwget.o ext/miniupnpc/minixml.o ext/miniupnpc/portlistingparse.o ext/miniupnpc/receivedata.o ext/miniupnpc/upnpcommands.o ext/miniupnpc/upnpdev.o ext/miniupnpc/upnperrors.o ext/miniupnpc/upnpreplyparse.o +endif +ifeq ($(wildcard /usr/include/natpmp.h),) + OBJS+=ext/libnatpmp/natpmp.o ext/libnatpmp/getgateway.o +else + LDLIBS+=-lnatpmp + DEFS+=-DZT_USE_SYSTEM_NATPMP endif ifeq ($(ZT_ENABLE_CLUSTER),1) @@ -137,7 +109,7 @@ endif ifeq ($(CC_MACH),aarch64) ZT_ARCHITECTURE=4 endif -DEFS+=-DZT_BUILD_PLATFORM=1 -DZT_BUILD_ARCHITECTURE=$(ZT_ARCHITECTURE) +DEFS+=-DZT_BUILD_PLATFORM=1 -DZT_BUILD_ARCHITECTURE=$(ZT_ARCHITECTURE) -DZT_SOFTWARE_UPDATE_DEFAULT="\"disable\"" all: one diff --git a/make-mac.mk b/make-mac.mk index e9de6f49..99a9d738 100644 --- a/make-mac.mk +++ b/make-mac.mk @@ -1,45 +1,39 @@ -ifeq ($(origin CC),default) - CC=$(shell if [ -e /usr/bin/clang ]; then echo clang; else echo gcc; fi) -endif -ifeq ($(origin CXX),default) - CXX=$(shell if [ -e /usr/bin/clang++ ]; then echo clang++; else echo g++; fi) -endif - +CC=clang +CXX=clang++ INCLUDES= DEFS= LIBS= -ARCH_FLAGS=-arch x86_64 - -include objects.mk -OBJS+=osdep/OSXEthernetTap.o ext/lz4/lz4.o ext/http-parser/http_parser.o - -# Disable codesign since open source users will not have ZeroTier's certs +ARCH_FLAGS= CODESIGN=echo PRODUCTSIGN=echo CODESIGN_APP_CERT= CODESIGN_INSTALLER_CERT= -# Build with libminiupnpc by default for Mac -- desktops/laptops almost always want this -ZT_USE_MINIUPNPC?=1 +# 3 == MacOS, 2 == X64 (the only arch for MacOS right now) +DEFS+=-DZT_BUILD_PLATFORM=3 -DZT_BUILD_ARCHITECTURE=2 -# For internal use only -- signs everything with ZeroTier's developer cert +include objects.mk +OBJS+=osdep/OSXEthernetTap.o ext/lz4/lz4.o ext/http-parser/http_parser.o + +# Official releases are signed with our Apple cert and apply software updates by default ifeq ($(ZT_OFFICIAL_RELEASE),1) - DEFS+=-DZT_OFFICIAL_RELEASE + DEFS+=-DZT_SOFTWARE_UPDATE_DEFAULT="apply" ZT_USE_MINIUPNPC=1 CODESIGN=codesign PRODUCTSIGN=productsign CODESIGN_APP_CERT="Developer ID Application: ZeroTier, Inc (8ZD9JUCZ4V)" CODESIGN_INSTALLER_CERT="Developer ID Installer: ZeroTier, Inc (8ZD9JUCZ4V)" +else + DEFS+=-DZT_SOFTWARE_UPDATE_DEFAULT="download" endif ifeq ($(ZT_ENABLE_CLUSTER),1) DEFS+=-DZT_ENABLE_CLUSTER endif -ifeq ($(ZT_USE_MINIUPNPC),1) - DEFS+=-DMACOSX -DZT_USE_MINIUPNPC -DMINIUPNP_STATICLIB -D_DARWIN_C_SOURCE -DMINIUPNPC_SET_SOCKET_TIMEOUT -DMINIUPNPC_GET_SRC_ADDR -D_BSD_SOURCE -D_DEFAULT_SOURCE -DOS_STRING=\"Darwin/15.0.0\" -DMINIUPNPC_VERSION_STRING=\"2.0\" -DUPNP_VERSION_STRING=\"UPnP/1.1\" -DENABLE_STRNATPMPERR - OBJS+=ext/libnatpmp/natpmp.o ext/libnatpmp/getgateway.o ext/miniupnpc/connecthostport.o ext/miniupnpc/igd_desc_parse.o ext/miniupnpc/minisoap.o ext/miniupnpc/minissdpc.o ext/miniupnpc/miniupnpc.o ext/miniupnpc/miniwget.o ext/miniupnpc/minixml.o ext/miniupnpc/portlistingparse.o ext/miniupnpc/receivedata.o ext/miniupnpc/upnpcommands.o ext/miniupnpc/upnpdev.o ext/miniupnpc/upnperrors.o ext/miniupnpc/upnpreplyparse.o osdep/PortMapper.o -endif +# Build miniupnpc and nat-pmp as included libraries -- extra defs are required for these sources +DEFS+=-DMACOSX -DZT_USE_MINIUPNPC -DMINIUPNP_STATICLIB -D_DARWIN_C_SOURCE -DMINIUPNPC_SET_SOCKET_TIMEOUT -DMINIUPNPC_GET_SRC_ADDR -D_BSD_SOURCE -D_DEFAULT_SOURCE -DOS_STRING=\"Darwin/15.0.0\" -DMINIUPNPC_VERSION_STRING=\"2.0\" -DUPNP_VERSION_STRING=\"UPnP/1.1\" -DENABLE_STRNATPMPERR +OBJS+=ext/libnatpmp/natpmp.o ext/libnatpmp/getgateway.o ext/miniupnpc/connecthostport.o ext/miniupnpc/igd_desc_parse.o ext/miniupnpc/minisoap.o ext/miniupnpc/minissdpc.o ext/miniupnpc/miniupnpc.o ext/miniupnpc/miniwget.o ext/miniupnpc/minixml.o ext/miniupnpc/portlistingparse.o ext/miniupnpc/receivedata.o ext/miniupnpc/upnpcommands.o ext/miniupnpc/upnpdev.o ext/miniupnpc/upnperrors.o ext/miniupnpc/upnpreplyparse.o osdep/PortMapper.o # Debug mode -- dump trace output, build binary with -g ifeq ($(ZT_DEBUG),1) @@ -55,10 +49,7 @@ else STRIP=strip endif -CXXFLAGS=$(CFLAGS) -mmacosx-version-min=10.7 -std=c++11 -stdlib=libc++ - -# 3 == MacOS, 2 == X64 -DEFS+=-DZT_BUILD_PLATFORM=3 -DZT_BUILD_ARCHITECTURE=2 +CXXFLAGS=$(CFLAGS) -std=c++11 -stdlib=libc++ all: one macui diff --git a/service/OneService.cpp b/service/OneService.cpp index 9ee05f96..697be24a 100644 --- a/service/OneService.cpp +++ b/service/OneService.cpp @@ -264,6 +264,7 @@ public: Phy _phy; Node *_node; SoftwareUpdater *_updater; + bool _updateAutoApply; unsigned int _primaryPort; // Local configuration and memo-ized static path definitions @@ -367,6 +368,7 @@ public: ,_phy(this,false,true) ,_node((Node *)0) ,_updater((SoftwareUpdater *)0) + ,_updateAutoApply(false) ,_primaryPort(port) ,_controlPlane((ControlPlane *)0) ,_lastDirectReceiveFromGlobal(0) @@ -454,9 +456,6 @@ public: _node = new Node(this,&cb,OSUtils::now()); } - _updater = new SoftwareUpdater(*_node,_homePath); - _updater->loadUpdatesToDistribute(); - // Read local configuration { uint64_t trustedPathIds[ZT_MAX_TRUSTED_PATHS]; @@ -982,6 +981,20 @@ public: _node->setRelayPolicy(ZT_RELAY_POLICY_NEVER); else _node->setRelayPolicy(ZT_RELAY_POLICY_TRUSTED); + const std::string up(OSUtils::jsonString(settings["softwareUpdate"],ZT_SOFTWARE_UPDATE_DEFAULT)); + const bool udist = OSUtils::jsonBool(settings["softwareUpdateDist"],false); + if (((up == "apply")||(up == "download"))||(udist)) { + if (!_updater) + _updater = new SoftwareUpdater(*_node,_homePath); + _updateAutoApply = (up == "apply"); + _updater->setUpdateDistribution(udist); + _updater->setChannel(OSUtils::jsonString(settings["softwareUpdateChannel"],ZT_SOFTWARE_UPDATE_DEFAULT_CHANNEL)); + } else { + delete _updater; + _updater = (SoftwareUpdater *)0; + _updateAutoApply = false; + } + json &ignoreIfs = settings["interfacePrefixBlacklist"]; if (ignoreIfs.is_array()) { for(unsigned long i=0;i ud(OSUtils::listDirectory(udd.c_str())); - for(std::vector::iterator u(ud.begin());u!=ud.end();++u) { - // Each update has a companion .json file describing it. Other files are ignored. - if ((u->length() > 5)&&(u->substr(u->length() - 5,5) == ".json")) { - std::string buf; - if (OSUtils::readFile((udd + ZT_PATH_SEPARATOR_S + *u).c_str(),buf)) { - try { - _D d; - d.meta = OSUtils::jsonParse(buf); - std::string metaHash = OSUtils::jsonBinFromHex(d.meta[ZT_SOFTWARE_UPDATE_JSON_UPDATE_HASH]); - if ((metaHash.length() == ZT_SHA512_DIGEST_LEN)&&(OSUtils::readFile((udd + ZT_PATH_SEPARATOR_S + u->substr(0,u->length() - 5)).c_str(),d.bin))) { - uint8_t sha512[ZT_SHA512_DIGEST_LEN]; - SHA512::hash(sha512,d.bin.data(),(unsigned int)d.bin.length()); - if (!memcmp(sha512,metaHash.data(),ZT_SHA512_DIGEST_LEN)) { // double check that hash in JSON is correct - _dist[Array(sha512)] = d; - printf("update-dist.d: %s\n",u->c_str()); + _dist.clear(); + if (distribute) { + std::string udd(_homePath + ZT_PATH_SEPARATOR_S + "update-dist.d"); + std::vector ud(OSUtils::listDirectory(udd.c_str())); + for(std::vector::iterator u(ud.begin());u!=ud.end();++u) { + // Each update has a companion .json file describing it. Other files are ignored. + if ((u->length() > 5)&&(u->substr(u->length() - 5,5) == ".json")) { + std::string buf; + if (OSUtils::readFile((udd + ZT_PATH_SEPARATOR_S + *u).c_str(),buf)) { + try { + _D d; + d.meta = OSUtils::jsonParse(buf); + std::string metaHash = OSUtils::jsonBinFromHex(d.meta[ZT_SOFTWARE_UPDATE_JSON_UPDATE_HASH]); + if ((metaHash.length() == ZT_SHA512_DIGEST_LEN)&&(OSUtils::readFile((udd + ZT_PATH_SEPARATOR_S + u->substr(0,u->length() - 5)).c_str(),d.bin))) { + uint8_t sha512[ZT_SHA512_DIGEST_LEN]; + SHA512::hash(sha512,d.bin.data(),(unsigned int)d.bin.length()); + if (!memcmp(sha512,metaHash.data(),ZT_SHA512_DIGEST_LEN)) { // double check that hash in JSON is correct + _dist[Array(sha512)] = d; + printf("update-dist.d: %s\n",u->c_str()); + } } - } - } catch ( ... ) {} // ignore bad meta JSON, etc. + } catch ( ... ) {} // ignore bad meta JSON, etc. + } } } } @@ -393,18 +397,17 @@ void SoftwareUpdater::handleSoftwareUpdateUserMessage(uint64_t origin,const void idx |= (unsigned long)*(reinterpret_cast(data) + 18) << 16; idx |= (unsigned long)*(reinterpret_cast(data) + 19) << 8; idx |= (unsigned long)*(reinterpret_cast(data) + 20); + printf("<< DATA @%u / %u bytes (we now have %u bytes)\n",(unsigned int)idx,(unsigned int)(len - 21),(unsigned int)_latestBin.length()); if (idx == _latestBin.length()) { _latestBin.append(reinterpret_cast(data) + 21,len - 21); - } - printf("<< DATA @%u / %u bytes (we now have %u bytes)\n",(unsigned int)idx,(unsigned int)(len - 21),(unsigned int)_latestBin.length()); - - if (_latestBin.length() < _latestBinLength) { - Buffer<128> gd; - gd.append((uint8_t)VERB_GET_DATA); - gd.append(_latestBinHashPrefix.data,16); - gd.append((uint32_t)_latestBin.length()); - _node.sendUserMessage(ZT_SOFTWARE_UPDATE_SERVICE,ZT_SOFTWARE_UPDATE_USER_MESSAGE_TYPE,gd.data(),gd.size()); - printf(">> GET_DATA @%u\n",(unsigned int)_latestBin.length()); + if (_latestBin.length() < _latestBinLength) { + Buffer<128> gd; + gd.append((uint8_t)VERB_GET_DATA); + gd.append(_latestBinHashPrefix.data,16); + gd.append((uint32_t)_latestBin.length()); + _node.sendUserMessage(ZT_SOFTWARE_UPDATE_SERVICE,ZT_SOFTWARE_UPDATE_USER_MESSAGE_TYPE,gd.data(),gd.size()); + printf(">> GET_DATA @%u\n",(unsigned int)_latestBin.length()); + } } } break; @@ -417,7 +420,7 @@ void SoftwareUpdater::handleSoftwareUpdateUserMessage(uint64_t origin,const void } } -nlohmann::json SoftwareUpdater::check(const uint64_t now) +bool SoftwareUpdater::check(const uint64_t now) { if ((now - _lastCheckTime) >= ZT_SOFTWARE_UPDATE_CHECK_PERIOD) { _lastCheckTime = now; @@ -439,7 +442,7 @@ nlohmann::json SoftwareUpdater::check(const uint64_t now) ZT_BUILD_PLATFORM, ZT_BUILD_ARCHITECTURE, (int)ZT_VENDOR_ZEROTIER, - "release"); + _channel.c_str()); _node.sendUserMessage(ZT_SOFTWARE_UPDATE_SERVICE,ZT_SOFTWARE_UPDATE_USER_MESSAGE_TYPE,tmp,len); printf(">> GET_LATEST\n"); } @@ -449,13 +452,14 @@ nlohmann::json SoftwareUpdater::check(const uint64_t now) if (_latestBinValid) { return _latestMeta; } else { - // This is the important security verification part! + // This is the very important security validation part that makes sure + // this software update doesn't have cooties. try { // (1) Check the hash itself to make sure the image is basically okay uint8_t sha512[ZT_SHA512_DIGEST_LEN]; SHA512::hash(sha512,_latestBin.data(),(unsigned int)_latestBin.length()); - if (Utils::hex(sha512,ZT_SHA512_DIGEST_LEN) == OSUtils::jsonString(_latestMeta[ZT_SOFTWARE_UPDATE_JSON_UPDATE_HASH],"")) { + if (Utils::hex(sha512,ZT_SHA512_DIGEST_LEN) == OSUtils::jsonString(_latestMeta[ZT_SOFTWARE_UPDATE_JSON_UPDATE_HASH],"~")) { // (2) Check signature by signing authority std::string sig(OSUtils::jsonBinFromHex(_latestMeta[ZT_SOFTWARE_UPDATE_JSON_UPDATE_SIGNATURE])); if (Identity(ZT_SOFTWARE_UPDATE_SIGNING_AUTHORITY).verify(_latestBin.data(),(unsigned int)_latestBin.length(),sig.data(),(unsigned int)sig.length())) { @@ -489,7 +493,7 @@ nlohmann::json SoftwareUpdater::check(const uint64_t now) void SoftwareUpdater::apply() { - if ((_latestBin.length() == _latestBinLength)&&(_latestBinLength > 0)&&(_latestBinValid)) { + if ((_latestBin.length() > 0)&&(_latestBinValid)) { } } diff --git a/service/SoftwareUpdater.hpp b/service/SoftwareUpdater.hpp index 0501ff23..b9cabf23 100644 --- a/service/SoftwareUpdater.hpp +++ b/service/SoftwareUpdater.hpp @@ -64,19 +64,25 @@ //#define ZT_SOFTWARE_UPDATE_CHECK_PERIOD (60 * 60 * 1000) #define ZT_SOFTWARE_UPDATE_CHECK_PERIOD 5000 +/** + * Default update channel + */ +#define ZT_SOFTWARE_UPDATE_DEFAULT_CHANNEL "release" + #define ZT_SOFTWARE_UPDATE_JSON_VERSION_MAJOR "versionMajor" #define ZT_SOFTWARE_UPDATE_JSON_VERSION_MINOR "versionMinor" #define ZT_SOFTWARE_UPDATE_JSON_VERSION_REVISION "versionRev" -#define ZT_SOFTWARE_UPDATE_JSON_EXPECT_SIGNED_BY "expectedSigner" #define ZT_SOFTWARE_UPDATE_JSON_PLATFORM "platform" #define ZT_SOFTWARE_UPDATE_JSON_ARCHITECTURE "arch" #define ZT_SOFTWARE_UPDATE_JSON_VENDOR "vendor" #define ZT_SOFTWARE_UPDATE_JSON_CHANNEL "channel" +#define ZT_SOFTWARE_UPDATE_JSON_EXPECT_SIGNED_BY "expectedSigner" #define ZT_SOFTWARE_UPDATE_JSON_UPDATE_SIGNED_BY "updateSigner" #define ZT_SOFTWARE_UPDATE_JSON_UPDATE_SIGNATURE "updateSig" #define ZT_SOFTWARE_UPDATE_JSON_UPDATE_HASH "updateHash" #define ZT_SOFTWARE_UPDATE_JSON_UPDATE_SIZE "updateSize" -#define ZT_SOFTWARE_UPDATE_JSON_EXEC_ARGS "updateExecArgs" +#define ZT_SOFTWARE_UPDATE_JSON_UPDATE_EXEC_ARGS "updateExecArgs" +#define ZT_SOFTWARE_UPDATE_JSON_UPDATE_URL "updateUrl" namespace ZeroTier { @@ -123,9 +129,11 @@ public: ~SoftwareUpdater(); /** - * Load update-dist.d if it exists + * Set whether or not we will distribute updates + * + * @param distribute If true, scan update-dist.d now and distribute updates found there -- if false, clear and stop distributing */ - void loadUpdatesToDistribute(); + void setUpdateDistribution(bool distribute); /** * Handle a software update user message @@ -141,9 +149,14 @@ public: * * It should be called about every 10 seconds. * - * @return Null JSON object or update information if there is an update downloaded and ready + * @return True if we've downloaded and verified an update */ - nlohmann::json check(const uint64_t now); + bool check(const uint64_t now); + + /** + * @return Meta-data for downloaded update or NULL if none + */ + inline const nlohmann::json &pending() const { return _latestMeta; } /** * Apply any ready update now @@ -153,10 +166,18 @@ public: */ void apply(); + /** + * Set software update channel + * + * @param channel 'release', 'beta', etc. + */ + inline void setChannel(const std::string &channel) { _channel = channel; } + private: Node &_node; uint64_t _lastCheckTime; std::string _homePath; + std::string _channel; // Offered software updates if we are an update host (we have update-dist.d and update hosting is enabled) struct _D -- cgit v1.2.3 From ab25c968212e402a2b2b8d562f3affac9e8717b9 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Thu, 12 Jan 2017 13:51:06 -0800 Subject: Build fixes. --- make-freebsd.mk | 2 +- make-mac.mk | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) (limited to 'make-mac.mk') diff --git a/make-freebsd.mk b/make-freebsd.mk index 7de85c8c..7ac50895 100644 --- a/make-freebsd.mk +++ b/make-freebsd.mk @@ -52,7 +52,7 @@ endif ifeq ($(CC_MACH),aarch64) ZT_ARCHITECTURE=4 endif -DEFS+=-DZT_BUILD_PLATFORM=7 -DZT_BUILD_ARCHITECTURE=$(ZT_ARCHITECTURE) -DZT_SOFTWARE_UPDATE_DEFAULT="disable" +DEFS+=-DZT_BUILD_PLATFORM=7 -DZT_BUILD_ARCHITECTURE=$(ZT_ARCHITECTURE) -DZT_SOFTWARE_UPDATE_DEFAULT="\"disable\"" CXXFLAGS+=$(CFLAGS) -fno-rtti diff --git a/make-mac.mk b/make-mac.mk index 99a9d738..1f130a1b 100644 --- a/make-mac.mk +++ b/make-mac.mk @@ -17,14 +17,14 @@ OBJS+=osdep/OSXEthernetTap.o ext/lz4/lz4.o ext/http-parser/http_parser.o # Official releases are signed with our Apple cert and apply software updates by default ifeq ($(ZT_OFFICIAL_RELEASE),1) - DEFS+=-DZT_SOFTWARE_UPDATE_DEFAULT="apply" + DEFS+=-DZT_SOFTWARE_UPDATE_DEFAULT="\"apply\"" ZT_USE_MINIUPNPC=1 CODESIGN=codesign PRODUCTSIGN=productsign CODESIGN_APP_CERT="Developer ID Application: ZeroTier, Inc (8ZD9JUCZ4V)" CODESIGN_INSTALLER_CERT="Developer ID Installer: ZeroTier, Inc (8ZD9JUCZ4V)" else - DEFS+=-DZT_SOFTWARE_UPDATE_DEFAULT="download" + DEFS+=-DZT_SOFTWARE_UPDATE_DEFAULT="\"download\"" endif ifeq ($(ZT_ENABLE_CLUSTER),1) -- cgit v1.2.3 From e1727d6297033881c9a379480aeacff806df2f2a Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Thu, 12 Jan 2017 14:34:52 -0800 Subject: Fix to software update repeat downloads. --- make-mac.mk | 2 +- service/SoftwareUpdater.cpp | 121 ++++++++++++++++++++++++-------------------- service/SoftwareUpdater.hpp | 19 +++++-- 3 files changed, 82 insertions(+), 60 deletions(-) (limited to 'make-mac.mk') diff --git a/make-mac.mk b/make-mac.mk index 1f130a1b..03c2f374 100644 --- a/make-mac.mk +++ b/make-mac.mk @@ -24,7 +24,7 @@ ifeq ($(ZT_OFFICIAL_RELEASE),1) CODESIGN_APP_CERT="Developer ID Application: ZeroTier, Inc (8ZD9JUCZ4V)" CODESIGN_INSTALLER_CERT="Developer ID Installer: ZeroTier, Inc (8ZD9JUCZ4V)" else - DEFS+=-DZT_SOFTWARE_UPDATE_DEFAULT="\"download\"" + DEFS+=-DZT_SOFTWARE_UPDATE_DEFAULT="\"disable\"" endif ifeq ($(ZT_ENABLE_CLUSTER),1) diff --git a/service/SoftwareUpdater.cpp b/service/SoftwareUpdater.cpp index 0f5b4fca..c47a5faf 100644 --- a/service/SoftwareUpdater.cpp +++ b/service/SoftwareUpdater.cpp @@ -258,9 +258,10 @@ SoftwareUpdater::SoftwareUpdater(Node &node,const std::string &homePath) : _lastCheckTime(0), _homePath(homePath), _channel(ZT_SOFTWARE_UPDATE_DEFAULT_CHANNEL), - _latestBinLength(0), - _latestBinValid(false) + _latestValid(false), + _downloadLength(0) { + // Check for a cached newer update. If there's a cached update that is not newer, delete. } SoftwareUpdater::~SoftwareUpdater() @@ -286,6 +287,7 @@ void SoftwareUpdater::setUpdateDistribution(bool distribute) uint8_t sha512[ZT_SHA512_DIGEST_LEN]; SHA512::hash(sha512,d.bin.data(),(unsigned int)d.bin.length()); if (!memcmp(sha512,metaHash.data(),ZT_SHA512_DIGEST_LEN)) { // double check that hash in JSON is correct + d.meta[ZT_SOFTWARE_UPDATE_JSON_UPDATE_SIZE] = d.bin.length(); // override with correct value -- setting this in meta json is optional _dist[Array(sha512)] = d; printf("update-dist.d: %s\n",u->c_str()); } @@ -351,19 +353,24 @@ void SoftwareUpdater::handleSoftwareUpdateUserMessage(uint64_t origin,const void if ((len <= ZT_SOFTWARE_UPDATE_MAX_SIZE)&&(hash.length() >= 16)) { if (_latestMeta != req) { _latestMeta = req; - _latestBin = ""; - memcpy(_latestBinHashPrefix.data,hash.data(),16); - _latestBinLength = len; - _latestBinValid = false; - printf("<< LATEST\n%s\n",OSUtils::jsonDump(req).c_str()); + _latestValid = false; + + OSUtils::rm((_homePath + ZT_PATH_SEPARATOR_S + ZT_SOFTWARE_UPDATE_META_FILENAME).c_str()); + OSUtils::rm((_homePath + ZT_PATH_SEPARATOR_S + ZT_SOFTWARE_UPDATE_BIN_FILENAME).c_str()); + + _download = std::string(); + memcpy(_downloadHashPrefix.data,hash.data(),16); + _downloadLength = len; } - Buffer<128> gd; - gd.append((uint8_t)VERB_GET_DATA); - gd.append(_latestBinHashPrefix.data,16); - gd.append((uint32_t)_latestBin.length()); - _node.sendUserMessage(ZT_SOFTWARE_UPDATE_SERVICE,ZT_SOFTWARE_UPDATE_USER_MESSAGE_TYPE,gd.data(),gd.size()); - printf(">> GET_DATA @%u\n",(unsigned int)_latestBin.length()); + if ((_downloadLength > 0)&&(_download.length() < _downloadLength)) { + Buffer<128> gd; + gd.append((uint8_t)VERB_GET_DATA); + gd.append(_downloadHashPrefix.data,16); + gd.append((uint32_t)_download.length()); + _node.sendUserMessage(ZT_SOFTWARE_UPDATE_SERVICE,ZT_SOFTWARE_UPDATE_USER_MESSAGE_TYPE,gd.data(),gd.size()); + printf(">> GET_DATA @%u\n",(unsigned int)_download.length()); + } } } @@ -379,7 +386,7 @@ void SoftwareUpdater::handleSoftwareUpdateUserMessage(uint64_t origin,const void idx |= (unsigned long)*(reinterpret_cast(data) + 20); printf("<< GET_DATA @%u from %.10llx for %s\n",(unsigned int)idx,origin,Utils::hex(reinterpret_cast(data) + 1,16).c_str()); std::map< Array,_D >::iterator d(_dist.find(Array(reinterpret_cast(data) + 1))); - if ((d != _dist.end())&&(idx < d->second.bin.length())) { + if ((d != _dist.end())&&(idx < (unsigned long)d->second.bin.length())) { Buffer buf; buf.append((uint8_t)VERB_DATA); buf.append(reinterpret_cast(data) + 1,16); @@ -392,21 +399,21 @@ void SoftwareUpdater::handleSoftwareUpdateUserMessage(uint64_t origin,const void break; case VERB_DATA: - if ((len >= 21)&&(!memcmp(_latestBinHashPrefix.data,reinterpret_cast(data) + 1,16))) { + if ((len >= 21)&&(_downloadLength > 0)&&(!memcmp(_downloadHashPrefix.data,reinterpret_cast(data) + 1,16))) { unsigned long idx = (unsigned long)*(reinterpret_cast(data) + 17) << 24; idx |= (unsigned long)*(reinterpret_cast(data) + 18) << 16; idx |= (unsigned long)*(reinterpret_cast(data) + 19) << 8; idx |= (unsigned long)*(reinterpret_cast(data) + 20); - printf("<< DATA @%u / %u bytes (we now have %u bytes)\n",(unsigned int)idx,(unsigned int)(len - 21),(unsigned int)_latestBin.length()); - if (idx == _latestBin.length()) { - _latestBin.append(reinterpret_cast(data) + 21,len - 21); - if (_latestBin.length() < _latestBinLength) { + printf("<< DATA @%u / %u bytes (we now have %u bytes)\n",(unsigned int)idx,(unsigned int)(len - 21),(unsigned int)_download.length()); + if (idx == (unsigned long)_download.length()) { + _download.append(reinterpret_cast(data) + 21,len - 21); + if (_download.length() < _downloadLength) { Buffer<128> gd; gd.append((uint8_t)VERB_GET_DATA); - gd.append(_latestBinHashPrefix.data,16); - gd.append((uint32_t)_latestBin.length()); + gd.append(_downloadHashPrefix.data,16); + gd.append((uint32_t)_download.length()); _node.sendUserMessage(ZT_SOFTWARE_UPDATE_SERVICE,ZT_SOFTWARE_UPDATE_USER_MESSAGE_TYPE,gd.data(),gd.size()); - printf(">> GET_DATA @%u\n",(unsigned int)_latestBin.length()); + printf(">> GET_DATA @%u\n",(unsigned int)_download.length()); } } } @@ -447,44 +454,50 @@ bool SoftwareUpdater::check(const uint64_t now) printf(">> GET_LATEST\n"); } - if (_latestBinLength > 0) { - if (_latestBin.length() >= _latestBinLength) { - if (_latestBinValid) { - return true; - } else { - // This is the very important security validation part that makes sure - // this software update doesn't have cooties. - - try { - // (1) Check the hash itself to make sure the image is basically okay - uint8_t sha512[ZT_SHA512_DIGEST_LEN]; - SHA512::hash(sha512,_latestBin.data(),(unsigned int)_latestBin.length()); - if (Utils::hex(sha512,ZT_SHA512_DIGEST_LEN) == OSUtils::jsonString(_latestMeta[ZT_SOFTWARE_UPDATE_JSON_UPDATE_HASH],"~")) { - // (2) Check signature by signing authority - std::string sig(OSUtils::jsonBinFromHex(_latestMeta[ZT_SOFTWARE_UPDATE_JSON_UPDATE_SIGNATURE])); - if (Identity(ZT_SOFTWARE_UPDATE_SIGNING_AUTHORITY).verify(_latestBin.data(),(unsigned int)_latestBin.length(),sig.data(),(unsigned int)sig.length())) { - // If we passed both of these, the update is good! - _latestBinValid = true; + if (_latestValid) + return true; + + if (_downloadLength > 0) { + if (_download.length() >= _downloadLength) { + // This is the very important security validation part that makes sure + // this software update doesn't have cooties. + + try { + // (1) Check the hash itself to make sure the image is basically okay + uint8_t sha512[ZT_SHA512_DIGEST_LEN]; + SHA512::hash(sha512,_download.data(),(unsigned int)_download.length()); + if (Utils::hex(sha512,ZT_SHA512_DIGEST_LEN) == OSUtils::jsonString(_latestMeta[ZT_SOFTWARE_UPDATE_JSON_UPDATE_HASH],"~")) { + // (2) Check signature by signing authority + std::string sig(OSUtils::jsonBinFromHex(_latestMeta[ZT_SOFTWARE_UPDATE_JSON_UPDATE_SIGNATURE])); + if (Identity(ZT_SOFTWARE_UPDATE_SIGNING_AUTHORITY).verify(_download.data(),(unsigned int)_download.length(),sig.data(),(unsigned int)sig.length())) { + // If we passed both of these, the update is good! + if (OSUtils::writeFile((_homePath + ZT_PATH_SEPARATOR_S + ZT_SOFTWARE_UPDATE_META_FILENAME).c_str(),OSUtils::jsonDump(_latestMeta)) && OSUtils::writeFile((_homePath + ZT_PATH_SEPARATOR_S + ZT_SOFTWARE_UPDATE_BIN_FILENAME).c_str(),_download)) { + _latestValid = true; printf("VALID UPDATE\n%s\n",OSUtils::jsonDump(_latestMeta).c_str()); - return true; + } else { + _latestMeta = nlohmann::json(); + _latestValid = false; } + _download = std::string(); + _downloadLength = 0; + return _latestValid; } - } catch ( ... ) {} // any exception equals verification failure - printf("INVALID UPDATE (!!!)\n%s\n",OSUtils::jsonDump(_latestMeta).c_str()); - - // If we get here, checks failed. - _latestMeta = nlohmann::json(); - _latestBin = ""; - _latestBinLength = 0; - _latestBinValid = false; - } + } + } catch ( ... ) {} // any exception equals verification failure + + // If we get here, checks failed. + printf("INVALID UPDATE (!!!)\n%s\n",OSUtils::jsonDump(_latestMeta).c_str()); + _latestMeta = nlohmann::json(); + _latestValid = false; + _download = std::string(); + _downloadLength = 0; } else { Buffer<128> gd; gd.append((uint8_t)VERB_GET_DATA); - gd.append(_latestBinHashPrefix.data,16); - gd.append((uint32_t)_latestBin.length()); + gd.append(_downloadHashPrefix.data,16); + gd.append((uint32_t)_download.length()); _node.sendUserMessage(ZT_SOFTWARE_UPDATE_SERVICE,ZT_SOFTWARE_UPDATE_USER_MESSAGE_TYPE,gd.data(),gd.size()); - printf(">> GET_DATA @%u\n",(unsigned int)_latestBin.length()); + printf(">> GET_DATA @%u\n",(unsigned int)_download.length()); } } @@ -493,8 +506,6 @@ bool SoftwareUpdater::check(const uint64_t now) void SoftwareUpdater::apply() { - if ((_latestBin.length() > 0)&&(_latestBinValid)) { - } } } // namespace ZeroTier diff --git a/service/SoftwareUpdater.hpp b/service/SoftwareUpdater.hpp index b9cabf23..26e44f86 100644 --- a/service/SoftwareUpdater.hpp +++ b/service/SoftwareUpdater.hpp @@ -69,6 +69,16 @@ */ #define ZT_SOFTWARE_UPDATE_DEFAULT_CHANNEL "release" +/** + * Filename for latest update's meta JSON + */ +#define ZT_SOFTWARE_UPDATE_META_FILENAME "latest-update.json" + +/** + * Filename for latest update's binary image + */ +#define ZT_SOFTWARE_UPDATE_BIN_FILENAME "latest-update.exe" + #define ZT_SOFTWARE_UPDATE_JSON_VERSION_MAJOR "versionMajor" #define ZT_SOFTWARE_UPDATE_JSON_VERSION_MINOR "versionMinor" #define ZT_SOFTWARE_UPDATE_JSON_VERSION_REVISION "versionRev" @@ -188,10 +198,11 @@ private: std::map< Array,_D > _dist; // key is first 16 bytes of hash nlohmann::json _latestMeta; - std::string _latestBin; - Array _latestBinHashPrefix; - unsigned long _latestBinLength; - bool _latestBinValid; + bool _latestValid; + + std::string _download; + Array _downloadHashPrefix; + unsigned long _downloadLength; }; } // namespace ZeroTier -- cgit v1.2.3 From 84bfec8329f0319dad2eae91271b879cdfb68682 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Thu, 12 Jan 2017 14:35:52 -0800 Subject: . --- make-mac.mk | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'make-mac.mk') diff --git a/make-mac.mk b/make-mac.mk index 03c2f374..1f130a1b 100644 --- a/make-mac.mk +++ b/make-mac.mk @@ -24,7 +24,7 @@ ifeq ($(ZT_OFFICIAL_RELEASE),1) CODESIGN_APP_CERT="Developer ID Application: ZeroTier, Inc (8ZD9JUCZ4V)" CODESIGN_INSTALLER_CERT="Developer ID Installer: ZeroTier, Inc (8ZD9JUCZ4V)" else - DEFS+=-DZT_SOFTWARE_UPDATE_DEFAULT="\"disable\"" + DEFS+=-DZT_SOFTWARE_UPDATE_DEFAULT="\"download\"" endif ifeq ($(ZT_ENABLE_CLUSTER),1) -- cgit v1.2.3 From 0e4d3244b295f1a067767dbad345c7223708f85c Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Fri, 13 Jan 2017 13:09:42 -0800 Subject: "make official" for Mac now builds update image as well as pkg (update image is just PKG + SFX script) --- ext/installfiles/mac-update/updater.tmpl.sh | 2 +- make-mac.mk | 26 +++++++++++++++++--------- 2 files changed, 18 insertions(+), 10 deletions(-) (limited to 'make-mac.mk') diff --git a/ext/installfiles/mac-update/updater.tmpl.sh b/ext/installfiles/mac-update/updater.tmpl.sh index b340ef13..0b07f6d9 100644 --- a/ext/installfiles/mac-update/updater.tmpl.sh +++ b/ext/installfiles/mac-update/updater.tmpl.sh @@ -12,7 +12,7 @@ scriptPath="`dirname "$0"`/`basename "$0"`" if [ ! -s "$scriptPath" ]; then scriptPath="$0" if [ ! -s "$scriptPath" ]; then - echo "Installer cannot determine its own path; $scriptPath is not readable." + echo "*** Auto-updater cannot determine its own path; $scriptPath is not readable." exit 2 fi fi diff --git a/make-mac.mk b/make-mac.mk index 1f130a1b..51fdd7ec 100644 --- a/make-mac.mk +++ b/make-mac.mk @@ -9,8 +9,13 @@ PRODUCTSIGN=echo CODESIGN_APP_CERT= CODESIGN_INSTALLER_CERT= -# 3 == MacOS, 2 == X64 (the only arch for MacOS right now) -DEFS+=-DZT_BUILD_PLATFORM=3 -DZT_BUILD_ARCHITECTURE=2 +ZT_BUILD_PLATFORM=3 +ZT_BUILD_ARCHITECTURE=2 +ZT_VERSION_MAJOR=$(shell cat version.h | grep -F VERSION_MAJOR | cut -d ' ' -f 3) +ZT_VERSION_MINOR=$(shell cat version.h | grep -F VERSION_MINOR | cut -d ' ' -f 3) +ZT_VERSION_REV=$(shell cat version.h | grep -F VERSION_REVISION | cut -d ' ' -f 3) + +DEFS+=-DZT_BUILD_PLATFORM=$(ZT_BUILD_PLATFORM) -DZT_BUILD_ARCHITECTURE=$(ZT_BUILD_ARCHITECTURE) include objects.mk OBJS+=osdep/OSXEthernetTap.o ext/lz4/lz4.o ext/http-parser/http_parser.o @@ -59,16 +64,14 @@ one: $(OBJS) service/OneService.o one.o ln -sf zerotier-one zerotier-idtool ln -sf zerotier-one zerotier-cli $(CODESIGN) -f -s $(CODESIGN_APP_CERT) zerotier-one - #$(CODESIGN) -vvv zerotier-one macui: FORCE cd macui && xcodebuild -target "ZeroTier One" -configuration Release $(CODESIGN) -f -s $(CODESIGN_APP_CERT) "macui/build/Release/ZeroTier One.app" - #$(CODESIGN) -vvv "macui/build/Release/ZeroTier One.app" -cli: FORCE - $(CXX) $(CXXFLAGS) -o zerotier cli/zerotier.cpp osdep/OSUtils.cpp node/InetAddress.cpp node/Utils.cpp node/Salsa20.cpp node/Identity.cpp node/SHA512.cpp node/C25519.cpp -lcurl - $(STRIP) zerotier +#cli: FORCE +# $(CXX) $(CXXFLAGS) -o zerotier cli/zerotier.cpp osdep/OSUtils.cpp node/InetAddress.cpp node/Utils.cpp node/Salsa20.cpp node/Identity.cpp node/SHA512.cpp node/C25519.cpp -lcurl +# $(STRIP) zerotier selftest: $(OBJS) selftest.o $(CXX) $(CXXFLAGS) -o zerotier-selftest selftest.o $(OBJS) $(LIBS) @@ -80,18 +83,23 @@ mac-dist-pkg: FORCE rm -f "ZeroTier One Signed.pkg" $(PRODUCTSIGN) --sign $(CODESIGN_INSTALLER_CERT) "ZeroTier One.pkg" "ZeroTier One Signed.pkg" if [ -f "ZeroTier One Signed.pkg" ]; then mv -f "ZeroTier One Signed.pkg" "ZeroTier One.pkg"; fi + rm -f zt1_update_$(ZT_BUILD_PLATFORM)_$(ZT_BUILD_ARCHITECTURE)_* + cat ext/installfiles/mac-update/updater.tmpl.sh "ZeroTier One.pkg" >zt1_update_$(ZT_BUILD_PLATFORM)_$(ZT_BUILD_ARCHITECTURE)_$(ZT_VERSION_MAJOR).$(ZT_VERSION_MINOR).$(ZT_VERSION_REV) # For ZeroTier, Inc. to build official signed packages official: FORCE make clean - make ZT_OFFICIAL_RELEASE=1 -j 4 one macui + make ZT_OFFICIAL_RELEASE=1 -j 4 one + make ZT_OFFICIAL_RELEASE=1 macui make ZT_OFFICIAL_RELEASE=1 mac-dist-pkg clean: - rm -rf *.dSYM build-* *.pkg *.dmg *.o node/*.o controller/*.o service/*.o osdep/*.o ext/http-parser/*.o ext/lz4/*.o ext/json-parser/*.o $(OBJS) zerotier-one zerotier-idtool zerotier-selftest zerotier-cli zerotier ZeroTierOneInstaller-* mkworld doc/node_modules macui/build + rm -rf *.dSYM build-* *.pkg *.dmg *.o node/*.o controller/*.o service/*.o osdep/*.o ext/http-parser/*.o ext/lz4/*.o ext/json-parser/*.o $(OBJS) zerotier-one zerotier-idtool zerotier-selftest zerotier-cli zerotier mkworld doc/node_modules macui/build zt1_update_$(ZT_BUILD_PLATFORM)_$(ZT_BUILD_ARCHITECTURE)_* distclean: clean +realclean: clean + # For those building from source -- installs signed binary tap driver in system ZT home install-mac-tap: FORCE mkdir -p /Library/Application\ Support/ZeroTier/One -- cgit v1.2.3 From 0fb3d1d58239837ae058c43d7269ce7fc33f5a7c Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Wed, 18 Jan 2017 09:16:23 -0800 Subject: Add a build version for software update use so we can do very minor updates within a version. --- make-mac.mk | 3 ++- node/Utils.hpp | 11 ++++++++--- service/SoftwareUpdater.cpp | 16 ++++++++++++---- service/SoftwareUpdater.hpp | 1 + version.h | 9 +++++++++ 5 files changed, 32 insertions(+), 8 deletions(-) (limited to 'make-mac.mk') diff --git a/make-mac.mk b/make-mac.mk index 51fdd7ec..298d1a2e 100644 --- a/make-mac.mk +++ b/make-mac.mk @@ -14,6 +14,7 @@ ZT_BUILD_ARCHITECTURE=2 ZT_VERSION_MAJOR=$(shell cat version.h | grep -F VERSION_MAJOR | cut -d ' ' -f 3) ZT_VERSION_MINOR=$(shell cat version.h | grep -F VERSION_MINOR | cut -d ' ' -f 3) ZT_VERSION_REV=$(shell cat version.h | grep -F VERSION_REVISION | cut -d ' ' -f 3) +ZT_VERSION_BUILD=$(shell cat version.h | grep -F VERSION_BUILD | cut -d ' ' -f 3) DEFS+=-DZT_BUILD_PLATFORM=$(ZT_BUILD_PLATFORM) -DZT_BUILD_ARCHITECTURE=$(ZT_BUILD_ARCHITECTURE) @@ -84,7 +85,7 @@ mac-dist-pkg: FORCE $(PRODUCTSIGN) --sign $(CODESIGN_INSTALLER_CERT) "ZeroTier One.pkg" "ZeroTier One Signed.pkg" if [ -f "ZeroTier One Signed.pkg" ]; then mv -f "ZeroTier One Signed.pkg" "ZeroTier One.pkg"; fi rm -f zt1_update_$(ZT_BUILD_PLATFORM)_$(ZT_BUILD_ARCHITECTURE)_* - cat ext/installfiles/mac-update/updater.tmpl.sh "ZeroTier One.pkg" >zt1_update_$(ZT_BUILD_PLATFORM)_$(ZT_BUILD_ARCHITECTURE)_$(ZT_VERSION_MAJOR).$(ZT_VERSION_MINOR).$(ZT_VERSION_REV) + cat ext/installfiles/mac-update/updater.tmpl.sh "ZeroTier One.pkg" >zt1_update_$(ZT_BUILD_PLATFORM)_$(ZT_BUILD_ARCHITECTURE)_$(ZT_VERSION_MAJOR).$(ZT_VERSION_MINOR).$(ZT_VERSION_REV)_$(ZT_VERSION_BUILD) # For ZeroTier, Inc. to build official signed packages official: FORCE diff --git a/node/Utils.hpp b/node/Utils.hpp index 48c43da3..48cf799e 100644 --- a/node/Utils.hpp +++ b/node/Utils.hpp @@ -335,8 +335,7 @@ public: * * @return -1, 0, or 1 based on whether first tuple is less than, equal to, or greater than second */ - static inline int compareVersion(unsigned int maj1,unsigned int min1,unsigned int rev1,unsigned int maj2,unsigned int min2,unsigned int rev2) - throw() + static inline int compareVersion(unsigned int maj1,unsigned int min1,unsigned int rev1,unsigned int b1,unsigned int maj2,unsigned int min2,unsigned int rev2,unsigned int b2) { if (maj1 > maj2) return 1; @@ -352,7 +351,13 @@ public: return 1; else if (rev1 < rev2) return -1; - else return 0; + else { + if (b1 > b2) + return 1; + else if (b1 < b2) + return -1; + else return 0; + } } } } diff --git a/service/SoftwareUpdater.cpp b/service/SoftwareUpdater.cpp index f6572cfc..c650ce42 100644 --- a/service/SoftwareUpdater.cpp +++ b/service/SoftwareUpdater.cpp @@ -74,7 +74,9 @@ SoftwareUpdater::SoftwareUpdater(Node &node,const std::string &homePath) : const unsigned int rvMaj = (unsigned int)OSUtils::jsonInt(meta[ZT_SOFTWARE_UPDATE_JSON_VERSION_MAJOR],0); const unsigned int rvMin = (unsigned int)OSUtils::jsonInt(meta[ZT_SOFTWARE_UPDATE_JSON_VERSION_MINOR],0); const unsigned int rvRev = (unsigned int)OSUtils::jsonInt(meta[ZT_SOFTWARE_UPDATE_JSON_VERSION_REVISION],0); - if ((Utils::compareVersion(rvMaj,rvMin,rvRev,ZEROTIER_ONE_VERSION_MAJOR,ZEROTIER_ONE_VERSION_MINOR,ZEROTIER_ONE_VERSION_REVISION) > 0)&&(OSUtils::readFile((_homePath + ZT_PATH_SEPARATOR_S ZT_SOFTWARE_UPDATE_BIN_FILENAME).c_str(),buf))) { + const unsigned int rvBld = (unsigned int)OSUtils::jsonInt(meta[ZT_SOFTWARE_UPDATE_JSON_VERSION_BUILD],0); + if ((Utils::compareVersion(rvMaj,rvMin,rvRev,rvBld,ZEROTIER_ONE_VERSION_MAJOR,ZEROTIER_ONE_VERSION_MINOR,ZEROTIER_ONE_VERSION_REVISION,ZEROTIER_ONE_VERSION_BUILD) > 0)&& + (OSUtils::readFile((_homePath + ZT_PATH_SEPARATOR_S ZT_SOFTWARE_UPDATE_BIN_FILENAME).c_str(),buf))) { if ((uint64_t)buf.length() == OSUtils::jsonInt(meta[ZT_SOFTWARE_UPDATE_JSON_UPDATE_SIZE],0)) { _latestMeta = meta; _latestValid = true; @@ -149,6 +151,7 @@ void SoftwareUpdater::handleSoftwareUpdateUserMessage(uint64_t origin,const void const unsigned int rvMaj = (unsigned int)OSUtils::jsonInt(req[ZT_SOFTWARE_UPDATE_JSON_VERSION_MAJOR],0); const unsigned int rvMin = (unsigned int)OSUtils::jsonInt(req[ZT_SOFTWARE_UPDATE_JSON_VERSION_MINOR],0); const unsigned int rvRev = (unsigned int)OSUtils::jsonInt(req[ZT_SOFTWARE_UPDATE_JSON_VERSION_REVISION],0); + const unsigned int rvBld = (unsigned int)OSUtils::jsonInt(req[ZT_SOFTWARE_UPDATE_JSON_VERSION_BUILD],0); const unsigned int rvPlatform = (unsigned int)OSUtils::jsonInt(req[ZT_SOFTWARE_UPDATE_JSON_PLATFORM],0); const unsigned int rvArch = (unsigned int)OSUtils::jsonInt(req[ZT_SOFTWARE_UPDATE_JSON_ARCHITECTURE],0); const unsigned int rvVendor = (unsigned int)OSUtils::jsonInt(req[ZT_SOFTWARE_UPDATE_JSON_VENDOR],0); @@ -161,6 +164,7 @@ void SoftwareUpdater::handleSoftwareUpdateUserMessage(uint64_t origin,const void unsigned int bestVMaj = rvMaj; unsigned int bestVMin = rvMin; unsigned int bestVRev = rvRev; + unsigned int bestVBld = rvBld; for(std::map< Array,_D >::const_iterator d(_dist.begin());d!=_dist.end();++d) { if ((OSUtils::jsonInt(d->second.meta[ZT_SOFTWARE_UPDATE_JSON_PLATFORM],0) == rvPlatform)&& (OSUtils::jsonInt(d->second.meta[ZT_SOFTWARE_UPDATE_JSON_ARCHITECTURE],0) == rvArch)&& @@ -170,11 +174,13 @@ void SoftwareUpdater::handleSoftwareUpdateUserMessage(uint64_t origin,const void const unsigned int dvMaj = (unsigned int)OSUtils::jsonInt(d->second.meta[ZT_SOFTWARE_UPDATE_JSON_VERSION_MAJOR],0); const unsigned int dvMin = (unsigned int)OSUtils::jsonInt(d->second.meta[ZT_SOFTWARE_UPDATE_JSON_VERSION_MINOR],0); const unsigned int dvRev = (unsigned int)OSUtils::jsonInt(d->second.meta[ZT_SOFTWARE_UPDATE_JSON_VERSION_REVISION],0); - if (Utils::compareVersion(dvMaj,dvMin,dvRev,bestVMaj,bestVMin,bestVRev) > 0) { + const unsigned int dvBld = (unsigned int)OSUtils::jsonInt(d->second.meta[ZT_SOFTWARE_UPDATE_JSON_VERSION_BUILD],0); + if (Utils::compareVersion(dvMaj,dvMin,dvRev,dvBld,bestVMaj,bestVMin,bestVRev,bestVBld) > 0) { latest = &(d->second.meta); bestVMaj = dvMaj; bestVMin = dvMin; bestVRev = dvRev; + bestVBld = dvBld; } } } @@ -184,7 +190,7 @@ void SoftwareUpdater::handleSoftwareUpdateUserMessage(uint64_t origin,const void lj.append(OSUtils::jsonDump(*latest)); _node.sendUserMessage(origin,ZT_SOFTWARE_UPDATE_USER_MESSAGE_TYPE,lj.data(),(unsigned int)lj.length()); if (_distLog) { - fprintf(_distLog,"%.10llx GET_LATEST %u.%u.%u platform %u arch %u vendor %u channel %s -> LATEST %u.%u.%u" ZT_EOL_S,(unsigned long long)origin,rvMaj,rvMin,rvRev,rvPlatform,rvArch,rvVendor,rvChannel.c_str(),bestVMaj,bestVMin,bestVRev); + fprintf(_distLog,"%.10llx GET_LATEST %u.%u.%u_%u platform %u arch %u vendor %u channel %s -> LATEST %u.%u.%u_%u" ZT_EOL_S,(unsigned long long)origin,rvMaj,rvMin,rvRev,rvBld,rvPlatform,rvArch,rvVendor,rvChannel.c_str(),bestVMaj,bestVMin,bestVRev,bestVBld); fflush(_distLog); } } @@ -193,7 +199,7 @@ void SoftwareUpdater::handleSoftwareUpdateUserMessage(uint64_t origin,const void } else { // VERB_LATEST if ((origin == ZT_SOFTWARE_UPDATE_SERVICE)&& - (Utils::compareVersion(rvMaj,rvMin,rvRev,ZEROTIER_ONE_VERSION_MAJOR,ZEROTIER_ONE_VERSION_MINOR,ZEROTIER_ONE_VERSION_REVISION) > 0)&& + (Utils::compareVersion(rvMaj,rvMin,rvRev,rvBld,ZEROTIER_ONE_VERSION_MAJOR,ZEROTIER_ONE_VERSION_MINOR,ZEROTIER_ONE_VERSION_REVISION,ZEROTIER_ONE_VERSION_BUILD) > 0)&& (OSUtils::jsonString(req[ZT_SOFTWARE_UPDATE_JSON_UPDATE_SIGNED_BY],"") == ZT_SOFTWARE_UPDATE_SIGNING_AUTHORITY)) { const unsigned long len = (unsigned long)OSUtils::jsonInt(req[ZT_SOFTWARE_UPDATE_JSON_UPDATE_SIZE],0); const std::string hash = OSUtils::jsonBinFromHex(req[ZT_SOFTWARE_UPDATE_JSON_UPDATE_HASH]); @@ -283,6 +289,7 @@ bool SoftwareUpdater::check(const uint64_t now) "%c{\"" ZT_SOFTWARE_UPDATE_JSON_VERSION_MAJOR "\":%d," "\"" ZT_SOFTWARE_UPDATE_JSON_VERSION_MINOR "\":%d," "\"" ZT_SOFTWARE_UPDATE_JSON_VERSION_REVISION "\":%d," + "\"" ZT_SOFTWARE_UPDATE_JSON_VERSION_BUILD "\":%d," "\"" ZT_SOFTWARE_UPDATE_JSON_EXPECT_SIGNED_BY "\":\"%s\"," "\"" ZT_SOFTWARE_UPDATE_JSON_PLATFORM "\":%d," "\"" ZT_SOFTWARE_UPDATE_JSON_ARCHITECTURE "\":%d," @@ -292,6 +299,7 @@ bool SoftwareUpdater::check(const uint64_t now) ZEROTIER_ONE_VERSION_MAJOR, ZEROTIER_ONE_VERSION_MINOR, ZEROTIER_ONE_VERSION_REVISION, + ZEROTIER_ONE_VERSION_BUILD, ZT_SOFTWARE_UPDATE_SIGNING_AUTHORITY, ZT_BUILD_PLATFORM, ZT_BUILD_ARCHITECTURE, diff --git a/service/SoftwareUpdater.hpp b/service/SoftwareUpdater.hpp index 0a7b6789..e6e6b0c7 100644 --- a/service/SoftwareUpdater.hpp +++ b/service/SoftwareUpdater.hpp @@ -85,6 +85,7 @@ #define ZT_SOFTWARE_UPDATE_JSON_VERSION_MAJOR "versionMajor" #define ZT_SOFTWARE_UPDATE_JSON_VERSION_MINOR "versionMinor" #define ZT_SOFTWARE_UPDATE_JSON_VERSION_REVISION "versionRev" +#define ZT_SOFTWARE_UPDATE_JSON_VERSION_BUILD "versionBuild" #define ZT_SOFTWARE_UPDATE_JSON_PLATFORM "platform" #define ZT_SOFTWARE_UPDATE_JSON_ARCHITECTURE "arch" #define ZT_SOFTWARE_UPDATE_JSON_VENDOR "vendor" diff --git a/version.h b/version.h index 118ac761..9f616664 100644 --- a/version.h +++ b/version.h @@ -34,4 +34,13 @@ */ #define ZEROTIER_ONE_VERSION_REVISION 17 +/** + * Build version + * + * This starts at 0 for each major.minor.rev tuple and can be incremented + * to force a minor update without an actual version number change. It's + * not part of the actual release version number. + */ +#define ZEROTIER_ONE_VERSION_BUILD 0 + #endif -- cgit v1.2.3 From 0995c1dcaa02a61a65e447f56aec1353885530df Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Thu, 19 Jan 2017 15:16:04 -0800 Subject: Encapsulate LZ4 in Packet.cpp to eliminate dependency. --- ext/lz4/LICENSE | 24 - ext/lz4/README.md | 73 --- ext/lz4/lz4.c | 1463 ----------------------------------------- ext/lz4/lz4.h | 463 ------------- make-freebsd.mk | 6 +- make-linux.mk | 12 +- make-mac.mk | 6 +- node/Packet.cpp | 1891 ++++++++++++++++++++++++++++++++++++++++++++++++++++- node/Packet.hpp | 2 +- 9 files changed, 1900 insertions(+), 2040 deletions(-) delete mode 100644 ext/lz4/LICENSE delete mode 100644 ext/lz4/README.md delete mode 100644 ext/lz4/lz4.c delete mode 100644 ext/lz4/lz4.h (limited to 'make-mac.mk') diff --git a/ext/lz4/LICENSE b/ext/lz4/LICENSE deleted file mode 100644 index 74c2cdd7..00000000 --- a/ext/lz4/LICENSE +++ /dev/null @@ -1,24 +0,0 @@ -LZ4 Library -Copyright (c) 2011-2016, Yann Collet -All rights reserved. - -Redistribution and use in source and binary forms, with or without modification, -are permitted provided that the following conditions are met: - -* Redistributions of source code must retain the above copyright notice, this - list of conditions and the following disclaimer. - -* Redistributions in binary form must reproduce the above copyright notice, this - list of conditions and the following disclaimer in the documentation and/or - other materials provided with the distribution. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR -ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES -(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; -LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON -ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/ext/lz4/README.md b/ext/lz4/README.md deleted file mode 100644 index b40442c4..00000000 --- a/ext/lz4/README.md +++ /dev/null @@ -1,73 +0,0 @@ -LZ4 - Library Files -================================ - -The directory contains many files, but depending on project's objectives, -not all of them are necessary. - -#### Minimal LZ4 build - -The minimum required is **`lz4.c`** and **`lz4.h`**, -which will provide the fast compression and decompression algorithm. - - -#### The High Compression variant of LZ4 - -For more compression at the cost of compression speed, -the High Compression variant **lz4hc** is available. -It's necessary to add **`lz4hc.c`** and **`lz4hc.h`**. -The variant still depends on regular `lz4` source files. -In particular, the decompression is still provided by `lz4.c`. - - -#### Compatibility issues - -In order to produce files or streams compatible with `lz4` command line utility, -it's necessary to encode lz4-compressed blocks using the [official interoperable frame format]. -This format is generated and decoded automatically by the **lz4frame** library. -In order to work properly, lz4frame needs lz4 and lz4hc, and also **xxhash**, -which provides error detection. -(_Advanced stuff_ : It's possible to hide xxhash symbols into a local namespace. -This is what `liblz4` does, to avoid symbol duplication -in case a user program would link to several libraries containing xxhash symbols.) - - -#### Advanced API - -A more complex `lz4frame_static.h` is also provided. -It contains definitions which are not guaranteed to remain stable within future versions. -It must be used with static linking ***only***. - - -#### Using MinGW+MSYS to create DLL - -DLL can be created using MinGW+MSYS with the `make liblz4` command. -This command creates `dll\liblz4.dll` and the import library `dll\liblz4.lib`. -The import library is only required with Visual C++. -The header files `lz4.h`, `lz4hc.h`, `lz4frame.h` and the dynamic library -`dll\liblz4.dll` are required to compile a project using gcc/MinGW. -The dynamic library has to be added to linking options. -It means that if a project that uses LZ4 consists of a single `test-dll.c` -file it should be linked with `dll\liblz4.dll`. For example: -``` - gcc $(CFLAGS) -Iinclude/ test-dll.c -o test-dll dll\liblz4.dll -``` -The compiled executable will require LZ4 DLL which is available at `dll\liblz4.dll`. - - -#### Miscellaneous - -Other files present in the directory are not source code. There are : - - - LICENSE : contains the BSD license text - - Makefile : script to compile or install lz4 library (static or dynamic) - - liblz4.pc.in : for pkg-config (make install) - - README.md : this file - -[official interoperable frame format]: ../doc/lz4_Frame_format.md - - -#### License - -All source material within __lib__ directory are BSD 2-Clause licensed. -See [LICENSE](LICENSE) for details. -The license is also repeated at the top of each source file. diff --git a/ext/lz4/lz4.c b/ext/lz4/lz4.c deleted file mode 100644 index 143c36e1..00000000 --- a/ext/lz4/lz4.c +++ /dev/null @@ -1,1463 +0,0 @@ -/* - LZ4 - Fast LZ compression algorithm - Copyright (C) 2011-2016, Yann Collet. - - BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions are - met: - - * Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above - copyright notice, this list of conditions and the following disclaimer - in the documentation and/or other materials provided with the - distribution. - - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - You can contact the author at : - - LZ4 homepage : http://www.lz4.org - - LZ4 source repository : https://github.com/lz4/lz4 -*/ - - -/*-************************************ -* Tuning parameters -**************************************/ -/* - * HEAPMODE : - * Select how default compression functions will allocate memory for their hash table, - * in memory stack (0:default, fastest), or in memory heap (1:requires malloc()). - */ -#ifndef HEAPMODE -# define HEAPMODE 0 -#endif - -/* - * ACCELERATION_DEFAULT : - * Select "acceleration" for LZ4_compress_fast() when parameter value <= 0 - */ -#define ACCELERATION_DEFAULT 1 - - -/*-************************************ -* CPU Feature Detection -**************************************/ -/* LZ4_FORCE_MEMORY_ACCESS - * By default, access to unaligned memory is controlled by `memcpy()`, which is safe and portable. - * Unfortunately, on some target/compiler combinations, the generated assembly is sub-optimal. - * The below switch allow to select different access method for improved performance. - * Method 0 (default) : use `memcpy()`. Safe and portable. - * Method 1 : `__packed` statement. It depends on compiler extension (ie, not portable). - * This method is safe if your compiler supports it, and *generally* as fast or faster than `memcpy`. - * Method 2 : direct access. This method is portable but violate C standard. - * It can generate buggy code on targets which generate assembly depending on alignment. - * But in some circumstances, it's the only known way to get the most performance (ie GCC + ARMv6) - * See https://fastcompression.blogspot.fr/2015/08/accessing-unaligned-memory.html for details. - * Prefer these methods in priority order (0 > 1 > 2) - */ -#ifndef LZ4_FORCE_MEMORY_ACCESS /* can be defined externally, on command line for example */ -# if defined(__GNUC__) && ( defined(__ARM_ARCH_6__) || defined(__ARM_ARCH_6J__) || defined(__ARM_ARCH_6K__) || defined(__ARM_ARCH_6Z__) || defined(__ARM_ARCH_6ZK__) || defined(__ARM_ARCH_6T2__) ) -# define LZ4_FORCE_MEMORY_ACCESS 2 -# elif defined(__INTEL_COMPILER) || \ - (defined(__GNUC__) && ( defined(__ARM_ARCH_7__) || defined(__ARM_ARCH_7A__) || defined(__ARM_ARCH_7R__) || defined(__ARM_ARCH_7M__) || defined(__ARM_ARCH_7S__) )) -# define LZ4_FORCE_MEMORY_ACCESS 1 -# endif -#endif - -/* - * LZ4_FORCE_SW_BITCOUNT - * Define this parameter if your target system or compiler does not support hardware bit count - */ -#if defined(_MSC_VER) && defined(_WIN32_WCE) /* Visual Studio for Windows CE does not support Hardware bit count */ -# define LZ4_FORCE_SW_BITCOUNT -#endif - - -/*-************************************ -* Dependency -**************************************/ -#include "lz4.h" -/* see also "memory routines" below */ - - -/*-************************************ -* Compiler Options -**************************************/ -#ifdef _MSC_VER /* Visual Studio */ -# define FORCE_INLINE static __forceinline -# include -# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ -# pragma warning(disable : 4293) /* disable: C4293: too large shift (32-bits) */ -#else -# if defined(__GNUC__) || defined(__clang__) -# define FORCE_INLINE static inline __attribute__((always_inline)) -# elif defined(__cplusplus) || (defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) -# define FORCE_INLINE static inline -# else -# define FORCE_INLINE static -# endif -#endif /* _MSC_VER */ - -#if (defined(__GNUC__) && (__GNUC__ >= 3)) || (defined(__INTEL_COMPILER) && (__INTEL_COMPILER >= 800)) || defined(__clang__) -# define expect(expr,value) (__builtin_expect ((expr),(value)) ) -#else -# define expect(expr,value) (expr) -#endif - -#define likely(expr) expect((expr) != 0, 1) -#define unlikely(expr) expect((expr) != 0, 0) - - -/*-************************************ -* Memory routines -**************************************/ -#include /* malloc, calloc, free */ -#define ALLOCATOR(n,s) calloc(n,s) -#define FREEMEM free -#include /* memset, memcpy */ -#define MEM_INIT memset - - -/*-************************************ -* Basic Types -**************************************/ -#if defined(__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) -# include - typedef uint8_t BYTE; - typedef uint16_t U16; - typedef uint32_t U32; - typedef int32_t S32; - typedef uint64_t U64; - typedef uintptr_t uptrval; -#else - typedef unsigned char BYTE; - typedef unsigned short U16; - typedef unsigned int U32; - typedef signed int S32; - typedef unsigned long long U64; - typedef size_t uptrval; /* generally true, except OpenVMS-64 */ -#endif - -#if defined(__x86_64__) - typedef U64 reg_t; /* 64-bits in x32 mode */ -#else - typedef size_t reg_t; /* 32-bits in x32 mode */ -#endif - -/*-************************************ -* Reading and writing into memory -**************************************/ -static unsigned LZ4_isLittleEndian(void) -{ - const union { U32 u; BYTE c[4]; } one = { 1 }; /* don't use static : performance detrimental */ - return one.c[0]; -} - - -#if defined(LZ4_FORCE_MEMORY_ACCESS) && (LZ4_FORCE_MEMORY_ACCESS==2) -/* lie to the compiler about data alignment; use with caution */ - -static U16 LZ4_read16(const void* memPtr) { return *(const U16*) memPtr; } -static U32 LZ4_read32(const void* memPtr) { return *(const U32*) memPtr; } -static reg_t LZ4_read_ARCH(const void* memPtr) { return *(const reg_t*) memPtr; } - -static void LZ4_write16(void* memPtr, U16 value) { *(U16*)memPtr = value; } -static void LZ4_write32(void* memPtr, U32 value) { *(U32*)memPtr = value; } - -#elif defined(LZ4_FORCE_MEMORY_ACCESS) && (LZ4_FORCE_MEMORY_ACCESS==1) - -/* __pack instructions are safer, but compiler specific, hence potentially problematic for some compilers */ -/* currently only defined for gcc and icc */ -typedef union { U16 u16; U32 u32; reg_t uArch; } __attribute__((packed)) unalign; - -static U16 LZ4_read16(const void* ptr) { return ((const unalign*)ptr)->u16; } -static U32 LZ4_read32(const void* ptr) { return ((const unalign*)ptr)->u32; } -static reg_t LZ4_read_ARCH(const void* ptr) { return ((const unalign*)ptr)->uArch; } - -static void LZ4_write16(void* memPtr, U16 value) { ((unalign*)memPtr)->u16 = value; } -static void LZ4_write32(void* memPtr, U32 value) { ((unalign*)memPtr)->u32 = value; } - -#else /* safe and portable access through memcpy() */ - -static U16 LZ4_read16(const void* memPtr) -{ - U16 val; memcpy(&val, memPtr, sizeof(val)); return val; -} - -static U32 LZ4_read32(const void* memPtr) -{ - U32 val; memcpy(&val, memPtr, sizeof(val)); return val; -} - -static reg_t LZ4_read_ARCH(const void* memPtr) -{ - reg_t val; memcpy(&val, memPtr, sizeof(val)); return val; -} - -static void LZ4_write16(void* memPtr, U16 value) -{ - memcpy(memPtr, &value, sizeof(value)); -} - -static void LZ4_write32(void* memPtr, U32 value) -{ - memcpy(memPtr, &value, sizeof(value)); -} - -#endif /* LZ4_FORCE_MEMORY_ACCESS */ - - -static U16 LZ4_readLE16(const void* memPtr) -{ - if (LZ4_isLittleEndian()) { - return LZ4_read16(memPtr); - } else { - const BYTE* p = (const BYTE*)memPtr; - return (U16)((U16)p[0] + (p[1]<<8)); - } -} - -static void LZ4_writeLE16(void* memPtr, U16 value) -{ - if (LZ4_isLittleEndian()) { - LZ4_write16(memPtr, value); - } else { - BYTE* p = (BYTE*)memPtr; - p[0] = (BYTE) value; - p[1] = (BYTE)(value>>8); - } -} - -static void LZ4_copy8(void* dst, const void* src) -{ - memcpy(dst,src,8); -} - -/* customized variant of memcpy, which can overwrite up to 8 bytes beyond dstEnd */ -static void LZ4_wildCopy(void* dstPtr, const void* srcPtr, void* dstEnd) -{ - BYTE* d = (BYTE*)dstPtr; - const BYTE* s = (const BYTE*)srcPtr; - BYTE* const e = (BYTE*)dstEnd; - - do { LZ4_copy8(d,s); d+=8; s+=8; } while (d>3); -# elif (defined(__clang__) || (defined(__GNUC__) && (__GNUC__>=3))) && !defined(LZ4_FORCE_SW_BITCOUNT) - return (__builtin_ctzll((U64)val) >> 3); -# else - static const int DeBruijnBytePos[64] = { 0, 0, 0, 0, 0, 1, 1, 2, 0, 3, 1, 3, 1, 4, 2, 7, 0, 2, 3, 6, 1, 5, 3, 5, 1, 3, 4, 4, 2, 5, 6, 7, 7, 0, 1, 2, 3, 3, 4, 6, 2, 6, 5, 5, 3, 4, 5, 6, 7, 1, 2, 4, 6, 4, 4, 5, 7, 2, 6, 5, 7, 6, 7, 7 }; - return DeBruijnBytePos[((U64)((val & -(long long)val) * 0x0218A392CDABBD3FULL)) >> 58]; -# endif - } else /* 32 bits */ { -# if defined(_MSC_VER) && !defined(LZ4_FORCE_SW_BITCOUNT) - unsigned long r; - _BitScanForward( &r, (U32)val ); - return (int)(r>>3); -# elif (defined(__clang__) || (defined(__GNUC__) && (__GNUC__>=3))) && !defined(LZ4_FORCE_SW_BITCOUNT) - return (__builtin_ctz((U32)val) >> 3); -# else - static const int DeBruijnBytePos[32] = { 0, 0, 3, 0, 3, 1, 3, 0, 3, 2, 2, 1, 3, 2, 0, 1, 3, 3, 1, 2, 2, 2, 2, 0, 3, 1, 2, 0, 1, 0, 1, 1 }; - return DeBruijnBytePos[((U32)((val & -(S32)val) * 0x077CB531U)) >> 27]; -# endif - } - } else /* Big Endian CPU */ { - if (sizeof(val)==8) { -# if defined(_MSC_VER) && defined(_WIN64) && !defined(LZ4_FORCE_SW_BITCOUNT) - unsigned long r = 0; - _BitScanReverse64( &r, val ); - return (unsigned)(r>>3); -# elif (defined(__clang__) || (defined(__GNUC__) && (__GNUC__>=3))) && !defined(LZ4_FORCE_SW_BITCOUNT) - return (__builtin_clzll((U64)val) >> 3); -# else - unsigned r; - if (!(val>>32)) { r=4; } else { r=0; val>>=32; } - if (!(val>>16)) { r+=2; val>>=8; } else { val>>=24; } - r += (!val); - return r; -# endif - } else /* 32 bits */ { -# if defined(_MSC_VER) && !defined(LZ4_FORCE_SW_BITCOUNT) - unsigned long r = 0; - _BitScanReverse( &r, (unsigned long)val ); - return (unsigned)(r>>3); -# elif (defined(__clang__) || (defined(__GNUC__) && (__GNUC__>=3))) && !defined(LZ4_FORCE_SW_BITCOUNT) - return (__builtin_clz((U32)val) >> 3); -# else - unsigned r; - if (!(val>>16)) { r=2; val>>=8; } else { r=0; val>>=24; } - r += (!val); - return r; -# endif - } - } -} - -#define STEPSIZE sizeof(reg_t) -static unsigned LZ4_count(const BYTE* pIn, const BYTE* pMatch, const BYTE* pInLimit) -{ - const BYTE* const pStart = pIn; - - while (likely(pIn compression run slower on incompressible data */ - - -/*-************************************ -* Local Structures and types -**************************************/ -typedef enum { notLimited = 0, limitedOutput = 1 } limitedOutput_directive; -typedef enum { byPtr, byU32, byU16 } tableType_t; - -typedef enum { noDict = 0, withPrefix64k, usingExtDict } dict_directive; -typedef enum { noDictIssue = 0, dictSmall } dictIssue_directive; - -typedef enum { endOnOutputSize = 0, endOnInputSize = 1 } endCondition_directive; -typedef enum { full = 0, partial = 1 } earlyEnd_directive; - - -/*-************************************ -* Local Utils -**************************************/ -int LZ4_versionNumber (void) { return LZ4_VERSION_NUMBER; } -const char* LZ4_versionString(void) { return LZ4_VERSION_STRING; } -int LZ4_compressBound(int isize) { return LZ4_COMPRESSBOUND(isize); } -int LZ4_sizeofState() { return LZ4_STREAMSIZE; } - - -/*-****************************** -* Compression functions -********************************/ -static U32 LZ4_hash4(U32 sequence, tableType_t const tableType) -{ - if (tableType == byU16) - return ((sequence * 2654435761U) >> ((MINMATCH*8)-(LZ4_HASHLOG+1))); - else - return ((sequence * 2654435761U) >> ((MINMATCH*8)-LZ4_HASHLOG)); -} - -static U32 LZ4_hash5(U64 sequence, tableType_t const tableType) -{ - static const U64 prime5bytes = 889523592379ULL; - static const U64 prime8bytes = 11400714785074694791ULL; - const U32 hashLog = (tableType == byU16) ? LZ4_HASHLOG+1 : LZ4_HASHLOG; - if (LZ4_isLittleEndian()) - return (U32)(((sequence << 24) * prime5bytes) >> (64 - hashLog)); - else - return (U32)(((sequence >> 24) * prime8bytes) >> (64 - hashLog)); -} - -FORCE_INLINE U32 LZ4_hashPosition(const void* const p, tableType_t const tableType) -{ - if ((sizeof(reg_t)==8) && (tableType != byU16)) return LZ4_hash5(LZ4_read_ARCH(p), tableType); - return LZ4_hash4(LZ4_read32(p), tableType); -} - -static void LZ4_putPositionOnHash(const BYTE* p, U32 h, void* tableBase, tableType_t const tableType, const BYTE* srcBase) -{ - switch (tableType) - { - case byPtr: { const BYTE** hashTable = (const BYTE**)tableBase; hashTable[h] = p; return; } - case byU32: { U32* hashTable = (U32*) tableBase; hashTable[h] = (U32)(p-srcBase); return; } - case byU16: { U16* hashTable = (U16*) tableBase; hashTable[h] = (U16)(p-srcBase); return; } - } -} - -FORCE_INLINE void LZ4_putPosition(const BYTE* p, void* tableBase, tableType_t tableType, const BYTE* srcBase) -{ - U32 const h = LZ4_hashPosition(p, tableType); - LZ4_putPositionOnHash(p, h, tableBase, tableType, srcBase); -} - -static const BYTE* LZ4_getPositionOnHash(U32 h, void* tableBase, tableType_t tableType, const BYTE* srcBase) -{ - if (tableType == byPtr) { const BYTE** hashTable = (const BYTE**) tableBase; return hashTable[h]; } - if (tableType == byU32) { const U32* const hashTable = (U32*) tableBase; return hashTable[h] + srcBase; } - { const U16* const hashTable = (U16*) tableBase; return hashTable[h] + srcBase; } /* default, to ensure a return */ -} - -FORCE_INLINE const BYTE* LZ4_getPosition(const BYTE* p, void* tableBase, tableType_t tableType, const BYTE* srcBase) -{ - U32 const h = LZ4_hashPosition(p, tableType); - return LZ4_getPositionOnHash(h, tableBase, tableType, srcBase); -} - - -/** LZ4_compress_generic() : - inlined, to ensure branches are decided at compilation time */ -FORCE_INLINE int LZ4_compress_generic( - LZ4_stream_t_internal* const cctx, - const char* const source, - char* const dest, - const int inputSize, - const int maxOutputSize, - const limitedOutput_directive outputLimited, - const tableType_t tableType, - const dict_directive dict, - const dictIssue_directive dictIssue, - const U32 acceleration) -{ - const BYTE* ip = (const BYTE*) source; - const BYTE* base; - const BYTE* lowLimit; - const BYTE* const lowRefLimit = ip - cctx->dictSize; - const BYTE* const dictionary = cctx->dictionary; - const BYTE* const dictEnd = dictionary + cctx->dictSize; - const ptrdiff_t dictDelta = dictEnd - (const BYTE*)source; - const BYTE* anchor = (const BYTE*) source; - const BYTE* const iend = ip + inputSize; - const BYTE* const mflimit = iend - MFLIMIT; - const BYTE* const matchlimit = iend - LASTLITERALS; - - BYTE* op = (BYTE*) dest; - BYTE* const olimit = op + maxOutputSize; - - U32 forwardH; - - /* Init conditions */ - if ((U32)inputSize > (U32)LZ4_MAX_INPUT_SIZE) return 0; /* Unsupported inputSize, too large (or negative) */ - switch(dict) - { - case noDict: - default: - base = (const BYTE*)source; - lowLimit = (const BYTE*)source; - break; - case withPrefix64k: - base = (const BYTE*)source - cctx->currentOffset; - lowLimit = (const BYTE*)source - cctx->dictSize; - break; - case usingExtDict: - base = (const BYTE*)source - cctx->currentOffset; - lowLimit = (const BYTE*)source; - break; - } - if ((tableType == byU16) && (inputSize>=LZ4_64Klimit)) return 0; /* Size too large (not within 64K limit) */ - if (inputSizehashTable, tableType, base); - ip++; forwardH = LZ4_hashPosition(ip, tableType); - - /* Main Loop */ - for ( ; ; ) { - ptrdiff_t refDelta = 0; - const BYTE* match; - BYTE* token; - - /* Find a match */ - { const BYTE* forwardIp = ip; - unsigned step = 1; - unsigned searchMatchNb = acceleration << LZ4_skipTrigger; - do { - U32 const h = forwardH; - ip = forwardIp; - forwardIp += step; - step = (searchMatchNb++ >> LZ4_skipTrigger); - - if (unlikely(forwardIp > mflimit)) goto _last_literals; - - match = LZ4_getPositionOnHash(h, cctx->hashTable, tableType, base); - if (dict==usingExtDict) { - if (match < (const BYTE*)source) { - refDelta = dictDelta; - lowLimit = dictionary; - } else { - refDelta = 0; - lowLimit = (const BYTE*)source; - } } - forwardH = LZ4_hashPosition(forwardIp, tableType); - LZ4_putPositionOnHash(ip, h, cctx->hashTable, tableType, base); - - } while ( ((dictIssue==dictSmall) ? (match < lowRefLimit) : 0) - || ((tableType==byU16) ? 0 : (match + MAX_DISTANCE < ip)) - || (LZ4_read32(match+refDelta) != LZ4_read32(ip)) ); - } - - /* Catch up */ - while (((ip>anchor) & (match+refDelta > lowLimit)) && (unlikely(ip[-1]==match[refDelta-1]))) { ip--; match--; } - - /* Encode Literals */ - { unsigned const litLength = (unsigned)(ip - anchor); - token = op++; - if ((outputLimited) && /* Check output buffer overflow */ - (unlikely(op + litLength + (2 + 1 + LASTLITERALS) + (litLength/255) > olimit))) - return 0; - if (litLength >= RUN_MASK) { - int len = (int)litLength-RUN_MASK; - *token = (RUN_MASK<= 255 ; len-=255) *op++ = 255; - *op++ = (BYTE)len; - } - else *token = (BYTE)(litLength< matchlimit) limit = matchlimit; - matchCode = LZ4_count(ip+MINMATCH, match+MINMATCH, limit); - ip += MINMATCH + matchCode; - if (ip==limit) { - unsigned const more = LZ4_count(ip, (const BYTE*)source, matchlimit); - matchCode += more; - ip += more; - } - } else { - matchCode = LZ4_count(ip+MINMATCH, match+MINMATCH, matchlimit); - ip += MINMATCH + matchCode; - } - - if ( outputLimited && /* Check output buffer overflow */ - (unlikely(op + (1 + LASTLITERALS) + (matchCode>>8) > olimit)) ) - return 0; - if (matchCode >= ML_MASK) { - *token += ML_MASK; - matchCode -= ML_MASK; - LZ4_write32(op, 0xFFFFFFFF); - while (matchCode >= 4*255) op+=4, LZ4_write32(op, 0xFFFFFFFF), matchCode -= 4*255; - op += matchCode / 255; - *op++ = (BYTE)(matchCode % 255); - } else - *token += (BYTE)(matchCode); - } - - anchor = ip; - - /* Test end of chunk */ - if (ip > mflimit) break; - - /* Fill table */ - LZ4_putPosition(ip-2, cctx->hashTable, tableType, base); - - /* Test next position */ - match = LZ4_getPosition(ip, cctx->hashTable, tableType, base); - if (dict==usingExtDict) { - if (match < (const BYTE*)source) { - refDelta = dictDelta; - lowLimit = dictionary; - } else { - refDelta = 0; - lowLimit = (const BYTE*)source; - } } - LZ4_putPosition(ip, cctx->hashTable, tableType, base); - if ( ((dictIssue==dictSmall) ? (match>=lowRefLimit) : 1) - && (match+MAX_DISTANCE>=ip) - && (LZ4_read32(match+refDelta)==LZ4_read32(ip)) ) - { token=op++; *token=0; goto _next_match; } - - /* Prepare next loop */ - forwardH = LZ4_hashPosition(++ip, tableType); - } - -_last_literals: - /* Encode Last Literals */ - { size_t const lastRun = (size_t)(iend - anchor); - if ( (outputLimited) && /* Check output buffer overflow */ - ((op - (BYTE*)dest) + lastRun + 1 + ((lastRun+255-RUN_MASK)/255) > (U32)maxOutputSize) ) - return 0; - if (lastRun >= RUN_MASK) { - size_t accumulator = lastRun - RUN_MASK; - *op++ = RUN_MASK << ML_BITS; - for(; accumulator >= 255 ; accumulator-=255) *op++ = 255; - *op++ = (BYTE) accumulator; - } else { - *op++ = (BYTE)(lastRun<internal_donotuse; - LZ4_resetStream((LZ4_stream_t*)state); - if (acceleration < 1) acceleration = ACCELERATION_DEFAULT; - - if (maxOutputSize >= LZ4_compressBound(inputSize)) { - if (inputSize < LZ4_64Klimit) - return LZ4_compress_generic(ctx, source, dest, inputSize, 0, notLimited, byU16, noDict, noDictIssue, acceleration); - else - return LZ4_compress_generic(ctx, source, dest, inputSize, 0, notLimited, (sizeof(void*)==8) ? byU32 : byPtr, noDict, noDictIssue, acceleration); - } else { - if (inputSize < LZ4_64Klimit) - return LZ4_compress_generic(ctx, source, dest, inputSize, maxOutputSize, limitedOutput, byU16, noDict, noDictIssue, acceleration); - else - return LZ4_compress_generic(ctx, source, dest, inputSize, maxOutputSize, limitedOutput, (sizeof(void*)==8) ? byU32 : byPtr, noDict, noDictIssue, acceleration); - } -} - - -int LZ4_compress_fast(const char* source, char* dest, int inputSize, int maxOutputSize, int acceleration) -{ -#if (HEAPMODE) - void* ctxPtr = ALLOCATOR(1, sizeof(LZ4_stream_t)); /* malloc-calloc always properly aligned */ -#else - LZ4_stream_t ctx; - void* const ctxPtr = &ctx; -#endif - - int const result = LZ4_compress_fast_extState(ctxPtr, source, dest, inputSize, maxOutputSize, acceleration); - -#if (HEAPMODE) - FREEMEM(ctxPtr); -#endif - return result; -} - - -int LZ4_compress_default(const char* source, char* dest, int inputSize, int maxOutputSize) -{ - return LZ4_compress_fast(source, dest, inputSize, maxOutputSize, 1); -} - - -/* hidden debug function */ -/* strangely enough, gcc generates faster code when this function is uncommented, even if unused */ -int LZ4_compress_fast_force(const char* source, char* dest, int inputSize, int maxOutputSize, int acceleration) -{ - LZ4_stream_t ctx; - LZ4_resetStream(&ctx); - - if (inputSize < LZ4_64Klimit) - return LZ4_compress_generic(&ctx.internal_donotuse, source, dest, inputSize, maxOutputSize, limitedOutput, byU16, noDict, noDictIssue, acceleration); - else - return LZ4_compress_generic(&ctx.internal_donotuse, source, dest, inputSize, maxOutputSize, limitedOutput, sizeof(void*)==8 ? byU32 : byPtr, noDict, noDictIssue, acceleration); -} - - -/*-****************************** -* *_destSize() variant -********************************/ - -static int LZ4_compress_destSize_generic( - LZ4_stream_t_internal* const ctx, - const char* const src, - char* const dst, - int* const srcSizePtr, - const int targetDstSize, - const tableType_t tableType) -{ - const BYTE* ip = (const BYTE*) src; - const BYTE* base = (const BYTE*) src; - const BYTE* lowLimit = (const BYTE*) src; - const BYTE* anchor = ip; - const BYTE* const iend = ip + *srcSizePtr; - const BYTE* const mflimit = iend - MFLIMIT; - const BYTE* const matchlimit = iend - LASTLITERALS; - - BYTE* op = (BYTE*) dst; - BYTE* const oend = op + targetDstSize; - BYTE* const oMaxLit = op + targetDstSize - 2 /* offset */ - 8 /* because 8+MINMATCH==MFLIMIT */ - 1 /* token */; - BYTE* const oMaxMatch = op + targetDstSize - (LASTLITERALS + 1 /* token */); - BYTE* const oMaxSeq = oMaxLit - 1 /* token */; - - U32 forwardH; - - - /* Init conditions */ - if (targetDstSize < 1) return 0; /* Impossible to store anything */ - if ((U32)*srcSizePtr > (U32)LZ4_MAX_INPUT_SIZE) return 0; /* Unsupported input size, too large (or negative) */ - if ((tableType == byU16) && (*srcSizePtr>=LZ4_64Klimit)) return 0; /* Size too large (not within 64K limit) */ - if (*srcSizePtrhashTable, tableType, base); - ip++; forwardH = LZ4_hashPosition(ip, tableType); - - /* Main Loop */ - for ( ; ; ) { - const BYTE* match; - BYTE* token; - - /* Find a match */ - { const BYTE* forwardIp = ip; - unsigned step = 1; - unsigned searchMatchNb = 1 << LZ4_skipTrigger; - - do { - U32 h = forwardH; - ip = forwardIp; - forwardIp += step; - step = (searchMatchNb++ >> LZ4_skipTrigger); - - if (unlikely(forwardIp > mflimit)) goto _last_literals; - - match = LZ4_getPositionOnHash(h, ctx->hashTable, tableType, base); - forwardH = LZ4_hashPosition(forwardIp, tableType); - LZ4_putPositionOnHash(ip, h, ctx->hashTable, tableType, base); - - } while ( ((tableType==byU16) ? 0 : (match + MAX_DISTANCE < ip)) - || (LZ4_read32(match) != LZ4_read32(ip)) ); - } - - /* Catch up */ - while ((ip>anchor) && (match > lowLimit) && (unlikely(ip[-1]==match[-1]))) { ip--; match--; } - - /* Encode Literal length */ - { unsigned litLength = (unsigned)(ip - anchor); - token = op++; - if (op + ((litLength+240)/255) + litLength > oMaxLit) { - /* Not enough space for a last match */ - op--; - goto _last_literals; - } - if (litLength>=RUN_MASK) { - unsigned len = litLength - RUN_MASK; - *token=(RUN_MASK<= 255 ; len-=255) *op++ = 255; - *op++ = (BYTE)len; - } - else *token = (BYTE)(litLength< oMaxMatch) { - /* Match description too long : reduce it */ - matchLength = (15-1) + (oMaxMatch-op) * 255; - } - ip += MINMATCH + matchLength; - - if (matchLength>=ML_MASK) { - *token += ML_MASK; - matchLength -= ML_MASK; - while (matchLength >= 255) { matchLength-=255; *op++ = 255; } - *op++ = (BYTE)matchLength; - } - else *token += (BYTE)(matchLength); - } - - anchor = ip; - - /* Test end of block */ - if (ip > mflimit) break; - if (op > oMaxSeq) break; - - /* Fill table */ - LZ4_putPosition(ip-2, ctx->hashTable, tableType, base); - - /* Test next position */ - match = LZ4_getPosition(ip, ctx->hashTable, tableType, base); - LZ4_putPosition(ip, ctx->hashTable, tableType, base); - if ( (match+MAX_DISTANCE>=ip) - && (LZ4_read32(match)==LZ4_read32(ip)) ) - { token=op++; *token=0; goto _next_match; } - - /* Prepare next loop */ - forwardH = LZ4_hashPosition(++ip, tableType); - } - -_last_literals: - /* Encode Last Literals */ - { size_t lastRunSize = (size_t)(iend - anchor); - if (op + 1 /* token */ + ((lastRunSize+240)/255) /* litLength */ + lastRunSize /* literals */ > oend) { - /* adapt lastRunSize to fill 'dst' */ - lastRunSize = (oend-op) - 1; - lastRunSize -= (lastRunSize+240)/255; - } - ip = anchor + lastRunSize; - - if (lastRunSize >= RUN_MASK) { - size_t accumulator = lastRunSize - RUN_MASK; - *op++ = RUN_MASK << ML_BITS; - for(; accumulator >= 255 ; accumulator-=255) *op++ = 255; - *op++ = (BYTE) accumulator; - } else { - *op++ = (BYTE)(lastRunSize<= LZ4_compressBound(*srcSizePtr)) { /* compression success is guaranteed */ - return LZ4_compress_fast_extState(state, src, dst, *srcSizePtr, targetDstSize, 1); - } else { - if (*srcSizePtr < LZ4_64Klimit) - return LZ4_compress_destSize_generic(&state->internal_donotuse, src, dst, srcSizePtr, targetDstSize, byU16); - else - return LZ4_compress_destSize_generic(&state->internal_donotuse, src, dst, srcSizePtr, targetDstSize, sizeof(void*)==8 ? byU32 : byPtr); - } -} - - -int LZ4_compress_destSize(const char* src, char* dst, int* srcSizePtr, int targetDstSize) -{ -#if (HEAPMODE) - LZ4_stream_t* ctx = (LZ4_stream_t*)ALLOCATOR(1, sizeof(LZ4_stream_t)); /* malloc-calloc always properly aligned */ -#else - LZ4_stream_t ctxBody; - LZ4_stream_t* ctx = &ctxBody; -#endif - - int result = LZ4_compress_destSize_extState(ctx, src, dst, srcSizePtr, targetDstSize); - -#if (HEAPMODE) - FREEMEM(ctx); -#endif - return result; -} - - - -/*-****************************** -* Streaming functions -********************************/ - -LZ4_stream_t* LZ4_createStream(void) -{ - LZ4_stream_t* lz4s = (LZ4_stream_t*)ALLOCATOR(8, LZ4_STREAMSIZE_U64); - LZ4_STATIC_ASSERT(LZ4_STREAMSIZE >= sizeof(LZ4_stream_t_internal)); /* A compilation error here means LZ4_STREAMSIZE is not large enough */ - LZ4_resetStream(lz4s); - return lz4s; -} - -void LZ4_resetStream (LZ4_stream_t* LZ4_stream) -{ - MEM_INIT(LZ4_stream, 0, sizeof(LZ4_stream_t)); -} - -int LZ4_freeStream (LZ4_stream_t* LZ4_stream) -{ - FREEMEM(LZ4_stream); - return (0); -} - - -#define HASH_UNIT sizeof(reg_t) -int LZ4_loadDict (LZ4_stream_t* LZ4_dict, const char* dictionary, int dictSize) -{ - LZ4_stream_t_internal* dict = &LZ4_dict->internal_donotuse; - const BYTE* p = (const BYTE*)dictionary; - const BYTE* const dictEnd = p + dictSize; - const BYTE* base; - - if ((dict->initCheck) || (dict->currentOffset > 1 GB)) /* Uninitialized structure, or reuse overflow */ - LZ4_resetStream(LZ4_dict); - - if (dictSize < (int)HASH_UNIT) { - dict->dictionary = NULL; - dict->dictSize = 0; - return 0; - } - - if ((dictEnd - p) > 64 KB) p = dictEnd - 64 KB; - dict->currentOffset += 64 KB; - base = p - dict->currentOffset; - dict->dictionary = p; - dict->dictSize = (U32)(dictEnd - p); - dict->currentOffset += dict->dictSize; - - while (p <= dictEnd-HASH_UNIT) { - LZ4_putPosition(p, dict->hashTable, byU32, base); - p+=3; - } - - return dict->dictSize; -} - - -static void LZ4_renormDictT(LZ4_stream_t_internal* LZ4_dict, const BYTE* src) -{ - if ((LZ4_dict->currentOffset > 0x80000000) || - ((uptrval)LZ4_dict->currentOffset > (uptrval)src)) { /* address space overflow */ - /* rescale hash table */ - U32 const delta = LZ4_dict->currentOffset - 64 KB; - const BYTE* dictEnd = LZ4_dict->dictionary + LZ4_dict->dictSize; - int i; - for (i=0; ihashTable[i] < delta) LZ4_dict->hashTable[i]=0; - else LZ4_dict->hashTable[i] -= delta; - } - LZ4_dict->currentOffset = 64 KB; - if (LZ4_dict->dictSize > 64 KB) LZ4_dict->dictSize = 64 KB; - LZ4_dict->dictionary = dictEnd - LZ4_dict->dictSize; - } -} - - -int LZ4_compress_fast_continue (LZ4_stream_t* LZ4_stream, const char* source, char* dest, int inputSize, int maxOutputSize, int acceleration) -{ - LZ4_stream_t_internal* streamPtr = &LZ4_stream->internal_donotuse; - const BYTE* const dictEnd = streamPtr->dictionary + streamPtr->dictSize; - - const BYTE* smallest = (const BYTE*) source; - if (streamPtr->initCheck) return 0; /* Uninitialized structure detected */ - if ((streamPtr->dictSize>0) && (smallest>dictEnd)) smallest = dictEnd; - LZ4_renormDictT(streamPtr, smallest); - if (acceleration < 1) acceleration = ACCELERATION_DEFAULT; - - /* Check overlapping input/dictionary space */ - { const BYTE* sourceEnd = (const BYTE*) source + inputSize; - if ((sourceEnd > streamPtr->dictionary) && (sourceEnd < dictEnd)) { - streamPtr->dictSize = (U32)(dictEnd - sourceEnd); - if (streamPtr->dictSize > 64 KB) streamPtr->dictSize = 64 KB; - if (streamPtr->dictSize < 4) streamPtr->dictSize = 0; - streamPtr->dictionary = dictEnd - streamPtr->dictSize; - } - } - - /* prefix mode : source data follows dictionary */ - if (dictEnd == (const BYTE*)source) { - int result; - if ((streamPtr->dictSize < 64 KB) && (streamPtr->dictSize < streamPtr->currentOffset)) - result = LZ4_compress_generic(streamPtr, source, dest, inputSize, maxOutputSize, limitedOutput, byU32, withPrefix64k, dictSmall, acceleration); - else - result = LZ4_compress_generic(streamPtr, source, dest, inputSize, maxOutputSize, limitedOutput, byU32, withPrefix64k, noDictIssue, acceleration); - streamPtr->dictSize += (U32)inputSize; - streamPtr->currentOffset += (U32)inputSize; - return result; - } - - /* external dictionary mode */ - { int result; - if ((streamPtr->dictSize < 64 KB) && (streamPtr->dictSize < streamPtr->currentOffset)) - result = LZ4_compress_generic(streamPtr, source, dest, inputSize, maxOutputSize, limitedOutput, byU32, usingExtDict, dictSmall, acceleration); - else - result = LZ4_compress_generic(streamPtr, source, dest, inputSize, maxOutputSize, limitedOutput, byU32, usingExtDict, noDictIssue, acceleration); - streamPtr->dictionary = (const BYTE*)source; - streamPtr->dictSize = (U32)inputSize; - streamPtr->currentOffset += (U32)inputSize; - return result; - } -} - - -/* Hidden debug function, to force external dictionary mode */ -int LZ4_compress_forceExtDict (LZ4_stream_t* LZ4_dict, const char* source, char* dest, int inputSize) -{ - LZ4_stream_t_internal* streamPtr = &LZ4_dict->internal_donotuse; - int result; - const BYTE* const dictEnd = streamPtr->dictionary + streamPtr->dictSize; - - const BYTE* smallest = dictEnd; - if (smallest > (const BYTE*) source) smallest = (const BYTE*) source; - LZ4_renormDictT(streamPtr, smallest); - - result = LZ4_compress_generic(streamPtr, source, dest, inputSize, 0, notLimited, byU32, usingExtDict, noDictIssue, 1); - - streamPtr->dictionary = (const BYTE*)source; - streamPtr->dictSize = (U32)inputSize; - streamPtr->currentOffset += (U32)inputSize; - - return result; -} - - -/*! LZ4_saveDict() : - * If previously compressed data block is not guaranteed to remain available at its memory location, - * save it into a safer place (char* safeBuffer). - * Note : you don't need to call LZ4_loadDict() afterwards, - * dictionary is immediately usable, you can therefore call LZ4_compress_fast_continue(). - * Return : saved dictionary size in bytes (necessarily <= dictSize), or 0 if error. - */ -int LZ4_saveDict (LZ4_stream_t* LZ4_dict, char* safeBuffer, int dictSize) -{ - LZ4_stream_t_internal* const dict = &LZ4_dict->internal_donotuse; - const BYTE* const previousDictEnd = dict->dictionary + dict->dictSize; - - if ((U32)dictSize > 64 KB) dictSize = 64 KB; /* useless to define a dictionary > 64 KB */ - if ((U32)dictSize > dict->dictSize) dictSize = dict->dictSize; - - memmove(safeBuffer, previousDictEnd - dictSize, dictSize); - - dict->dictionary = (const BYTE*)safeBuffer; - dict->dictSize = (U32)dictSize; - - return dictSize; -} - - - -/*-***************************** -* Decompression functions -*******************************/ -/*! LZ4_decompress_generic() : - * This generic decompression function cover all use cases. - * It shall be instantiated several times, using different sets of directives - * Note that it is important this generic function is really inlined, - * in order to remove useless branches during compilation optimization. - */ -FORCE_INLINE int LZ4_decompress_generic( - const char* const source, - char* const dest, - int inputSize, - int outputSize, /* If endOnInput==endOnInputSize, this value is the max size of Output Buffer. */ - - int endOnInput, /* endOnOutputSize, endOnInputSize */ - int partialDecoding, /* full, partial */ - int targetOutputSize, /* only used if partialDecoding==partial */ - int dict, /* noDict, withPrefix64k, usingExtDict */ - const BYTE* const lowPrefix, /* == dest when no prefix */ - const BYTE* const dictStart, /* only if dict==usingExtDict */ - const size_t dictSize /* note : = 0 if noDict */ - ) -{ - /* Local Variables */ - const BYTE* ip = (const BYTE*) source; - const BYTE* const iend = ip + inputSize; - - BYTE* op = (BYTE*) dest; - BYTE* const oend = op + outputSize; - BYTE* cpy; - BYTE* oexit = op + targetOutputSize; - const BYTE* const lowLimit = lowPrefix - dictSize; - - const BYTE* const dictEnd = (const BYTE*)dictStart + dictSize; - const unsigned dec32table[] = {0, 1, 2, 1, 4, 4, 4, 4}; - const int dec64table[] = {0, 0, 0, -1, 0, 1, 2, 3}; - - const int safeDecode = (endOnInput==endOnInputSize); - const int checkOffset = ((safeDecode) && (dictSize < (int)(64 KB))); - - - /* Special cases */ - if ((partialDecoding) && (oexit > oend-MFLIMIT)) oexit = oend-MFLIMIT; /* targetOutputSize too high => decode everything */ - if ((endOnInput) && (unlikely(outputSize==0))) return ((inputSize==1) && (*ip==0)) ? 0 : -1; /* Empty output buffer */ - if ((!endOnInput) && (unlikely(outputSize==0))) return (*ip==0?1:-1); - - /* Main Loop : decode sequences */ - while (1) { - size_t length; - const BYTE* match; - size_t offset; - - /* get literal length */ - unsigned const token = *ip++; - if ((length=(token>>ML_BITS)) == RUN_MASK) { - unsigned s; - do { - s = *ip++; - length += s; - } while ( likely(endOnInput ? ip(partialDecoding?oexit:oend-MFLIMIT)) || (ip+length>iend-(2+1+LASTLITERALS))) ) - || ((!endOnInput) && (cpy>oend-WILDCOPYLENGTH)) ) - { - if (partialDecoding) { - if (cpy > oend) goto _output_error; /* Error : write attempt beyond end of output buffer */ - if ((endOnInput) && (ip+length > iend)) goto _output_error; /* Error : read attempt beyond end of input buffer */ - } else { - if ((!endOnInput) && (cpy != oend)) goto _output_error; /* Error : block decoding must stop exactly there */ - if ((endOnInput) && ((ip+length != iend) || (cpy > oend))) goto _output_error; /* Error : input must be consumed */ - } - memcpy(op, ip, length); - ip += length; - op += length; - break; /* Necessarily EOF, due to parsing restrictions */ - } - LZ4_wildCopy(op, ip, cpy); - ip += length; op = cpy; - - /* get offset */ - offset = LZ4_readLE16(ip); ip+=2; - match = op - offset; - if ((checkOffset) && (unlikely(match < lowLimit))) goto _output_error; /* Error : offset outside buffers */ - LZ4_write32(op, (U32)offset); /* costs ~1%; silence an msan warning when offset==0 */ - - /* get matchlength */ - length = token & ML_MASK; - if (length == ML_MASK) { - unsigned s; - do { - s = *ip++; - if ((endOnInput) && (ip > iend-LASTLITERALS)) goto _output_error; - length += s; - } while (s==255); - if ((safeDecode) && unlikely((uptrval)(op)+length<(uptrval)op)) goto _output_error; /* overflow detection */ - } - length += MINMATCH; - - /* check external dictionary */ - if ((dict==usingExtDict) && (match < lowPrefix)) { - if (unlikely(op+length > oend-LASTLITERALS)) goto _output_error; /* doesn't respect parsing restriction */ - - if (length <= (size_t)(lowPrefix-match)) { - /* match can be copied as a single segment from external dictionary */ - memmove(op, dictEnd - (lowPrefix-match), length); - op += length; - } else { - /* match encompass external dictionary and current block */ - size_t const copySize = (size_t)(lowPrefix-match); - size_t const restSize = length - copySize; - memcpy(op, dictEnd - copySize, copySize); - op += copySize; - if (restSize > (size_t)(op-lowPrefix)) { /* overlap copy */ - BYTE* const endOfMatch = op + restSize; - const BYTE* copyFrom = lowPrefix; - while (op < endOfMatch) *op++ = *copyFrom++; - } else { - memcpy(op, lowPrefix, restSize); - op += restSize; - } } - continue; - } - - /* copy match within block */ - cpy = op + length; - if (unlikely(offset<8)) { - const int dec64 = dec64table[offset]; - op[0] = match[0]; - op[1] = match[1]; - op[2] = match[2]; - op[3] = match[3]; - match += dec32table[offset]; - memcpy(op+4, match, 4); - match -= dec64; - } else { LZ4_copy8(op, match); match+=8; } - op += 8; - - if (unlikely(cpy>oend-12)) { - BYTE* const oCopyLimit = oend-(WILDCOPYLENGTH-1); - if (cpy > oend-LASTLITERALS) goto _output_error; /* Error : last LASTLITERALS bytes must be literals (uncompressed) */ - if (op < oCopyLimit) { - LZ4_wildCopy(op, match, oCopyLimit); - match += oCopyLimit - op; - op = oCopyLimit; - } - while (op16) LZ4_wildCopy(op+8, match+8, cpy); - } - op=cpy; /* correction */ - } - - /* end of decoding */ - if (endOnInput) - return (int) (((char*)op)-dest); /* Nb of output bytes decoded */ - else - return (int) (((const char*)ip)-source); /* Nb of input bytes read */ - - /* Overflow error detected */ -_output_error: - return (int) (-(((const char*)ip)-source))-1; -} - - -int LZ4_decompress_safe(const char* source, char* dest, int compressedSize, int maxDecompressedSize) -{ - return LZ4_decompress_generic(source, dest, compressedSize, maxDecompressedSize, endOnInputSize, full, 0, noDict, (BYTE*)dest, NULL, 0); -} - -int LZ4_decompress_safe_partial(const char* source, char* dest, int compressedSize, int targetOutputSize, int maxDecompressedSize) -{ - return LZ4_decompress_generic(source, dest, compressedSize, maxDecompressedSize, endOnInputSize, partial, targetOutputSize, noDict, (BYTE*)dest, NULL, 0); -} - -int LZ4_decompress_fast(const char* source, char* dest, int originalSize) -{ - return LZ4_decompress_generic(source, dest, 0, originalSize, endOnOutputSize, full, 0, withPrefix64k, (BYTE*)(dest - 64 KB), NULL, 64 KB); -} - - -/*===== streaming decompression functions =====*/ - -/* - * If you prefer dynamic allocation methods, - * LZ4_createStreamDecode() - * provides a pointer (void*) towards an initialized LZ4_streamDecode_t structure. - */ -LZ4_streamDecode_t* LZ4_createStreamDecode(void) -{ - LZ4_streamDecode_t* lz4s = (LZ4_streamDecode_t*) ALLOCATOR(1, sizeof(LZ4_streamDecode_t)); - return lz4s; -} - -int LZ4_freeStreamDecode (LZ4_streamDecode_t* LZ4_stream) -{ - FREEMEM(LZ4_stream); - return 0; -} - -/*! - * LZ4_setStreamDecode() : - * Use this function to instruct where to find the dictionary. - * This function is not necessary if previous data is still available where it was decoded. - * Loading a size of 0 is allowed (same effect as no dictionary). - * Return : 1 if OK, 0 if error - */ -int LZ4_setStreamDecode (LZ4_streamDecode_t* LZ4_streamDecode, const char* dictionary, int dictSize) -{ - LZ4_streamDecode_t_internal* lz4sd = &LZ4_streamDecode->internal_donotuse; - lz4sd->prefixSize = (size_t) dictSize; - lz4sd->prefixEnd = (const BYTE*) dictionary + dictSize; - lz4sd->externalDict = NULL; - lz4sd->extDictSize = 0; - return 1; -} - -/* -*_continue() : - These decoding functions allow decompression of multiple blocks in "streaming" mode. - Previously decoded blocks must still be available at the memory position where they were decoded. - If it's not possible, save the relevant part of decoded data into a safe buffer, - and indicate where it stands using LZ4_setStreamDecode() -*/ -int LZ4_decompress_safe_continue (LZ4_streamDecode_t* LZ4_streamDecode, const char* source, char* dest, int compressedSize, int maxOutputSize) -{ - LZ4_streamDecode_t_internal* lz4sd = &LZ4_streamDecode->internal_donotuse; - int result; - - if (lz4sd->prefixEnd == (BYTE*)dest) { - result = LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, - endOnInputSize, full, 0, - usingExtDict, lz4sd->prefixEnd - lz4sd->prefixSize, lz4sd->externalDict, lz4sd->extDictSize); - if (result <= 0) return result; - lz4sd->prefixSize += result; - lz4sd->prefixEnd += result; - } else { - lz4sd->extDictSize = lz4sd->prefixSize; - lz4sd->externalDict = lz4sd->prefixEnd - lz4sd->extDictSize; - result = LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, - endOnInputSize, full, 0, - usingExtDict, (BYTE*)dest, lz4sd->externalDict, lz4sd->extDictSize); - if (result <= 0) return result; - lz4sd->prefixSize = result; - lz4sd->prefixEnd = (BYTE*)dest + result; - } - - return result; -} - -int LZ4_decompress_fast_continue (LZ4_streamDecode_t* LZ4_streamDecode, const char* source, char* dest, int originalSize) -{ - LZ4_streamDecode_t_internal* lz4sd = &LZ4_streamDecode->internal_donotuse; - int result; - - if (lz4sd->prefixEnd == (BYTE*)dest) { - result = LZ4_decompress_generic(source, dest, 0, originalSize, - endOnOutputSize, full, 0, - usingExtDict, lz4sd->prefixEnd - lz4sd->prefixSize, lz4sd->externalDict, lz4sd->extDictSize); - if (result <= 0) return result; - lz4sd->prefixSize += originalSize; - lz4sd->prefixEnd += originalSize; - } else { - lz4sd->extDictSize = lz4sd->prefixSize; - lz4sd->externalDict = lz4sd->prefixEnd - lz4sd->extDictSize; - result = LZ4_decompress_generic(source, dest, 0, originalSize, - endOnOutputSize, full, 0, - usingExtDict, (BYTE*)dest, lz4sd->externalDict, lz4sd->extDictSize); - if (result <= 0) return result; - lz4sd->prefixSize = originalSize; - lz4sd->prefixEnd = (BYTE*)dest + originalSize; - } - - return result; -} - - -/* -Advanced decoding functions : -*_usingDict() : - These decoding functions work the same as "_continue" ones, - the dictionary must be explicitly provided within parameters -*/ - -FORCE_INLINE int LZ4_decompress_usingDict_generic(const char* source, char* dest, int compressedSize, int maxOutputSize, int safe, const char* dictStart, int dictSize) -{ - if (dictSize==0) - return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, safe, full, 0, noDict, (BYTE*)dest, NULL, 0); - if (dictStart+dictSize == dest) { - if (dictSize >= (int)(64 KB - 1)) - return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, safe, full, 0, withPrefix64k, (BYTE*)dest-64 KB, NULL, 0); - return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, safe, full, 0, noDict, (BYTE*)dest-dictSize, NULL, 0); - } - return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, safe, full, 0, usingExtDict, (BYTE*)dest, (const BYTE*)dictStart, dictSize); -} - -int LZ4_decompress_safe_usingDict(const char* source, char* dest, int compressedSize, int maxOutputSize, const char* dictStart, int dictSize) -{ - return LZ4_decompress_usingDict_generic(source, dest, compressedSize, maxOutputSize, 1, dictStart, dictSize); -} - -int LZ4_decompress_fast_usingDict(const char* source, char* dest, int originalSize, const char* dictStart, int dictSize) -{ - return LZ4_decompress_usingDict_generic(source, dest, 0, originalSize, 0, dictStart, dictSize); -} - -/* debug function */ -int LZ4_decompress_safe_forceExtDict(const char* source, char* dest, int compressedSize, int maxOutputSize, const char* dictStart, int dictSize) -{ - return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, endOnInputSize, full, 0, usingExtDict, (BYTE*)dest, (const BYTE*)dictStart, dictSize); -} - - -/*=************************************************* -* Obsolete Functions -***************************************************/ -/* obsolete compression functions */ -int LZ4_compress_limitedOutput(const char* source, char* dest, int inputSize, int maxOutputSize) { return LZ4_compress_default(source, dest, inputSize, maxOutputSize); } -int LZ4_compress(const char* source, char* dest, int inputSize) { return LZ4_compress_default(source, dest, inputSize, LZ4_compressBound(inputSize)); } -int LZ4_compress_limitedOutput_withState (void* state, const char* src, char* dst, int srcSize, int dstSize) { return LZ4_compress_fast_extState(state, src, dst, srcSize, dstSize, 1); } -int LZ4_compress_withState (void* state, const char* src, char* dst, int srcSize) { return LZ4_compress_fast_extState(state, src, dst, srcSize, LZ4_compressBound(srcSize), 1); } -int LZ4_compress_limitedOutput_continue (LZ4_stream_t* LZ4_stream, const char* src, char* dst, int srcSize, int maxDstSize) { return LZ4_compress_fast_continue(LZ4_stream, src, dst, srcSize, maxDstSize, 1); } -int LZ4_compress_continue (LZ4_stream_t* LZ4_stream, const char* source, char* dest, int inputSize) { return LZ4_compress_fast_continue(LZ4_stream, source, dest, inputSize, LZ4_compressBound(inputSize), 1); } - -/* -These function names are deprecated and should no longer be used. -They are only provided here for compatibility with older user programs. -- LZ4_uncompress is totally equivalent to LZ4_decompress_fast -- LZ4_uncompress_unknownOutputSize is totally equivalent to LZ4_decompress_safe -*/ -int LZ4_uncompress (const char* source, char* dest, int outputSize) { return LZ4_decompress_fast(source, dest, outputSize); } -int LZ4_uncompress_unknownOutputSize (const char* source, char* dest, int isize, int maxOutputSize) { return LZ4_decompress_safe(source, dest, isize, maxOutputSize); } - - -/* Obsolete Streaming functions */ - -int LZ4_sizeofStreamState() { return LZ4_STREAMSIZE; } - -static void LZ4_init(LZ4_stream_t* lz4ds, BYTE* base) -{ - MEM_INIT(lz4ds, 0, sizeof(LZ4_stream_t)); - lz4ds->internal_donotuse.bufferStart = base; -} - -int LZ4_resetStreamState(void* state, char* inputBuffer) -{ - if ((((uptrval)state) & 3) != 0) return 1; /* Error : pointer is not aligned on 4-bytes boundary */ - LZ4_init((LZ4_stream_t*)state, (BYTE*)inputBuffer); - return 0; -} - -void* LZ4_create (char* inputBuffer) -{ - LZ4_stream_t* lz4ds = (LZ4_stream_t*)ALLOCATOR(8, sizeof(LZ4_stream_t)); - LZ4_init (lz4ds, (BYTE*)inputBuffer); - return lz4ds; -} - -char* LZ4_slideInputBuffer (void* LZ4_Data) -{ - LZ4_stream_t_internal* ctx = &((LZ4_stream_t*)LZ4_Data)->internal_donotuse; - int dictSize = LZ4_saveDict((LZ4_stream_t*)LZ4_Data, (char*)ctx->bufferStart, 64 KB); - return (char*)(ctx->bufferStart + dictSize); -} - -/* Obsolete streaming decompression functions */ - -int LZ4_decompress_safe_withPrefix64k(const char* source, char* dest, int compressedSize, int maxOutputSize) -{ - return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, endOnInputSize, full, 0, withPrefix64k, (BYTE*)dest - 64 KB, NULL, 64 KB); -} - -int LZ4_decompress_fast_withPrefix64k(const char* source, char* dest, int originalSize) -{ - return LZ4_decompress_generic(source, dest, 0, originalSize, endOnOutputSize, full, 0, withPrefix64k, (BYTE*)dest - 64 KB, NULL, 64 KB); -} - -#endif /* LZ4_COMMONDEFS_ONLY */ diff --git a/ext/lz4/lz4.h b/ext/lz4/lz4.h deleted file mode 100644 index 0aae19c9..00000000 --- a/ext/lz4/lz4.h +++ /dev/null @@ -1,463 +0,0 @@ -/* - * LZ4 - Fast LZ compression algorithm - * Header File - * Copyright (C) 2011-2016, Yann Collet. - - BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions are - met: - - * Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above - copyright notice, this list of conditions and the following disclaimer - in the documentation and/or other materials provided with the - distribution. - - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - You can contact the author at : - - LZ4 homepage : http://www.lz4.org - - LZ4 source repository : https://github.com/lz4/lz4 -*/ -#ifndef LZ4_H_2983827168210 -#define LZ4_H_2983827168210 - -#if defined (__cplusplus) -extern "C" { -#endif - -/* --- Dependency --- */ -#include /* size_t */ - - -/** - Introduction - - LZ4 is lossless compression algorithm, providing compression speed at 400 MB/s per core, - scalable with multi-cores CPU. It features an extremely fast decoder, with speed in - multiple GB/s per core, typically reaching RAM speed limits on multi-core systems. - - The LZ4 compression library provides in-memory compression and decompression functions. - Compression can be done in: - - a single step (described as Simple Functions) - - a single step, reusing a context (described in Advanced Functions) - - unbounded multiple steps (described as Streaming compression) - - lz4.h provides block compression functions. It gives full buffer control to user. - Decompressing an lz4-compressed block also requires metadata (such as compressed size). - Each application is free to encode such metadata in whichever way it wants. - - An additional format, called LZ4 frame specification (doc/lz4_Frame_format.md), - take care of encoding standard metadata alongside LZ4-compressed blocks. - If your application requires interoperability, it's recommended to use it. - A library is provided to take care of it, see lz4frame.h. -*/ - -/*^*************************************************************** -* Export parameters -*****************************************************************/ -/* -* LZ4_DLL_EXPORT : -* Enable exporting of functions when building a Windows DLL -*/ -#if defined(LZ4_DLL_EXPORT) && (LZ4_DLL_EXPORT==1) -# define LZ4LIB_API __declspec(dllexport) -#elif defined(LZ4_DLL_IMPORT) && (LZ4_DLL_IMPORT==1) -# define LZ4LIB_API __declspec(dllimport) /* It isn't required but allows to generate better code, saving a function pointer load from the IAT and an indirect jump.*/ -#else -# define LZ4LIB_API -#endif - - -/*========== Version =========== */ -#define LZ4_VERSION_MAJOR 1 /* for breaking interface changes */ -#define LZ4_VERSION_MINOR 7 /* for new (non-breaking) interface capabilities */ -#define LZ4_VERSION_RELEASE 5 /* for tweaks, bug-fixes, or development */ - -#define LZ4_VERSION_NUMBER (LZ4_VERSION_MAJOR *100*100 + LZ4_VERSION_MINOR *100 + LZ4_VERSION_RELEASE) - -#define LZ4_LIB_VERSION LZ4_VERSION_MAJOR.LZ4_VERSION_MINOR.LZ4_VERSION_RELEASE -#define LZ4_QUOTE(str) #str -#define LZ4_EXPAND_AND_QUOTE(str) LZ4_QUOTE(str) -#define LZ4_VERSION_STRING LZ4_EXPAND_AND_QUOTE(LZ4_LIB_VERSION) - -LZ4LIB_API int LZ4_versionNumber (void); -LZ4LIB_API const char* LZ4_versionString (void); - - -/*-************************************ -* Tuning parameter -**************************************/ -/*! - * LZ4_MEMORY_USAGE : - * Memory usage formula : N->2^N Bytes (examples : 10 -> 1KB; 12 -> 4KB ; 16 -> 64KB; 20 -> 1MB; etc.) - * Increasing memory usage improves compression ratio - * Reduced memory usage can improve speed, due to cache effect - * Default value is 14, for 16KB, which nicely fits into Intel x86 L1 cache - */ -#define LZ4_MEMORY_USAGE 14 - - -/*-************************************ -* Simple Functions -**************************************/ -/*! LZ4_compress_default() : - Compresses 'sourceSize' bytes from buffer 'source' - into already allocated 'dest' buffer of size 'maxDestSize'. - Compression is guaranteed to succeed if 'maxDestSize' >= LZ4_compressBound(sourceSize). - It also runs faster, so it's a recommended setting. - If the function cannot compress 'source' into a more limited 'dest' budget, - compression stops *immediately*, and the function result is zero. - As a consequence, 'dest' content is not valid. - This function never writes outside 'dest' buffer, nor read outside 'source' buffer. - sourceSize : Max supported value is LZ4_MAX_INPUT_VALUE - maxDestSize : full or partial size of buffer 'dest' (which must be already allocated) - return : the number of bytes written into buffer 'dest' (necessarily <= maxOutputSize) - or 0 if compression fails */ -LZ4LIB_API int LZ4_compress_default(const char* source, char* dest, int sourceSize, int maxDestSize); - -/*! LZ4_decompress_safe() : - compressedSize : is the precise full size of the compressed block. - maxDecompressedSize : is the size of destination buffer, which must be already allocated. - return : the number of bytes decompressed into destination buffer (necessarily <= maxDecompressedSize) - If destination buffer is not large enough, decoding will stop and output an error code (<0). - If the source stream is detected malformed, the function will stop decoding and return a negative result. - This function is protected against buffer overflow exploits, including malicious data packets. - It never writes outside output buffer, nor reads outside input buffer. -*/ -LZ4LIB_API int LZ4_decompress_safe (const char* source, char* dest, int compressedSize, int maxDecompressedSize); - - -/*-************************************ -* Advanced Functions -**************************************/ -#define LZ4_MAX_INPUT_SIZE 0x7E000000 /* 2 113 929 216 bytes */ -#define LZ4_COMPRESSBOUND(isize) ((unsigned)(isize) > (unsigned)LZ4_MAX_INPUT_SIZE ? 0 : (isize) + ((isize)/255) + 16) - -/*! -LZ4_compressBound() : - Provides the maximum size that LZ4 compression may output in a "worst case" scenario (input data not compressible) - This function is primarily useful for memory allocation purposes (destination buffer size). - Macro LZ4_COMPRESSBOUND() is also provided for compilation-time evaluation (stack memory allocation for example). - Note that LZ4_compress_default() compress faster when dest buffer size is >= LZ4_compressBound(srcSize) - inputSize : max supported value is LZ4_MAX_INPUT_SIZE - return : maximum output size in a "worst case" scenario - or 0, if input size is too large ( > LZ4_MAX_INPUT_SIZE) -*/ -LZ4LIB_API int LZ4_compressBound(int inputSize); - -/*! -LZ4_compress_fast() : - Same as LZ4_compress_default(), but allows to select an "acceleration" factor. - The larger the acceleration value, the faster the algorithm, but also the lesser the compression. - It's a trade-off. It can be fine tuned, with each successive value providing roughly +~3% to speed. - An acceleration value of "1" is the same as regular LZ4_compress_default() - Values <= 0 will be replaced by ACCELERATION_DEFAULT (see lz4.c), which is 1. -*/ -LZ4LIB_API int LZ4_compress_fast (const char* source, char* dest, int sourceSize, int maxDestSize, int acceleration); - - -/*! -LZ4_compress_fast_extState() : - Same compression function, just using an externally allocated memory space to store compression state. - Use LZ4_sizeofState() to know how much memory must be allocated, - and allocate it on 8-bytes boundaries (using malloc() typically). - Then, provide it as 'void* state' to compression function. -*/ -LZ4LIB_API int LZ4_sizeofState(void); -LZ4LIB_API int LZ4_compress_fast_extState (void* state, const char* source, char* dest, int inputSize, int maxDestSize, int acceleration); - - -/*! -LZ4_compress_destSize() : - Reverse the logic, by compressing as much data as possible from 'source' buffer - into already allocated buffer 'dest' of size 'targetDestSize'. - This function either compresses the entire 'source' content into 'dest' if it's large enough, - or fill 'dest' buffer completely with as much data as possible from 'source'. - *sourceSizePtr : will be modified to indicate how many bytes where read from 'source' to fill 'dest'. - New value is necessarily <= old value. - return : Nb bytes written into 'dest' (necessarily <= targetDestSize) - or 0 if compression fails -*/ -LZ4LIB_API int LZ4_compress_destSize (const char* source, char* dest, int* sourceSizePtr, int targetDestSize); - - -/*! -LZ4_decompress_fast() : - originalSize : is the original and therefore uncompressed size - return : the number of bytes read from the source buffer (in other words, the compressed size) - If the source stream is detected malformed, the function will stop decoding and return a negative result. - Destination buffer must be already allocated. Its size must be a minimum of 'originalSize' bytes. - note : This function fully respect memory boundaries for properly formed compressed data. - It is a bit faster than LZ4_decompress_safe(). - However, it does not provide any protection against intentionally modified data stream (malicious input). - Use this function in trusted environment only (data to decode comes from a trusted source). -*/ -LZ4LIB_API int LZ4_decompress_fast (const char* source, char* dest, int originalSize); - -/*! -LZ4_decompress_safe_partial() : - This function decompress a compressed block of size 'compressedSize' at position 'source' - into destination buffer 'dest' of size 'maxDecompressedSize'. - The function tries to stop decompressing operation as soon as 'targetOutputSize' has been reached, - reducing decompression time. - return : the number of bytes decoded in the destination buffer (necessarily <= maxDecompressedSize) - Note : this number can be < 'targetOutputSize' should the compressed block to decode be smaller. - Always control how many bytes were decoded. - If the source stream is detected malformed, the function will stop decoding and return a negative result. - This function never writes outside of output buffer, and never reads outside of input buffer. It is therefore protected against malicious data packets -*/ -LZ4LIB_API int LZ4_decompress_safe_partial (const char* source, char* dest, int compressedSize, int targetOutputSize, int maxDecompressedSize); - - -/*-********************************************* -* Streaming Compression Functions -***********************************************/ -typedef union LZ4_stream_u LZ4_stream_t; /* incomplete type (defined later) */ - -/*! LZ4_createStream() and LZ4_freeStream() : - * LZ4_createStream() will allocate and initialize an `LZ4_stream_t` structure. - * LZ4_freeStream() releases its memory. - */ -LZ4LIB_API LZ4_stream_t* LZ4_createStream(void); -LZ4LIB_API int LZ4_freeStream (LZ4_stream_t* streamPtr); - -/*! LZ4_resetStream() : - * An LZ4_stream_t structure can be allocated once and re-used multiple times. - * Use this function to init an allocated `LZ4_stream_t` structure and start a new compression. - */ -LZ4LIB_API void LZ4_resetStream (LZ4_stream_t* streamPtr); - -/*! LZ4_loadDict() : - * Use this function to load a static dictionary into LZ4_stream. - * Any previous data will be forgotten, only 'dictionary' will remain in memory. - * Loading a size of 0 is allowed. - * Return : dictionary size, in bytes (necessarily <= 64 KB) - */ -LZ4LIB_API int LZ4_loadDict (LZ4_stream_t* streamPtr, const char* dictionary, int dictSize); - -/*! LZ4_compress_fast_continue() : - * Compress buffer content 'src', using data from previously compressed blocks as dictionary to improve compression ratio. - * Important : Previous data blocks are assumed to still be present and unmodified ! - * 'dst' buffer must be already allocated. - * If maxDstSize >= LZ4_compressBound(srcSize), compression is guaranteed to succeed, and runs faster. - * If not, and if compressed data cannot fit into 'dst' buffer size, compression stops, and function returns a zero. - */ -LZ4LIB_API int LZ4_compress_fast_continue (LZ4_stream_t* streamPtr, const char* src, char* dst, int srcSize, int maxDstSize, int acceleration); - -/*! LZ4_saveDict() : - * If previously compressed data block is not guaranteed to remain available at its memory location, - * save it into a safer place (char* safeBuffer). - * Note : you don't need to call LZ4_loadDict() afterwards, - * dictionary is immediately usable, you can therefore call LZ4_compress_fast_continue(). - * Return : saved dictionary size in bytes (necessarily <= dictSize), or 0 if error. - */ -LZ4LIB_API int LZ4_saveDict (LZ4_stream_t* streamPtr, char* safeBuffer, int dictSize); - - -/*-********************************************** -* Streaming Decompression Functions -* Bufferless synchronous API -************************************************/ -typedef union LZ4_streamDecode_u LZ4_streamDecode_t; /* incomplete type (defined later) */ - -/* creation / destruction of streaming decompression tracking structure */ -LZ4LIB_API LZ4_streamDecode_t* LZ4_createStreamDecode(void); -LZ4LIB_API int LZ4_freeStreamDecode (LZ4_streamDecode_t* LZ4_stream); - -/*! LZ4_setStreamDecode() : - * Use this function to instruct where to find the dictionary. - * Setting a size of 0 is allowed (same effect as reset). - * @return : 1 if OK, 0 if error - */ -LZ4LIB_API int LZ4_setStreamDecode (LZ4_streamDecode_t* LZ4_streamDecode, const char* dictionary, int dictSize); - -/*! -LZ4_decompress_*_continue() : - These decoding functions allow decompression of multiple blocks in "streaming" mode. - Previously decoded blocks *must* remain available at the memory position where they were decoded (up to 64 KB) - In the case of a ring buffers, decoding buffer must be either : - - Exactly same size as encoding buffer, with same update rule (block boundaries at same positions) - In which case, the decoding & encoding ring buffer can have any size, including very small ones ( < 64 KB). - - Larger than encoding buffer, by a minimum of maxBlockSize more bytes. - maxBlockSize is implementation dependent. It's the maximum size you intend to compress into a single block. - In which case, encoding and decoding buffers do not need to be synchronized, - and encoding ring buffer can have any size, including small ones ( < 64 KB). - - _At least_ 64 KB + 8 bytes + maxBlockSize. - In which case, encoding and decoding buffers do not need to be synchronized, - and encoding ring buffer can have any size, including larger than decoding buffer. - Whenever these conditions are not possible, save the last 64KB of decoded data into a safe buffer, - and indicate where it is saved using LZ4_setStreamDecode() -*/ -LZ4LIB_API int LZ4_decompress_safe_continue (LZ4_streamDecode_t* LZ4_streamDecode, const char* source, char* dest, int compressedSize, int maxDecompressedSize); -LZ4LIB_API int LZ4_decompress_fast_continue (LZ4_streamDecode_t* LZ4_streamDecode, const char* source, char* dest, int originalSize); - - -/*! LZ4_decompress_*_usingDict() : - * These decoding functions work the same as - * a combination of LZ4_setStreamDecode() followed by LZ4_decompress_*_continue() - * They are stand-alone, and don't need an LZ4_streamDecode_t structure. - */ -LZ4LIB_API int LZ4_decompress_safe_usingDict (const char* source, char* dest, int compressedSize, int maxDecompressedSize, const char* dictStart, int dictSize); -LZ4LIB_API int LZ4_decompress_fast_usingDict (const char* source, char* dest, int originalSize, const char* dictStart, int dictSize); - - -/*^********************************************** - * !!!!!! STATIC LINKING ONLY !!!!!! - ***********************************************/ -/*-************************************ - * Private definitions - ************************************** - * Do not use these definitions. - * They are exposed to allow static allocation of `LZ4_stream_t` and `LZ4_streamDecode_t`. - * Using these definitions will expose code to API and/or ABI break in future versions of the library. - **************************************/ -#define LZ4_HASHLOG (LZ4_MEMORY_USAGE-2) -#define LZ4_HASHTABLESIZE (1 << LZ4_MEMORY_USAGE) -#define LZ4_HASH_SIZE_U32 (1 << LZ4_HASHLOG) /* required as macro for static allocation */ - -#if defined(__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) -#include - -typedef struct { - uint32_t hashTable[LZ4_HASH_SIZE_U32]; - uint32_t currentOffset; - uint32_t initCheck; - const uint8_t* dictionary; - uint8_t* bufferStart; /* obsolete, used for slideInputBuffer */ - uint32_t dictSize; -} LZ4_stream_t_internal; - -typedef struct { - const uint8_t* externalDict; - size_t extDictSize; - const uint8_t* prefixEnd; - size_t prefixSize; -} LZ4_streamDecode_t_internal; - -#else - -typedef struct { - unsigned int hashTable[LZ4_HASH_SIZE_U32]; - unsigned int currentOffset; - unsigned int initCheck; - const unsigned char* dictionary; - unsigned char* bufferStart; /* obsolete, used for slideInputBuffer */ - unsigned int dictSize; -} LZ4_stream_t_internal; - -typedef struct { - const unsigned char* externalDict; - size_t extDictSize; - const unsigned char* prefixEnd; - size_t prefixSize; -} LZ4_streamDecode_t_internal; - -#endif - -/*! - * LZ4_stream_t : - * information structure to track an LZ4 stream. - * init this structure before first use. - * note : only use in association with static linking ! - * this definition is not API/ABI safe, - * and may change in a future version ! - */ -#define LZ4_STREAMSIZE_U64 ((1 << (LZ4_MEMORY_USAGE-3)) + 4) -#define LZ4_STREAMSIZE (LZ4_STREAMSIZE_U64 * sizeof(unsigned long long)) -union LZ4_stream_u { - unsigned long long table[LZ4_STREAMSIZE_U64]; - LZ4_stream_t_internal internal_donotuse; -} ; /* previously typedef'd to LZ4_stream_t */ - - -/*! - * LZ4_streamDecode_t : - * information structure to track an LZ4 stream during decompression. - * init this structure using LZ4_setStreamDecode (or memset()) before first use - * note : only use in association with static linking ! - * this definition is not API/ABI safe, - * and may change in a future version ! - */ -#define LZ4_STREAMDECODESIZE_U64 4 -#define LZ4_STREAMDECODESIZE (LZ4_STREAMDECODESIZE_U64 * sizeof(unsigned long long)) -union LZ4_streamDecode_u { - unsigned long long table[LZ4_STREAMDECODESIZE_U64]; - LZ4_streamDecode_t_internal internal_donotuse; -} ; /* previously typedef'd to LZ4_streamDecode_t */ - - -/*=************************************ -* Obsolete Functions -**************************************/ -/* Deprecation warnings */ -/* Should these warnings be a problem, - it is generally possible to disable them, - typically with -Wno-deprecated-declarations for gcc - or _CRT_SECURE_NO_WARNINGS in Visual. - Otherwise, it's also possible to define LZ4_DISABLE_DEPRECATE_WARNINGS */ -#ifdef LZ4_DISABLE_DEPRECATE_WARNINGS -# define LZ4_DEPRECATED(message) /* disable deprecation warnings */ -#else -# define LZ4_GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__) -# if defined (__cplusplus) && (__cplusplus >= 201402) /* C++14 or greater */ -# define LZ4_DEPRECATED(message) [[deprecated(message)]] -# elif (LZ4_GCC_VERSION >= 405) || defined(__clang__) -# define LZ4_DEPRECATED(message) __attribute__((deprecated(message))) -# elif (LZ4_GCC_VERSION >= 301) -# define LZ4_DEPRECATED(message) __attribute__((deprecated)) -# elif defined(_MSC_VER) -# define LZ4_DEPRECATED(message) __declspec(deprecated(message)) -# else -# pragma message("WARNING: You need to implement LZ4_DEPRECATED for this compiler") -# define LZ4_DEPRECATED(message) -# endif -#endif /* LZ4_DISABLE_DEPRECATE_WARNINGS */ - -/* Obsolete compression functions */ -LZ4_DEPRECATED("use LZ4_compress_default() instead") int LZ4_compress (const char* source, char* dest, int sourceSize); -LZ4_DEPRECATED("use LZ4_compress_default() instead") int LZ4_compress_limitedOutput (const char* source, char* dest, int sourceSize, int maxOutputSize); -LZ4_DEPRECATED("use LZ4_compress_fast_extState() instead") int LZ4_compress_withState (void* state, const char* source, char* dest, int inputSize); -LZ4_DEPRECATED("use LZ4_compress_fast_extState() instead") int LZ4_compress_limitedOutput_withState (void* state, const char* source, char* dest, int inputSize, int maxOutputSize); -LZ4_DEPRECATED("use LZ4_compress_fast_continue() instead") int LZ4_compress_continue (LZ4_stream_t* LZ4_streamPtr, const char* source, char* dest, int inputSize); -LZ4_DEPRECATED("use LZ4_compress_fast_continue() instead") int LZ4_compress_limitedOutput_continue (LZ4_stream_t* LZ4_streamPtr, const char* source, char* dest, int inputSize, int maxOutputSize); - -/* Obsolete decompression functions */ -/* These function names are completely deprecated and must no longer be used. - They are only provided in lz4.c for compatibility with older programs. - - LZ4_uncompress is the same as LZ4_decompress_fast - - LZ4_uncompress_unknownOutputSize is the same as LZ4_decompress_safe - These function prototypes are now disabled; uncomment them only if you really need them. - It is highly recommended to stop using these prototypes and migrate to maintained ones */ -/* int LZ4_uncompress (const char* source, char* dest, int outputSize); */ -/* int LZ4_uncompress_unknownOutputSize (const char* source, char* dest, int isize, int maxOutputSize); */ - -/* Obsolete streaming functions; use new streaming interface whenever possible */ -LZ4_DEPRECATED("use LZ4_createStream() instead") void* LZ4_create (char* inputBuffer); -LZ4_DEPRECATED("use LZ4_createStream() instead") int LZ4_sizeofStreamState(void); -LZ4_DEPRECATED("use LZ4_resetStream() instead") int LZ4_resetStreamState(void* state, char* inputBuffer); -LZ4_DEPRECATED("use LZ4_saveDict() instead") char* LZ4_slideInputBuffer (void* state); - -/* Obsolete streaming decoding functions */ -LZ4_DEPRECATED("use LZ4_decompress_safe_usingDict() instead") int LZ4_decompress_safe_withPrefix64k (const char* src, char* dst, int compressedSize, int maxDstSize); -LZ4_DEPRECATED("use LZ4_decompress_fast_usingDict() instead") int LZ4_decompress_fast_withPrefix64k (const char* src, char* dst, int originalSize); - - -#if defined (__cplusplus) -} -#endif - -#endif /* LZ4_H_2983827168210 */ diff --git a/make-freebsd.mk b/make-freebsd.mk index 5b2a7766..23ad278b 100644 --- a/make-freebsd.mk +++ b/make-freebsd.mk @@ -5,7 +5,7 @@ DEFS= LIBS= include objects.mk -OBJS+=osdep/BSDEthernetTap.o ext/lz4/lz4.o ext/http-parser/http_parser.o +OBJS+=osdep/BSDEthernetTap.o ext/http-parser/http_parser.o # Build with ZT_ENABLE_CLUSTER=1 to build with cluster support ifeq ($(ZT_ENABLE_CLUSTER),1) @@ -20,7 +20,7 @@ ifeq ($(ZT_DEBUG),1) STRIP=echo # The following line enables optimization for the crypto code, since # C25519 in particular is almost UNUSABLE in heavy testing without it. -ext/lz4/lz4.o node/Salsa20.o node/SHA512.o node/C25519.o node/Poly1305.o: CFLAGS = -Wall -O2 -g -pthread $(INCLUDES) $(DEFS) +node/Salsa20.o node/SHA512.o node/C25519.o node/Poly1305.o: CFLAGS = -Wall -O2 -g -pthread $(INCLUDES) $(DEFS) else CFLAGS?=-O3 -fstack-protector CFLAGS+=-Wall -fPIE -fvisibility=hidden -fstack-protector -pthread $(INCLUDES) -DNDEBUG $(DEFS) @@ -69,7 +69,7 @@ selftest: $(OBJS) selftest.o $(STRIP) zerotier-selftest clean: - rm -rf *.o node/*.o controller/*.o osdep/*.o service/*.o ext/http-parser/*.o ext/lz4/*.o ext/json-parser/*.o build-* zerotier-one zerotier-idtool zerotier-selftest zerotier-cli ZeroTierOneInstaller-* + rm -rf *.o node/*.o controller/*.o osdep/*.o service/*.o ext/http-parser/*.o build-* zerotier-one zerotier-idtool zerotier-selftest zerotier-cli ZeroTierOneInstaller-* debug: FORCE make -j 4 ZT_DEBUG=1 diff --git a/make-linux.mk b/make-linux.mk index 6f116bfc..5121d604 100644 --- a/make-linux.mk +++ b/make-linux.mk @@ -18,19 +18,13 @@ include objects.mk # since it caused too many damn problems. The http-parser library in particular # is basically broken between versions. Fark the Debian policies about including # libraries. It's better if things work. -#ifeq ($(wildcard /usr/include/lz4.h),) -# OBJS+=ext/lz4/lz4.o -#else -# LDLIBS+=-llz4 -# DEFS+=-DZT_USE_SYSTEM_LZ4 -#endif #ifeq ($(wildcard /usr/include/http_parser.h),) # OBJS+=ext/http-parser/http_parser.o #else # LDLIBS+=-lhttp_parser # DEFS+=-DZT_USE_SYSTEM_HTTP_PARSER #endif -OBJS+=ext/lz4/lz4.o ext/http-parser/http_parser.o +OBJS+=ext/http-parser/http_parser.o # Auto-detect miniupnpc and nat-pmp as well and use system libs if present, # otherwise build into binary as done on Mac and Windows. @@ -71,7 +65,7 @@ ifeq ($(ZT_DEBUG),1) STRIP?=echo # The following line enables optimization for the crypto code, since # C25519 in particular is almost UNUSABLE in -O0 even on a 3ghz box! -ext/lz4/lz4.o node/Salsa20.o node/SHA512.o node/C25519.o node/Poly1305.o: CFLAGS = -Wall -O2 -g -pthread $(INCLUDES) $(DEFS) +node/Salsa20.o node/SHA512.o node/C25519.o node/Poly1305.o: CFLAGS = -Wall -O2 -g -pthread $(INCLUDES) $(DEFS) else CFLAGS?=-O3 -fstack-protector override CFLAGS+=-Wall -fPIE -pthread $(INCLUDES) -DNDEBUG $(DEFS) @@ -132,7 +126,7 @@ manpages: FORCE doc: manpages clean: FORCE - rm -rf *.so *.o node/*.o controller/*.o osdep/*.o service/*.o ext/http-parser/*.o ext/lz4/*.o ext/json-parser/*.o ext/miniupnpc/*.o ext/libnatpmp/*.o $(OBJS) zerotier-one zerotier-idtool zerotier-cli zerotier-selftest build-* ZeroTierOneInstaller-* *.deb *.rpm .depend debian/files debian/zerotier-one*.debhelper debian/zerotier-one.substvars debian/*.log debian/zerotier-one doc/node_modules + rm -rf *.so *.o node/*.o controller/*.o osdep/*.o service/*.o ext/http-parser/*.o ext/miniupnpc/*.o ext/libnatpmp/*.o $(OBJS) zerotier-one zerotier-idtool zerotier-cli zerotier-selftest build-* ZeroTierOneInstaller-* *.deb *.rpm .depend debian/files debian/zerotier-one*.debhelper debian/zerotier-one.substvars debian/*.log debian/zerotier-one doc/node_modules distclean: clean diff --git a/make-mac.mk b/make-mac.mk index 298d1a2e..6c388a73 100644 --- a/make-mac.mk +++ b/make-mac.mk @@ -19,7 +19,7 @@ ZT_VERSION_BUILD=$(shell cat version.h | grep -F VERSION_BUILD | cut -d ' ' -f 3 DEFS+=-DZT_BUILD_PLATFORM=$(ZT_BUILD_PLATFORM) -DZT_BUILD_ARCHITECTURE=$(ZT_BUILD_ARCHITECTURE) include objects.mk -OBJS+=osdep/OSXEthernetTap.o ext/lz4/lz4.o ext/http-parser/http_parser.o +OBJS+=osdep/OSXEthernetTap.o ext/http-parser/http_parser.o # Official releases are signed with our Apple cert and apply software updates by default ifeq ($(ZT_OFFICIAL_RELEASE),1) @@ -48,7 +48,7 @@ ifeq ($(ZT_DEBUG),1) STRIP=echo # The following line enables optimization for the crypto code, since # C25519 in particular is almost UNUSABLE in heavy testing without it. -ext/lz4/lz4.o node/Salsa20.o node/SHA512.o node/C25519.o node/Poly1305.o: CFLAGS = -Wall -O2 -g -pthread $(INCLUDES) $(DEFS) +node/Salsa20.o node/SHA512.o node/C25519.o node/Poly1305.o: CFLAGS = -Wall -O2 -g -pthread $(INCLUDES) $(DEFS) else CFLAGS?=-Ofast -fstack-protector-strong CFLAGS+=$(ARCH_FLAGS) -Wall -flto -fPIE -pthread -mmacosx-version-min=10.7 -DNDEBUG -Wno-unused-private-field $(INCLUDES) $(DEFS) @@ -95,7 +95,7 @@ official: FORCE make ZT_OFFICIAL_RELEASE=1 mac-dist-pkg clean: - rm -rf *.dSYM build-* *.pkg *.dmg *.o node/*.o controller/*.o service/*.o osdep/*.o ext/http-parser/*.o ext/lz4/*.o ext/json-parser/*.o $(OBJS) zerotier-one zerotier-idtool zerotier-selftest zerotier-cli zerotier mkworld doc/node_modules macui/build zt1_update_$(ZT_BUILD_PLATFORM)_$(ZT_BUILD_ARCHITECTURE)_* + rm -rf *.dSYM build-* *.pkg *.dmg *.o node/*.o controller/*.o service/*.o osdep/*.o ext/http-parser/*.o $(OBJS) zerotier-one zerotier-idtool zerotier-selftest zerotier-cli zerotier mkworld doc/node_modules macui/build zt1_update_$(ZT_BUILD_PLATFORM)_$(ZT_BUILD_ARCHITECTURE)_* distclean: clean diff --git a/node/Packet.cpp b/node/Packet.cpp index 6d1f49e8..05fe8dd9 100644 --- a/node/Packet.cpp +++ b/node/Packet.cpp @@ -16,10 +16,1899 @@ * along with this program. If not, see . */ +#include +#include + #include "Packet.hpp" namespace ZeroTier { +/************************************************************************** */ +/************************************************************************** */ + +/* LZ4 is shipped encapsulated into Packet in an anonymous namespace. + * + * We're doing this as a deliberate workaround for various Linux distribution + * policies that forbid static linking of support libraries. + * + * The reason is that relying on distribution versions of LZ4 has been too + * big a source of bugs and compatibility issues. The LZ4 API is not stable + * enough across versions, and dependency hell ensues. So fark it. */ + +/* Needless to say the code in this anonymous namespace should be considered + * BSD 2-clause licensed. */ + +namespace { + +/* lz4.h ------------------------------------------------------------------ */ + +/* + * LZ4 - Fast LZ compression algorithm + * Header File + * Copyright (C) 2011-2016, Yann Collet. + + BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following disclaimer + in the documentation and/or other materials provided with the + distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + You can contact the author at : + - LZ4 homepage : http://www.lz4.org + - LZ4 source repository : https://github.com/lz4/lz4 +*/ + +/* --- Dependency --- */ +//#include /* size_t */ + +/** + Introduction + + LZ4 is lossless compression algorithm, providing compression speed at 400 MB/s per core, + scalable with multi-cores CPU. It features an extremely fast decoder, with speed in + multiple GB/s per core, typically reaching RAM speed limits on multi-core systems. + + The LZ4 compression library provides in-memory compression and decompression functions. + Compression can be done in: + - a single step (described as Simple Functions) + - a single step, reusing a context (described in Advanced Functions) + - unbounded multiple steps (described as Streaming compression) + + lz4.h provides block compression functions. It gives full buffer control to user. + Decompressing an lz4-compressed block also requires metadata (such as compressed size). + Each application is free to encode such metadata in whichever way it wants. + + An additional format, called LZ4 frame specification (doc/lz4_Frame_format.md), + take care of encoding standard metadata alongside LZ4-compressed blocks. + If your application requires interoperability, it's recommended to use it. + A library is provided to take care of it, see lz4frame.h. +*/ + +/*^*************************************************************** +* Export parameters +*****************************************************************/ +/* +* LZ4_DLL_EXPORT : +* Enable exporting of functions when building a Windows DLL +*/ +#if defined(LZ4_DLL_EXPORT) && (LZ4_DLL_EXPORT==1) +# define LZ4LIB_API __declspec(dllexport) +#elif defined(LZ4_DLL_IMPORT) && (LZ4_DLL_IMPORT==1) +# define LZ4LIB_API __declspec(dllimport) /* It isn't required but allows to generate better code, saving a function pointer load from the IAT and an indirect jump.*/ +#else +# define LZ4LIB_API +#endif + + +/*========== Version =========== */ +#define LZ4_VERSION_MAJOR 1 /* for breaking interface changes */ +#define LZ4_VERSION_MINOR 7 /* for new (non-breaking) interface capabilities */ +#define LZ4_VERSION_RELEASE 5 /* for tweaks, bug-fixes, or development */ + +#define LZ4_VERSION_NUMBER (LZ4_VERSION_MAJOR *100*100 + LZ4_VERSION_MINOR *100 + LZ4_VERSION_RELEASE) + +#define LZ4_LIB_VERSION LZ4_VERSION_MAJOR.LZ4_VERSION_MINOR.LZ4_VERSION_RELEASE +#define LZ4_QUOTE(str) #str +#define LZ4_EXPAND_AND_QUOTE(str) LZ4_QUOTE(str) +#define LZ4_VERSION_STRING LZ4_EXPAND_AND_QUOTE(LZ4_LIB_VERSION) + +//LZ4LIB_API int LZ4_versionNumber (void); +//LZ4LIB_API const char* LZ4_versionString (void); + + +/*-************************************ +* Tuning parameter +**************************************/ +/*! + * LZ4_MEMORY_USAGE : + * Memory usage formula : N->2^N Bytes (examples : 10 -> 1KB; 12 -> 4KB ; 16 -> 64KB; 20 -> 1MB; etc.) + * Increasing memory usage improves compression ratio + * Reduced memory usage can improve speed, due to cache effect + * Default value is 14, for 16KB, which nicely fits into Intel x86 L1 cache + */ +#define LZ4_MEMORY_USAGE 14 + + +/*-************************************ +* Simple Functions +**************************************/ +/*! LZ4_compress_default() : + Compresses 'sourceSize' bytes from buffer 'source' + into already allocated 'dest' buffer of size 'maxDestSize'. + Compression is guaranteed to succeed if 'maxDestSize' >= LZ4_compressBound(sourceSize). + It also runs faster, so it's a recommended setting. + If the function cannot compress 'source' into a more limited 'dest' budget, + compression stops *immediately*, and the function result is zero. + As a consequence, 'dest' content is not valid. + This function never writes outside 'dest' buffer, nor read outside 'source' buffer. + sourceSize : Max supported value is LZ4_MAX_INPUT_VALUE + maxDestSize : full or partial size of buffer 'dest' (which must be already allocated) + return : the number of bytes written into buffer 'dest' (necessarily <= maxOutputSize) + or 0 if compression fails */ +//LZ4LIB_API int LZ4_compress_default(const char* source, char* dest, int sourceSize, int maxDestSize); + +/*! LZ4_decompress_safe() : + compressedSize : is the precise full size of the compressed block. + maxDecompressedSize : is the size of destination buffer, which must be already allocated. + return : the number of bytes decompressed into destination buffer (necessarily <= maxDecompressedSize) + If destination buffer is not large enough, decoding will stop and output an error code (<0). + If the source stream is detected malformed, the function will stop decoding and return a negative result. + This function is protected against buffer overflow exploits, including malicious data packets. + It never writes outside output buffer, nor reads outside input buffer. +*/ +LZ4LIB_API int LZ4_decompress_safe (const char* source, char* dest, int compressedSize, int maxDecompressedSize); + + +/*-************************************ +* Advanced Functions +**************************************/ +#define LZ4_MAX_INPUT_SIZE 0x7E000000 /* 2 113 929 216 bytes */ +#define LZ4_COMPRESSBOUND(isize) ((unsigned)(isize) > (unsigned)LZ4_MAX_INPUT_SIZE ? 0 : (isize) + ((isize)/255) + 16) + +/*! +LZ4_compressBound() : + Provides the maximum size that LZ4 compression may output in a "worst case" scenario (input data not compressible) + This function is primarily useful for memory allocation purposes (destination buffer size). + Macro LZ4_COMPRESSBOUND() is also provided for compilation-time evaluation (stack memory allocation for example). + Note that LZ4_compress_default() compress faster when dest buffer size is >= LZ4_compressBound(srcSize) + inputSize : max supported value is LZ4_MAX_INPUT_SIZE + return : maximum output size in a "worst case" scenario + or 0, if input size is too large ( > LZ4_MAX_INPUT_SIZE) +*/ +LZ4LIB_API int LZ4_compressBound(int inputSize); + +/*! +LZ4_compress_fast() : + Same as LZ4_compress_default(), but allows to select an "acceleration" factor. + The larger the acceleration value, the faster the algorithm, but also the lesser the compression. + It's a trade-off. It can be fine tuned, with each successive value providing roughly +~3% to speed. + An acceleration value of "1" is the same as regular LZ4_compress_default() + Values <= 0 will be replaced by ACCELERATION_DEFAULT (see lz4.c), which is 1. +*/ +LZ4LIB_API int LZ4_compress_fast (const char* source, char* dest, int sourceSize, int maxDestSize, int acceleration); + + +/*! +LZ4_compress_fast_extState() : + Same compression function, just using an externally allocated memory space to store compression state. + Use LZ4_sizeofState() to know how much memory must be allocated, + and allocate it on 8-bytes boundaries (using malloc() typically). + Then, provide it as 'void* state' to compression function. +*/ +//LZ4LIB_API int LZ4_sizeofState(void); +LZ4LIB_API int LZ4_compress_fast_extState (void* state, const char* source, char* dest, int inputSize, int maxDestSize, int acceleration); + + +/*! +LZ4_compress_destSize() : + Reverse the logic, by compressing as much data as possible from 'source' buffer + into already allocated buffer 'dest' of size 'targetDestSize'. + This function either compresses the entire 'source' content into 'dest' if it's large enough, + or fill 'dest' buffer completely with as much data as possible from 'source'. + *sourceSizePtr : will be modified to indicate how many bytes where read from 'source' to fill 'dest'. + New value is necessarily <= old value. + return : Nb bytes written into 'dest' (necessarily <= targetDestSize) + or 0 if compression fails +*/ +//LZ4LIB_API int LZ4_compress_destSize (const char* source, char* dest, int* sourceSizePtr, int targetDestSize); + + +/*! +LZ4_decompress_fast() : + originalSize : is the original and therefore uncompressed size + return : the number of bytes read from the source buffer (in other words, the compressed size) + If the source stream is detected malformed, the function will stop decoding and return a negative result. + Destination buffer must be already allocated. Its size must be a minimum of 'originalSize' bytes. + note : This function fully respect memory boundaries for properly formed compressed data. + It is a bit faster than LZ4_decompress_safe(). + However, it does not provide any protection against intentionally modified data stream (malicious input). + Use this function in trusted environment only (data to decode comes from a trusted source). +*/ +//LZ4LIB_API int LZ4_decompress_fast (const char* source, char* dest, int originalSize); + +/*! +LZ4_decompress_safe_partial() : + This function decompress a compressed block of size 'compressedSize' at position 'source' + into destination buffer 'dest' of size 'maxDecompressedSize'. + The function tries to stop decompressing operation as soon as 'targetOutputSize' has been reached, + reducing decompression time. + return : the number of bytes decoded in the destination buffer (necessarily <= maxDecompressedSize) + Note : this number can be < 'targetOutputSize' should the compressed block to decode be smaller. + Always control how many bytes were decoded. + If the source stream is detected malformed, the function will stop decoding and return a negative result. + This function never writes outside of output buffer, and never reads outside of input buffer. It is therefore protected against malicious data packets +*/ +//LZ4LIB_API int LZ4_decompress_safe_partial (const char* source, char* dest, int compressedSize, int targetOutputSize, int maxDecompressedSize); + + +/*-********************************************* +* Streaming Compression Functions +***********************************************/ +typedef union LZ4_stream_u LZ4_stream_t; /* incomplete type (defined later) */ + +/*! LZ4_createStream() and LZ4_freeStream() : + * LZ4_createStream() will allocate and initialize an `LZ4_stream_t` structure. + * LZ4_freeStream() releases its memory. + */ +//LZ4LIB_API LZ4_stream_t* LZ4_createStream(void); +//LZ4LIB_API int LZ4_freeStream (LZ4_stream_t* streamPtr); + +/*! LZ4_resetStream() : + * An LZ4_stream_t structure can be allocated once and re-used multiple times. + * Use this function to init an allocated `LZ4_stream_t` structure and start a new compression. + */ +LZ4LIB_API void LZ4_resetStream (LZ4_stream_t* streamPtr); + +/*! LZ4_loadDict() : + * Use this function to load a static dictionary into LZ4_stream. + * Any previous data will be forgotten, only 'dictionary' will remain in memory. + * Loading a size of 0 is allowed. + * Return : dictionary size, in bytes (necessarily <= 64 KB) + */ +//LZ4LIB_API int LZ4_loadDict (LZ4_stream_t* streamPtr, const char* dictionary, int dictSize); + +/*! LZ4_compress_fast_continue() : + * Compress buffer content 'src', using data from previously compressed blocks as dictionary to improve compression ratio. + * Important : Previous data blocks are assumed to still be present and unmodified ! + * 'dst' buffer must be already allocated. + * If maxDstSize >= LZ4_compressBound(srcSize), compression is guaranteed to succeed, and runs faster. + * If not, and if compressed data cannot fit into 'dst' buffer size, compression stops, and function returns a zero. + */ +//LZ4LIB_API int LZ4_compress_fast_continue (LZ4_stream_t* streamPtr, const char* src, char* dst, int srcSize, int maxDstSize, int acceleration); + +/*! LZ4_saveDict() : + * If previously compressed data block is not guaranteed to remain available at its memory location, + * save it into a safer place (char* safeBuffer). + * Note : you don't need to call LZ4_loadDict() afterwards, + * dictionary is immediately usable, you can therefore call LZ4_compress_fast_continue(). + * Return : saved dictionary size in bytes (necessarily <= dictSize), or 0 if error. + */ +//LZ4LIB_API int LZ4_saveDict (LZ4_stream_t* streamPtr, char* safeBuffer, int dictSize); + + +/*-********************************************** +* Streaming Decompression Functions +* Bufferless synchronous API +************************************************/ +typedef union LZ4_streamDecode_u LZ4_streamDecode_t; /* incomplete type (defined later) */ + +/* creation / destruction of streaming decompression tracking structure */ +//LZ4LIB_API LZ4_streamDecode_t* LZ4_createStreamDecode(void); +//LZ4LIB_API int LZ4_freeStreamDecode (LZ4_streamDecode_t* LZ4_stream); + +/*! LZ4_setStreamDecode() : + * Use this function to instruct where to find the dictionary. + * Setting a size of 0 is allowed (same effect as reset). + * @return : 1 if OK, 0 if error + */ +//LZ4LIB_API int LZ4_setStreamDecode (LZ4_streamDecode_t* LZ4_streamDecode, const char* dictionary, int dictSize); + +/*! +LZ4_decompress_*_continue() : + These decoding functions allow decompression of multiple blocks in "streaming" mode. + Previously decoded blocks *must* remain available at the memory position where they were decoded (up to 64 KB) + In the case of a ring buffers, decoding buffer must be either : + - Exactly same size as encoding buffer, with same update rule (block boundaries at same positions) + In which case, the decoding & encoding ring buffer can have any size, including very small ones ( < 64 KB). + - Larger than encoding buffer, by a minimum of maxBlockSize more bytes. + maxBlockSize is implementation dependent. It's the maximum size you intend to compress into a single block. + In which case, encoding and decoding buffers do not need to be synchronized, + and encoding ring buffer can have any size, including small ones ( < 64 KB). + - _At least_ 64 KB + 8 bytes + maxBlockSize. + In which case, encoding and decoding buffers do not need to be synchronized, + and encoding ring buffer can have any size, including larger than decoding buffer. + Whenever these conditions are not possible, save the last 64KB of decoded data into a safe buffer, + and indicate where it is saved using LZ4_setStreamDecode() +*/ +//LZ4LIB_API int LZ4_decompress_safe_continue (LZ4_streamDecode_t* LZ4_streamDecode, const char* source, char* dest, int compressedSize, int maxDecompressedSize); +//LZ4LIB_API int LZ4_decompress_fast_continue (LZ4_streamDecode_t* LZ4_streamDecode, const char* source, char* dest, int originalSize); + + +/*! LZ4_decompress_*_usingDict() : + * These decoding functions work the same as + * a combination of LZ4_setStreamDecode() followed by LZ4_decompress_*_continue() + * They are stand-alone, and don't need an LZ4_streamDecode_t structure. + */ +//LZ4LIB_API int LZ4_decompress_safe_usingDict (const char* source, char* dest, int compressedSize, int maxDecompressedSize, const char* dictStart, int dictSize); +//LZ4LIB_API int LZ4_decompress_fast_usingDict (const char* source, char* dest, int originalSize, const char* dictStart, int dictSize); + + +/*^********************************************** + * !!!!!! STATIC LINKING ONLY !!!!!! + ***********************************************/ +/*-************************************ + * Private definitions + ************************************** + * Do not use these definitions. + * They are exposed to allow static allocation of `LZ4_stream_t` and `LZ4_streamDecode_t`. + * Using these definitions will expose code to API and/or ABI break in future versions of the library. + **************************************/ +#define LZ4_HASHLOG (LZ4_MEMORY_USAGE-2) +#define LZ4_HASHTABLESIZE (1 << LZ4_MEMORY_USAGE) +#define LZ4_HASH_SIZE_U32 (1 << LZ4_HASHLOG) /* required as macro for static allocation */ + +#if defined(__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) +#include + +typedef struct { + uint32_t hashTable[LZ4_HASH_SIZE_U32]; + uint32_t currentOffset; + uint32_t initCheck; + const uint8_t* dictionary; + uint8_t* bufferStart; /* obsolete, used for slideInputBuffer */ + uint32_t dictSize; +} LZ4_stream_t_internal; + +typedef struct { + const uint8_t* externalDict; + size_t extDictSize; + const uint8_t* prefixEnd; + size_t prefixSize; +} LZ4_streamDecode_t_internal; + +#else + +typedef struct { + unsigned int hashTable[LZ4_HASH_SIZE_U32]; + unsigned int currentOffset; + unsigned int initCheck; + const unsigned char* dictionary; + unsigned char* bufferStart; /* obsolete, used for slideInputBuffer */ + unsigned int dictSize; +} LZ4_stream_t_internal; + +typedef struct { + const unsigned char* externalDict; + size_t extDictSize; + const unsigned char* prefixEnd; + size_t prefixSize; +} LZ4_streamDecode_t_internal; + +#endif + +/*! + * LZ4_stream_t : + * information structure to track an LZ4 stream. + * init this structure before first use. + * note : only use in association with static linking ! + * this definition is not API/ABI safe, + * and may change in a future version ! + */ +#define LZ4_STREAMSIZE_U64 ((1 << (LZ4_MEMORY_USAGE-3)) + 4) +#define LZ4_STREAMSIZE (LZ4_STREAMSIZE_U64 * sizeof(unsigned long long)) +union LZ4_stream_u { + unsigned long long table[LZ4_STREAMSIZE_U64]; + LZ4_stream_t_internal internal_donotuse; +} ; /* previously typedef'd to LZ4_stream_t */ + + +/*! + * LZ4_streamDecode_t : + * information structure to track an LZ4 stream during decompression. + * init this structure using LZ4_setStreamDecode (or memset()) before first use + * note : only use in association with static linking ! + * this definition is not API/ABI safe, + * and may change in a future version ! + */ +#define LZ4_STREAMDECODESIZE_U64 4 +#define LZ4_STREAMDECODESIZE (LZ4_STREAMDECODESIZE_U64 * sizeof(unsigned long long)) +union LZ4_streamDecode_u { + unsigned long long table[LZ4_STREAMDECODESIZE_U64]; + LZ4_streamDecode_t_internal internal_donotuse; +} ; /* previously typedef'd to LZ4_streamDecode_t */ + +/* lz4.c ------------------------------------------------------------------ */ + +/* + LZ4 - Fast LZ compression algorithm + Copyright (C) 2011-2016, Yann Collet. + + BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following disclaimer + in the documentation and/or other materials provided with the + distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + You can contact the author at : + - LZ4 homepage : http://www.lz4.org + - LZ4 source repository : https://github.com/lz4/lz4 +*/ + + +/*-************************************ +* Tuning parameters +**************************************/ +/* + * HEAPMODE : + * Select how default compression functions will allocate memory for their hash table, + * in memory stack (0:default, fastest), or in memory heap (1:requires malloc()). + */ +#ifndef HEAPMODE +# define HEAPMODE 0 +#endif + +/* + * ACCELERATION_DEFAULT : + * Select "acceleration" for LZ4_compress_fast() when parameter value <= 0 + */ +#define ACCELERATION_DEFAULT 1 + + +/*-************************************ +* CPU Feature Detection +**************************************/ +/* LZ4_FORCE_MEMORY_ACCESS + * By default, access to unaligned memory is controlled by `memcpy()`, which is safe and portable. + * Unfortunately, on some target/compiler combinations, the generated assembly is sub-optimal. + * The below switch allow to select different access method for improved performance. + * Method 0 (default) : use `memcpy()`. Safe and portable. + * Method 1 : `__packed` statement. It depends on compiler extension (ie, not portable). + * This method is safe if your compiler supports it, and *generally* as fast or faster than `memcpy`. + * Method 2 : direct access. This method is portable but violate C standard. + * It can generate buggy code on targets which generate assembly depending on alignment. + * But in some circumstances, it's the only known way to get the most performance (ie GCC + ARMv6) + * See https://fastcompression.blogspot.fr/2015/08/accessing-unaligned-memory.html for details. + * Prefer these methods in priority order (0 > 1 > 2) + */ +#ifndef LZ4_FORCE_MEMORY_ACCESS /* can be defined externally, on command line for example */ +# if defined(__GNUC__) && ( defined(__ARM_ARCH_6__) || defined(__ARM_ARCH_6J__) || defined(__ARM_ARCH_6K__) || defined(__ARM_ARCH_6Z__) || defined(__ARM_ARCH_6ZK__) || defined(__ARM_ARCH_6T2__) ) +# define LZ4_FORCE_MEMORY_ACCESS 2 +# elif defined(__INTEL_COMPILER) || \ + (defined(__GNUC__) && ( defined(__ARM_ARCH_7__) || defined(__ARM_ARCH_7A__) || defined(__ARM_ARCH_7R__) || defined(__ARM_ARCH_7M__) || defined(__ARM_ARCH_7S__) )) +# define LZ4_FORCE_MEMORY_ACCESS 1 +# endif +#endif + +/* + * LZ4_FORCE_SW_BITCOUNT + * Define this parameter if your target system or compiler does not support hardware bit count + */ +#if defined(_MSC_VER) && defined(_WIN32_WCE) /* Visual Studio for Windows CE does not support Hardware bit count */ +# define LZ4_FORCE_SW_BITCOUNT +#endif + + +/*-************************************ +* Dependency +**************************************/ +//#include "lz4.h" +/* see also "memory routines" below */ + + +/*-************************************ +* Compiler Options +**************************************/ +#ifdef _MSC_VER /* Visual Studio */ +# define FORCE_INLINE static __forceinline +# include +# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ +# pragma warning(disable : 4293) /* disable: C4293: too large shift (32-bits) */ +#else +# if defined(__GNUC__) || defined(__clang__) +# define FORCE_INLINE static inline __attribute__((always_inline)) +# elif defined(__cplusplus) || (defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) +# define FORCE_INLINE static inline +# else +# define FORCE_INLINE static +# endif +#endif /* _MSC_VER */ + +#if (defined(__GNUC__) && (__GNUC__ >= 3)) || (defined(__INTEL_COMPILER) && (__INTEL_COMPILER >= 800)) || defined(__clang__) +# define expect(expr,value) (__builtin_expect ((expr),(value)) ) +#else +# define expect(expr,value) (expr) +#endif + +#define likely(expr) expect((expr) != 0, 1) +#define unlikely(expr) expect((expr) != 0, 0) + + +/*-************************************ +* Memory routines +**************************************/ +#include /* malloc, calloc, free */ +#define ALLOCATOR(n,s) calloc(n,s) +#define FREEMEM free +#include /* memset, memcpy */ +#define MEM_INIT memset + + +/*-************************************ +* Basic Types +**************************************/ +#if defined(__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) +# include + typedef uint8_t BYTE; + typedef uint16_t U16; + typedef uint32_t U32; + typedef int32_t S32; + typedef uint64_t U64; + typedef uintptr_t uptrval; +#else + typedef unsigned char BYTE; + typedef unsigned short U16; + typedef unsigned int U32; + typedef signed int S32; + typedef unsigned long long U64; + typedef size_t uptrval; /* generally true, except OpenVMS-64 */ +#endif + +#if defined(__x86_64__) + typedef U64 reg_t; /* 64-bits in x32 mode */ +#else + typedef size_t reg_t; /* 32-bits in x32 mode */ +#endif + +/*-************************************ +* Reading and writing into memory +**************************************/ +static unsigned LZ4_isLittleEndian(void) +{ + const union { U32 u; BYTE c[4]; } one = { 1 }; /* don't use static : performance detrimental */ + return one.c[0]; +} + + +#if defined(LZ4_FORCE_MEMORY_ACCESS) && (LZ4_FORCE_MEMORY_ACCESS==2) +/* lie to the compiler about data alignment; use with caution */ + +static U16 LZ4_read16(const void* memPtr) { return *(const U16*) memPtr; } +static U32 LZ4_read32(const void* memPtr) { return *(const U32*) memPtr; } +static reg_t LZ4_read_ARCH(const void* memPtr) { return *(const reg_t*) memPtr; } + +static void LZ4_write16(void* memPtr, U16 value) { *(U16*)memPtr = value; } +static void LZ4_write32(void* memPtr, U32 value) { *(U32*)memPtr = value; } + +#elif defined(LZ4_FORCE_MEMORY_ACCESS) && (LZ4_FORCE_MEMORY_ACCESS==1) + +/* __pack instructions are safer, but compiler specific, hence potentially problematic for some compilers */ +/* currently only defined for gcc and icc */ +typedef union { U16 u16; U32 u32; reg_t uArch; } __attribute__((packed)) unalign; + +static U16 LZ4_read16(const void* ptr) { return ((const unalign*)ptr)->u16; } +static U32 LZ4_read32(const void* ptr) { return ((const unalign*)ptr)->u32; } +static reg_t LZ4_read_ARCH(const void* ptr) { return ((const unalign*)ptr)->uArch; } + +static void LZ4_write16(void* memPtr, U16 value) { ((unalign*)memPtr)->u16 = value; } +static void LZ4_write32(void* memPtr, U32 value) { ((unalign*)memPtr)->u32 = value; } + +#else /* safe and portable access through memcpy() */ + +static U16 LZ4_read16(const void* memPtr) +{ + U16 val; memcpy(&val, memPtr, sizeof(val)); return val; +} + +static U32 LZ4_read32(const void* memPtr) +{ + U32 val; memcpy(&val, memPtr, sizeof(val)); return val; +} + +static reg_t LZ4_read_ARCH(const void* memPtr) +{ + reg_t val; memcpy(&val, memPtr, sizeof(val)); return val; +} + +static void LZ4_write16(void* memPtr, U16 value) +{ + memcpy(memPtr, &value, sizeof(value)); +} + +static void LZ4_write32(void* memPtr, U32 value) +{ + memcpy(memPtr, &value, sizeof(value)); +} + +#endif /* LZ4_FORCE_MEMORY_ACCESS */ + + +static U16 LZ4_readLE16(const void* memPtr) +{ + if (LZ4_isLittleEndian()) { + return LZ4_read16(memPtr); + } else { + const BYTE* p = (const BYTE*)memPtr; + return (U16)((U16)p[0] + (p[1]<<8)); + } +} + +static void LZ4_writeLE16(void* memPtr, U16 value) +{ + if (LZ4_isLittleEndian()) { + LZ4_write16(memPtr, value); + } else { + BYTE* p = (BYTE*)memPtr; + p[0] = (BYTE) value; + p[1] = (BYTE)(value>>8); + } +} + +static void LZ4_copy8(void* dst, const void* src) +{ + memcpy(dst,src,8); +} + +/* customized variant of memcpy, which can overwrite up to 8 bytes beyond dstEnd */ +static void LZ4_wildCopy(void* dstPtr, const void* srcPtr, void* dstEnd) +{ + BYTE* d = (BYTE*)dstPtr; + const BYTE* s = (const BYTE*)srcPtr; + BYTE* const e = (BYTE*)dstEnd; + + do { LZ4_copy8(d,s); d+=8; s+=8; } while (d>3); +# elif (defined(__clang__) || (defined(__GNUC__) && (__GNUC__>=3))) && !defined(LZ4_FORCE_SW_BITCOUNT) + return (__builtin_ctzll((U64)val) >> 3); +# else + static const int DeBruijnBytePos[64] = { 0, 0, 0, 0, 0, 1, 1, 2, 0, 3, 1, 3, 1, 4, 2, 7, 0, 2, 3, 6, 1, 5, 3, 5, 1, 3, 4, 4, 2, 5, 6, 7, 7, 0, 1, 2, 3, 3, 4, 6, 2, 6, 5, 5, 3, 4, 5, 6, 7, 1, 2, 4, 6, 4, 4, 5, 7, 2, 6, 5, 7, 6, 7, 7 }; + return DeBruijnBytePos[((U64)((val & -(long long)val) * 0x0218A392CDABBD3FULL)) >> 58]; +# endif + } else /* 32 bits */ { +# if defined(_MSC_VER) && !defined(LZ4_FORCE_SW_BITCOUNT) + unsigned long r; + _BitScanForward( &r, (U32)val ); + return (int)(r>>3); +# elif (defined(__clang__) || (defined(__GNUC__) && (__GNUC__>=3))) && !defined(LZ4_FORCE_SW_BITCOUNT) + return (__builtin_ctz((U32)val) >> 3); +# else + static const int DeBruijnBytePos[32] = { 0, 0, 3, 0, 3, 1, 3, 0, 3, 2, 2, 1, 3, 2, 0, 1, 3, 3, 1, 2, 2, 2, 2, 0, 3, 1, 2, 0, 1, 0, 1, 1 }; + return DeBruijnBytePos[((U32)((val & -(S32)val) * 0x077CB531U)) >> 27]; +# endif + } + } else /* Big Endian CPU */ { + if (sizeof(val)==8) { +# if defined(_MSC_VER) && defined(_WIN64) && !defined(LZ4_FORCE_SW_BITCOUNT) + unsigned long r = 0; + _BitScanReverse64( &r, val ); + return (unsigned)(r>>3); +# elif (defined(__clang__) || (defined(__GNUC__) && (__GNUC__>=3))) && !defined(LZ4_FORCE_SW_BITCOUNT) + return (__builtin_clzll((U64)val) >> 3); +# else + unsigned r; + if (!(val>>32)) { r=4; } else { r=0; val>>=32; } + if (!(val>>16)) { r+=2; val>>=8; } else { val>>=24; } + r += (!val); + return r; +# endif + } else /* 32 bits */ { +# if defined(_MSC_VER) && !defined(LZ4_FORCE_SW_BITCOUNT) + unsigned long r = 0; + _BitScanReverse( &r, (unsigned long)val ); + return (unsigned)(r>>3); +# elif (defined(__clang__) || (defined(__GNUC__) && (__GNUC__>=3))) && !defined(LZ4_FORCE_SW_BITCOUNT) + return (__builtin_clz((U32)val) >> 3); +# else + unsigned r; + if (!(val>>16)) { r=2; val>>=8; } else { r=0; val>>=24; } + r += (!val); + return r; +# endif + } + } +} + +#define STEPSIZE sizeof(reg_t) +static unsigned LZ4_count(const BYTE* pIn, const BYTE* pMatch, const BYTE* pInLimit) +{ + const BYTE* const pStart = pIn; + + while (likely(pIn compression run slower on incompressible data */ + + +/*-************************************ +* Local Structures and types +**************************************/ +typedef enum { notLimited = 0, limitedOutput = 1 } limitedOutput_directive; +typedef enum { byPtr, byU32, byU16 } tableType_t; + +typedef enum { noDict = 0, withPrefix64k, usingExtDict } dict_directive; +typedef enum { noDictIssue = 0, dictSmall } dictIssue_directive; + +typedef enum { endOnOutputSize = 0, endOnInputSize = 1 } endCondition_directive; +typedef enum { full = 0, partial = 1 } earlyEnd_directive; + + +/*-************************************ +* Local Utils +**************************************/ +//int LZ4_versionNumber (void) { return LZ4_VERSION_NUMBER; } +//const char* LZ4_versionString(void) { return LZ4_VERSION_STRING; } +int LZ4_compressBound(int isize) { return LZ4_COMPRESSBOUND(isize); } +//int LZ4_sizeofState() { return LZ4_STREAMSIZE; } + + +/*-****************************** +* Compression functions +********************************/ +static U32 LZ4_hash4(U32 sequence, tableType_t const tableType) +{ + if (tableType == byU16) + return ((sequence * 2654435761U) >> ((MINMATCH*8)-(LZ4_HASHLOG+1))); + else + return ((sequence * 2654435761U) >> ((MINMATCH*8)-LZ4_HASHLOG)); +} + +static U32 LZ4_hash5(U64 sequence, tableType_t const tableType) +{ + static const U64 prime5bytes = 889523592379ULL; + static const U64 prime8bytes = 11400714785074694791ULL; + const U32 hashLog = (tableType == byU16) ? LZ4_HASHLOG+1 : LZ4_HASHLOG; + if (LZ4_isLittleEndian()) + return (U32)(((sequence << 24) * prime5bytes) >> (64 - hashLog)); + else + return (U32)(((sequence >> 24) * prime8bytes) >> (64 - hashLog)); +} + +FORCE_INLINE U32 LZ4_hashPosition(const void* const p, tableType_t const tableType) +{ + if ((sizeof(reg_t)==8) && (tableType != byU16)) return LZ4_hash5(LZ4_read_ARCH(p), tableType); + return LZ4_hash4(LZ4_read32(p), tableType); +} + +static void LZ4_putPositionOnHash(const BYTE* p, U32 h, void* tableBase, tableType_t const tableType, const BYTE* srcBase) +{ + switch (tableType) + { + case byPtr: { const BYTE** hashTable = (const BYTE**)tableBase; hashTable[h] = p; return; } + case byU32: { U32* hashTable = (U32*) tableBase; hashTable[h] = (U32)(p-srcBase); return; } + case byU16: { U16* hashTable = (U16*) tableBase; hashTable[h] = (U16)(p-srcBase); return; } + } +} + +FORCE_INLINE void LZ4_putPosition(const BYTE* p, void* tableBase, tableType_t tableType, const BYTE* srcBase) +{ + U32 const h = LZ4_hashPosition(p, tableType); + LZ4_putPositionOnHash(p, h, tableBase, tableType, srcBase); +} + +static const BYTE* LZ4_getPositionOnHash(U32 h, void* tableBase, tableType_t tableType, const BYTE* srcBase) +{ + if (tableType == byPtr) { const BYTE** hashTable = (const BYTE**) tableBase; return hashTable[h]; } + if (tableType == byU32) { const U32* const hashTable = (U32*) tableBase; return hashTable[h] + srcBase; } + { const U16* const hashTable = (U16*) tableBase; return hashTable[h] + srcBase; } /* default, to ensure a return */ +} + +FORCE_INLINE const BYTE* LZ4_getPosition(const BYTE* p, void* tableBase, tableType_t tableType, const BYTE* srcBase) +{ + U32 const h = LZ4_hashPosition(p, tableType); + return LZ4_getPositionOnHash(h, tableBase, tableType, srcBase); +} + + +/** LZ4_compress_generic() : + inlined, to ensure branches are decided at compilation time */ +FORCE_INLINE int LZ4_compress_generic( + LZ4_stream_t_internal* const cctx, + const char* const source, + char* const dest, + const int inputSize, + const int maxOutputSize, + const limitedOutput_directive outputLimited, + const tableType_t tableType, + const dict_directive dict, + const dictIssue_directive dictIssue, + const U32 acceleration) +{ + const BYTE* ip = (const BYTE*) source; + const BYTE* base; + const BYTE* lowLimit; + const BYTE* const lowRefLimit = ip - cctx->dictSize; + const BYTE* const dictionary = cctx->dictionary; + const BYTE* const dictEnd = dictionary + cctx->dictSize; + const ptrdiff_t dictDelta = dictEnd - (const BYTE*)source; + const BYTE* anchor = (const BYTE*) source; + const BYTE* const iend = ip + inputSize; + const BYTE* const mflimit = iend - MFLIMIT; + const BYTE* const matchlimit = iend - LASTLITERALS; + + BYTE* op = (BYTE*) dest; + BYTE* const olimit = op + maxOutputSize; + + U32 forwardH; + + /* Init conditions */ + if ((U32)inputSize > (U32)LZ4_MAX_INPUT_SIZE) return 0; /* Unsupported inputSize, too large (or negative) */ + switch(dict) + { + case noDict: + default: + base = (const BYTE*)source; + lowLimit = (const BYTE*)source; + break; + case withPrefix64k: + base = (const BYTE*)source - cctx->currentOffset; + lowLimit = (const BYTE*)source - cctx->dictSize; + break; + case usingExtDict: + base = (const BYTE*)source - cctx->currentOffset; + lowLimit = (const BYTE*)source; + break; + } + if ((tableType == byU16) && (inputSize>=LZ4_64Klimit)) return 0; /* Size too large (not within 64K limit) */ + if (inputSizehashTable, tableType, base); + ip++; forwardH = LZ4_hashPosition(ip, tableType); + + /* Main Loop */ + for ( ; ; ) { + ptrdiff_t refDelta = 0; + const BYTE* match; + BYTE* token; + + /* Find a match */ + { const BYTE* forwardIp = ip; + unsigned step = 1; + unsigned searchMatchNb = acceleration << LZ4_skipTrigger; + do { + U32 const h = forwardH; + ip = forwardIp; + forwardIp += step; + step = (searchMatchNb++ >> LZ4_skipTrigger); + + if (unlikely(forwardIp > mflimit)) goto _last_literals; + + match = LZ4_getPositionOnHash(h, cctx->hashTable, tableType, base); + if (dict==usingExtDict) { + if (match < (const BYTE*)source) { + refDelta = dictDelta; + lowLimit = dictionary; + } else { + refDelta = 0; + lowLimit = (const BYTE*)source; + } } + forwardH = LZ4_hashPosition(forwardIp, tableType); + LZ4_putPositionOnHash(ip, h, cctx->hashTable, tableType, base); + + } while ( ((dictIssue==dictSmall) ? (match < lowRefLimit) : 0) + || ((tableType==byU16) ? 0 : (match + MAX_DISTANCE < ip)) + || (LZ4_read32(match+refDelta) != LZ4_read32(ip)) ); + } + + /* Catch up */ + while (((ip>anchor) & (match+refDelta > lowLimit)) && (unlikely(ip[-1]==match[refDelta-1]))) { ip--; match--; } + + /* Encode Literals */ + { unsigned const litLength = (unsigned)(ip - anchor); + token = op++; + if ((outputLimited) && /* Check output buffer overflow */ + (unlikely(op + litLength + (2 + 1 + LASTLITERALS) + (litLength/255) > olimit))) + return 0; + if (litLength >= RUN_MASK) { + int len = (int)litLength-RUN_MASK; + *token = (RUN_MASK<= 255 ; len-=255) *op++ = 255; + *op++ = (BYTE)len; + } + else *token = (BYTE)(litLength< matchlimit) limit = matchlimit; + matchCode = LZ4_count(ip+MINMATCH, match+MINMATCH, limit); + ip += MINMATCH + matchCode; + if (ip==limit) { + unsigned const more = LZ4_count(ip, (const BYTE*)source, matchlimit); + matchCode += more; + ip += more; + } + } else { + matchCode = LZ4_count(ip+MINMATCH, match+MINMATCH, matchlimit); + ip += MINMATCH + matchCode; + } + + if ( outputLimited && /* Check output buffer overflow */ + (unlikely(op + (1 + LASTLITERALS) + (matchCode>>8) > olimit)) ) + return 0; + if (matchCode >= ML_MASK) { + *token += ML_MASK; + matchCode -= ML_MASK; + LZ4_write32(op, 0xFFFFFFFF); + while (matchCode >= 4*255) op+=4, LZ4_write32(op, 0xFFFFFFFF), matchCode -= 4*255; + op += matchCode / 255; + *op++ = (BYTE)(matchCode % 255); + } else + *token += (BYTE)(matchCode); + } + + anchor = ip; + + /* Test end of chunk */ + if (ip > mflimit) break; + + /* Fill table */ + LZ4_putPosition(ip-2, cctx->hashTable, tableType, base); + + /* Test next position */ + match = LZ4_getPosition(ip, cctx->hashTable, tableType, base); + if (dict==usingExtDict) { + if (match < (const BYTE*)source) { + refDelta = dictDelta; + lowLimit = dictionary; + } else { + refDelta = 0; + lowLimit = (const BYTE*)source; + } } + LZ4_putPosition(ip, cctx->hashTable, tableType, base); + if ( ((dictIssue==dictSmall) ? (match>=lowRefLimit) : 1) + && (match+MAX_DISTANCE>=ip) + && (LZ4_read32(match+refDelta)==LZ4_read32(ip)) ) + { token=op++; *token=0; goto _next_match; } + + /* Prepare next loop */ + forwardH = LZ4_hashPosition(++ip, tableType); + } + +_last_literals: + /* Encode Last Literals */ + { size_t const lastRun = (size_t)(iend - anchor); + if ( (outputLimited) && /* Check output buffer overflow */ + ((op - (BYTE*)dest) + lastRun + 1 + ((lastRun+255-RUN_MASK)/255) > (U32)maxOutputSize) ) + return 0; + if (lastRun >= RUN_MASK) { + size_t accumulator = lastRun - RUN_MASK; + *op++ = RUN_MASK << ML_BITS; + for(; accumulator >= 255 ; accumulator-=255) *op++ = 255; + *op++ = (BYTE) accumulator; + } else { + *op++ = (BYTE)(lastRun<internal_donotuse; + LZ4_resetStream((LZ4_stream_t*)state); + if (acceleration < 1) acceleration = ACCELERATION_DEFAULT; + + if (maxOutputSize >= LZ4_compressBound(inputSize)) { + if (inputSize < LZ4_64Klimit) + return LZ4_compress_generic(ctx, source, dest, inputSize, 0, notLimited, byU16, noDict, noDictIssue, acceleration); + else + return LZ4_compress_generic(ctx, source, dest, inputSize, 0, notLimited, (sizeof(void*)==8) ? byU32 : byPtr, noDict, noDictIssue, acceleration); + } else { + if (inputSize < LZ4_64Klimit) + return LZ4_compress_generic(ctx, source, dest, inputSize, maxOutputSize, limitedOutput, byU16, noDict, noDictIssue, acceleration); + else + return LZ4_compress_generic(ctx, source, dest, inputSize, maxOutputSize, limitedOutput, (sizeof(void*)==8) ? byU32 : byPtr, noDict, noDictIssue, acceleration); + } +} + + +int LZ4_compress_fast(const char* source, char* dest, int inputSize, int maxOutputSize, int acceleration) +{ +#if (HEAPMODE) + void* ctxPtr = ALLOCATOR(1, sizeof(LZ4_stream_t)); /* malloc-calloc always properly aligned */ +#else + LZ4_stream_t ctx; + void* const ctxPtr = &ctx; +#endif + + int const result = LZ4_compress_fast_extState(ctxPtr, source, dest, inputSize, maxOutputSize, acceleration); + +#if (HEAPMODE) + FREEMEM(ctxPtr); +#endif + return result; +} + +#if 0 +int LZ4_compress_default(const char* source, char* dest, int inputSize, int maxOutputSize) +{ + return LZ4_compress_fast(source, dest, inputSize, maxOutputSize, 1); +} + +/* hidden debug function */ +/* strangely enough, gcc generates faster code when this function is uncommented, even if unused */ +int LZ4_compress_fast_force(const char* source, char* dest, int inputSize, int maxOutputSize, int acceleration) +{ + LZ4_stream_t ctx; + LZ4_resetStream(&ctx); + + if (inputSize < LZ4_64Klimit) + return LZ4_compress_generic(&ctx.internal_donotuse, source, dest, inputSize, maxOutputSize, limitedOutput, byU16, noDict, noDictIssue, acceleration); + else + return LZ4_compress_generic(&ctx.internal_donotuse, source, dest, inputSize, maxOutputSize, limitedOutput, sizeof(void*)==8 ? byU32 : byPtr, noDict, noDictIssue, acceleration); +} +#endif + +/*-****************************** +* *_destSize() variant +********************************/ + +#if 0 +static int LZ4_compress_destSize_generic( + LZ4_stream_t_internal* const ctx, + const char* const src, + char* const dst, + int* const srcSizePtr, + const int targetDstSize, + const tableType_t tableType) +{ + const BYTE* ip = (const BYTE*) src; + const BYTE* base = (const BYTE*) src; + const BYTE* lowLimit = (const BYTE*) src; + const BYTE* anchor = ip; + const BYTE* const iend = ip + *srcSizePtr; + const BYTE* const mflimit = iend - MFLIMIT; + const BYTE* const matchlimit = iend - LASTLITERALS; + + BYTE* op = (BYTE*) dst; + BYTE* const oend = op + targetDstSize; + BYTE* const oMaxLit = op + targetDstSize - 2 /* offset */ - 8 /* because 8+MINMATCH==MFLIMIT */ - 1 /* token */; + BYTE* const oMaxMatch = op + targetDstSize - (LASTLITERALS + 1 /* token */); + BYTE* const oMaxSeq = oMaxLit - 1 /* token */; + + U32 forwardH; + + + /* Init conditions */ + if (targetDstSize < 1) return 0; /* Impossible to store anything */ + if ((U32)*srcSizePtr > (U32)LZ4_MAX_INPUT_SIZE) return 0; /* Unsupported input size, too large (or negative) */ + if ((tableType == byU16) && (*srcSizePtr>=LZ4_64Klimit)) return 0; /* Size too large (not within 64K limit) */ + if (*srcSizePtrhashTable, tableType, base); + ip++; forwardH = LZ4_hashPosition(ip, tableType); + + /* Main Loop */ + for ( ; ; ) { + const BYTE* match; + BYTE* token; + + /* Find a match */ + { const BYTE* forwardIp = ip; + unsigned step = 1; + unsigned searchMatchNb = 1 << LZ4_skipTrigger; + + do { + U32 h = forwardH; + ip = forwardIp; + forwardIp += step; + step = (searchMatchNb++ >> LZ4_skipTrigger); + + if (unlikely(forwardIp > mflimit)) goto _last_literals; + + match = LZ4_getPositionOnHash(h, ctx->hashTable, tableType, base); + forwardH = LZ4_hashPosition(forwardIp, tableType); + LZ4_putPositionOnHash(ip, h, ctx->hashTable, tableType, base); + + } while ( ((tableType==byU16) ? 0 : (match + MAX_DISTANCE < ip)) + || (LZ4_read32(match) != LZ4_read32(ip)) ); + } + + /* Catch up */ + while ((ip>anchor) && (match > lowLimit) && (unlikely(ip[-1]==match[-1]))) { ip--; match--; } + + /* Encode Literal length */ + { unsigned litLength = (unsigned)(ip - anchor); + token = op++; + if (op + ((litLength+240)/255) + litLength > oMaxLit) { + /* Not enough space for a last match */ + op--; + goto _last_literals; + } + if (litLength>=RUN_MASK) { + unsigned len = litLength - RUN_MASK; + *token=(RUN_MASK<= 255 ; len-=255) *op++ = 255; + *op++ = (BYTE)len; + } + else *token = (BYTE)(litLength< oMaxMatch) { + /* Match description too long : reduce it */ + matchLength = (15-1) + (oMaxMatch-op) * 255; + } + ip += MINMATCH + matchLength; + + if (matchLength>=ML_MASK) { + *token += ML_MASK; + matchLength -= ML_MASK; + while (matchLength >= 255) { matchLength-=255; *op++ = 255; } + *op++ = (BYTE)matchLength; + } + else *token += (BYTE)(matchLength); + } + + anchor = ip; + + /* Test end of block */ + if (ip > mflimit) break; + if (op > oMaxSeq) break; + + /* Fill table */ + LZ4_putPosition(ip-2, ctx->hashTable, tableType, base); + + /* Test next position */ + match = LZ4_getPosition(ip, ctx->hashTable, tableType, base); + LZ4_putPosition(ip, ctx->hashTable, tableType, base); + if ( (match+MAX_DISTANCE>=ip) + && (LZ4_read32(match)==LZ4_read32(ip)) ) + { token=op++; *token=0; goto _next_match; } + + /* Prepare next loop */ + forwardH = LZ4_hashPosition(++ip, tableType); + } + +_last_literals: + /* Encode Last Literals */ + { size_t lastRunSize = (size_t)(iend - anchor); + if (op + 1 /* token */ + ((lastRunSize+240)/255) /* litLength */ + lastRunSize /* literals */ > oend) { + /* adapt lastRunSize to fill 'dst' */ + lastRunSize = (oend-op) - 1; + lastRunSize -= (lastRunSize+240)/255; + } + ip = anchor + lastRunSize; + + if (lastRunSize >= RUN_MASK) { + size_t accumulator = lastRunSize - RUN_MASK; + *op++ = RUN_MASK << ML_BITS; + for(; accumulator >= 255 ; accumulator-=255) *op++ = 255; + *op++ = (BYTE) accumulator; + } else { + *op++ = (BYTE)(lastRunSize<= LZ4_compressBound(*srcSizePtr)) { /* compression success is guaranteed */ + return LZ4_compress_fast_extState(state, src, dst, *srcSizePtr, targetDstSize, 1); + } else { + if (*srcSizePtr < LZ4_64Klimit) + return LZ4_compress_destSize_generic(&state->internal_donotuse, src, dst, srcSizePtr, targetDstSize, byU16); + else + return LZ4_compress_destSize_generic(&state->internal_donotuse, src, dst, srcSizePtr, targetDstSize, sizeof(void*)==8 ? byU32 : byPtr); + } +} + +int LZ4_compress_destSize(const char* src, char* dst, int* srcSizePtr, int targetDstSize) +{ +#if (HEAPMODE) + LZ4_stream_t* ctx = (LZ4_stream_t*)ALLOCATOR(1, sizeof(LZ4_stream_t)); /* malloc-calloc always properly aligned */ +#else + LZ4_stream_t ctxBody; + LZ4_stream_t* ctx = &ctxBody; +#endif + + int result = LZ4_compress_destSize_extState(ctx, src, dst, srcSizePtr, targetDstSize); + +#if (HEAPMODE) + FREEMEM(ctx); +#endif + return result; +} +#endif + +/*-****************************** +* Streaming functions +********************************/ + +#if 0 +LZ4_stream_t* LZ4_createStream(void) +{ + LZ4_stream_t* lz4s = (LZ4_stream_t*)ALLOCATOR(8, LZ4_STREAMSIZE_U64); + LZ4_STATIC_ASSERT(LZ4_STREAMSIZE >= sizeof(LZ4_stream_t_internal)); /* A compilation error here means LZ4_STREAMSIZE is not large enough */ + LZ4_resetStream(lz4s); + return lz4s; +} +#endif + +void LZ4_resetStream (LZ4_stream_t* LZ4_stream) +{ + MEM_INIT(LZ4_stream, 0, sizeof(LZ4_stream_t)); +} + +#if 0 +int LZ4_freeStream (LZ4_stream_t* LZ4_stream) +{ + FREEMEM(LZ4_stream); + return (0); +} +#endif + +#if 0 +#define HASH_UNIT sizeof(reg_t) +int LZ4_loadDict (LZ4_stream_t* LZ4_dict, const char* dictionary, int dictSize) +{ + LZ4_stream_t_internal* dict = &LZ4_dict->internal_donotuse; + const BYTE* p = (const BYTE*)dictionary; + const BYTE* const dictEnd = p + dictSize; + const BYTE* base; + + if ((dict->initCheck) || (dict->currentOffset > 1 GB)) /* Uninitialized structure, or reuse overflow */ + LZ4_resetStream(LZ4_dict); + + if (dictSize < (int)HASH_UNIT) { + dict->dictionary = NULL; + dict->dictSize = 0; + return 0; + } + + if ((dictEnd - p) > 64 KB) p = dictEnd - 64 KB; + dict->currentOffset += 64 KB; + base = p - dict->currentOffset; + dict->dictionary = p; + dict->dictSize = (U32)(dictEnd - p); + dict->currentOffset += dict->dictSize; + + while (p <= dictEnd-HASH_UNIT) { + LZ4_putPosition(p, dict->hashTable, byU32, base); + p+=3; + } + + return dict->dictSize; +} + +static void LZ4_renormDictT(LZ4_stream_t_internal* LZ4_dict, const BYTE* src) +{ + if ((LZ4_dict->currentOffset > 0x80000000) || + ((uptrval)LZ4_dict->currentOffset > (uptrval)src)) { /* address space overflow */ + /* rescale hash table */ + U32 const delta = LZ4_dict->currentOffset - 64 KB; + const BYTE* dictEnd = LZ4_dict->dictionary + LZ4_dict->dictSize; + int i; + for (i=0; ihashTable[i] < delta) LZ4_dict->hashTable[i]=0; + else LZ4_dict->hashTable[i] -= delta; + } + LZ4_dict->currentOffset = 64 KB; + if (LZ4_dict->dictSize > 64 KB) LZ4_dict->dictSize = 64 KB; + LZ4_dict->dictionary = dictEnd - LZ4_dict->dictSize; + } +} + +int LZ4_compress_fast_continue (LZ4_stream_t* LZ4_stream, const char* source, char* dest, int inputSize, int maxOutputSize, int acceleration) +{ + LZ4_stream_t_internal* streamPtr = &LZ4_stream->internal_donotuse; + const BYTE* const dictEnd = streamPtr->dictionary + streamPtr->dictSize; + + const BYTE* smallest = (const BYTE*) source; + if (streamPtr->initCheck) return 0; /* Uninitialized structure detected */ + if ((streamPtr->dictSize>0) && (smallest>dictEnd)) smallest = dictEnd; + LZ4_renormDictT(streamPtr, smallest); + if (acceleration < 1) acceleration = ACCELERATION_DEFAULT; + + /* Check overlapping input/dictionary space */ + { const BYTE* sourceEnd = (const BYTE*) source + inputSize; + if ((sourceEnd > streamPtr->dictionary) && (sourceEnd < dictEnd)) { + streamPtr->dictSize = (U32)(dictEnd - sourceEnd); + if (streamPtr->dictSize > 64 KB) streamPtr->dictSize = 64 KB; + if (streamPtr->dictSize < 4) streamPtr->dictSize = 0; + streamPtr->dictionary = dictEnd - streamPtr->dictSize; + } + } + + /* prefix mode : source data follows dictionary */ + if (dictEnd == (const BYTE*)source) { + int result; + if ((streamPtr->dictSize < 64 KB) && (streamPtr->dictSize < streamPtr->currentOffset)) + result = LZ4_compress_generic(streamPtr, source, dest, inputSize, maxOutputSize, limitedOutput, byU32, withPrefix64k, dictSmall, acceleration); + else + result = LZ4_compress_generic(streamPtr, source, dest, inputSize, maxOutputSize, limitedOutput, byU32, withPrefix64k, noDictIssue, acceleration); + streamPtr->dictSize += (U32)inputSize; + streamPtr->currentOffset += (U32)inputSize; + return result; + } + + /* external dictionary mode */ + { int result; + if ((streamPtr->dictSize < 64 KB) && (streamPtr->dictSize < streamPtr->currentOffset)) + result = LZ4_compress_generic(streamPtr, source, dest, inputSize, maxOutputSize, limitedOutput, byU32, usingExtDict, dictSmall, acceleration); + else + result = LZ4_compress_generic(streamPtr, source, dest, inputSize, maxOutputSize, limitedOutput, byU32, usingExtDict, noDictIssue, acceleration); + streamPtr->dictionary = (const BYTE*)source; + streamPtr->dictSize = (U32)inputSize; + streamPtr->currentOffset += (U32)inputSize; + return result; + } +} + +/* Hidden debug function, to force external dictionary mode */ +int LZ4_compress_forceExtDict (LZ4_stream_t* LZ4_dict, const char* source, char* dest, int inputSize) +{ + LZ4_stream_t_internal* streamPtr = &LZ4_dict->internal_donotuse; + int result; + const BYTE* const dictEnd = streamPtr->dictionary + streamPtr->dictSize; + + const BYTE* smallest = dictEnd; + if (smallest > (const BYTE*) source) smallest = (const BYTE*) source; + LZ4_renormDictT(streamPtr, smallest); + + result = LZ4_compress_generic(streamPtr, source, dest, inputSize, 0, notLimited, byU32, usingExtDict, noDictIssue, 1); + + streamPtr->dictionary = (const BYTE*)source; + streamPtr->dictSize = (U32)inputSize; + streamPtr->currentOffset += (U32)inputSize; + + return result; +} + +/*! LZ4_saveDict() : + * If previously compressed data block is not guaranteed to remain available at its memory location, + * save it into a safer place (char* safeBuffer). + * Note : you don't need to call LZ4_loadDict() afterwards, + * dictionary is immediately usable, you can therefore call LZ4_compress_fast_continue(). + * Return : saved dictionary size in bytes (necessarily <= dictSize), or 0 if error. + */ +int LZ4_saveDict (LZ4_stream_t* LZ4_dict, char* safeBuffer, int dictSize) +{ + LZ4_stream_t_internal* const dict = &LZ4_dict->internal_donotuse; + const BYTE* const previousDictEnd = dict->dictionary + dict->dictSize; + + if ((U32)dictSize > 64 KB) dictSize = 64 KB; /* useless to define a dictionary > 64 KB */ + if ((U32)dictSize > dict->dictSize) dictSize = dict->dictSize; + + memmove(safeBuffer, previousDictEnd - dictSize, dictSize); + + dict->dictionary = (const BYTE*)safeBuffer; + dict->dictSize = (U32)dictSize; + + return dictSize; +} + +#endif + +/*-***************************** +* Decompression functions +*******************************/ +/*! LZ4_decompress_generic() : + * This generic decompression function cover all use cases. + * It shall be instantiated several times, using different sets of directives + * Note that it is important this generic function is really inlined, + * in order to remove useless branches during compilation optimization. + */ +FORCE_INLINE int LZ4_decompress_generic( + const char* const source, + char* const dest, + int inputSize, + int outputSize, /* If endOnInput==endOnInputSize, this value is the max size of Output Buffer. */ + + int endOnInput, /* endOnOutputSize, endOnInputSize */ + int partialDecoding, /* full, partial */ + int targetOutputSize, /* only used if partialDecoding==partial */ + int dict, /* noDict, withPrefix64k, usingExtDict */ + const BYTE* const lowPrefix, /* == dest when no prefix */ + const BYTE* const dictStart, /* only if dict==usingExtDict */ + const size_t dictSize /* note : = 0 if noDict */ + ) +{ + /* Local Variables */ + const BYTE* ip = (const BYTE*) source; + const BYTE* const iend = ip + inputSize; + + BYTE* op = (BYTE*) dest; + BYTE* const oend = op + outputSize; + BYTE* cpy; + BYTE* oexit = op + targetOutputSize; + const BYTE* const lowLimit = lowPrefix - dictSize; + + const BYTE* const dictEnd = (const BYTE*)dictStart + dictSize; + const unsigned dec32table[] = {0, 1, 2, 1, 4, 4, 4, 4}; + const int dec64table[] = {0, 0, 0, -1, 0, 1, 2, 3}; + + const int safeDecode = (endOnInput==endOnInputSize); + const int checkOffset = ((safeDecode) && (dictSize < (int)(64 KB))); + + + /* Special cases */ + if ((partialDecoding) && (oexit > oend-MFLIMIT)) oexit = oend-MFLIMIT; /* targetOutputSize too high => decode everything */ + if ((endOnInput) && (unlikely(outputSize==0))) return ((inputSize==1) && (*ip==0)) ? 0 : -1; /* Empty output buffer */ + if ((!endOnInput) && (unlikely(outputSize==0))) return (*ip==0?1:-1); + + /* Main Loop : decode sequences */ + while (1) { + size_t length; + const BYTE* match; + size_t offset; + + /* get literal length */ + unsigned const token = *ip++; + if ((length=(token>>ML_BITS)) == RUN_MASK) { + unsigned s; + do { + s = *ip++; + length += s; + } while ( likely(endOnInput ? ip(partialDecoding?oexit:oend-MFLIMIT)) || (ip+length>iend-(2+1+LASTLITERALS))) ) + || ((!endOnInput) && (cpy>oend-WILDCOPYLENGTH)) ) + { + if (partialDecoding) { + if (cpy > oend) goto _output_error; /* Error : write attempt beyond end of output buffer */ + if ((endOnInput) && (ip+length > iend)) goto _output_error; /* Error : read attempt beyond end of input buffer */ + } else { + if ((!endOnInput) && (cpy != oend)) goto _output_error; /* Error : block decoding must stop exactly there */ + if ((endOnInput) && ((ip+length != iend) || (cpy > oend))) goto _output_error; /* Error : input must be consumed */ + } + memcpy(op, ip, length); + ip += length; + op += length; + break; /* Necessarily EOF, due to parsing restrictions */ + } + LZ4_wildCopy(op, ip, cpy); + ip += length; op = cpy; + + /* get offset */ + offset = LZ4_readLE16(ip); ip+=2; + match = op - offset; + if ((checkOffset) && (unlikely(match < lowLimit))) goto _output_error; /* Error : offset outside buffers */ + LZ4_write32(op, (U32)offset); /* costs ~1%; silence an msan warning when offset==0 */ + + /* get matchlength */ + length = token & ML_MASK; + if (length == ML_MASK) { + unsigned s; + do { + s = *ip++; + if ((endOnInput) && (ip > iend-LASTLITERALS)) goto _output_error; + length += s; + } while (s==255); + if ((safeDecode) && unlikely((uptrval)(op)+length<(uptrval)op)) goto _output_error; /* overflow detection */ + } + length += MINMATCH; + + /* check external dictionary */ + if ((dict==usingExtDict) && (match < lowPrefix)) { + if (unlikely(op+length > oend-LASTLITERALS)) goto _output_error; /* doesn't respect parsing restriction */ + + if (length <= (size_t)(lowPrefix-match)) { + /* match can be copied as a single segment from external dictionary */ + memmove(op, dictEnd - (lowPrefix-match), length); + op += length; + } else { + /* match encompass external dictionary and current block */ + size_t const copySize = (size_t)(lowPrefix-match); + size_t const restSize = length - copySize; + memcpy(op, dictEnd - copySize, copySize); + op += copySize; + if (restSize > (size_t)(op-lowPrefix)) { /* overlap copy */ + BYTE* const endOfMatch = op + restSize; + const BYTE* copyFrom = lowPrefix; + while (op < endOfMatch) *op++ = *copyFrom++; + } else { + memcpy(op, lowPrefix, restSize); + op += restSize; + } } + continue; + } + + /* copy match within block */ + cpy = op + length; + if (unlikely(offset<8)) { + const int dec64 = dec64table[offset]; + op[0] = match[0]; + op[1] = match[1]; + op[2] = match[2]; + op[3] = match[3]; + match += dec32table[offset]; + memcpy(op+4, match, 4); + match -= dec64; + } else { LZ4_copy8(op, match); match+=8; } + op += 8; + + if (unlikely(cpy>oend-12)) { + BYTE* const oCopyLimit = oend-(WILDCOPYLENGTH-1); + if (cpy > oend-LASTLITERALS) goto _output_error; /* Error : last LASTLITERALS bytes must be literals (uncompressed) */ + if (op < oCopyLimit) { + LZ4_wildCopy(op, match, oCopyLimit); + match += oCopyLimit - op; + op = oCopyLimit; + } + while (op16) LZ4_wildCopy(op+8, match+8, cpy); + } + op=cpy; /* correction */ + } + + /* end of decoding */ + if (endOnInput) + return (int) (((char*)op)-dest); /* Nb of output bytes decoded */ + else + return (int) (((const char*)ip)-source); /* Nb of input bytes read */ + + /* Overflow error detected */ +_output_error: + return (int) (-(((const char*)ip)-source))-1; +} + + +int LZ4_decompress_safe(const char* source, char* dest, int compressedSize, int maxDecompressedSize) +{ + return LZ4_decompress_generic(source, dest, compressedSize, maxDecompressedSize, endOnInputSize, full, 0, noDict, (BYTE*)dest, NULL, 0); +} + +#if 0 +int LZ4_decompress_safe_partial(const char* source, char* dest, int compressedSize, int targetOutputSize, int maxDecompressedSize) +{ + return LZ4_decompress_generic(source, dest, compressedSize, maxDecompressedSize, endOnInputSize, partial, targetOutputSize, noDict, (BYTE*)dest, NULL, 0); +} + +int LZ4_decompress_fast(const char* source, char* dest, int originalSize) +{ + return LZ4_decompress_generic(source, dest, 0, originalSize, endOnOutputSize, full, 0, withPrefix64k, (BYTE*)(dest - 64 KB), NULL, 64 KB); +} +#endif + +/*===== streaming decompression functions =====*/ + +#if 0 +/* + * If you prefer dynamic allocation methods, + * LZ4_createStreamDecode() + * provides a pointer (void*) towards an initialized LZ4_streamDecode_t structure. + */ +LZ4_streamDecode_t* LZ4_createStreamDecode(void) +{ + LZ4_streamDecode_t* lz4s = (LZ4_streamDecode_t*) ALLOCATOR(1, sizeof(LZ4_streamDecode_t)); + return lz4s; +} + +int LZ4_freeStreamDecode (LZ4_streamDecode_t* LZ4_stream) +{ + FREEMEM(LZ4_stream); + return 0; +} + +/*! + * LZ4_setStreamDecode() : + * Use this function to instruct where to find the dictionary. + * This function is not necessary if previous data is still available where it was decoded. + * Loading a size of 0 is allowed (same effect as no dictionary). + * Return : 1 if OK, 0 if error + */ +int LZ4_setStreamDecode (LZ4_streamDecode_t* LZ4_streamDecode, const char* dictionary, int dictSize) +{ + LZ4_streamDecode_t_internal* lz4sd = &LZ4_streamDecode->internal_donotuse; + lz4sd->prefixSize = (size_t) dictSize; + lz4sd->prefixEnd = (const BYTE*) dictionary + dictSize; + lz4sd->externalDict = NULL; + lz4sd->extDictSize = 0; + return 1; +} + +/* +*_continue() : + These decoding functions allow decompression of multiple blocks in "streaming" mode. + Previously decoded blocks must still be available at the memory position where they were decoded. + If it's not possible, save the relevant part of decoded data into a safe buffer, + and indicate where it stands using LZ4_setStreamDecode() +*/ +int LZ4_decompress_safe_continue (LZ4_streamDecode_t* LZ4_streamDecode, const char* source, char* dest, int compressedSize, int maxOutputSize) +{ + LZ4_streamDecode_t_internal* lz4sd = &LZ4_streamDecode->internal_donotuse; + int result; + + if (lz4sd->prefixEnd == (BYTE*)dest) { + result = LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, + endOnInputSize, full, 0, + usingExtDict, lz4sd->prefixEnd - lz4sd->prefixSize, lz4sd->externalDict, lz4sd->extDictSize); + if (result <= 0) return result; + lz4sd->prefixSize += result; + lz4sd->prefixEnd += result; + } else { + lz4sd->extDictSize = lz4sd->prefixSize; + lz4sd->externalDict = lz4sd->prefixEnd - lz4sd->extDictSize; + result = LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, + endOnInputSize, full, 0, + usingExtDict, (BYTE*)dest, lz4sd->externalDict, lz4sd->extDictSize); + if (result <= 0) return result; + lz4sd->prefixSize = result; + lz4sd->prefixEnd = (BYTE*)dest + result; + } + + return result; +} + +int LZ4_decompress_fast_continue (LZ4_streamDecode_t* LZ4_streamDecode, const char* source, char* dest, int originalSize) +{ + LZ4_streamDecode_t_internal* lz4sd = &LZ4_streamDecode->internal_donotuse; + int result; + + if (lz4sd->prefixEnd == (BYTE*)dest) { + result = LZ4_decompress_generic(source, dest, 0, originalSize, + endOnOutputSize, full, 0, + usingExtDict, lz4sd->prefixEnd - lz4sd->prefixSize, lz4sd->externalDict, lz4sd->extDictSize); + if (result <= 0) return result; + lz4sd->prefixSize += originalSize; + lz4sd->prefixEnd += originalSize; + } else { + lz4sd->extDictSize = lz4sd->prefixSize; + lz4sd->externalDict = lz4sd->prefixEnd - lz4sd->extDictSize; + result = LZ4_decompress_generic(source, dest, 0, originalSize, + endOnOutputSize, full, 0, + usingExtDict, (BYTE*)dest, lz4sd->externalDict, lz4sd->extDictSize); + if (result <= 0) return result; + lz4sd->prefixSize = originalSize; + lz4sd->prefixEnd = (BYTE*)dest + originalSize; + } + + return result; +} + + +/* +Advanced decoding functions : +*_usingDict() : + These decoding functions work the same as "_continue" ones, + the dictionary must be explicitly provided within parameters +*/ + +FORCE_INLINE int LZ4_decompress_usingDict_generic(const char* source, char* dest, int compressedSize, int maxOutputSize, int safe, const char* dictStart, int dictSize) +{ + if (dictSize==0) + return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, safe, full, 0, noDict, (BYTE*)dest, NULL, 0); + if (dictStart+dictSize == dest) { + if (dictSize >= (int)(64 KB - 1)) + return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, safe, full, 0, withPrefix64k, (BYTE*)dest-64 KB, NULL, 0); + return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, safe, full, 0, noDict, (BYTE*)dest-dictSize, NULL, 0); + } + return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, safe, full, 0, usingExtDict, (BYTE*)dest, (const BYTE*)dictStart, dictSize); +} + +int LZ4_decompress_safe_usingDict(const char* source, char* dest, int compressedSize, int maxOutputSize, const char* dictStart, int dictSize) +{ + return LZ4_decompress_usingDict_generic(source, dest, compressedSize, maxOutputSize, 1, dictStart, dictSize); +} + +int LZ4_decompress_fast_usingDict(const char* source, char* dest, int originalSize, const char* dictStart, int dictSize) +{ + return LZ4_decompress_usingDict_generic(source, dest, 0, originalSize, 0, dictStart, dictSize); +} + +/* debug function */ +int LZ4_decompress_safe_forceExtDict(const char* source, char* dest, int compressedSize, int maxOutputSize, const char* dictStart, int dictSize) +{ + return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, endOnInputSize, full, 0, usingExtDict, (BYTE*)dest, (const BYTE*)dictStart, dictSize); +} + +#endif + +#if 0 +/*=************************************************* +* Obsolete Functions +***************************************************/ +/* obsolete compression functions */ +int LZ4_compress_limitedOutput(const char* source, char* dest, int inputSize, int maxOutputSize) { return LZ4_compress_default(source, dest, inputSize, maxOutputSize); } +int LZ4_compress(const char* source, char* dest, int inputSize) { return LZ4_compress_default(source, dest, inputSize, LZ4_compressBound(inputSize)); } +int LZ4_compress_limitedOutput_withState (void* state, const char* src, char* dst, int srcSize, int dstSize) { return LZ4_compress_fast_extState(state, src, dst, srcSize, dstSize, 1); } +int LZ4_compress_withState (void* state, const char* src, char* dst, int srcSize) { return LZ4_compress_fast_extState(state, src, dst, srcSize, LZ4_compressBound(srcSize), 1); } +int LZ4_compress_limitedOutput_continue (LZ4_stream_t* LZ4_stream, const char* src, char* dst, int srcSize, int maxDstSize) { return LZ4_compress_fast_continue(LZ4_stream, src, dst, srcSize, maxDstSize, 1); } +int LZ4_compress_continue (LZ4_stream_t* LZ4_stream, const char* source, char* dest, int inputSize) { return LZ4_compress_fast_continue(LZ4_stream, source, dest, inputSize, LZ4_compressBound(inputSize), 1); } + +/* +These function names are deprecated and should no longer be used. +They are only provided here for compatibility with older user programs. +- LZ4_uncompress is totally equivalent to LZ4_decompress_fast +- LZ4_uncompress_unknownOutputSize is totally equivalent to LZ4_decompress_safe +*/ +int LZ4_uncompress (const char* source, char* dest, int outputSize) { return LZ4_decompress_fast(source, dest, outputSize); } +int LZ4_uncompress_unknownOutputSize (const char* source, char* dest, int isize, int maxOutputSize) { return LZ4_decompress_safe(source, dest, isize, maxOutputSize); } + + +/* Obsolete Streaming functions */ + +int LZ4_sizeofStreamState() { return LZ4_STREAMSIZE; } + +static void LZ4_init(LZ4_stream_t* lz4ds, BYTE* base) +{ + MEM_INIT(lz4ds, 0, sizeof(LZ4_stream_t)); + lz4ds->internal_donotuse.bufferStart = base; +} + +int LZ4_resetStreamState(void* state, char* inputBuffer) +{ + if ((((uptrval)state) & 3) != 0) return 1; /* Error : pointer is not aligned on 4-bytes boundary */ + LZ4_init((LZ4_stream_t*)state, (BYTE*)inputBuffer); + return 0; +} + +void* LZ4_create (char* inputBuffer) +{ + LZ4_stream_t* lz4ds = (LZ4_stream_t*)ALLOCATOR(8, sizeof(LZ4_stream_t)); + LZ4_init (lz4ds, (BYTE*)inputBuffer); + return lz4ds; +} + +char* LZ4_slideInputBuffer (void* LZ4_Data) +{ + LZ4_stream_t_internal* ctx = &((LZ4_stream_t*)LZ4_Data)->internal_donotuse; + int dictSize = LZ4_saveDict((LZ4_stream_t*)LZ4_Data, (char*)ctx->bufferStart, 64 KB); + return (char*)(ctx->bufferStart + dictSize); +} + +/* Obsolete streaming decompression functions */ + +int LZ4_decompress_safe_withPrefix64k(const char* source, char* dest, int compressedSize, int maxOutputSize) +{ + return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, endOnInputSize, full, 0, withPrefix64k, (BYTE*)dest - 64 KB, NULL, 64 KB); +} + +int LZ4_decompress_fast_withPrefix64k(const char* source, char* dest, int originalSize) +{ + return LZ4_decompress_generic(source, dest, 0, originalSize, endOnOutputSize, full, 0, withPrefix64k, (BYTE*)dest - 64 KB, NULL, 64 KB); +} +#endif + +#endif /* LZ4_COMMONDEFS_ONLY */ + +} // anonymous namespace + +/************************************************************************** */ +/************************************************************************** */ + const unsigned char Packet::ZERO_KEY[32] = { 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 }; #ifdef ZT_TRACE @@ -125,7 +2014,7 @@ bool Packet::compress() unsigned char buf[ZT_PROTO_MAX_PACKET_LENGTH * 2]; if ((!compressed())&&(size() > (ZT_PACKET_IDX_PAYLOAD + 32))) { int pl = (int)(size() - ZT_PACKET_IDX_PAYLOAD); - int cl = LZ4_compress_default((const char *)field(ZT_PACKET_IDX_PAYLOAD,(unsigned int)pl),(char *)buf,pl,ZT_PROTO_MAX_PACKET_LENGTH * 2); + int cl = LZ4_compress_fast((const char *)field(ZT_PACKET_IDX_PAYLOAD,(unsigned int)pl),(char *)buf,pl,ZT_PROTO_MAX_PACKET_LENGTH * 2,2); if ((cl > 0)&&(cl < pl)) { (*this)[ZT_PACKET_IDX_VERB] |= (char)ZT_PROTO_VERB_FLAG_COMPRESSED; setSize((unsigned int)cl + ZT_PACKET_IDX_PAYLOAD); diff --git a/node/Packet.hpp b/node/Packet.hpp index 0ac5d9d3..0be19f8a 100644 --- a/node/Packet.hpp +++ b/node/Packet.hpp @@ -37,7 +37,7 @@ //#ifdef ZT_USE_SYSTEM_LZ4 //#include //#else -#include "../ext/lz4/lz4.h" +//#include "../ext/lz4/lz4.h" //#endif /** -- cgit v1.2.3 From 712e8e828be368fc7e5e1c67ae4faa04a69a26cc Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Tue, 7 Mar 2017 11:58:17 -0800 Subject: use .exe naming convention --- make-mac.mk | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'make-mac.mk') diff --git a/make-mac.mk b/make-mac.mk index 6c388a73..8ff1b772 100644 --- a/make-mac.mk +++ b/make-mac.mk @@ -85,7 +85,7 @@ mac-dist-pkg: FORCE $(PRODUCTSIGN) --sign $(CODESIGN_INSTALLER_CERT) "ZeroTier One.pkg" "ZeroTier One Signed.pkg" if [ -f "ZeroTier One Signed.pkg" ]; then mv -f "ZeroTier One Signed.pkg" "ZeroTier One.pkg"; fi rm -f zt1_update_$(ZT_BUILD_PLATFORM)_$(ZT_BUILD_ARCHITECTURE)_* - cat ext/installfiles/mac-update/updater.tmpl.sh "ZeroTier One.pkg" >zt1_update_$(ZT_BUILD_PLATFORM)_$(ZT_BUILD_ARCHITECTURE)_$(ZT_VERSION_MAJOR).$(ZT_VERSION_MINOR).$(ZT_VERSION_REV)_$(ZT_VERSION_BUILD) + cat ext/installfiles/mac-update/updater.tmpl.sh "ZeroTier One.pkg" >zt1_update_$(ZT_BUILD_PLATFORM)_$(ZT_BUILD_ARCHITECTURE)_$(ZT_VERSION_MAJOR).$(ZT_VERSION_MINOR).$(ZT_VERSION_REV)_$(ZT_VERSION_BUILD).exe # For ZeroTier, Inc. to build official signed packages official: FORCE -- cgit v1.2.3 From 683ba79ff0ed4b80c6240fa84d43838027449acd Mon Sep 17 00:00:00 2001 From: Joseph Henry Date: Fri, 24 Mar 2017 17:00:14 -0700 Subject: Added ButtFlare stub code --- make-mac.mk | 4 + objects.mk | 3 +- service/ButtFlare.cpp | 194 +++++++++++++++++++++++++++++++++++++++++++++++++ service/ButtFlare.hpp | 94 ++++++++++++++++++++++++ service/OneService.cpp | 12 +++ 5 files changed, 306 insertions(+), 1 deletion(-) create mode 100644 service/ButtFlare.cpp create mode 100644 service/ButtFlare.hpp (limited to 'make-mac.mk') diff --git a/make-mac.mk b/make-mac.mk index 8ff1b772..e84f0746 100644 --- a/make-mac.mk +++ b/make-mac.mk @@ -37,6 +37,10 @@ ifeq ($(ZT_ENABLE_CLUSTER),1) DEFS+=-DZT_ENABLE_CLUSTER endif +ifeq ($(ZT_ENABLE_BUTTFLARE),1) + DEFS+=-DZT_ENABLE_BUTTFLARE +endif + # Build miniupnpc and nat-pmp as included libraries -- extra defs are required for these sources DEFS+=-DMACOSX -DZT_USE_MINIUPNPC -DMINIUPNP_STATICLIB -D_DARWIN_C_SOURCE -DMINIUPNPC_SET_SOCKET_TIMEOUT -DMINIUPNPC_GET_SRC_ADDR -D_BSD_SOURCE -D_DEFAULT_SOURCE -DOS_STRING=\"Darwin/15.0.0\" -DMINIUPNPC_VERSION_STRING=\"2.0\" -DUPNP_VERSION_STRING=\"UPnP/1.1\" -DENABLE_STRNATPMPERR OBJS+=ext/libnatpmp/natpmp.o ext/libnatpmp/getgateway.o ext/miniupnpc/connecthostport.o ext/miniupnpc/igd_desc_parse.o ext/miniupnpc/minisoap.o ext/miniupnpc/minissdpc.o ext/miniupnpc/miniupnpc.o ext/miniupnpc/miniwget.o ext/miniupnpc/minixml.o ext/miniupnpc/portlistingparse.o ext/miniupnpc/receivedata.o ext/miniupnpc/upnpcommands.o ext/miniupnpc/upnpdev.o ext/miniupnpc/upnperrors.o ext/miniupnpc/upnpreplyparse.o osdep/PortMapper.o diff --git a/objects.mk b/objects.mk index 74efc337..47ce65e6 100644 --- a/objects.mk +++ b/objects.mk @@ -31,4 +31,5 @@ OBJS=\ osdep/Http.o \ osdep/OSUtils.o \ service/ClusterGeoIpService.o \ - service/SoftwareUpdater.o + service/SoftwareUpdater.o \ + service/ButtFlare.o diff --git a/service/ButtFlare.cpp b/service/ButtFlare.cpp new file mode 100644 index 00000000..cda92913 --- /dev/null +++ b/service/ButtFlare.cpp @@ -0,0 +1,194 @@ +/* + * ZeroTier One - Network Virtualization Everywhere + * Copyright (C) 2011-2016 ZeroTier, Inc. https://www.zerotier.com/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +#include "../osdep/Thread.hpp" +#include "ButtFlare.hpp" + +namespace ZeroTier { + + typedef void PhySocket; + + ButtFlare::ButtFlare( + void (*handler)(void *,uint64_t,const MAC &,const MAC &,unsigned int,unsigned int,const void *,unsigned int), void *arg) + : + _handler(handler), + _phy(this,false,true), + _enabled(true), + _run(true) + { + // HTTP listen socket + struct sockaddr_in in4; + memset(&in4,0,sizeof(in4)); + in4.sin_family = AF_INET; + in4.sin_addr.s_addr = Utils::hton((uint32_t)(0x7f000001)); // right now we just listen for TCP @127.0.0.1 + in4.sin_port = Utils::hton((uint16_t)http_listen_port); + _tcpHttpListenSocket = _phy.tcpListen((const struct sockaddr *)&in4,this); + // SSL listen socket + in4.sin_port = Utils::hton((uint16_t)ssl_listen_port); + _tcpSSLListenSocket = _phy.tcpListen((const struct sockaddr *)&in4,this); + + /* + struct sockaddr_in6 in6; + memset((void *)&in6,0,sizeof(in6)); + in6.sin6_family = AF_INET6; + in6.sin6_port = in4.sin_port; + if (_allowManagementFrom.size() == 0) + in6.sin6_addr.s6_addr[15] = 1; // IPv6 localhost == ::1 + _v6TcpControlSocket = _phy.tcpListen((const struct sockaddr *)&in6,this); + + // We must bind one of IPv4 or IPv6 -- support either failing to support hosts that + // have only IPv4 or only IPv6 stacks. + if ((_v4TcpControlSocket)||(_v6TcpControlSocket)) { + _ports[0] = _primaryPort; + break; + } else { + if (_v4TcpControlSocket) + _phy.close(_v4TcpControlSocket,false); + if (_v6TcpControlSocket) + _phy.close(_v6TcpControlSocket,false); + _primaryPort = 0; + } + */ + + if(!_tcpHttpListenSocket) + printf("Error binding on port %d for HTTP listen socket\n", http_listen_port); + if(!_tcpSSLListenSocket) + printf("Error binding on port %d for SSL listen socket\n", ssl_listen_port); + _thread = Thread::start(this); + } + + ButtFlare::~ButtFlare() + { + _run = false; + _phy.whack(); + _phy.whack(); // TODO: Rationale? + Thread::join(_thread); + _phy.close(_tcpHttpListenSocket,false); + _phy.close(_tcpSSLListenSocket,false); + } + void ButtFlare::threadMain() + throw() + { + while(_run) { + _phy.poll(50); // in ms + } + } + + void ButtFlare::phyOnTcpData(PhySocket *sock,void **uptr,void *data,unsigned long len) + { + TcpConnection *conn = cmap[sock]; + unsigned char *buf = (unsigned char*)data; + std::string host = ""; + + for(int i=0; idestination_sock) { // no connection yet + + // Determine if HTTP or TLS/SSL + if(buf[0] == 22 && len > 100) // naive and incomplete way of checking for TLS, just for stub code + { + printf("TLS/SSL\n"); + host = "127.0.0.1"; + } + else + { + printf("HTTP\n"); + host = "127.0.0.1"; + } + + if(host != "") + { + bool connected; + struct sockaddr_in in4; + memset(&in4,0,sizeof(in4)); + in4.sin_family = AF_INET; + in4.sin_addr.s_addr = Utils::hton((uint32_t)(0x7f000001)); // right now we just listen for TCP @127.0.0.1 + in4.sin_port = Utils::hton((uint16_t)http_listen_port); + conn->destination_sock = _phy.tcpConnect((const struct sockaddr *)&in4, connected, this); + + if(!connected) { + printf("instant connect has occured\n"); + } + if(!conn->destination_sock) { + printf("there was an error connecting to the remote host\n"); + } + } + } + else // connection already established, just forward the data + { + int n = _phy.streamSend(conn->destination_sock, buf, len); + printf("wrote %d bytes (%p -> %p)\n", conn->origin_sock, conn->destination_sock); + } + } + + void ButtFlare::phyOnDatagram(PhySocket *sock,void **uptr,const struct sockaddr *localAddr,const struct sockaddr *from,void *data,unsigned long len) + { + } + void ButtFlare::phyOnTcpWritable(PhySocket *sock,void **uptr) + { + } + void ButtFlare::phyOnFileDescriptorActivity(PhySocket *sock,void **uptr,bool readable,bool writable) + { + } + + void ButtFlare::phyOnTcpConnect(PhySocket *sock,void **uptr,bool success) + { + printf("phyOnTcpConnect()\n"); + } + + void ButtFlare::phyOnTcpAccept(PhySocket *sockL,PhySocket *sockN,void **uptrL,void **uptrN,const struct sockaddr *from) + { + TcpConnection *conn = new TcpConnection(); + conn->origin_sock = sockN; + cmap[sockN]=conn; // add new connection + } + + void ButtFlare::phyOnUnixClose(PhySocket *sock,void **uptr) + { + } + void ButtFlare::phyOnUnixData(PhySocket *sock,void **uptr,void *data,ssize_t len) + { + } + void ButtFlare::phyOnUnixWritable(PhySocket *sock,void **uptr,bool lwip_invoked) + { + } + + + void ButtFlare::phyOnTcpClose(PhySocket *sock,void **uptr) + { + TcpConnection *conn = cmap[sock]; + if(conn == NULL) { + printf("No existing connection for this sock. Only closing origin sock\n"); + // _phy.close(sock); + return; + } + else + { + // Close both ends + // _phy.close(conn->destination_sock); + // _phy.close(conn->origin_sock); + } + } +} + +//#endif \ No newline at end of file diff --git a/service/ButtFlare.hpp b/service/ButtFlare.hpp new file mode 100644 index 00000000..6e8f0955 --- /dev/null +++ b/service/ButtFlare.hpp @@ -0,0 +1,94 @@ +/* + * ZeroTier One - Network Virtualization Everywhere + * Copyright (C) 2011-2016 ZeroTier, Inc. https://www.zerotier.com/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +#ifndef ZT_BUTTFLARE_HPP +#define ZT_BUTTFLARE_HPP + +#include "../node/InetAddress.hpp" +#include "../osdep/Phy.hpp" + +namespace ZeroTier { + + typedef void PhySocket; + class ButtFlare; + + struct TcpConnection + { + enum { + TCP_HTTP_INCOMING, + TCP_HTTP_OUTGOING, + TCP_TUNNEL_OUTGOING + } type; + + InetAddress from; + InetAddress realhost; + + PhySocket *origin_sock; + PhySocket *destination_sock; + }; + + class ButtFlare + { + friend class Phy; + + public: + ButtFlare( + void (*handler)(void *,uint64_t,const MAC &,const MAC &,unsigned int,unsigned int,const void *,unsigned int), + void *arg); + + ~ButtFlare(); + + // Send incoming data to intended host + void phyOnTcpData(PhySocket *sock,void **uptr,void *data,unsigned long len); + + void phyOnDatagram(PhySocket *sock,void **uptr,const struct sockaddr *localAddr,const struct sockaddr *from,void *data,unsigned long len); + void phyOnTcpWritable(PhySocket *sock,void **uptr); + void phyOnFileDescriptorActivity(PhySocket *sock,void **uptr,bool readable,bool writable); + + // Establish outgoing connection to intended host + void phyOnTcpConnect(PhySocket *sock,void **uptr,bool success); + // Accept connection + void phyOnTcpAccept(PhySocket *sockL,PhySocket *sockN,void **uptrL,void **uptrN,const struct sockaddr *from); + // Handle the closure of a Unix Domain socket + void phyOnUnixClose(PhySocket *sock,void **uptr); + void phyOnUnixData(PhySocket *sock,void **uptr,void *data,ssize_t len); + void phyOnUnixWritable(PhySocket *sock,void **uptr,bool lwip_invoked); + + // Handle the closure of a TCP connection + void phyOnTcpClose(PhySocket *sock,void **uptr); + + void threadMain() + throw(); + + void (*_handler)(void *,uint64_t,const MAC &,const MAC &,unsigned int,unsigned int,const void *,unsigned int); + + private: + int http_listen_port = 80; + int ssl_listen_port = 8899; + + volatile bool _enabled; + volatile bool _run; + + Thread _thread; + Phy _phy; + PhySocket *_tcpHttpListenSocket, *_tcpSSLListenSocket; + std::map cmap; + }; +} + +#endif \ No newline at end of file diff --git a/service/OneService.cpp b/service/OneService.cpp index 1c2fa05d..14f637b4 100644 --- a/service/OneService.cpp +++ b/service/OneService.cpp @@ -67,6 +67,10 @@ #include #endif +#ifdef ZT_ENABLE_BUTTFLARE +#include "ButtFlare.hpp" +#endif + #ifdef ZT_USE_SYSTEM_HTTP_PARSER #include #else @@ -474,6 +478,10 @@ public: unsigned int _clusterMemberId; #endif +#ifdef ZT_ENABLE_BUTTFLARE + ButtFlare *butt; +#endif + // Set to false to force service to stop volatile bool _run; Mutex _run_m; @@ -1914,6 +1922,10 @@ public: try { char friendlyName[128]; Utils::snprintf(friendlyName,sizeof(friendlyName),"ZeroTier One [%.16llx]",nwid); + +#ifdef ZT_ENABLE_BUTTFLARE + butt = new ButtFlare(StapFrameHandler, (void *)this); +#endif n.tap = new EthernetTap( _homePath.c_str(), MAC(nwc->mac), -- cgit v1.2.3 From 592cac58155fecf2e226b32c614e19064214cc1a Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Mon, 27 Mar 2017 12:09:37 -0700 Subject: Remove from a few other places. --- make-mac.mk | 4 ---- service/OneService.cpp | 11 ----------- 2 files changed, 15 deletions(-) (limited to 'make-mac.mk') diff --git a/make-mac.mk b/make-mac.mk index e84f0746..8ff1b772 100644 --- a/make-mac.mk +++ b/make-mac.mk @@ -37,10 +37,6 @@ ifeq ($(ZT_ENABLE_CLUSTER),1) DEFS+=-DZT_ENABLE_CLUSTER endif -ifeq ($(ZT_ENABLE_BUTTFLARE),1) - DEFS+=-DZT_ENABLE_BUTTFLARE -endif - # Build miniupnpc and nat-pmp as included libraries -- extra defs are required for these sources DEFS+=-DMACOSX -DZT_USE_MINIUPNPC -DMINIUPNP_STATICLIB -D_DARWIN_C_SOURCE -DMINIUPNPC_SET_SOCKET_TIMEOUT -DMINIUPNPC_GET_SRC_ADDR -D_BSD_SOURCE -D_DEFAULT_SOURCE -DOS_STRING=\"Darwin/15.0.0\" -DMINIUPNPC_VERSION_STRING=\"2.0\" -DUPNP_VERSION_STRING=\"UPnP/1.1\" -DENABLE_STRNATPMPERR OBJS+=ext/libnatpmp/natpmp.o ext/libnatpmp/getgateway.o ext/miniupnpc/connecthostport.o ext/miniupnpc/igd_desc_parse.o ext/miniupnpc/minisoap.o ext/miniupnpc/minissdpc.o ext/miniupnpc/miniupnpc.o ext/miniupnpc/miniwget.o ext/miniupnpc/minixml.o ext/miniupnpc/portlistingparse.o ext/miniupnpc/receivedata.o ext/miniupnpc/upnpcommands.o ext/miniupnpc/upnpdev.o ext/miniupnpc/upnperrors.o ext/miniupnpc/upnpreplyparse.o osdep/PortMapper.o diff --git a/service/OneService.cpp b/service/OneService.cpp index 14f637b4..22eefbb9 100644 --- a/service/OneService.cpp +++ b/service/OneService.cpp @@ -67,10 +67,6 @@ #include #endif -#ifdef ZT_ENABLE_BUTTFLARE -#include "ButtFlare.hpp" -#endif - #ifdef ZT_USE_SYSTEM_HTTP_PARSER #include #else @@ -478,10 +474,6 @@ public: unsigned int _clusterMemberId; #endif -#ifdef ZT_ENABLE_BUTTFLARE - ButtFlare *butt; -#endif - // Set to false to force service to stop volatile bool _run; Mutex _run_m; @@ -1923,9 +1915,6 @@ public: char friendlyName[128]; Utils::snprintf(friendlyName,sizeof(friendlyName),"ZeroTier One [%.16llx]",nwid); -#ifdef ZT_ENABLE_BUTTFLARE - butt = new ButtFlare(StapFrameHandler, (void *)this); -#endif n.tap = new EthernetTap( _homePath.c_str(), MAC(nwc->mac), -- cgit v1.2.3 From df48738ac96a6eab5e3baa03f6dd1fb62bdc8040 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Mon, 17 Apr 2017 16:43:03 -0700 Subject: Enable use of NaCl for faster X64 Salsa20 implementations. Also include binary for OSX for easy build. Blazingly fast. --- ext/bin/cnacl-osx-amd64/README.md | 53 ++++++++++++ .../include/sodium/crypto_auth_hmacsha256.h | 27 ++++++ .../include/sodium/crypto_auth_hmacsha512256.h | 27 ++++++ .../sodium/crypto_box_curve25519xsalsa20poly1305.h | 44 ++++++++++ .../include/sodium/crypto_core_hsalsa20.h | 27 ++++++ .../include/sodium/crypto_core_salsa20.h | 27 ++++++ .../include/sodium/crypto_core_salsa2012.h | 27 ++++++ .../include/sodium/crypto_core_salsa208.h | 27 ++++++ .../include/sodium/crypto_hash_sha256.h | 22 +++++ .../include/sodium/crypto_hash_sha512.h | 22 +++++ .../include/sodium/crypto_hashblocks_sha256.h | 23 ++++++ .../include/sodium/crypto_hashblocks_sha512.h | 23 ++++++ .../include/sodium/crypto_onetimeauth_poly1305.h | 27 ++++++ .../include/sodium/crypto_scalarmult_curve25519.h | 27 ++++++ .../sodium/crypto_secretbox_xsalsa20poly1305.h | 31 +++++++ .../sodium/crypto_sign_edwards25519sha512batch.h | 32 ++++++++ .../include/sodium/crypto_stream_aes128ctr.h | 35 ++++++++ .../include/sodium/crypto_stream_salsa20.h | 34 ++++++++ .../include/sodium/crypto_stream_salsa2012.h | 34 ++++++++ .../include/sodium/crypto_stream_salsa208.h | 34 ++++++++ .../include/sodium/crypto_stream_xsalsa20.h | 34 ++++++++ .../cnacl-osx-amd64/include/sodium/crypto_types.h | 11 +++ .../include/sodium/crypto_verify_16.h | 21 +++++ .../include/sodium/crypto_verify_32.h | 21 +++++ make-mac.mk | 6 ++ node/Identity.cpp | 2 +- node/Node.cpp | 6 +- node/Packet.cpp | 6 +- node/Salsa20.cpp | 29 +++---- node/Salsa20.hpp | 95 ++++++++++++++++++---- node/Utils.cpp | 2 +- selftest.cpp | 12 +-- 32 files changed, 801 insertions(+), 47 deletions(-) create mode 100644 ext/bin/cnacl-osx-amd64/README.md create mode 100644 ext/bin/cnacl-osx-amd64/include/sodium/crypto_auth_hmacsha256.h create mode 100644 ext/bin/cnacl-osx-amd64/include/sodium/crypto_auth_hmacsha512256.h create mode 100644 ext/bin/cnacl-osx-amd64/include/sodium/crypto_box_curve25519xsalsa20poly1305.h create mode 100644 ext/bin/cnacl-osx-amd64/include/sodium/crypto_core_hsalsa20.h create mode 100644 ext/bin/cnacl-osx-amd64/include/sodium/crypto_core_salsa20.h create mode 100644 ext/bin/cnacl-osx-amd64/include/sodium/crypto_core_salsa2012.h create mode 100644 ext/bin/cnacl-osx-amd64/include/sodium/crypto_core_salsa208.h create mode 100644 ext/bin/cnacl-osx-amd64/include/sodium/crypto_hash_sha256.h create mode 100644 ext/bin/cnacl-osx-amd64/include/sodium/crypto_hash_sha512.h create mode 100644 ext/bin/cnacl-osx-amd64/include/sodium/crypto_hashblocks_sha256.h create mode 100644 ext/bin/cnacl-osx-amd64/include/sodium/crypto_hashblocks_sha512.h create mode 100644 ext/bin/cnacl-osx-amd64/include/sodium/crypto_onetimeauth_poly1305.h create mode 100644 ext/bin/cnacl-osx-amd64/include/sodium/crypto_scalarmult_curve25519.h create mode 100644 ext/bin/cnacl-osx-amd64/include/sodium/crypto_secretbox_xsalsa20poly1305.h create mode 100644 ext/bin/cnacl-osx-amd64/include/sodium/crypto_sign_edwards25519sha512batch.h create mode 100644 ext/bin/cnacl-osx-amd64/include/sodium/crypto_stream_aes128ctr.h create mode 100644 ext/bin/cnacl-osx-amd64/include/sodium/crypto_stream_salsa20.h create mode 100644 ext/bin/cnacl-osx-amd64/include/sodium/crypto_stream_salsa2012.h create mode 100644 ext/bin/cnacl-osx-amd64/include/sodium/crypto_stream_salsa208.h create mode 100644 ext/bin/cnacl-osx-amd64/include/sodium/crypto_stream_xsalsa20.h create mode 100644 ext/bin/cnacl-osx-amd64/include/sodium/crypto_types.h create mode 100644 ext/bin/cnacl-osx-amd64/include/sodium/crypto_verify_16.h create mode 100644 ext/bin/cnacl-osx-amd64/include/sodium/crypto_verify_32.h (limited to 'make-mac.mk') diff --git a/ext/bin/cnacl-osx-amd64/README.md b/ext/bin/cnacl-osx-amd64/README.md new file mode 100644 index 00000000..35426286 --- /dev/null +++ b/ext/bin/cnacl-osx-amd64/README.md @@ -0,0 +1,53 @@ +# cNaCl + + If you would like to be confusing, you could pronounce it sea-salt + +This is a fork NaCl by Daniel J. Bernstein and Tanja Lange. +The build has been ported to cmake so it can be cross compiled and build output is reliable. +Since it uses cmake, it could theoretically be built on windows but this has not been tested. +It does compile using mingw32. + +## How do I make this thing work? + + mkdir cbuild + cd cbuild + cmake .. + make + +## Ok now how about cross compiling? + + mkdir cbuildw32 + cd cbuildw32 + cmake -DCMAKE_TOOLCHAIN_FILE=../CMakeWindows.txt .. + make + +## Why fork? + +NaCl builds using a shell script called `./do`. This script does compiling, testing, measuring +and selection of the best implementation of each algorithm for the given machine. It also generates +the header files which will be used. + +The problems with `./do` are it's slow, it tries compiling with multiple different compiler +profiles, it's very platform independent but it doesn't run on Windows and most importantly, with +compiling, testing and measuring so tightly bound, it is impossible to cross compile for a +different operating system. + + +## How it works + +The first time you build for a new ABI, it will trigger the traditional nacl `./do` script. +What cNaCl does is parse the resulting headers from the `./do` build and create a plan so that it +can repeat roughly the same build. + +If there is already a plan for the given ABI, the build uses this plan and the build is very fast. + +Plans are stored in `./cmake/plans/` and I will be adding plans as I find new ones. + + +## What else is new? + +There is a problem with the `./do` build which prevents it from running on some ARM based machines, +this was fixed by adding a more lax method for measuring CPU speed as a fall back. + + +`#EOF#` diff --git a/ext/bin/cnacl-osx-amd64/include/sodium/crypto_auth_hmacsha256.h b/ext/bin/cnacl-osx-amd64/include/sodium/crypto_auth_hmacsha256.h new file mode 100644 index 00000000..6b5600f3 --- /dev/null +++ b/ext/bin/cnacl-osx-amd64/include/sodium/crypto_auth_hmacsha256.h @@ -0,0 +1,27 @@ +#ifndef crypto_auth_hmacsha256_H +#define crypto_auth_hmacsha256_H + +#define crypto_auth_hmacsha256_ref_BYTES 32 +#define crypto_auth_hmacsha256_ref_KEYBYTES 32 +#ifdef __cplusplus +#include +extern std::string crypto_auth_hmacsha256_ref(const std::string &,const std::string &); +extern void crypto_auth_hmacsha256_ref_verify(const std::string &,const std::string &,const std::string &); +extern "C" { +#endif +extern int crypto_auth_hmacsha256_ref(unsigned char *,const unsigned char *,unsigned long long,const unsigned char *); +extern int crypto_auth_hmacsha256_ref_verify(const unsigned char *,const unsigned char *,unsigned long long,const unsigned char *); +#ifdef __cplusplus +} +#endif +#define crypto_auth_hmacsha256 crypto_auth_hmacsha256_ref +#define crypto_auth_hmacsha256_verify crypto_auth_hmacsha256_ref_verify +#define crypto_auth_hmacsha256_BYTES crypto_auth_hmacsha256_ref_BYTES +#define crypto_auth_hmacsha256_KEYBYTES crypto_auth_hmacsha256_ref_KEYBYTES +#define crypto_auth_hmacsha256_IMPLEMENTATION "crypto_auth/hmacsha256/ref" +#ifndef crypto_auth_hmacsha256_ref_VERSION +#define crypto_auth_hmacsha256_ref_VERSION "-" +#endif +#define crypto_auth_hmacsha256_VERSION crypto_auth_hmacsha256_ref_VERSION + +#endif diff --git a/ext/bin/cnacl-osx-amd64/include/sodium/crypto_auth_hmacsha512256.h b/ext/bin/cnacl-osx-amd64/include/sodium/crypto_auth_hmacsha512256.h new file mode 100644 index 00000000..c9bd96e4 --- /dev/null +++ b/ext/bin/cnacl-osx-amd64/include/sodium/crypto_auth_hmacsha512256.h @@ -0,0 +1,27 @@ +#ifndef crypto_auth_hmacsha512256_H +#define crypto_auth_hmacsha512256_H + +#define crypto_auth_hmacsha512256_ref_BYTES 32 +#define crypto_auth_hmacsha512256_ref_KEYBYTES 32 +#ifdef __cplusplus +#include +extern std::string crypto_auth_hmacsha512256_ref(const std::string &,const std::string &); +extern void crypto_auth_hmacsha512256_ref_verify(const std::string &,const std::string &,const std::string &); +extern "C" { +#endif +extern int crypto_auth_hmacsha512256_ref(unsigned char *,const unsigned char *,unsigned long long,const unsigned char *); +extern int crypto_auth_hmacsha512256_ref_verify(const unsigned char *,const unsigned char *,unsigned long long,const unsigned char *); +#ifdef __cplusplus +} +#endif +#define crypto_auth_hmacsha512256 crypto_auth_hmacsha512256_ref +#define crypto_auth_hmacsha512256_verify crypto_auth_hmacsha512256_ref_verify +#define crypto_auth_hmacsha512256_BYTES crypto_auth_hmacsha512256_ref_BYTES +#define crypto_auth_hmacsha512256_KEYBYTES crypto_auth_hmacsha512256_ref_KEYBYTES +#define crypto_auth_hmacsha512256_IMPLEMENTATION "crypto_auth/hmacsha512256/ref" +#ifndef crypto_auth_hmacsha512256_ref_VERSION +#define crypto_auth_hmacsha512256_ref_VERSION "-" +#endif +#define crypto_auth_hmacsha512256_VERSION crypto_auth_hmacsha512256_ref_VERSION + +#endif diff --git a/ext/bin/cnacl-osx-amd64/include/sodium/crypto_box_curve25519xsalsa20poly1305.h b/ext/bin/cnacl-osx-amd64/include/sodium/crypto_box_curve25519xsalsa20poly1305.h new file mode 100644 index 00000000..e2c3b4cc --- /dev/null +++ b/ext/bin/cnacl-osx-amd64/include/sodium/crypto_box_curve25519xsalsa20poly1305.h @@ -0,0 +1,44 @@ +#ifndef crypto_box_curve25519xsalsa20poly1305_H +#define crypto_box_curve25519xsalsa20poly1305_H + +#define crypto_box_curve25519xsalsa20poly1305_ref_PUBLICKEYBYTES 32 +#define crypto_box_curve25519xsalsa20poly1305_ref_SECRETKEYBYTES 32 +#define crypto_box_curve25519xsalsa20poly1305_ref_BEFORENMBYTES 32 +#define crypto_box_curve25519xsalsa20poly1305_ref_NONCEBYTES 24 +#define crypto_box_curve25519xsalsa20poly1305_ref_ZEROBYTES 32 +#define crypto_box_curve25519xsalsa20poly1305_ref_BOXZEROBYTES 16 +#ifdef __cplusplus +#include +extern std::string crypto_box_curve25519xsalsa20poly1305_ref(const std::string &,const std::string &,const std::string &,const std::string &); +extern std::string crypto_box_curve25519xsalsa20poly1305_ref_open(const std::string &,const std::string &,const std::string &,const std::string &); +extern std::string crypto_box_curve25519xsalsa20poly1305_ref_keypair(std::string *); +extern "C" { +#endif +extern int crypto_box_curve25519xsalsa20poly1305_ref(unsigned char *,const unsigned char *,unsigned long long,const unsigned char *,const unsigned char *,const unsigned char *); +extern int crypto_box_curve25519xsalsa20poly1305_ref_open(unsigned char *,const unsigned char *,unsigned long long,const unsigned char *,const unsigned char *,const unsigned char *); +extern int crypto_box_curve25519xsalsa20poly1305_ref_keypair(unsigned char *,unsigned char *); +extern int crypto_box_curve25519xsalsa20poly1305_ref_beforenm(unsigned char *,const unsigned char *,const unsigned char *); +extern int crypto_box_curve25519xsalsa20poly1305_ref_afternm(unsigned char *,const unsigned char *,unsigned long long,const unsigned char *,const unsigned char *); +extern int crypto_box_curve25519xsalsa20poly1305_ref_open_afternm(unsigned char *,const unsigned char *,unsigned long long,const unsigned char *,const unsigned char *); +#ifdef __cplusplus +} +#endif +#define crypto_box_curve25519xsalsa20poly1305 crypto_box_curve25519xsalsa20poly1305_ref +#define crypto_box_curve25519xsalsa20poly1305_open crypto_box_curve25519xsalsa20poly1305_ref_open +#define crypto_box_curve25519xsalsa20poly1305_keypair crypto_box_curve25519xsalsa20poly1305_ref_keypair +#define crypto_box_curve25519xsalsa20poly1305_beforenm crypto_box_curve25519xsalsa20poly1305_ref_beforenm +#define crypto_box_curve25519xsalsa20poly1305_afternm crypto_box_curve25519xsalsa20poly1305_ref_afternm +#define crypto_box_curve25519xsalsa20poly1305_open_afternm crypto_box_curve25519xsalsa20poly1305_ref_open_afternm +#define crypto_box_curve25519xsalsa20poly1305_PUBLICKEYBYTES crypto_box_curve25519xsalsa20poly1305_ref_PUBLICKEYBYTES +#define crypto_box_curve25519xsalsa20poly1305_SECRETKEYBYTES crypto_box_curve25519xsalsa20poly1305_ref_SECRETKEYBYTES +#define crypto_box_curve25519xsalsa20poly1305_BEFORENMBYTES crypto_box_curve25519xsalsa20poly1305_ref_BEFORENMBYTES +#define crypto_box_curve25519xsalsa20poly1305_NONCEBYTES crypto_box_curve25519xsalsa20poly1305_ref_NONCEBYTES +#define crypto_box_curve25519xsalsa20poly1305_ZEROBYTES crypto_box_curve25519xsalsa20poly1305_ref_ZEROBYTES +#define crypto_box_curve25519xsalsa20poly1305_BOXZEROBYTES crypto_box_curve25519xsalsa20poly1305_ref_BOXZEROBYTES +#define crypto_box_curve25519xsalsa20poly1305_IMPLEMENTATION "crypto_box/curve25519xsalsa20poly1305/ref" +#ifndef crypto_box_curve25519xsalsa20poly1305_ref_VERSION +#define crypto_box_curve25519xsalsa20poly1305_ref_VERSION "-" +#endif +#define crypto_box_curve25519xsalsa20poly1305_VERSION crypto_box_curve25519xsalsa20poly1305_ref_VERSION + +#endif diff --git a/ext/bin/cnacl-osx-amd64/include/sodium/crypto_core_hsalsa20.h b/ext/bin/cnacl-osx-amd64/include/sodium/crypto_core_hsalsa20.h new file mode 100644 index 00000000..abae188e --- /dev/null +++ b/ext/bin/cnacl-osx-amd64/include/sodium/crypto_core_hsalsa20.h @@ -0,0 +1,27 @@ +#ifndef crypto_core_hsalsa20_H +#define crypto_core_hsalsa20_H + +#define crypto_core_hsalsa20_ref_OUTPUTBYTES 32 +#define crypto_core_hsalsa20_ref_INPUTBYTES 16 +#define crypto_core_hsalsa20_ref_KEYBYTES 32 +#define crypto_core_hsalsa20_ref_CONSTBYTES 16 +#ifdef __cplusplus +#include +extern "C" { +#endif +extern int crypto_core_hsalsa20_ref(unsigned char *,const unsigned char *,const unsigned char *,const unsigned char *); +#ifdef __cplusplus +} +#endif +#define crypto_core_hsalsa20 crypto_core_hsalsa20_ref +#define crypto_core_hsalsa20_OUTPUTBYTES crypto_core_hsalsa20_ref_OUTPUTBYTES +#define crypto_core_hsalsa20_INPUTBYTES crypto_core_hsalsa20_ref_INPUTBYTES +#define crypto_core_hsalsa20_KEYBYTES crypto_core_hsalsa20_ref_KEYBYTES +#define crypto_core_hsalsa20_CONSTBYTES crypto_core_hsalsa20_ref_CONSTBYTES +#define crypto_core_hsalsa20_IMPLEMENTATION "crypto_core/hsalsa20/ref" +#ifndef crypto_core_hsalsa20_ref_VERSION +#define crypto_core_hsalsa20_ref_VERSION "-" +#endif +#define crypto_core_hsalsa20_VERSION crypto_core_hsalsa20_ref_VERSION + +#endif diff --git a/ext/bin/cnacl-osx-amd64/include/sodium/crypto_core_salsa20.h b/ext/bin/cnacl-osx-amd64/include/sodium/crypto_core_salsa20.h new file mode 100644 index 00000000..9737b101 --- /dev/null +++ b/ext/bin/cnacl-osx-amd64/include/sodium/crypto_core_salsa20.h @@ -0,0 +1,27 @@ +#ifndef crypto_core_salsa20_H +#define crypto_core_salsa20_H + +#define crypto_core_salsa20_ref_OUTPUTBYTES 64 +#define crypto_core_salsa20_ref_INPUTBYTES 16 +#define crypto_core_salsa20_ref_KEYBYTES 32 +#define crypto_core_salsa20_ref_CONSTBYTES 16 +#ifdef __cplusplus +#include +extern "C" { +#endif +extern int crypto_core_salsa20_ref(unsigned char *,const unsigned char *,const unsigned char *,const unsigned char *); +#ifdef __cplusplus +} +#endif +#define crypto_core_salsa20 crypto_core_salsa20_ref +#define crypto_core_salsa20_OUTPUTBYTES crypto_core_salsa20_ref_OUTPUTBYTES +#define crypto_core_salsa20_INPUTBYTES crypto_core_salsa20_ref_INPUTBYTES +#define crypto_core_salsa20_KEYBYTES crypto_core_salsa20_ref_KEYBYTES +#define crypto_core_salsa20_CONSTBYTES crypto_core_salsa20_ref_CONSTBYTES +#define crypto_core_salsa20_IMPLEMENTATION "crypto_core/salsa20/ref" +#ifndef crypto_core_salsa20_ref_VERSION +#define crypto_core_salsa20_ref_VERSION "-" +#endif +#define crypto_core_salsa20_VERSION crypto_core_salsa20_ref_VERSION + +#endif diff --git a/ext/bin/cnacl-osx-amd64/include/sodium/crypto_core_salsa2012.h b/ext/bin/cnacl-osx-amd64/include/sodium/crypto_core_salsa2012.h new file mode 100644 index 00000000..137cd47f --- /dev/null +++ b/ext/bin/cnacl-osx-amd64/include/sodium/crypto_core_salsa2012.h @@ -0,0 +1,27 @@ +#ifndef crypto_core_salsa2012_H +#define crypto_core_salsa2012_H + +#define crypto_core_salsa2012_ref_OUTPUTBYTES 64 +#define crypto_core_salsa2012_ref_INPUTBYTES 16 +#define crypto_core_salsa2012_ref_KEYBYTES 32 +#define crypto_core_salsa2012_ref_CONSTBYTES 16 +#ifdef __cplusplus +#include +extern "C" { +#endif +extern int crypto_core_salsa2012_ref(unsigned char *,const unsigned char *,const unsigned char *,const unsigned char *); +#ifdef __cplusplus +} +#endif +#define crypto_core_salsa2012 crypto_core_salsa2012_ref +#define crypto_core_salsa2012_OUTPUTBYTES crypto_core_salsa2012_ref_OUTPUTBYTES +#define crypto_core_salsa2012_INPUTBYTES crypto_core_salsa2012_ref_INPUTBYTES +#define crypto_core_salsa2012_KEYBYTES crypto_core_salsa2012_ref_KEYBYTES +#define crypto_core_salsa2012_CONSTBYTES crypto_core_salsa2012_ref_CONSTBYTES +#define crypto_core_salsa2012_IMPLEMENTATION "crypto_core/salsa2012/ref" +#ifndef crypto_core_salsa2012_ref_VERSION +#define crypto_core_salsa2012_ref_VERSION "-" +#endif +#define crypto_core_salsa2012_VERSION crypto_core_salsa2012_ref_VERSION + +#endif diff --git a/ext/bin/cnacl-osx-amd64/include/sodium/crypto_core_salsa208.h b/ext/bin/cnacl-osx-amd64/include/sodium/crypto_core_salsa208.h new file mode 100644 index 00000000..4895bbbe --- /dev/null +++ b/ext/bin/cnacl-osx-amd64/include/sodium/crypto_core_salsa208.h @@ -0,0 +1,27 @@ +#ifndef crypto_core_salsa208_H +#define crypto_core_salsa208_H + +#define crypto_core_salsa208_ref_OUTPUTBYTES 64 +#define crypto_core_salsa208_ref_INPUTBYTES 16 +#define crypto_core_salsa208_ref_KEYBYTES 32 +#define crypto_core_salsa208_ref_CONSTBYTES 16 +#ifdef __cplusplus +#include +extern "C" { +#endif +extern int crypto_core_salsa208_ref(unsigned char *,const unsigned char *,const unsigned char *,const unsigned char *); +#ifdef __cplusplus +} +#endif +#define crypto_core_salsa208 crypto_core_salsa208_ref +#define crypto_core_salsa208_OUTPUTBYTES crypto_core_salsa208_ref_OUTPUTBYTES +#define crypto_core_salsa208_INPUTBYTES crypto_core_salsa208_ref_INPUTBYTES +#define crypto_core_salsa208_KEYBYTES crypto_core_salsa208_ref_KEYBYTES +#define crypto_core_salsa208_CONSTBYTES crypto_core_salsa208_ref_CONSTBYTES +#define crypto_core_salsa208_IMPLEMENTATION "crypto_core/salsa208/ref" +#ifndef crypto_core_salsa208_ref_VERSION +#define crypto_core_salsa208_ref_VERSION "-" +#endif +#define crypto_core_salsa208_VERSION crypto_core_salsa208_ref_VERSION + +#endif diff --git a/ext/bin/cnacl-osx-amd64/include/sodium/crypto_hash_sha256.h b/ext/bin/cnacl-osx-amd64/include/sodium/crypto_hash_sha256.h new file mode 100644 index 00000000..20d18703 --- /dev/null +++ b/ext/bin/cnacl-osx-amd64/include/sodium/crypto_hash_sha256.h @@ -0,0 +1,22 @@ +#ifndef crypto_hash_sha256_H +#define crypto_hash_sha256_H + +#define crypto_hash_sha256_ref_BYTES 32 +#ifdef __cplusplus +#include +extern std::string crypto_hash_sha256_ref(const std::string &); +extern "C" { +#endif +extern int crypto_hash_sha256_ref(unsigned char *,const unsigned char *,unsigned long long); +#ifdef __cplusplus +} +#endif +#define crypto_hash_sha256 crypto_hash_sha256_ref +#define crypto_hash_sha256_BYTES crypto_hash_sha256_ref_BYTES +#define crypto_hash_sha256_IMPLEMENTATION "crypto_hash/sha256/ref" +#ifndef crypto_hash_sha256_ref_VERSION +#define crypto_hash_sha256_ref_VERSION "-" +#endif +#define crypto_hash_sha256_VERSION crypto_hash_sha256_ref_VERSION + +#endif diff --git a/ext/bin/cnacl-osx-amd64/include/sodium/crypto_hash_sha512.h b/ext/bin/cnacl-osx-amd64/include/sodium/crypto_hash_sha512.h new file mode 100644 index 00000000..fe19d2d9 --- /dev/null +++ b/ext/bin/cnacl-osx-amd64/include/sodium/crypto_hash_sha512.h @@ -0,0 +1,22 @@ +#ifndef crypto_hash_sha512_H +#define crypto_hash_sha512_H + +#define crypto_hash_sha512_ref_BYTES 64 +#ifdef __cplusplus +#include +extern std::string crypto_hash_sha512_ref(const std::string &); +extern "C" { +#endif +extern int crypto_hash_sha512_ref(unsigned char *,const unsigned char *,unsigned long long); +#ifdef __cplusplus +} +#endif +#define crypto_hash_sha512 crypto_hash_sha512_ref +#define crypto_hash_sha512_BYTES crypto_hash_sha512_ref_BYTES +#define crypto_hash_sha512_IMPLEMENTATION "crypto_hash/sha512/ref" +#ifndef crypto_hash_sha512_ref_VERSION +#define crypto_hash_sha512_ref_VERSION "-" +#endif +#define crypto_hash_sha512_VERSION crypto_hash_sha512_ref_VERSION + +#endif diff --git a/ext/bin/cnacl-osx-amd64/include/sodium/crypto_hashblocks_sha256.h b/ext/bin/cnacl-osx-amd64/include/sodium/crypto_hashblocks_sha256.h new file mode 100644 index 00000000..3b473e6c --- /dev/null +++ b/ext/bin/cnacl-osx-amd64/include/sodium/crypto_hashblocks_sha256.h @@ -0,0 +1,23 @@ +#ifndef crypto_hashblocks_sha256_H +#define crypto_hashblocks_sha256_H + +#define crypto_hashblocks_sha256_inplace_STATEBYTES 32 +#define crypto_hashblocks_sha256_inplace_BLOCKBYTES 64 +#ifdef __cplusplus +#include +extern "C" { +#endif +extern int crypto_hashblocks_sha256_inplace(unsigned char *,const unsigned char *,unsigned long long); +#ifdef __cplusplus +} +#endif +#define crypto_hashblocks_sha256 crypto_hashblocks_sha256_inplace +#define crypto_hashblocks_sha256_STATEBYTES crypto_hashblocks_sha256_inplace_STATEBYTES +#define crypto_hashblocks_sha256_BLOCKBYTES crypto_hashblocks_sha256_inplace_BLOCKBYTES +#define crypto_hashblocks_sha256_IMPLEMENTATION "crypto_hashblocks/sha256/inplace" +#ifndef crypto_hashblocks_sha256_inplace_VERSION +#define crypto_hashblocks_sha256_inplace_VERSION "-" +#endif +#define crypto_hashblocks_sha256_VERSION crypto_hashblocks_sha256_inplace_VERSION + +#endif diff --git a/ext/bin/cnacl-osx-amd64/include/sodium/crypto_hashblocks_sha512.h b/ext/bin/cnacl-osx-amd64/include/sodium/crypto_hashblocks_sha512.h new file mode 100644 index 00000000..f66edd09 --- /dev/null +++ b/ext/bin/cnacl-osx-amd64/include/sodium/crypto_hashblocks_sha512.h @@ -0,0 +1,23 @@ +#ifndef crypto_hashblocks_sha512_H +#define crypto_hashblocks_sha512_H + +#define crypto_hashblocks_sha512_ref_STATEBYTES 64 +#define crypto_hashblocks_sha512_ref_BLOCKBYTES 128 +#ifdef __cplusplus +#include +extern "C" { +#endif +extern int crypto_hashblocks_sha512_ref(unsigned char *,const unsigned char *,unsigned long long); +#ifdef __cplusplus +} +#endif +#define crypto_hashblocks_sha512 crypto_hashblocks_sha512_ref +#define crypto_hashblocks_sha512_STATEBYTES crypto_hashblocks_sha512_ref_STATEBYTES +#define crypto_hashblocks_sha512_BLOCKBYTES crypto_hashblocks_sha512_ref_BLOCKBYTES +#define crypto_hashblocks_sha512_IMPLEMENTATION "crypto_hashblocks/sha512/ref" +#ifndef crypto_hashblocks_sha512_ref_VERSION +#define crypto_hashblocks_sha512_ref_VERSION "-" +#endif +#define crypto_hashblocks_sha512_VERSION crypto_hashblocks_sha512_ref_VERSION + +#endif diff --git a/ext/bin/cnacl-osx-amd64/include/sodium/crypto_onetimeauth_poly1305.h b/ext/bin/cnacl-osx-amd64/include/sodium/crypto_onetimeauth_poly1305.h new file mode 100644 index 00000000..de08dc9f --- /dev/null +++ b/ext/bin/cnacl-osx-amd64/include/sodium/crypto_onetimeauth_poly1305.h @@ -0,0 +1,27 @@ +#ifndef crypto_onetimeauth_poly1305_H +#define crypto_onetimeauth_poly1305_H + +#define crypto_onetimeauth_poly1305_53_BYTES 16 +#define crypto_onetimeauth_poly1305_53_KEYBYTES 32 +#ifdef __cplusplus +#include +extern std::string crypto_onetimeauth_poly1305_53(const std::string &,const std::string &); +extern void crypto_onetimeauth_poly1305_53_verify(const std::string &,const std::string &,const std::string &); +extern "C" { +#endif +extern int crypto_onetimeauth_poly1305_53(unsigned char *,const unsigned char *,unsigned long long,const unsigned char *); +extern int crypto_onetimeauth_poly1305_53_verify(const unsigned char *,const unsigned char *,unsigned long long,const unsigned char *); +#ifdef __cplusplus +} +#endif +#define crypto_onetimeauth_poly1305 crypto_onetimeauth_poly1305_53 +#define crypto_onetimeauth_poly1305_verify crypto_onetimeauth_poly1305_53_verify +#define crypto_onetimeauth_poly1305_BYTES crypto_onetimeauth_poly1305_53_BYTES +#define crypto_onetimeauth_poly1305_KEYBYTES crypto_onetimeauth_poly1305_53_KEYBYTES +#define crypto_onetimeauth_poly1305_IMPLEMENTATION "crypto_onetimeauth/poly1305/53" +#ifndef crypto_onetimeauth_poly1305_53_VERSION +#define crypto_onetimeauth_poly1305_53_VERSION "-" +#endif +#define crypto_onetimeauth_poly1305_VERSION crypto_onetimeauth_poly1305_53_VERSION + +#endif diff --git a/ext/bin/cnacl-osx-amd64/include/sodium/crypto_scalarmult_curve25519.h b/ext/bin/cnacl-osx-amd64/include/sodium/crypto_scalarmult_curve25519.h new file mode 100644 index 00000000..550c4e3d --- /dev/null +++ b/ext/bin/cnacl-osx-amd64/include/sodium/crypto_scalarmult_curve25519.h @@ -0,0 +1,27 @@ +#ifndef crypto_scalarmult_curve25519_H +#define crypto_scalarmult_curve25519_H + +#define crypto_scalarmult_curve25519_donna_c64_BYTES 32 +#define crypto_scalarmult_curve25519_donna_c64_SCALARBYTES 32 +#ifdef __cplusplus +#include +extern std::string crypto_scalarmult_curve25519_donna_c64(const std::string &,const std::string &); +extern std::string crypto_scalarmult_curve25519_donna_c64_base(const std::string &); +extern "C" { +#endif +extern int crypto_scalarmult_curve25519_donna_c64(unsigned char *,const unsigned char *,const unsigned char *); +extern int crypto_scalarmult_curve25519_donna_c64_base(unsigned char *,const unsigned char *); +#ifdef __cplusplus +} +#endif +#define crypto_scalarmult_curve25519 crypto_scalarmult_curve25519_donna_c64 +#define crypto_scalarmult_curve25519_base crypto_scalarmult_curve25519_donna_c64_base +#define crypto_scalarmult_curve25519_BYTES crypto_scalarmult_curve25519_donna_c64_BYTES +#define crypto_scalarmult_curve25519_SCALARBYTES crypto_scalarmult_curve25519_donna_c64_SCALARBYTES +#define crypto_scalarmult_curve25519_IMPLEMENTATION "crypto_scalarmult/curve25519/donna_c64" +#ifndef crypto_scalarmult_curve25519_donna_c64_VERSION +#define crypto_scalarmult_curve25519_donna_c64_VERSION "-" +#endif +#define crypto_scalarmult_curve25519_VERSION crypto_scalarmult_curve25519_donna_c64_VERSION + +#endif diff --git a/ext/bin/cnacl-osx-amd64/include/sodium/crypto_secretbox_xsalsa20poly1305.h b/ext/bin/cnacl-osx-amd64/include/sodium/crypto_secretbox_xsalsa20poly1305.h new file mode 100644 index 00000000..c930b6f1 --- /dev/null +++ b/ext/bin/cnacl-osx-amd64/include/sodium/crypto_secretbox_xsalsa20poly1305.h @@ -0,0 +1,31 @@ +#ifndef crypto_secretbox_xsalsa20poly1305_H +#define crypto_secretbox_xsalsa20poly1305_H + +#define crypto_secretbox_xsalsa20poly1305_ref_KEYBYTES 32 +#define crypto_secretbox_xsalsa20poly1305_ref_NONCEBYTES 24 +#define crypto_secretbox_xsalsa20poly1305_ref_ZEROBYTES 32 +#define crypto_secretbox_xsalsa20poly1305_ref_BOXZEROBYTES 16 +#ifdef __cplusplus +#include +extern std::string crypto_secretbox_xsalsa20poly1305_ref(const std::string &,const std::string &,const std::string &); +extern std::string crypto_secretbox_xsalsa20poly1305_ref_open(const std::string &,const std::string &,const std::string &); +extern "C" { +#endif +extern int crypto_secretbox_xsalsa20poly1305_ref(unsigned char *,const unsigned char *,unsigned long long,const unsigned char *,const unsigned char *); +extern int crypto_secretbox_xsalsa20poly1305_ref_open(unsigned char *,const unsigned char *,unsigned long long,const unsigned char *,const unsigned char *); +#ifdef __cplusplus +} +#endif +#define crypto_secretbox_xsalsa20poly1305 crypto_secretbox_xsalsa20poly1305_ref +#define crypto_secretbox_xsalsa20poly1305_open crypto_secretbox_xsalsa20poly1305_ref_open +#define crypto_secretbox_xsalsa20poly1305_KEYBYTES crypto_secretbox_xsalsa20poly1305_ref_KEYBYTES +#define crypto_secretbox_xsalsa20poly1305_NONCEBYTES crypto_secretbox_xsalsa20poly1305_ref_NONCEBYTES +#define crypto_secretbox_xsalsa20poly1305_ZEROBYTES crypto_secretbox_xsalsa20poly1305_ref_ZEROBYTES +#define crypto_secretbox_xsalsa20poly1305_BOXZEROBYTES crypto_secretbox_xsalsa20poly1305_ref_BOXZEROBYTES +#define crypto_secretbox_xsalsa20poly1305_IMPLEMENTATION "crypto_secretbox/xsalsa20poly1305/ref" +#ifndef crypto_secretbox_xsalsa20poly1305_ref_VERSION +#define crypto_secretbox_xsalsa20poly1305_ref_VERSION "-" +#endif +#define crypto_secretbox_xsalsa20poly1305_VERSION crypto_secretbox_xsalsa20poly1305_ref_VERSION + +#endif diff --git a/ext/bin/cnacl-osx-amd64/include/sodium/crypto_sign_edwards25519sha512batch.h b/ext/bin/cnacl-osx-amd64/include/sodium/crypto_sign_edwards25519sha512batch.h new file mode 100644 index 00000000..936108ef --- /dev/null +++ b/ext/bin/cnacl-osx-amd64/include/sodium/crypto_sign_edwards25519sha512batch.h @@ -0,0 +1,32 @@ +#ifndef crypto_sign_edwards25519sha512batch_H +#define crypto_sign_edwards25519sha512batch_H + +#define crypto_sign_edwards25519sha512batch_ref_SECRETKEYBYTES 64 +#define crypto_sign_edwards25519sha512batch_ref_PUBLICKEYBYTES 32 +#define crypto_sign_edwards25519sha512batch_ref_BYTES 64 +#ifdef __cplusplus +#include +extern std::string crypto_sign_edwards25519sha512batch_ref(const std::string &,const std::string &); +extern std::string crypto_sign_edwards25519sha512batch_ref_open(const std::string &,const std::string &); +extern std::string crypto_sign_edwards25519sha512batch_ref_keypair(std::string *); +extern "C" { +#endif +extern int crypto_sign_edwards25519sha512batch_ref(unsigned char *,unsigned long long *,const unsigned char *,unsigned long long,const unsigned char *); +extern int crypto_sign_edwards25519sha512batch_ref_open(unsigned char *,unsigned long long *,const unsigned char *,unsigned long long,const unsigned char *); +extern int crypto_sign_edwards25519sha512batch_ref_keypair(unsigned char *,unsigned char *); +#ifdef __cplusplus +} +#endif +#define crypto_sign_edwards25519sha512batch crypto_sign_edwards25519sha512batch_ref +#define crypto_sign_edwards25519sha512batch_open crypto_sign_edwards25519sha512batch_ref_open +#define crypto_sign_edwards25519sha512batch_keypair crypto_sign_edwards25519sha512batch_ref_keypair +#define crypto_sign_edwards25519sha512batch_BYTES crypto_sign_edwards25519sha512batch_ref_BYTES +#define crypto_sign_edwards25519sha512batch_PUBLICKEYBYTES crypto_sign_edwards25519sha512batch_ref_PUBLICKEYBYTES +#define crypto_sign_edwards25519sha512batch_SECRETKEYBYTES crypto_sign_edwards25519sha512batch_ref_SECRETKEYBYTES +#define crypto_sign_edwards25519sha512batch_IMPLEMENTATION "crypto_sign/edwards25519sha512batch/ref" +#ifndef crypto_sign_edwards25519sha512batch_ref_VERSION +#define crypto_sign_edwards25519sha512batch_ref_VERSION "-" +#endif +#define crypto_sign_edwards25519sha512batch_VERSION crypto_sign_edwards25519sha512batch_ref_VERSION + +#endif diff --git a/ext/bin/cnacl-osx-amd64/include/sodium/crypto_stream_aes128ctr.h b/ext/bin/cnacl-osx-amd64/include/sodium/crypto_stream_aes128ctr.h new file mode 100644 index 00000000..76bf9137 --- /dev/null +++ b/ext/bin/cnacl-osx-amd64/include/sodium/crypto_stream_aes128ctr.h @@ -0,0 +1,35 @@ +#ifndef crypto_stream_aes128ctr_H +#define crypto_stream_aes128ctr_H + +#define crypto_stream_aes128ctr_portable_KEYBYTES 16 +#define crypto_stream_aes128ctr_portable_NONCEBYTES 16 +#define crypto_stream_aes128ctr_portable_BEFORENMBYTES 1408 +#ifdef __cplusplus +#include +extern std::string crypto_stream_aes128ctr_portable(size_t,const std::string &,const std::string &); +extern std::string crypto_stream_aes128ctr_portable_xor(const std::string &,const std::string &,const std::string &); +extern "C" { +#endif +extern int crypto_stream_aes128ctr_portable(unsigned char *,unsigned long long,const unsigned char *,const unsigned char *); +extern int crypto_stream_aes128ctr_portable_xor(unsigned char *,const unsigned char *,unsigned long long,const unsigned char *,const unsigned char *); +extern int crypto_stream_aes128ctr_portable_beforenm(unsigned char *,const unsigned char *); +extern int crypto_stream_aes128ctr_portable_afternm(unsigned char *,unsigned long long,const unsigned char *,const unsigned char *); +extern int crypto_stream_aes128ctr_portable_xor_afternm(unsigned char *,const unsigned char *,unsigned long long,const unsigned char *,const unsigned char *); +#ifdef __cplusplus +} +#endif +#define crypto_stream_aes128ctr crypto_stream_aes128ctr_portable +#define crypto_stream_aes128ctr_xor crypto_stream_aes128ctr_portable_xor +#define crypto_stream_aes128ctr_beforenm crypto_stream_aes128ctr_portable_beforenm +#define crypto_stream_aes128ctr_afternm crypto_stream_aes128ctr_portable_afternm +#define crypto_stream_aes128ctr_xor_afternm crypto_stream_aes128ctr_portable_xor_afternm +#define crypto_stream_aes128ctr_KEYBYTES crypto_stream_aes128ctr_portable_KEYBYTES +#define crypto_stream_aes128ctr_NONCEBYTES crypto_stream_aes128ctr_portable_NONCEBYTES +#define crypto_stream_aes128ctr_BEFORENMBYTES crypto_stream_aes128ctr_portable_BEFORENMBYTES +#define crypto_stream_aes128ctr_IMPLEMENTATION "crypto_stream/aes128ctr/portable" +#ifndef crypto_stream_aes128ctr_portable_VERSION +#define crypto_stream_aes128ctr_portable_VERSION "-" +#endif +#define crypto_stream_aes128ctr_VERSION crypto_stream_aes128ctr_portable_VERSION + +#endif diff --git a/ext/bin/cnacl-osx-amd64/include/sodium/crypto_stream_salsa20.h b/ext/bin/cnacl-osx-amd64/include/sodium/crypto_stream_salsa20.h new file mode 100644 index 00000000..c96d20b4 --- /dev/null +++ b/ext/bin/cnacl-osx-amd64/include/sodium/crypto_stream_salsa20.h @@ -0,0 +1,34 @@ +#ifndef crypto_stream_salsa20_H +#define crypto_stream_salsa20_H + +#define crypto_stream_salsa20_amd64_xmm6_KEYBYTES 32 +#define crypto_stream_salsa20_amd64_xmm6_NONCEBYTES 8 +#ifdef __cplusplus +#include +extern std::string crypto_stream_salsa20_amd64_xmm6(size_t,const std::string &,const std::string &); +extern std::string crypto_stream_salsa20_amd64_xmm6_xor(const std::string &,const std::string &,const std::string &); +extern "C" { +#endif +extern int crypto_stream_salsa20_amd64_xmm6(unsigned char *,unsigned long long,const unsigned char *,const unsigned char *); +extern int crypto_stream_salsa20_amd64_xmm6_xor(unsigned char *,const unsigned char *,unsigned long long,const unsigned char *,const unsigned char *); +extern int crypto_stream_salsa20_amd64_xmm6_beforenm(unsigned char *,const unsigned char *); +extern int crypto_stream_salsa20_amd64_xmm6_afternm(unsigned char *,unsigned long long,const unsigned char *,const unsigned char *); +extern int crypto_stream_salsa20_amd64_xmm6_xor_afternm(unsigned char *,const unsigned char *,unsigned long long,const unsigned char *,const unsigned char *); +#ifdef __cplusplus +} +#endif +#define crypto_stream_salsa20 crypto_stream_salsa20_amd64_xmm6 +#define crypto_stream_salsa20_xor crypto_stream_salsa20_amd64_xmm6_xor +#define crypto_stream_salsa20_beforenm crypto_stream_salsa20_amd64_xmm6_beforenm +#define crypto_stream_salsa20_afternm crypto_stream_salsa20_amd64_xmm6_afternm +#define crypto_stream_salsa20_xor_afternm crypto_stream_salsa20_amd64_xmm6_xor_afternm +#define crypto_stream_salsa20_KEYBYTES crypto_stream_salsa20_amd64_xmm6_KEYBYTES +#define crypto_stream_salsa20_NONCEBYTES crypto_stream_salsa20_amd64_xmm6_NONCEBYTES +#define crypto_stream_salsa20_BEFORENMBYTES crypto_stream_salsa20_amd64_xmm6_BEFORENMBYTES +#define crypto_stream_salsa20_IMPLEMENTATION "crypto_stream/salsa20/amd64_xmm6" +#ifndef crypto_stream_salsa20_amd64_xmm6_VERSION +#define crypto_stream_salsa20_amd64_xmm6_VERSION "-" +#endif +#define crypto_stream_salsa20_VERSION crypto_stream_salsa20_amd64_xmm6_VERSION + +#endif diff --git a/ext/bin/cnacl-osx-amd64/include/sodium/crypto_stream_salsa2012.h b/ext/bin/cnacl-osx-amd64/include/sodium/crypto_stream_salsa2012.h new file mode 100644 index 00000000..051e4e39 --- /dev/null +++ b/ext/bin/cnacl-osx-amd64/include/sodium/crypto_stream_salsa2012.h @@ -0,0 +1,34 @@ +#ifndef crypto_stream_salsa2012_H +#define crypto_stream_salsa2012_H + +#define crypto_stream_salsa2012_amd64_xmm6_KEYBYTES 32 +#define crypto_stream_salsa2012_amd64_xmm6_NONCEBYTES 8 +#ifdef __cplusplus +#include +extern std::string crypto_stream_salsa2012_amd64_xmm6(size_t,const std::string &,const std::string &); +extern std::string crypto_stream_salsa2012_amd64_xmm6_xor(const std::string &,const std::string &,const std::string &); +extern "C" { +#endif +extern int crypto_stream_salsa2012_amd64_xmm6(unsigned char *,unsigned long long,const unsigned char *,const unsigned char *); +extern int crypto_stream_salsa2012_amd64_xmm6_xor(unsigned char *,const unsigned char *,unsigned long long,const unsigned char *,const unsigned char *); +extern int crypto_stream_salsa2012_amd64_xmm6_beforenm(unsigned char *,const unsigned char *); +extern int crypto_stream_salsa2012_amd64_xmm6_afternm(unsigned char *,unsigned long long,const unsigned char *,const unsigned char *); +extern int crypto_stream_salsa2012_amd64_xmm6_xor_afternm(unsigned char *,const unsigned char *,unsigned long long,const unsigned char *,const unsigned char *); +#ifdef __cplusplus +} +#endif +#define crypto_stream_salsa2012 crypto_stream_salsa2012_amd64_xmm6 +#define crypto_stream_salsa2012_xor crypto_stream_salsa2012_amd64_xmm6_xor +#define crypto_stream_salsa2012_beforenm crypto_stream_salsa2012_amd64_xmm6_beforenm +#define crypto_stream_salsa2012_afternm crypto_stream_salsa2012_amd64_xmm6_afternm +#define crypto_stream_salsa2012_xor_afternm crypto_stream_salsa2012_amd64_xmm6_xor_afternm +#define crypto_stream_salsa2012_KEYBYTES crypto_stream_salsa2012_amd64_xmm6_KEYBYTES +#define crypto_stream_salsa2012_NONCEBYTES crypto_stream_salsa2012_amd64_xmm6_NONCEBYTES +#define crypto_stream_salsa2012_BEFORENMBYTES crypto_stream_salsa2012_amd64_xmm6_BEFORENMBYTES +#define crypto_stream_salsa2012_IMPLEMENTATION "crypto_stream/salsa2012/amd64_xmm6" +#ifndef crypto_stream_salsa2012_amd64_xmm6_VERSION +#define crypto_stream_salsa2012_amd64_xmm6_VERSION "-" +#endif +#define crypto_stream_salsa2012_VERSION crypto_stream_salsa2012_amd64_xmm6_VERSION + +#endif diff --git a/ext/bin/cnacl-osx-amd64/include/sodium/crypto_stream_salsa208.h b/ext/bin/cnacl-osx-amd64/include/sodium/crypto_stream_salsa208.h new file mode 100644 index 00000000..4bd470c3 --- /dev/null +++ b/ext/bin/cnacl-osx-amd64/include/sodium/crypto_stream_salsa208.h @@ -0,0 +1,34 @@ +#ifndef crypto_stream_salsa208_H +#define crypto_stream_salsa208_H + +#define crypto_stream_salsa208_amd64_xmm6_KEYBYTES 32 +#define crypto_stream_salsa208_amd64_xmm6_NONCEBYTES 8 +#ifdef __cplusplus +#include +extern std::string crypto_stream_salsa208_amd64_xmm6(size_t,const std::string &,const std::string &); +extern std::string crypto_stream_salsa208_amd64_xmm6_xor(const std::string &,const std::string &,const std::string &); +extern "C" { +#endif +extern int crypto_stream_salsa208_amd64_xmm6(unsigned char *,unsigned long long,const unsigned char *,const unsigned char *); +extern int crypto_stream_salsa208_amd64_xmm6_xor(unsigned char *,const unsigned char *,unsigned long long,const unsigned char *,const unsigned char *); +extern int crypto_stream_salsa208_amd64_xmm6_beforenm(unsigned char *,const unsigned char *); +extern int crypto_stream_salsa208_amd64_xmm6_afternm(unsigned char *,unsigned long long,const unsigned char *,const unsigned char *); +extern int crypto_stream_salsa208_amd64_xmm6_xor_afternm(unsigned char *,const unsigned char *,unsigned long long,const unsigned char *,const unsigned char *); +#ifdef __cplusplus +} +#endif +#define crypto_stream_salsa208 crypto_stream_salsa208_amd64_xmm6 +#define crypto_stream_salsa208_xor crypto_stream_salsa208_amd64_xmm6_xor +#define crypto_stream_salsa208_beforenm crypto_stream_salsa208_amd64_xmm6_beforenm +#define crypto_stream_salsa208_afternm crypto_stream_salsa208_amd64_xmm6_afternm +#define crypto_stream_salsa208_xor_afternm crypto_stream_salsa208_amd64_xmm6_xor_afternm +#define crypto_stream_salsa208_KEYBYTES crypto_stream_salsa208_amd64_xmm6_KEYBYTES +#define crypto_stream_salsa208_NONCEBYTES crypto_stream_salsa208_amd64_xmm6_NONCEBYTES +#define crypto_stream_salsa208_BEFORENMBYTES crypto_stream_salsa208_amd64_xmm6_BEFORENMBYTES +#define crypto_stream_salsa208_IMPLEMENTATION "crypto_stream/salsa208/amd64_xmm6" +#ifndef crypto_stream_salsa208_amd64_xmm6_VERSION +#define crypto_stream_salsa208_amd64_xmm6_VERSION "-" +#endif +#define crypto_stream_salsa208_VERSION crypto_stream_salsa208_amd64_xmm6_VERSION + +#endif diff --git a/ext/bin/cnacl-osx-amd64/include/sodium/crypto_stream_xsalsa20.h b/ext/bin/cnacl-osx-amd64/include/sodium/crypto_stream_xsalsa20.h new file mode 100644 index 00000000..d75268c6 --- /dev/null +++ b/ext/bin/cnacl-osx-amd64/include/sodium/crypto_stream_xsalsa20.h @@ -0,0 +1,34 @@ +#ifndef crypto_stream_xsalsa20_H +#define crypto_stream_xsalsa20_H + +#define crypto_stream_xsalsa20_ref_KEYBYTES 32 +#define crypto_stream_xsalsa20_ref_NONCEBYTES 24 +#ifdef __cplusplus +#include +extern std::string crypto_stream_xsalsa20_ref(size_t,const std::string &,const std::string &); +extern std::string crypto_stream_xsalsa20_ref_xor(const std::string &,const std::string &,const std::string &); +extern "C" { +#endif +extern int crypto_stream_xsalsa20_ref(unsigned char *,unsigned long long,const unsigned char *,const unsigned char *); +extern int crypto_stream_xsalsa20_ref_xor(unsigned char *,const unsigned char *,unsigned long long,const unsigned char *,const unsigned char *); +extern int crypto_stream_xsalsa20_ref_beforenm(unsigned char *,const unsigned char *); +extern int crypto_stream_xsalsa20_ref_afternm(unsigned char *,unsigned long long,const unsigned char *,const unsigned char *); +extern int crypto_stream_xsalsa20_ref_xor_afternm(unsigned char *,const unsigned char *,unsigned long long,const unsigned char *,const unsigned char *); +#ifdef __cplusplus +} +#endif +#define crypto_stream_xsalsa20 crypto_stream_xsalsa20_ref +#define crypto_stream_xsalsa20_xor crypto_stream_xsalsa20_ref_xor +#define crypto_stream_xsalsa20_beforenm crypto_stream_xsalsa20_ref_beforenm +#define crypto_stream_xsalsa20_afternm crypto_stream_xsalsa20_ref_afternm +#define crypto_stream_xsalsa20_xor_afternm crypto_stream_xsalsa20_ref_xor_afternm +#define crypto_stream_xsalsa20_KEYBYTES crypto_stream_xsalsa20_ref_KEYBYTES +#define crypto_stream_xsalsa20_NONCEBYTES crypto_stream_xsalsa20_ref_NONCEBYTES +#define crypto_stream_xsalsa20_BEFORENMBYTES crypto_stream_xsalsa20_ref_BEFORENMBYTES +#define crypto_stream_xsalsa20_IMPLEMENTATION "crypto_stream/xsalsa20/ref" +#ifndef crypto_stream_xsalsa20_ref_VERSION +#define crypto_stream_xsalsa20_ref_VERSION "-" +#endif +#define crypto_stream_xsalsa20_VERSION crypto_stream_xsalsa20_ref_VERSION + +#endif diff --git a/ext/bin/cnacl-osx-amd64/include/sodium/crypto_types.h b/ext/bin/cnacl-osx-amd64/include/sodium/crypto_types.h new file mode 100644 index 00000000..b0ce9656 --- /dev/null +++ b/ext/bin/cnacl-osx-amd64/include/sodium/crypto_types.h @@ -0,0 +1,11 @@ +#ifndef crypto_types_h +#define crypto_types_h +typedef short crypto_int16; +typedef int crypto_int32; +typedef long long crypto_int64; +typedef signed char crypto_int8; +typedef unsigned short crypto_uint16; +typedef unsigned int crypto_uint32; +typedef unsigned long long crypto_uint64; +typedef unsigned char crypto_uint8; +#endif diff --git a/ext/bin/cnacl-osx-amd64/include/sodium/crypto_verify_16.h b/ext/bin/cnacl-osx-amd64/include/sodium/crypto_verify_16.h new file mode 100644 index 00000000..6bf6ca11 --- /dev/null +++ b/ext/bin/cnacl-osx-amd64/include/sodium/crypto_verify_16.h @@ -0,0 +1,21 @@ +#ifndef crypto_verify_16_H +#define crypto_verify_16_H + +#define crypto_verify_16_ref_BYTES 16 +#ifdef __cplusplus +#include +extern "C" { +#endif +extern int crypto_verify_16_ref(const unsigned char *,const unsigned char *); +#ifdef __cplusplus +} +#endif +#define crypto_verify_16 crypto_verify_16_ref +#define crypto_verify_16_BYTES crypto_verify_16_ref_BYTES +#define crypto_verify_16_IMPLEMENTATION "crypto_verify/16/ref" +#ifndef crypto_verify_16_ref_VERSION +#define crypto_verify_16_ref_VERSION "-" +#endif +#define crypto_verify_16_VERSION crypto_verify_16_ref_VERSION + +#endif diff --git a/ext/bin/cnacl-osx-amd64/include/sodium/crypto_verify_32.h b/ext/bin/cnacl-osx-amd64/include/sodium/crypto_verify_32.h new file mode 100644 index 00000000..bd5fc644 --- /dev/null +++ b/ext/bin/cnacl-osx-amd64/include/sodium/crypto_verify_32.h @@ -0,0 +1,21 @@ +#ifndef crypto_verify_32_H +#define crypto_verify_32_H + +#define crypto_verify_32_ref_BYTES 32 +#ifdef __cplusplus +#include +extern "C" { +#endif +extern int crypto_verify_32_ref(const unsigned char *,const unsigned char *); +#ifdef __cplusplus +} +#endif +#define crypto_verify_32 crypto_verify_32_ref +#define crypto_verify_32_BYTES crypto_verify_32_ref_BYTES +#define crypto_verify_32_IMPLEMENTATION "crypto_verify/32/ref" +#ifndef crypto_verify_32_ref_VERSION +#define crypto_verify_32_ref_VERSION "-" +#endif +#define crypto_verify_32_VERSION crypto_verify_32_ref_VERSION + +#endif diff --git a/make-mac.mk b/make-mac.mk index 8ff1b772..b71ca2fe 100644 --- a/make-mac.mk +++ b/make-mac.mk @@ -33,6 +33,12 @@ else DEFS+=-DZT_SOFTWARE_UPDATE_DEFAULT="\"download\"" endif +# Use precompiled extremely fast Salsa20/12 from "cnacl" included in ext/bin +# See https://github.com/cjdelisle/cnacl +DEFS+=-DZT_USE_LIBSODIUM +CFLAGS+=-Iext/bin/cnacl-osx-amd64/include +LIBS+=ext/bin/cnacl-osx-amd64/libnacl.a + ifeq ($(ZT_ENABLE_CLUSTER),1) DEFS+=-DZT_ENABLE_CLUSTER endif diff --git a/node/Identity.cpp b/node/Identity.cpp index 89fdb836..d1b21e9c 100644 --- a/node/Identity.cpp +++ b/node/Identity.cpp @@ -45,7 +45,7 @@ static inline void _computeMemoryHardHash(const void *publicKey,unsigned int pub // ordinary Salsa20 is randomly seekable. This is good for a cipher // but is not what we want for sequential memory-harndess. memset(genmem,0,ZT_IDENTITY_GEN_MEMORY); - Salsa20 s20(digest,256,(char *)digest + 32); + Salsa20 s20(digest,(char *)digest + 32); s20.crypt20((char *)genmem,(char *)genmem,64); for(unsigned long i=64;i> (32 - (c)))) #define XOR(v,w) ((v) ^ (w)) #define PLUS(v,w) ((uint32_t)((v) + (w))) @@ -66,8 +68,7 @@ static const _s20sseconsts _S20SSECONSTANTS; namespace ZeroTier { -void Salsa20::init(const void *key,unsigned int kbits,const void *iv) - throw() +void Salsa20::init(const void *key,const void *iv) { #ifdef ZT_SALSA20_SSE const uint32_t *k = (const uint32_t *)key; @@ -78,14 +79,9 @@ void Salsa20::init(const void *key,unsigned int kbits,const void *iv) _state.i[10] = k[1]; _state.i[7] = k[2]; _state.i[4] = k[3]; - if (kbits == 256) { - k += 4; - _state.i[1] = 0x3320646e; - _state.i[2] = 0x79622d32; - } else { - _state.i[1] = 0x3120646e; - _state.i[2] = 0x79622d36; - } + k += 4; + _state.i[1] = 0x3320646e; + _state.i[2] = 0x79622d32; _state.i[15] = k[0]; _state.i[12] = k[1]; _state.i[9] = k[2]; @@ -95,19 +91,14 @@ void Salsa20::init(const void *key,unsigned int kbits,const void *iv) _state.i[5] = 0; _state.i[8] = 0; #else - const char *constants; + const char *const constants = "expand 32-byte k"; const uint8_t *k = (const uint8_t *)key; _state.i[1] = U8TO32_LITTLE(k + 0); _state.i[2] = U8TO32_LITTLE(k + 4); _state.i[3] = U8TO32_LITTLE(k + 8); _state.i[4] = U8TO32_LITTLE(k + 12); - if (kbits == 256) { /* recommended */ - k += 16; - constants = "expand 32-byte k"; - } else { /* kbits == 128 */ - constants = "expand 16-byte k"; - } + k += 16; _state.i[5] = U8TO32_LITTLE(constants + 4); _state.i[6] = U8TO32_LITTLE(((const uint8_t *)iv) + 0); _state.i[7] = U8TO32_LITTLE(((const uint8_t *)iv) + 4); @@ -124,7 +115,6 @@ void Salsa20::init(const void *key,unsigned int kbits,const void *iv) } void Salsa20::crypt12(const void *in,void *out,unsigned int bytes) - throw() { uint8_t tmp[64]; const uint8_t *m = (const uint8_t *)in; @@ -624,7 +614,6 @@ void Salsa20::crypt12(const void *in,void *out,unsigned int bytes) } void Salsa20::crypt20(const void *in,void *out,unsigned int bytes) - throw() { uint8_t tmp[64]; const uint8_t *m = (const uint8_t *)in; @@ -1356,3 +1345,5 @@ void Salsa20::crypt20(const void *in,void *out,unsigned int bytes) } } // namespace ZeroTier + +#endif // !ZT_USE_LIBSODIUM diff --git a/node/Salsa20.hpp b/node/Salsa20.hpp index 6405d450..5e4c68be 100644 --- a/node/Salsa20.hpp +++ b/node/Salsa20.hpp @@ -10,10 +10,82 @@ #include #include #include +#include #include "Constants.hpp" #include "Utils.hpp" +#ifdef ZT_USE_LIBSODIUM + +#include +#include + +namespace ZeroTier { + +/** + * Salsa20 stream cipher + */ +class Salsa20 +{ +public: + Salsa20() {} + ~Salsa20() { Utils::burn(_k,sizeof(_k)); } + + /** + * @param key 256-bit (32 byte) key + * @param iv 64-bit initialization vector + */ + Salsa20(const void *key,const void *iv) + { + memcpy(_k,key,32); + memcpy(&_iv,iv,8); + } + + /** + * Initialize cipher + * + * @param key Key bits + * @param iv 64-bit initialization vector + */ + inline void init(const void *key,const void *iv) + { + memcpy(_k,key,32); + memcpy(&_iv,iv,8); + } + + /** + * Encrypt/decrypt data using Salsa20/12 + * + * @param in Input data + * @param out Output buffer + * @param bytes Length of data + */ + inline void crypt12(const void *in,void *out,unsigned int bytes) + { + crypto_stream_salsa2012_xor(reinterpret_cast(out),reinterpret_cast(in),bytes,reinterpret_cast(&_iv),reinterpret_cast(_k)); + } + + /** + * Encrypt/decrypt data using Salsa20/20 + * + * @param in Input data + * @param out Output buffer + * @param bytes Length of data + */ + inline void crypt20(const void *in,void *out,unsigned int bytes) + { + crypto_stream_salsa20_xor(reinterpret_cast(out),reinterpret_cast(in),bytes,reinterpret_cast(&_iv),reinterpret_cast(_k)); + } + +private: + uint64_t _k[4]; + uint64_t _iv; +}; + +} // namespace ZeroTier + +#else // !ZT_USE_LIBSODIUM + #if (!defined(ZT_SALSA20_SSE)) && (defined(__SSE2__) || defined(__WINDOWS__)) #define ZT_SALSA20_SSE 1 #endif @@ -30,30 +102,25 @@ namespace ZeroTier { class Salsa20 { public: - Salsa20() throw() {} - + Salsa20() {} ~Salsa20() { Utils::burn(&_state,sizeof(_state)); } /** - * @param key Key bits - * @param kbits Number of key bits: 128 or 256 (recommended) + * @param key 256-bit (32 byte) key * @param iv 64-bit initialization vector */ - Salsa20(const void *key,unsigned int kbits,const void *iv) - throw() + Salsa20(const void *key,const void *iv) { - init(key,kbits,iv); + init(key,iv); } /** * Initialize cipher * * @param key Key bits - * @param kbits Number of key bits: 128 or 256 (recommended) * @param iv 64-bit initialization vector */ - void init(const void *key,unsigned int kbits,const void *iv) - throw(); + void init(const void *key,const void *iv); /** * Encrypt/decrypt data using Salsa20/12 @@ -62,8 +129,7 @@ public: * @param out Output buffer * @param bytes Length of data */ - void crypt12(const void *in,void *out,unsigned int bytes) - throw(); + void crypt12(const void *in,void *out,unsigned int bytes); /** * Encrypt/decrypt data using Salsa20/20 @@ -72,8 +138,7 @@ public: * @param out Output buffer * @param bytes Length of data */ - void crypt20(const void *in,void *out,unsigned int bytes) - throw(); + void crypt20(const void *in,void *out,unsigned int bytes); private: union { @@ -86,4 +151,6 @@ private: } // namespace ZeroTier +#endif // ZT_USE_LIBSODIUM + #endif diff --git a/node/Utils.cpp b/node/Utils.cpp index fb448dd6..92d14d19 100644 --- a/node/Utils.cpp +++ b/node/Utils.cpp @@ -156,7 +156,7 @@ void Utils::getSecureRandom(void *buf,unsigned int bytes) s20Key[1] = (uint64_t)buf; // address of buf s20Key[2] = (uint64_t)s20Key; // address of s20Key[] s20Key[3] = (uint64_t)&s20; // address of s20 - s20.init(s20Key,256,s20Key); + s20.init(s20Key,s20Key); } #ifdef __WINDOWS__ diff --git a/selftest.cpp b/selftest.cpp index 48625d53..fe0aa933 100644 --- a/selftest.cpp +++ b/selftest.cpp @@ -153,16 +153,16 @@ static int testCrypto() memset(buf2,0,sizeof(buf2)); memset(buf3,0,sizeof(buf3)); Salsa20 s20; - s20.init("12345678123456781234567812345678",256,"12345678"); + s20.init("12345678123456781234567812345678","12345678"); s20.crypt20(buf1,buf2,sizeof(buf1)); - s20.init("12345678123456781234567812345678",256,"12345678"); + s20.init("12345678123456781234567812345678","12345678"); s20.crypt20(buf2,buf3,sizeof(buf2)); if (memcmp(buf1,buf3,sizeof(buf1))) { std::cout << "FAIL (encrypt/decrypt test)" << std::endl; return -1; } } - Salsa20 s20(s20TV0Key,256,s20TV0Iv); + Salsa20 s20(s20TV0Key,s20TV0Iv); memset(buf1,0,sizeof(buf1)); memset(buf2,0,sizeof(buf2)); s20.crypt20(buf1,buf2,64); @@ -170,7 +170,7 @@ static int testCrypto() std::cout << "FAIL (test vector 0)" << std::endl; return -1; } - s20.init(s2012TV0Key,256,s2012TV0Iv); + s20.init(s2012TV0Key,s2012TV0Iv); memset(buf1,0,sizeof(buf1)); memset(buf2,0,sizeof(buf2)); s20.crypt12(buf1,buf2,64); @@ -191,7 +191,7 @@ static int testCrypto() unsigned char *bb = (unsigned char *)::malloc(1234567); for(unsigned int i=0;i<1234567;++i) bb[i] = (unsigned char)i; - Salsa20 s20(s20TV0Key,256,s20TV0Iv); + Salsa20 s20(s20TV0Key,s20TV0Iv); double bytes = 0.0; uint64_t start = OSUtils::now(); for(unsigned int i=0;i<200;++i) { @@ -209,7 +209,7 @@ static int testCrypto() unsigned char *bb = (unsigned char *)::malloc(1234567); for(unsigned int i=0;i<1234567;++i) bb[i] = (unsigned char)i; - Salsa20 s20(s20TV0Key,256,s20TV0Iv); + Salsa20 s20(s20TV0Key,s20TV0Iv); double bytes = 0.0; uint64_t start = OSUtils::now(); for(unsigned int i=0;i<200;++i) { -- cgit v1.2.3 From 7a94f6305812b7ea5748283a6ec9503f4ea9c7e1 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Mon, 17 Apr 2017 17:54:12 -0700 Subject: Back out NaCl since the old one with xmm6 salsa2012 does not support multi-block use and the new one is slower. --- ext/bin/cnacl-osx-amd64/README.md | 53 -------------- .../include/sodium/crypto_auth_hmacsha256.h | 27 ------- .../include/sodium/crypto_auth_hmacsha512256.h | 27 ------- .../sodium/crypto_box_curve25519xsalsa20poly1305.h | 44 ------------ .../include/sodium/crypto_core_hsalsa20.h | 27 ------- .../include/sodium/crypto_core_salsa20.h | 27 ------- .../include/sodium/crypto_core_salsa2012.h | 27 ------- .../include/sodium/crypto_core_salsa208.h | 27 ------- .../include/sodium/crypto_hash_sha256.h | 22 ------ .../include/sodium/crypto_hash_sha512.h | 22 ------ .../include/sodium/crypto_hashblocks_sha256.h | 23 ------ .../include/sodium/crypto_hashblocks_sha512.h | 23 ------ .../include/sodium/crypto_onetimeauth_poly1305.h | 27 ------- .../include/sodium/crypto_scalarmult_curve25519.h | 27 ------- .../sodium/crypto_secretbox_xsalsa20poly1305.h | 31 -------- .../sodium/crypto_sign_edwards25519sha512batch.h | 32 --------- .../include/sodium/crypto_stream_aes128ctr.h | 35 --------- .../include/sodium/crypto_stream_salsa20.h | 34 --------- .../include/sodium/crypto_stream_salsa2012.h | 34 --------- .../include/sodium/crypto_stream_salsa208.h | 34 --------- .../include/sodium/crypto_stream_xsalsa20.h | 34 --------- .../cnacl-osx-amd64/include/sodium/crypto_types.h | 11 --- .../include/sodium/crypto_verify_16.h | 21 ------ .../include/sodium/crypto_verify_32.h | 21 ------ ext/bin/cnacl-osx-amd64/libnacl.a | Bin 401672 -> 0 bytes make-mac.mk | 6 -- node/Node.cpp | 22 +++--- node/Node.hpp | 8 +-- node/Salsa20.cpp | 4 -- node/Salsa20.hpp | 78 ++------------------- node/Utils.cpp | 2 + 31 files changed, 18 insertions(+), 792 deletions(-) delete mode 100644 ext/bin/cnacl-osx-amd64/README.md delete mode 100644 ext/bin/cnacl-osx-amd64/include/sodium/crypto_auth_hmacsha256.h delete mode 100644 ext/bin/cnacl-osx-amd64/include/sodium/crypto_auth_hmacsha512256.h delete mode 100644 ext/bin/cnacl-osx-amd64/include/sodium/crypto_box_curve25519xsalsa20poly1305.h delete mode 100644 ext/bin/cnacl-osx-amd64/include/sodium/crypto_core_hsalsa20.h delete mode 100644 ext/bin/cnacl-osx-amd64/include/sodium/crypto_core_salsa20.h delete mode 100644 ext/bin/cnacl-osx-amd64/include/sodium/crypto_core_salsa2012.h delete mode 100644 ext/bin/cnacl-osx-amd64/include/sodium/crypto_core_salsa208.h delete mode 100644 ext/bin/cnacl-osx-amd64/include/sodium/crypto_hash_sha256.h delete mode 100644 ext/bin/cnacl-osx-amd64/include/sodium/crypto_hash_sha512.h delete mode 100644 ext/bin/cnacl-osx-amd64/include/sodium/crypto_hashblocks_sha256.h delete mode 100644 ext/bin/cnacl-osx-amd64/include/sodium/crypto_hashblocks_sha512.h delete mode 100644 ext/bin/cnacl-osx-amd64/include/sodium/crypto_onetimeauth_poly1305.h delete mode 100644 ext/bin/cnacl-osx-amd64/include/sodium/crypto_scalarmult_curve25519.h delete mode 100644 ext/bin/cnacl-osx-amd64/include/sodium/crypto_secretbox_xsalsa20poly1305.h delete mode 100644 ext/bin/cnacl-osx-amd64/include/sodium/crypto_sign_edwards25519sha512batch.h delete mode 100644 ext/bin/cnacl-osx-amd64/include/sodium/crypto_stream_aes128ctr.h delete mode 100644 ext/bin/cnacl-osx-amd64/include/sodium/crypto_stream_salsa20.h delete mode 100644 ext/bin/cnacl-osx-amd64/include/sodium/crypto_stream_salsa2012.h delete mode 100644 ext/bin/cnacl-osx-amd64/include/sodium/crypto_stream_salsa208.h delete mode 100644 ext/bin/cnacl-osx-amd64/include/sodium/crypto_stream_xsalsa20.h delete mode 100644 ext/bin/cnacl-osx-amd64/include/sodium/crypto_types.h delete mode 100644 ext/bin/cnacl-osx-amd64/include/sodium/crypto_verify_16.h delete mode 100644 ext/bin/cnacl-osx-amd64/include/sodium/crypto_verify_32.h delete mode 100644 ext/bin/cnacl-osx-amd64/libnacl.a (limited to 'make-mac.mk') diff --git a/ext/bin/cnacl-osx-amd64/README.md b/ext/bin/cnacl-osx-amd64/README.md deleted file mode 100644 index 35426286..00000000 --- a/ext/bin/cnacl-osx-amd64/README.md +++ /dev/null @@ -1,53 +0,0 @@ -# cNaCl - - If you would like to be confusing, you could pronounce it sea-salt - -This is a fork NaCl by Daniel J. Bernstein and Tanja Lange. -The build has been ported to cmake so it can be cross compiled and build output is reliable. -Since it uses cmake, it could theoretically be built on windows but this has not been tested. -It does compile using mingw32. - -## How do I make this thing work? - - mkdir cbuild - cd cbuild - cmake .. - make - -## Ok now how about cross compiling? - - mkdir cbuildw32 - cd cbuildw32 - cmake -DCMAKE_TOOLCHAIN_FILE=../CMakeWindows.txt .. - make - -## Why fork? - -NaCl builds using a shell script called `./do`. This script does compiling, testing, measuring -and selection of the best implementation of each algorithm for the given machine. It also generates -the header files which will be used. - -The problems with `./do` are it's slow, it tries compiling with multiple different compiler -profiles, it's very platform independent but it doesn't run on Windows and most importantly, with -compiling, testing and measuring so tightly bound, it is impossible to cross compile for a -different operating system. - - -## How it works - -The first time you build for a new ABI, it will trigger the traditional nacl `./do` script. -What cNaCl does is parse the resulting headers from the `./do` build and create a plan so that it -can repeat roughly the same build. - -If there is already a plan for the given ABI, the build uses this plan and the build is very fast. - -Plans are stored in `./cmake/plans/` and I will be adding plans as I find new ones. - - -## What else is new? - -There is a problem with the `./do` build which prevents it from running on some ARM based machines, -this was fixed by adding a more lax method for measuring CPU speed as a fall back. - - -`#EOF#` diff --git a/ext/bin/cnacl-osx-amd64/include/sodium/crypto_auth_hmacsha256.h b/ext/bin/cnacl-osx-amd64/include/sodium/crypto_auth_hmacsha256.h deleted file mode 100644 index 6b5600f3..00000000 --- a/ext/bin/cnacl-osx-amd64/include/sodium/crypto_auth_hmacsha256.h +++ /dev/null @@ -1,27 +0,0 @@ -#ifndef crypto_auth_hmacsha256_H -#define crypto_auth_hmacsha256_H - -#define crypto_auth_hmacsha256_ref_BYTES 32 -#define crypto_auth_hmacsha256_ref_KEYBYTES 32 -#ifdef __cplusplus -#include -extern std::string crypto_auth_hmacsha256_ref(const std::string &,const std::string &); -extern void crypto_auth_hmacsha256_ref_verify(const std::string &,const std::string &,const std::string &); -extern "C" { -#endif -extern int crypto_auth_hmacsha256_ref(unsigned char *,const unsigned char *,unsigned long long,const unsigned char *); -extern int crypto_auth_hmacsha256_ref_verify(const unsigned char *,const unsigned char *,unsigned long long,const unsigned char *); -#ifdef __cplusplus -} -#endif -#define crypto_auth_hmacsha256 crypto_auth_hmacsha256_ref -#define crypto_auth_hmacsha256_verify crypto_auth_hmacsha256_ref_verify -#define crypto_auth_hmacsha256_BYTES crypto_auth_hmacsha256_ref_BYTES -#define crypto_auth_hmacsha256_KEYBYTES crypto_auth_hmacsha256_ref_KEYBYTES -#define crypto_auth_hmacsha256_IMPLEMENTATION "crypto_auth/hmacsha256/ref" -#ifndef crypto_auth_hmacsha256_ref_VERSION -#define crypto_auth_hmacsha256_ref_VERSION "-" -#endif -#define crypto_auth_hmacsha256_VERSION crypto_auth_hmacsha256_ref_VERSION - -#endif diff --git a/ext/bin/cnacl-osx-amd64/include/sodium/crypto_auth_hmacsha512256.h b/ext/bin/cnacl-osx-amd64/include/sodium/crypto_auth_hmacsha512256.h deleted file mode 100644 index c9bd96e4..00000000 --- a/ext/bin/cnacl-osx-amd64/include/sodium/crypto_auth_hmacsha512256.h +++ /dev/null @@ -1,27 +0,0 @@ -#ifndef crypto_auth_hmacsha512256_H -#define crypto_auth_hmacsha512256_H - -#define crypto_auth_hmacsha512256_ref_BYTES 32 -#define crypto_auth_hmacsha512256_ref_KEYBYTES 32 -#ifdef __cplusplus -#include -extern std::string crypto_auth_hmacsha512256_ref(const std::string &,const std::string &); -extern void crypto_auth_hmacsha512256_ref_verify(const std::string &,const std::string &,const std::string &); -extern "C" { -#endif -extern int crypto_auth_hmacsha512256_ref(unsigned char *,const unsigned char *,unsigned long long,const unsigned char *); -extern int crypto_auth_hmacsha512256_ref_verify(const unsigned char *,const unsigned char *,unsigned long long,const unsigned char *); -#ifdef __cplusplus -} -#endif -#define crypto_auth_hmacsha512256 crypto_auth_hmacsha512256_ref -#define crypto_auth_hmacsha512256_verify crypto_auth_hmacsha512256_ref_verify -#define crypto_auth_hmacsha512256_BYTES crypto_auth_hmacsha512256_ref_BYTES -#define crypto_auth_hmacsha512256_KEYBYTES crypto_auth_hmacsha512256_ref_KEYBYTES -#define crypto_auth_hmacsha512256_IMPLEMENTATION "crypto_auth/hmacsha512256/ref" -#ifndef crypto_auth_hmacsha512256_ref_VERSION -#define crypto_auth_hmacsha512256_ref_VERSION "-" -#endif -#define crypto_auth_hmacsha512256_VERSION crypto_auth_hmacsha512256_ref_VERSION - -#endif diff --git a/ext/bin/cnacl-osx-amd64/include/sodium/crypto_box_curve25519xsalsa20poly1305.h b/ext/bin/cnacl-osx-amd64/include/sodium/crypto_box_curve25519xsalsa20poly1305.h deleted file mode 100644 index e2c3b4cc..00000000 --- a/ext/bin/cnacl-osx-amd64/include/sodium/crypto_box_curve25519xsalsa20poly1305.h +++ /dev/null @@ -1,44 +0,0 @@ -#ifndef crypto_box_curve25519xsalsa20poly1305_H -#define crypto_box_curve25519xsalsa20poly1305_H - -#define crypto_box_curve25519xsalsa20poly1305_ref_PUBLICKEYBYTES 32 -#define crypto_box_curve25519xsalsa20poly1305_ref_SECRETKEYBYTES 32 -#define crypto_box_curve25519xsalsa20poly1305_ref_BEFORENMBYTES 32 -#define crypto_box_curve25519xsalsa20poly1305_ref_NONCEBYTES 24 -#define crypto_box_curve25519xsalsa20poly1305_ref_ZEROBYTES 32 -#define crypto_box_curve25519xsalsa20poly1305_ref_BOXZEROBYTES 16 -#ifdef __cplusplus -#include -extern std::string crypto_box_curve25519xsalsa20poly1305_ref(const std::string &,const std::string &,const std::string &,const std::string &); -extern std::string crypto_box_curve25519xsalsa20poly1305_ref_open(const std::string &,const std::string &,const std::string &,const std::string &); -extern std::string crypto_box_curve25519xsalsa20poly1305_ref_keypair(std::string *); -extern "C" { -#endif -extern int crypto_box_curve25519xsalsa20poly1305_ref(unsigned char *,const unsigned char *,unsigned long long,const unsigned char *,const unsigned char *,const unsigned char *); -extern int crypto_box_curve25519xsalsa20poly1305_ref_open(unsigned char *,const unsigned char *,unsigned long long,const unsigned char *,const unsigned char *,const unsigned char *); -extern int crypto_box_curve25519xsalsa20poly1305_ref_keypair(unsigned char *,unsigned char *); -extern int crypto_box_curve25519xsalsa20poly1305_ref_beforenm(unsigned char *,const unsigned char *,const unsigned char *); -extern int crypto_box_curve25519xsalsa20poly1305_ref_afternm(unsigned char *,const unsigned char *,unsigned long long,const unsigned char *,const unsigned char *); -extern int crypto_box_curve25519xsalsa20poly1305_ref_open_afternm(unsigned char *,const unsigned char *,unsigned long long,const unsigned char *,const unsigned char *); -#ifdef __cplusplus -} -#endif -#define crypto_box_curve25519xsalsa20poly1305 crypto_box_curve25519xsalsa20poly1305_ref -#define crypto_box_curve25519xsalsa20poly1305_open crypto_box_curve25519xsalsa20poly1305_ref_open -#define crypto_box_curve25519xsalsa20poly1305_keypair crypto_box_curve25519xsalsa20poly1305_ref_keypair -#define crypto_box_curve25519xsalsa20poly1305_beforenm crypto_box_curve25519xsalsa20poly1305_ref_beforenm -#define crypto_box_curve25519xsalsa20poly1305_afternm crypto_box_curve25519xsalsa20poly1305_ref_afternm -#define crypto_box_curve25519xsalsa20poly1305_open_afternm crypto_box_curve25519xsalsa20poly1305_ref_open_afternm -#define crypto_box_curve25519xsalsa20poly1305_PUBLICKEYBYTES crypto_box_curve25519xsalsa20poly1305_ref_PUBLICKEYBYTES -#define crypto_box_curve25519xsalsa20poly1305_SECRETKEYBYTES crypto_box_curve25519xsalsa20poly1305_ref_SECRETKEYBYTES -#define crypto_box_curve25519xsalsa20poly1305_BEFORENMBYTES crypto_box_curve25519xsalsa20poly1305_ref_BEFORENMBYTES -#define crypto_box_curve25519xsalsa20poly1305_NONCEBYTES crypto_box_curve25519xsalsa20poly1305_ref_NONCEBYTES -#define crypto_box_curve25519xsalsa20poly1305_ZEROBYTES crypto_box_curve25519xsalsa20poly1305_ref_ZEROBYTES -#define crypto_box_curve25519xsalsa20poly1305_BOXZEROBYTES crypto_box_curve25519xsalsa20poly1305_ref_BOXZEROBYTES -#define crypto_box_curve25519xsalsa20poly1305_IMPLEMENTATION "crypto_box/curve25519xsalsa20poly1305/ref" -#ifndef crypto_box_curve25519xsalsa20poly1305_ref_VERSION -#define crypto_box_curve25519xsalsa20poly1305_ref_VERSION "-" -#endif -#define crypto_box_curve25519xsalsa20poly1305_VERSION crypto_box_curve25519xsalsa20poly1305_ref_VERSION - -#endif diff --git a/ext/bin/cnacl-osx-amd64/include/sodium/crypto_core_hsalsa20.h b/ext/bin/cnacl-osx-amd64/include/sodium/crypto_core_hsalsa20.h deleted file mode 100644 index abae188e..00000000 --- a/ext/bin/cnacl-osx-amd64/include/sodium/crypto_core_hsalsa20.h +++ /dev/null @@ -1,27 +0,0 @@ -#ifndef crypto_core_hsalsa20_H -#define crypto_core_hsalsa20_H - -#define crypto_core_hsalsa20_ref_OUTPUTBYTES 32 -#define crypto_core_hsalsa20_ref_INPUTBYTES 16 -#define crypto_core_hsalsa20_ref_KEYBYTES 32 -#define crypto_core_hsalsa20_ref_CONSTBYTES 16 -#ifdef __cplusplus -#include -extern "C" { -#endif -extern int crypto_core_hsalsa20_ref(unsigned char *,const unsigned char *,const unsigned char *,const unsigned char *); -#ifdef __cplusplus -} -#endif -#define crypto_core_hsalsa20 crypto_core_hsalsa20_ref -#define crypto_core_hsalsa20_OUTPUTBYTES crypto_core_hsalsa20_ref_OUTPUTBYTES -#define crypto_core_hsalsa20_INPUTBYTES crypto_core_hsalsa20_ref_INPUTBYTES -#define crypto_core_hsalsa20_KEYBYTES crypto_core_hsalsa20_ref_KEYBYTES -#define crypto_core_hsalsa20_CONSTBYTES crypto_core_hsalsa20_ref_CONSTBYTES -#define crypto_core_hsalsa20_IMPLEMENTATION "crypto_core/hsalsa20/ref" -#ifndef crypto_core_hsalsa20_ref_VERSION -#define crypto_core_hsalsa20_ref_VERSION "-" -#endif -#define crypto_core_hsalsa20_VERSION crypto_core_hsalsa20_ref_VERSION - -#endif diff --git a/ext/bin/cnacl-osx-amd64/include/sodium/crypto_core_salsa20.h b/ext/bin/cnacl-osx-amd64/include/sodium/crypto_core_salsa20.h deleted file mode 100644 index 9737b101..00000000 --- a/ext/bin/cnacl-osx-amd64/include/sodium/crypto_core_salsa20.h +++ /dev/null @@ -1,27 +0,0 @@ -#ifndef crypto_core_salsa20_H -#define crypto_core_salsa20_H - -#define crypto_core_salsa20_ref_OUTPUTBYTES 64 -#define crypto_core_salsa20_ref_INPUTBYTES 16 -#define crypto_core_salsa20_ref_KEYBYTES 32 -#define crypto_core_salsa20_ref_CONSTBYTES 16 -#ifdef __cplusplus -#include -extern "C" { -#endif -extern int crypto_core_salsa20_ref(unsigned char *,const unsigned char *,const unsigned char *,const unsigned char *); -#ifdef __cplusplus -} -#endif -#define crypto_core_salsa20 crypto_core_salsa20_ref -#define crypto_core_salsa20_OUTPUTBYTES crypto_core_salsa20_ref_OUTPUTBYTES -#define crypto_core_salsa20_INPUTBYTES crypto_core_salsa20_ref_INPUTBYTES -#define crypto_core_salsa20_KEYBYTES crypto_core_salsa20_ref_KEYBYTES -#define crypto_core_salsa20_CONSTBYTES crypto_core_salsa20_ref_CONSTBYTES -#define crypto_core_salsa20_IMPLEMENTATION "crypto_core/salsa20/ref" -#ifndef crypto_core_salsa20_ref_VERSION -#define crypto_core_salsa20_ref_VERSION "-" -#endif -#define crypto_core_salsa20_VERSION crypto_core_salsa20_ref_VERSION - -#endif diff --git a/ext/bin/cnacl-osx-amd64/include/sodium/crypto_core_salsa2012.h b/ext/bin/cnacl-osx-amd64/include/sodium/crypto_core_salsa2012.h deleted file mode 100644 index 137cd47f..00000000 --- a/ext/bin/cnacl-osx-amd64/include/sodium/crypto_core_salsa2012.h +++ /dev/null @@ -1,27 +0,0 @@ -#ifndef crypto_core_salsa2012_H -#define crypto_core_salsa2012_H - -#define crypto_core_salsa2012_ref_OUTPUTBYTES 64 -#define crypto_core_salsa2012_ref_INPUTBYTES 16 -#define crypto_core_salsa2012_ref_KEYBYTES 32 -#define crypto_core_salsa2012_ref_CONSTBYTES 16 -#ifdef __cplusplus -#include -extern "C" { -#endif -extern int crypto_core_salsa2012_ref(unsigned char *,const unsigned char *,const unsigned char *,const unsigned char *); -#ifdef __cplusplus -} -#endif -#define crypto_core_salsa2012 crypto_core_salsa2012_ref -#define crypto_core_salsa2012_OUTPUTBYTES crypto_core_salsa2012_ref_OUTPUTBYTES -#define crypto_core_salsa2012_INPUTBYTES crypto_core_salsa2012_ref_INPUTBYTES -#define crypto_core_salsa2012_KEYBYTES crypto_core_salsa2012_ref_KEYBYTES -#define crypto_core_salsa2012_CONSTBYTES crypto_core_salsa2012_ref_CONSTBYTES -#define crypto_core_salsa2012_IMPLEMENTATION "crypto_core/salsa2012/ref" -#ifndef crypto_core_salsa2012_ref_VERSION -#define crypto_core_salsa2012_ref_VERSION "-" -#endif -#define crypto_core_salsa2012_VERSION crypto_core_salsa2012_ref_VERSION - -#endif diff --git a/ext/bin/cnacl-osx-amd64/include/sodium/crypto_core_salsa208.h b/ext/bin/cnacl-osx-amd64/include/sodium/crypto_core_salsa208.h deleted file mode 100644 index 4895bbbe..00000000 --- a/ext/bin/cnacl-osx-amd64/include/sodium/crypto_core_salsa208.h +++ /dev/null @@ -1,27 +0,0 @@ -#ifndef crypto_core_salsa208_H -#define crypto_core_salsa208_H - -#define crypto_core_salsa208_ref_OUTPUTBYTES 64 -#define crypto_core_salsa208_ref_INPUTBYTES 16 -#define crypto_core_salsa208_ref_KEYBYTES 32 -#define crypto_core_salsa208_ref_CONSTBYTES 16 -#ifdef __cplusplus -#include -extern "C" { -#endif -extern int crypto_core_salsa208_ref(unsigned char *,const unsigned char *,const unsigned char *,const unsigned char *); -#ifdef __cplusplus -} -#endif -#define crypto_core_salsa208 crypto_core_salsa208_ref -#define crypto_core_salsa208_OUTPUTBYTES crypto_core_salsa208_ref_OUTPUTBYTES -#define crypto_core_salsa208_INPUTBYTES crypto_core_salsa208_ref_INPUTBYTES -#define crypto_core_salsa208_KEYBYTES crypto_core_salsa208_ref_KEYBYTES -#define crypto_core_salsa208_CONSTBYTES crypto_core_salsa208_ref_CONSTBYTES -#define crypto_core_salsa208_IMPLEMENTATION "crypto_core/salsa208/ref" -#ifndef crypto_core_salsa208_ref_VERSION -#define crypto_core_salsa208_ref_VERSION "-" -#endif -#define crypto_core_salsa208_VERSION crypto_core_salsa208_ref_VERSION - -#endif diff --git a/ext/bin/cnacl-osx-amd64/include/sodium/crypto_hash_sha256.h b/ext/bin/cnacl-osx-amd64/include/sodium/crypto_hash_sha256.h deleted file mode 100644 index 20d18703..00000000 --- a/ext/bin/cnacl-osx-amd64/include/sodium/crypto_hash_sha256.h +++ /dev/null @@ -1,22 +0,0 @@ -#ifndef crypto_hash_sha256_H -#define crypto_hash_sha256_H - -#define crypto_hash_sha256_ref_BYTES 32 -#ifdef __cplusplus -#include -extern std::string crypto_hash_sha256_ref(const std::string &); -extern "C" { -#endif -extern int crypto_hash_sha256_ref(unsigned char *,const unsigned char *,unsigned long long); -#ifdef __cplusplus -} -#endif -#define crypto_hash_sha256 crypto_hash_sha256_ref -#define crypto_hash_sha256_BYTES crypto_hash_sha256_ref_BYTES -#define crypto_hash_sha256_IMPLEMENTATION "crypto_hash/sha256/ref" -#ifndef crypto_hash_sha256_ref_VERSION -#define crypto_hash_sha256_ref_VERSION "-" -#endif -#define crypto_hash_sha256_VERSION crypto_hash_sha256_ref_VERSION - -#endif diff --git a/ext/bin/cnacl-osx-amd64/include/sodium/crypto_hash_sha512.h b/ext/bin/cnacl-osx-amd64/include/sodium/crypto_hash_sha512.h deleted file mode 100644 index fe19d2d9..00000000 --- a/ext/bin/cnacl-osx-amd64/include/sodium/crypto_hash_sha512.h +++ /dev/null @@ -1,22 +0,0 @@ -#ifndef crypto_hash_sha512_H -#define crypto_hash_sha512_H - -#define crypto_hash_sha512_ref_BYTES 64 -#ifdef __cplusplus -#include -extern std::string crypto_hash_sha512_ref(const std::string &); -extern "C" { -#endif -extern int crypto_hash_sha512_ref(unsigned char *,const unsigned char *,unsigned long long); -#ifdef __cplusplus -} -#endif -#define crypto_hash_sha512 crypto_hash_sha512_ref -#define crypto_hash_sha512_BYTES crypto_hash_sha512_ref_BYTES -#define crypto_hash_sha512_IMPLEMENTATION "crypto_hash/sha512/ref" -#ifndef crypto_hash_sha512_ref_VERSION -#define crypto_hash_sha512_ref_VERSION "-" -#endif -#define crypto_hash_sha512_VERSION crypto_hash_sha512_ref_VERSION - -#endif diff --git a/ext/bin/cnacl-osx-amd64/include/sodium/crypto_hashblocks_sha256.h b/ext/bin/cnacl-osx-amd64/include/sodium/crypto_hashblocks_sha256.h deleted file mode 100644 index 3b473e6c..00000000 --- a/ext/bin/cnacl-osx-amd64/include/sodium/crypto_hashblocks_sha256.h +++ /dev/null @@ -1,23 +0,0 @@ -#ifndef crypto_hashblocks_sha256_H -#define crypto_hashblocks_sha256_H - -#define crypto_hashblocks_sha256_inplace_STATEBYTES 32 -#define crypto_hashblocks_sha256_inplace_BLOCKBYTES 64 -#ifdef __cplusplus -#include -extern "C" { -#endif -extern int crypto_hashblocks_sha256_inplace(unsigned char *,const unsigned char *,unsigned long long); -#ifdef __cplusplus -} -#endif -#define crypto_hashblocks_sha256 crypto_hashblocks_sha256_inplace -#define crypto_hashblocks_sha256_STATEBYTES crypto_hashblocks_sha256_inplace_STATEBYTES -#define crypto_hashblocks_sha256_BLOCKBYTES crypto_hashblocks_sha256_inplace_BLOCKBYTES -#define crypto_hashblocks_sha256_IMPLEMENTATION "crypto_hashblocks/sha256/inplace" -#ifndef crypto_hashblocks_sha256_inplace_VERSION -#define crypto_hashblocks_sha256_inplace_VERSION "-" -#endif -#define crypto_hashblocks_sha256_VERSION crypto_hashblocks_sha256_inplace_VERSION - -#endif diff --git a/ext/bin/cnacl-osx-amd64/include/sodium/crypto_hashblocks_sha512.h b/ext/bin/cnacl-osx-amd64/include/sodium/crypto_hashblocks_sha512.h deleted file mode 100644 index f66edd09..00000000 --- a/ext/bin/cnacl-osx-amd64/include/sodium/crypto_hashblocks_sha512.h +++ /dev/null @@ -1,23 +0,0 @@ -#ifndef crypto_hashblocks_sha512_H -#define crypto_hashblocks_sha512_H - -#define crypto_hashblocks_sha512_ref_STATEBYTES 64 -#define crypto_hashblocks_sha512_ref_BLOCKBYTES 128 -#ifdef __cplusplus -#include -extern "C" { -#endif -extern int crypto_hashblocks_sha512_ref(unsigned char *,const unsigned char *,unsigned long long); -#ifdef __cplusplus -} -#endif -#define crypto_hashblocks_sha512 crypto_hashblocks_sha512_ref -#define crypto_hashblocks_sha512_STATEBYTES crypto_hashblocks_sha512_ref_STATEBYTES -#define crypto_hashblocks_sha512_BLOCKBYTES crypto_hashblocks_sha512_ref_BLOCKBYTES -#define crypto_hashblocks_sha512_IMPLEMENTATION "crypto_hashblocks/sha512/ref" -#ifndef crypto_hashblocks_sha512_ref_VERSION -#define crypto_hashblocks_sha512_ref_VERSION "-" -#endif -#define crypto_hashblocks_sha512_VERSION crypto_hashblocks_sha512_ref_VERSION - -#endif diff --git a/ext/bin/cnacl-osx-amd64/include/sodium/crypto_onetimeauth_poly1305.h b/ext/bin/cnacl-osx-amd64/include/sodium/crypto_onetimeauth_poly1305.h deleted file mode 100644 index de08dc9f..00000000 --- a/ext/bin/cnacl-osx-amd64/include/sodium/crypto_onetimeauth_poly1305.h +++ /dev/null @@ -1,27 +0,0 @@ -#ifndef crypto_onetimeauth_poly1305_H -#define crypto_onetimeauth_poly1305_H - -#define crypto_onetimeauth_poly1305_53_BYTES 16 -#define crypto_onetimeauth_poly1305_53_KEYBYTES 32 -#ifdef __cplusplus -#include -extern std::string crypto_onetimeauth_poly1305_53(const std::string &,const std::string &); -extern void crypto_onetimeauth_poly1305_53_verify(const std::string &,const std::string &,const std::string &); -extern "C" { -#endif -extern int crypto_onetimeauth_poly1305_53(unsigned char *,const unsigned char *,unsigned long long,const unsigned char *); -extern int crypto_onetimeauth_poly1305_53_verify(const unsigned char *,const unsigned char *,unsigned long long,const unsigned char *); -#ifdef __cplusplus -} -#endif -#define crypto_onetimeauth_poly1305 crypto_onetimeauth_poly1305_53 -#define crypto_onetimeauth_poly1305_verify crypto_onetimeauth_poly1305_53_verify -#define crypto_onetimeauth_poly1305_BYTES crypto_onetimeauth_poly1305_53_BYTES -#define crypto_onetimeauth_poly1305_KEYBYTES crypto_onetimeauth_poly1305_53_KEYBYTES -#define crypto_onetimeauth_poly1305_IMPLEMENTATION "crypto_onetimeauth/poly1305/53" -#ifndef crypto_onetimeauth_poly1305_53_VERSION -#define crypto_onetimeauth_poly1305_53_VERSION "-" -#endif -#define crypto_onetimeauth_poly1305_VERSION crypto_onetimeauth_poly1305_53_VERSION - -#endif diff --git a/ext/bin/cnacl-osx-amd64/include/sodium/crypto_scalarmult_curve25519.h b/ext/bin/cnacl-osx-amd64/include/sodium/crypto_scalarmult_curve25519.h deleted file mode 100644 index 550c4e3d..00000000 --- a/ext/bin/cnacl-osx-amd64/include/sodium/crypto_scalarmult_curve25519.h +++ /dev/null @@ -1,27 +0,0 @@ -#ifndef crypto_scalarmult_curve25519_H -#define crypto_scalarmult_curve25519_H - -#define crypto_scalarmult_curve25519_donna_c64_BYTES 32 -#define crypto_scalarmult_curve25519_donna_c64_SCALARBYTES 32 -#ifdef __cplusplus -#include -extern std::string crypto_scalarmult_curve25519_donna_c64(const std::string &,const std::string &); -extern std::string crypto_scalarmult_curve25519_donna_c64_base(const std::string &); -extern "C" { -#endif -extern int crypto_scalarmult_curve25519_donna_c64(unsigned char *,const unsigned char *,const unsigned char *); -extern int crypto_scalarmult_curve25519_donna_c64_base(unsigned char *,const unsigned char *); -#ifdef __cplusplus -} -#endif -#define crypto_scalarmult_curve25519 crypto_scalarmult_curve25519_donna_c64 -#define crypto_scalarmult_curve25519_base crypto_scalarmult_curve25519_donna_c64_base -#define crypto_scalarmult_curve25519_BYTES crypto_scalarmult_curve25519_donna_c64_BYTES -#define crypto_scalarmult_curve25519_SCALARBYTES crypto_scalarmult_curve25519_donna_c64_SCALARBYTES -#define crypto_scalarmult_curve25519_IMPLEMENTATION "crypto_scalarmult/curve25519/donna_c64" -#ifndef crypto_scalarmult_curve25519_donna_c64_VERSION -#define crypto_scalarmult_curve25519_donna_c64_VERSION "-" -#endif -#define crypto_scalarmult_curve25519_VERSION crypto_scalarmult_curve25519_donna_c64_VERSION - -#endif diff --git a/ext/bin/cnacl-osx-amd64/include/sodium/crypto_secretbox_xsalsa20poly1305.h b/ext/bin/cnacl-osx-amd64/include/sodium/crypto_secretbox_xsalsa20poly1305.h deleted file mode 100644 index c930b6f1..00000000 --- a/ext/bin/cnacl-osx-amd64/include/sodium/crypto_secretbox_xsalsa20poly1305.h +++ /dev/null @@ -1,31 +0,0 @@ -#ifndef crypto_secretbox_xsalsa20poly1305_H -#define crypto_secretbox_xsalsa20poly1305_H - -#define crypto_secretbox_xsalsa20poly1305_ref_KEYBYTES 32 -#define crypto_secretbox_xsalsa20poly1305_ref_NONCEBYTES 24 -#define crypto_secretbox_xsalsa20poly1305_ref_ZEROBYTES 32 -#define crypto_secretbox_xsalsa20poly1305_ref_BOXZEROBYTES 16 -#ifdef __cplusplus -#include -extern std::string crypto_secretbox_xsalsa20poly1305_ref(const std::string &,const std::string &,const std::string &); -extern std::string crypto_secretbox_xsalsa20poly1305_ref_open(const std::string &,const std::string &,const std::string &); -extern "C" { -#endif -extern int crypto_secretbox_xsalsa20poly1305_ref(unsigned char *,const unsigned char *,unsigned long long,const unsigned char *,const unsigned char *); -extern int crypto_secretbox_xsalsa20poly1305_ref_open(unsigned char *,const unsigned char *,unsigned long long,const unsigned char *,const unsigned char *); -#ifdef __cplusplus -} -#endif -#define crypto_secretbox_xsalsa20poly1305 crypto_secretbox_xsalsa20poly1305_ref -#define crypto_secretbox_xsalsa20poly1305_open crypto_secretbox_xsalsa20poly1305_ref_open -#define crypto_secretbox_xsalsa20poly1305_KEYBYTES crypto_secretbox_xsalsa20poly1305_ref_KEYBYTES -#define crypto_secretbox_xsalsa20poly1305_NONCEBYTES crypto_secretbox_xsalsa20poly1305_ref_NONCEBYTES -#define crypto_secretbox_xsalsa20poly1305_ZEROBYTES crypto_secretbox_xsalsa20poly1305_ref_ZEROBYTES -#define crypto_secretbox_xsalsa20poly1305_BOXZEROBYTES crypto_secretbox_xsalsa20poly1305_ref_BOXZEROBYTES -#define crypto_secretbox_xsalsa20poly1305_IMPLEMENTATION "crypto_secretbox/xsalsa20poly1305/ref" -#ifndef crypto_secretbox_xsalsa20poly1305_ref_VERSION -#define crypto_secretbox_xsalsa20poly1305_ref_VERSION "-" -#endif -#define crypto_secretbox_xsalsa20poly1305_VERSION crypto_secretbox_xsalsa20poly1305_ref_VERSION - -#endif diff --git a/ext/bin/cnacl-osx-amd64/include/sodium/crypto_sign_edwards25519sha512batch.h b/ext/bin/cnacl-osx-amd64/include/sodium/crypto_sign_edwards25519sha512batch.h deleted file mode 100644 index 936108ef..00000000 --- a/ext/bin/cnacl-osx-amd64/include/sodium/crypto_sign_edwards25519sha512batch.h +++ /dev/null @@ -1,32 +0,0 @@ -#ifndef crypto_sign_edwards25519sha512batch_H -#define crypto_sign_edwards25519sha512batch_H - -#define crypto_sign_edwards25519sha512batch_ref_SECRETKEYBYTES 64 -#define crypto_sign_edwards25519sha512batch_ref_PUBLICKEYBYTES 32 -#define crypto_sign_edwards25519sha512batch_ref_BYTES 64 -#ifdef __cplusplus -#include -extern std::string crypto_sign_edwards25519sha512batch_ref(const std::string &,const std::string &); -extern std::string crypto_sign_edwards25519sha512batch_ref_open(const std::string &,const std::string &); -extern std::string crypto_sign_edwards25519sha512batch_ref_keypair(std::string *); -extern "C" { -#endif -extern int crypto_sign_edwards25519sha512batch_ref(unsigned char *,unsigned long long *,const unsigned char *,unsigned long long,const unsigned char *); -extern int crypto_sign_edwards25519sha512batch_ref_open(unsigned char *,unsigned long long *,const unsigned char *,unsigned long long,const unsigned char *); -extern int crypto_sign_edwards25519sha512batch_ref_keypair(unsigned char *,unsigned char *); -#ifdef __cplusplus -} -#endif -#define crypto_sign_edwards25519sha512batch crypto_sign_edwards25519sha512batch_ref -#define crypto_sign_edwards25519sha512batch_open crypto_sign_edwards25519sha512batch_ref_open -#define crypto_sign_edwards25519sha512batch_keypair crypto_sign_edwards25519sha512batch_ref_keypair -#define crypto_sign_edwards25519sha512batch_BYTES crypto_sign_edwards25519sha512batch_ref_BYTES -#define crypto_sign_edwards25519sha512batch_PUBLICKEYBYTES crypto_sign_edwards25519sha512batch_ref_PUBLICKEYBYTES -#define crypto_sign_edwards25519sha512batch_SECRETKEYBYTES crypto_sign_edwards25519sha512batch_ref_SECRETKEYBYTES -#define crypto_sign_edwards25519sha512batch_IMPLEMENTATION "crypto_sign/edwards25519sha512batch/ref" -#ifndef crypto_sign_edwards25519sha512batch_ref_VERSION -#define crypto_sign_edwards25519sha512batch_ref_VERSION "-" -#endif -#define crypto_sign_edwards25519sha512batch_VERSION crypto_sign_edwards25519sha512batch_ref_VERSION - -#endif diff --git a/ext/bin/cnacl-osx-amd64/include/sodium/crypto_stream_aes128ctr.h b/ext/bin/cnacl-osx-amd64/include/sodium/crypto_stream_aes128ctr.h deleted file mode 100644 index 76bf9137..00000000 --- a/ext/bin/cnacl-osx-amd64/include/sodium/crypto_stream_aes128ctr.h +++ /dev/null @@ -1,35 +0,0 @@ -#ifndef crypto_stream_aes128ctr_H -#define crypto_stream_aes128ctr_H - -#define crypto_stream_aes128ctr_portable_KEYBYTES 16 -#define crypto_stream_aes128ctr_portable_NONCEBYTES 16 -#define crypto_stream_aes128ctr_portable_BEFORENMBYTES 1408 -#ifdef __cplusplus -#include -extern std::string crypto_stream_aes128ctr_portable(size_t,const std::string &,const std::string &); -extern std::string crypto_stream_aes128ctr_portable_xor(const std::string &,const std::string &,const std::string &); -extern "C" { -#endif -extern int crypto_stream_aes128ctr_portable(unsigned char *,unsigned long long,const unsigned char *,const unsigned char *); -extern int crypto_stream_aes128ctr_portable_xor(unsigned char *,const unsigned char *,unsigned long long,const unsigned char *,const unsigned char *); -extern int crypto_stream_aes128ctr_portable_beforenm(unsigned char *,const unsigned char *); -extern int crypto_stream_aes128ctr_portable_afternm(unsigned char *,unsigned long long,const unsigned char *,const unsigned char *); -extern int crypto_stream_aes128ctr_portable_xor_afternm(unsigned char *,const unsigned char *,unsigned long long,const unsigned char *,const unsigned char *); -#ifdef __cplusplus -} -#endif -#define crypto_stream_aes128ctr crypto_stream_aes128ctr_portable -#define crypto_stream_aes128ctr_xor crypto_stream_aes128ctr_portable_xor -#define crypto_stream_aes128ctr_beforenm crypto_stream_aes128ctr_portable_beforenm -#define crypto_stream_aes128ctr_afternm crypto_stream_aes128ctr_portable_afternm -#define crypto_stream_aes128ctr_xor_afternm crypto_stream_aes128ctr_portable_xor_afternm -#define crypto_stream_aes128ctr_KEYBYTES crypto_stream_aes128ctr_portable_KEYBYTES -#define crypto_stream_aes128ctr_NONCEBYTES crypto_stream_aes128ctr_portable_NONCEBYTES -#define crypto_stream_aes128ctr_BEFORENMBYTES crypto_stream_aes128ctr_portable_BEFORENMBYTES -#define crypto_stream_aes128ctr_IMPLEMENTATION "crypto_stream/aes128ctr/portable" -#ifndef crypto_stream_aes128ctr_portable_VERSION -#define crypto_stream_aes128ctr_portable_VERSION "-" -#endif -#define crypto_stream_aes128ctr_VERSION crypto_stream_aes128ctr_portable_VERSION - -#endif diff --git a/ext/bin/cnacl-osx-amd64/include/sodium/crypto_stream_salsa20.h b/ext/bin/cnacl-osx-amd64/include/sodium/crypto_stream_salsa20.h deleted file mode 100644 index c96d20b4..00000000 --- a/ext/bin/cnacl-osx-amd64/include/sodium/crypto_stream_salsa20.h +++ /dev/null @@ -1,34 +0,0 @@ -#ifndef crypto_stream_salsa20_H -#define crypto_stream_salsa20_H - -#define crypto_stream_salsa20_amd64_xmm6_KEYBYTES 32 -#define crypto_stream_salsa20_amd64_xmm6_NONCEBYTES 8 -#ifdef __cplusplus -#include -extern std::string crypto_stream_salsa20_amd64_xmm6(size_t,const std::string &,const std::string &); -extern std::string crypto_stream_salsa20_amd64_xmm6_xor(const std::string &,const std::string &,const std::string &); -extern "C" { -#endif -extern int crypto_stream_salsa20_amd64_xmm6(unsigned char *,unsigned long long,const unsigned char *,const unsigned char *); -extern int crypto_stream_salsa20_amd64_xmm6_xor(unsigned char *,const unsigned char *,unsigned long long,const unsigned char *,const unsigned char *); -extern int crypto_stream_salsa20_amd64_xmm6_beforenm(unsigned char *,const unsigned char *); -extern int crypto_stream_salsa20_amd64_xmm6_afternm(unsigned char *,unsigned long long,const unsigned char *,const unsigned char *); -extern int crypto_stream_salsa20_amd64_xmm6_xor_afternm(unsigned char *,const unsigned char *,unsigned long long,const unsigned char *,const unsigned char *); -#ifdef __cplusplus -} -#endif -#define crypto_stream_salsa20 crypto_stream_salsa20_amd64_xmm6 -#define crypto_stream_salsa20_xor crypto_stream_salsa20_amd64_xmm6_xor -#define crypto_stream_salsa20_beforenm crypto_stream_salsa20_amd64_xmm6_beforenm -#define crypto_stream_salsa20_afternm crypto_stream_salsa20_amd64_xmm6_afternm -#define crypto_stream_salsa20_xor_afternm crypto_stream_salsa20_amd64_xmm6_xor_afternm -#define crypto_stream_salsa20_KEYBYTES crypto_stream_salsa20_amd64_xmm6_KEYBYTES -#define crypto_stream_salsa20_NONCEBYTES crypto_stream_salsa20_amd64_xmm6_NONCEBYTES -#define crypto_stream_salsa20_BEFORENMBYTES crypto_stream_salsa20_amd64_xmm6_BEFORENMBYTES -#define crypto_stream_salsa20_IMPLEMENTATION "crypto_stream/salsa20/amd64_xmm6" -#ifndef crypto_stream_salsa20_amd64_xmm6_VERSION -#define crypto_stream_salsa20_amd64_xmm6_VERSION "-" -#endif -#define crypto_stream_salsa20_VERSION crypto_stream_salsa20_amd64_xmm6_VERSION - -#endif diff --git a/ext/bin/cnacl-osx-amd64/include/sodium/crypto_stream_salsa2012.h b/ext/bin/cnacl-osx-amd64/include/sodium/crypto_stream_salsa2012.h deleted file mode 100644 index 051e4e39..00000000 --- a/ext/bin/cnacl-osx-amd64/include/sodium/crypto_stream_salsa2012.h +++ /dev/null @@ -1,34 +0,0 @@ -#ifndef crypto_stream_salsa2012_H -#define crypto_stream_salsa2012_H - -#define crypto_stream_salsa2012_amd64_xmm6_KEYBYTES 32 -#define crypto_stream_salsa2012_amd64_xmm6_NONCEBYTES 8 -#ifdef __cplusplus -#include -extern std::string crypto_stream_salsa2012_amd64_xmm6(size_t,const std::string &,const std::string &); -extern std::string crypto_stream_salsa2012_amd64_xmm6_xor(const std::string &,const std::string &,const std::string &); -extern "C" { -#endif -extern int crypto_stream_salsa2012_amd64_xmm6(unsigned char *,unsigned long long,const unsigned char *,const unsigned char *); -extern int crypto_stream_salsa2012_amd64_xmm6_xor(unsigned char *,const unsigned char *,unsigned long long,const unsigned char *,const unsigned char *); -extern int crypto_stream_salsa2012_amd64_xmm6_beforenm(unsigned char *,const unsigned char *); -extern int crypto_stream_salsa2012_amd64_xmm6_afternm(unsigned char *,unsigned long long,const unsigned char *,const unsigned char *); -extern int crypto_stream_salsa2012_amd64_xmm6_xor_afternm(unsigned char *,const unsigned char *,unsigned long long,const unsigned char *,const unsigned char *); -#ifdef __cplusplus -} -#endif -#define crypto_stream_salsa2012 crypto_stream_salsa2012_amd64_xmm6 -#define crypto_stream_salsa2012_xor crypto_stream_salsa2012_amd64_xmm6_xor -#define crypto_stream_salsa2012_beforenm crypto_stream_salsa2012_amd64_xmm6_beforenm -#define crypto_stream_salsa2012_afternm crypto_stream_salsa2012_amd64_xmm6_afternm -#define crypto_stream_salsa2012_xor_afternm crypto_stream_salsa2012_amd64_xmm6_xor_afternm -#define crypto_stream_salsa2012_KEYBYTES crypto_stream_salsa2012_amd64_xmm6_KEYBYTES -#define crypto_stream_salsa2012_NONCEBYTES crypto_stream_salsa2012_amd64_xmm6_NONCEBYTES -#define crypto_stream_salsa2012_BEFORENMBYTES crypto_stream_salsa2012_amd64_xmm6_BEFORENMBYTES -#define crypto_stream_salsa2012_IMPLEMENTATION "crypto_stream/salsa2012/amd64_xmm6" -#ifndef crypto_stream_salsa2012_amd64_xmm6_VERSION -#define crypto_stream_salsa2012_amd64_xmm6_VERSION "-" -#endif -#define crypto_stream_salsa2012_VERSION crypto_stream_salsa2012_amd64_xmm6_VERSION - -#endif diff --git a/ext/bin/cnacl-osx-amd64/include/sodium/crypto_stream_salsa208.h b/ext/bin/cnacl-osx-amd64/include/sodium/crypto_stream_salsa208.h deleted file mode 100644 index 4bd470c3..00000000 --- a/ext/bin/cnacl-osx-amd64/include/sodium/crypto_stream_salsa208.h +++ /dev/null @@ -1,34 +0,0 @@ -#ifndef crypto_stream_salsa208_H -#define crypto_stream_salsa208_H - -#define crypto_stream_salsa208_amd64_xmm6_KEYBYTES 32 -#define crypto_stream_salsa208_amd64_xmm6_NONCEBYTES 8 -#ifdef __cplusplus -#include -extern std::string crypto_stream_salsa208_amd64_xmm6(size_t,const std::string &,const std::string &); -extern std::string crypto_stream_salsa208_amd64_xmm6_xor(const std::string &,const std::string &,const std::string &); -extern "C" { -#endif -extern int crypto_stream_salsa208_amd64_xmm6(unsigned char *,unsigned long long,const unsigned char *,const unsigned char *); -extern int crypto_stream_salsa208_amd64_xmm6_xor(unsigned char *,const unsigned char *,unsigned long long,const unsigned char *,const unsigned char *); -extern int crypto_stream_salsa208_amd64_xmm6_beforenm(unsigned char *,const unsigned char *); -extern int crypto_stream_salsa208_amd64_xmm6_afternm(unsigned char *,unsigned long long,const unsigned char *,const unsigned char *); -extern int crypto_stream_salsa208_amd64_xmm6_xor_afternm(unsigned char *,const unsigned char *,unsigned long long,const unsigned char *,const unsigned char *); -#ifdef __cplusplus -} -#endif -#define crypto_stream_salsa208 crypto_stream_salsa208_amd64_xmm6 -#define crypto_stream_salsa208_xor crypto_stream_salsa208_amd64_xmm6_xor -#define crypto_stream_salsa208_beforenm crypto_stream_salsa208_amd64_xmm6_beforenm -#define crypto_stream_salsa208_afternm crypto_stream_salsa208_amd64_xmm6_afternm -#define crypto_stream_salsa208_xor_afternm crypto_stream_salsa208_amd64_xmm6_xor_afternm -#define crypto_stream_salsa208_KEYBYTES crypto_stream_salsa208_amd64_xmm6_KEYBYTES -#define crypto_stream_salsa208_NONCEBYTES crypto_stream_salsa208_amd64_xmm6_NONCEBYTES -#define crypto_stream_salsa208_BEFORENMBYTES crypto_stream_salsa208_amd64_xmm6_BEFORENMBYTES -#define crypto_stream_salsa208_IMPLEMENTATION "crypto_stream/salsa208/amd64_xmm6" -#ifndef crypto_stream_salsa208_amd64_xmm6_VERSION -#define crypto_stream_salsa208_amd64_xmm6_VERSION "-" -#endif -#define crypto_stream_salsa208_VERSION crypto_stream_salsa208_amd64_xmm6_VERSION - -#endif diff --git a/ext/bin/cnacl-osx-amd64/include/sodium/crypto_stream_xsalsa20.h b/ext/bin/cnacl-osx-amd64/include/sodium/crypto_stream_xsalsa20.h deleted file mode 100644 index d75268c6..00000000 --- a/ext/bin/cnacl-osx-amd64/include/sodium/crypto_stream_xsalsa20.h +++ /dev/null @@ -1,34 +0,0 @@ -#ifndef crypto_stream_xsalsa20_H -#define crypto_stream_xsalsa20_H - -#define crypto_stream_xsalsa20_ref_KEYBYTES 32 -#define crypto_stream_xsalsa20_ref_NONCEBYTES 24 -#ifdef __cplusplus -#include -extern std::string crypto_stream_xsalsa20_ref(size_t,const std::string &,const std::string &); -extern std::string crypto_stream_xsalsa20_ref_xor(const std::string &,const std::string &,const std::string &); -extern "C" { -#endif -extern int crypto_stream_xsalsa20_ref(unsigned char *,unsigned long long,const unsigned char *,const unsigned char *); -extern int crypto_stream_xsalsa20_ref_xor(unsigned char *,const unsigned char *,unsigned long long,const unsigned char *,const unsigned char *); -extern int crypto_stream_xsalsa20_ref_beforenm(unsigned char *,const unsigned char *); -extern int crypto_stream_xsalsa20_ref_afternm(unsigned char *,unsigned long long,const unsigned char *,const unsigned char *); -extern int crypto_stream_xsalsa20_ref_xor_afternm(unsigned char *,const unsigned char *,unsigned long long,const unsigned char *,const unsigned char *); -#ifdef __cplusplus -} -#endif -#define crypto_stream_xsalsa20 crypto_stream_xsalsa20_ref -#define crypto_stream_xsalsa20_xor crypto_stream_xsalsa20_ref_xor -#define crypto_stream_xsalsa20_beforenm crypto_stream_xsalsa20_ref_beforenm -#define crypto_stream_xsalsa20_afternm crypto_stream_xsalsa20_ref_afternm -#define crypto_stream_xsalsa20_xor_afternm crypto_stream_xsalsa20_ref_xor_afternm -#define crypto_stream_xsalsa20_KEYBYTES crypto_stream_xsalsa20_ref_KEYBYTES -#define crypto_stream_xsalsa20_NONCEBYTES crypto_stream_xsalsa20_ref_NONCEBYTES -#define crypto_stream_xsalsa20_BEFORENMBYTES crypto_stream_xsalsa20_ref_BEFORENMBYTES -#define crypto_stream_xsalsa20_IMPLEMENTATION "crypto_stream/xsalsa20/ref" -#ifndef crypto_stream_xsalsa20_ref_VERSION -#define crypto_stream_xsalsa20_ref_VERSION "-" -#endif -#define crypto_stream_xsalsa20_VERSION crypto_stream_xsalsa20_ref_VERSION - -#endif diff --git a/ext/bin/cnacl-osx-amd64/include/sodium/crypto_types.h b/ext/bin/cnacl-osx-amd64/include/sodium/crypto_types.h deleted file mode 100644 index b0ce9656..00000000 --- a/ext/bin/cnacl-osx-amd64/include/sodium/crypto_types.h +++ /dev/null @@ -1,11 +0,0 @@ -#ifndef crypto_types_h -#define crypto_types_h -typedef short crypto_int16; -typedef int crypto_int32; -typedef long long crypto_int64; -typedef signed char crypto_int8; -typedef unsigned short crypto_uint16; -typedef unsigned int crypto_uint32; -typedef unsigned long long crypto_uint64; -typedef unsigned char crypto_uint8; -#endif diff --git a/ext/bin/cnacl-osx-amd64/include/sodium/crypto_verify_16.h b/ext/bin/cnacl-osx-amd64/include/sodium/crypto_verify_16.h deleted file mode 100644 index 6bf6ca11..00000000 --- a/ext/bin/cnacl-osx-amd64/include/sodium/crypto_verify_16.h +++ /dev/null @@ -1,21 +0,0 @@ -#ifndef crypto_verify_16_H -#define crypto_verify_16_H - -#define crypto_verify_16_ref_BYTES 16 -#ifdef __cplusplus -#include -extern "C" { -#endif -extern int crypto_verify_16_ref(const unsigned char *,const unsigned char *); -#ifdef __cplusplus -} -#endif -#define crypto_verify_16 crypto_verify_16_ref -#define crypto_verify_16_BYTES crypto_verify_16_ref_BYTES -#define crypto_verify_16_IMPLEMENTATION "crypto_verify/16/ref" -#ifndef crypto_verify_16_ref_VERSION -#define crypto_verify_16_ref_VERSION "-" -#endif -#define crypto_verify_16_VERSION crypto_verify_16_ref_VERSION - -#endif diff --git a/ext/bin/cnacl-osx-amd64/include/sodium/crypto_verify_32.h b/ext/bin/cnacl-osx-amd64/include/sodium/crypto_verify_32.h deleted file mode 100644 index bd5fc644..00000000 --- a/ext/bin/cnacl-osx-amd64/include/sodium/crypto_verify_32.h +++ /dev/null @@ -1,21 +0,0 @@ -#ifndef crypto_verify_32_H -#define crypto_verify_32_H - -#define crypto_verify_32_ref_BYTES 32 -#ifdef __cplusplus -#include -extern "C" { -#endif -extern int crypto_verify_32_ref(const unsigned char *,const unsigned char *); -#ifdef __cplusplus -} -#endif -#define crypto_verify_32 crypto_verify_32_ref -#define crypto_verify_32_BYTES crypto_verify_32_ref_BYTES -#define crypto_verify_32_IMPLEMENTATION "crypto_verify/32/ref" -#ifndef crypto_verify_32_ref_VERSION -#define crypto_verify_32_ref_VERSION "-" -#endif -#define crypto_verify_32_VERSION crypto_verify_32_ref_VERSION - -#endif diff --git a/ext/bin/cnacl-osx-amd64/libnacl.a b/ext/bin/cnacl-osx-amd64/libnacl.a deleted file mode 100644 index d46fffab..00000000 Binary files a/ext/bin/cnacl-osx-amd64/libnacl.a and /dev/null differ diff --git a/make-mac.mk b/make-mac.mk index b71ca2fe..8ff1b772 100644 --- a/make-mac.mk +++ b/make-mac.mk @@ -33,12 +33,6 @@ else DEFS+=-DZT_SOFTWARE_UPDATE_DEFAULT="\"download\"" endif -# Use precompiled extremely fast Salsa20/12 from "cnacl" included in ext/bin -# See https://github.com/cjdelisle/cnacl -DEFS+=-DZT_USE_LIBSODIUM -CFLAGS+=-Iext/bin/cnacl-osx-amd64/include -LIBS+=ext/bin/cnacl-osx-amd64/libnacl.a - ifeq ($(ZT_ENABLE_CLUSTER),1) DEFS+=-DZT_ENABLE_CLUSTER endif diff --git a/node/Node.cpp b/node/Node.cpp index 55fb4e72..2b3f7996 100644 --- a/node/Node.cpp +++ b/node/Node.cpp @@ -50,7 +50,6 @@ Node::Node(void *uptr,void *tptr,const struct ZT_Node_Callbacks *callbacks,uint6 _RR(this), RR(&_RR), _uPtr(uptr), - _prngStreamPtr(0), _now(now), _lastPingCheck(0), _lastHousekeepingRun(0) @@ -59,19 +58,14 @@ Node::Node(void *uptr,void *tptr,const struct ZT_Node_Callbacks *callbacks,uint6 throw std::runtime_error("callbacks struct version mismatch"); memcpy(&_cb,callbacks,sizeof(ZT_Node_Callbacks)); + Utils::getSecureRandom((void *)_prngState,sizeof(_prngState)); + _online = false; memset(_expectingRepliesToBucketPtr,0,sizeof(_expectingRepliesToBucketPtr)); memset(_expectingRepliesTo,0,sizeof(_expectingRepliesTo)); memset(_lastIdentityVerification,0,sizeof(_lastIdentityVerification)); - // Use Salsa20 alone as a high-quality non-crypto PRNG - char foo[64]; - Utils::getSecureRandom(foo,64); - _prng.init(foo,foo + 32); - memset(_prngStream,0,sizeof(_prngStream)); - _prng.crypt12(_prngStream,_prngStream,sizeof(_prngStream)); - std::string idtmp(dataStoreGet(tptr,"identity.secret")); if ((!idtmp.length())||(!RR->identity.fromString(idtmp))||(!RR->identity.hasPrivate())) { TRACE("identity.secret not found, generating..."); @@ -701,10 +695,14 @@ void Node::postTrace(const char *module,unsigned int line,const char *fmt,...) uint64_t Node::prng() { - unsigned int p = (++_prngStreamPtr % ZT_NODE_PRNG_BUF_SIZE); - if (!p) - _prng.crypt12(_prngStream,_prngStream,sizeof(_prngStream)); - return _prngStream[p]; + // https://en.wikipedia.org/wiki/Xorshift#xorshift.2B + uint64_t x = _prngState[0]; + const uint64_t y = _prngState[1]; + _prngState[0] = y; + x ^= x << 23; + const uint64_t z = x ^ y ^ (x >> 17) ^ (y >> 26); + _prngState[1] = z; + return z + y; } void Node::postCircuitTestReport(const ZT_CircuitTestReport *report) diff --git a/node/Node.hpp b/node/Node.hpp index 03bd7a8c..d25a619b 100644 --- a/node/Node.hpp +++ b/node/Node.hpp @@ -50,9 +50,6 @@ #define ZT_EXPECTING_REPLIES_BUCKET_MASK1 255 #define ZT_EXPECTING_REPLIES_BUCKET_MASK2 31 -// Size of PRNG stream buffer -#define ZT_NODE_PRNG_BUF_SIZE 64 - namespace ZeroTier { class World; @@ -312,13 +309,10 @@ private: Mutex _backgroundTasksLock; - unsigned int _prngStreamPtr; - Salsa20 _prng; - uint64_t _prngStream[ZT_NODE_PRNG_BUF_SIZE]; // repeatedly encrypted with _prng to yield a high-quality non-crypto PRNG stream - uint64_t _now; uint64_t _lastPingCheck; uint64_t _lastHousekeepingRun; + volatile uint64_t _prngState[2]; bool _online; }; diff --git a/node/Salsa20.cpp b/node/Salsa20.cpp index 1e2b4b0f..2a802555 100644 --- a/node/Salsa20.cpp +++ b/node/Salsa20.cpp @@ -10,8 +10,6 @@ #include "Constants.hpp" #include "Salsa20.hpp" -#ifndef ZT_USE_LIBSODIUM - #define ROTATE(v,c) (((v) << (c)) | ((v) >> (32 - (c)))) #define XOR(v,w) ((v) ^ (w)) #define PLUS(v,w) ((uint32_t)((v) + (w))) @@ -1345,5 +1343,3 @@ void Salsa20::crypt20(const void *in,void *out,unsigned int bytes) } } // namespace ZeroTier - -#endif // !ZT_USE_LIBSODIUM diff --git a/node/Salsa20.hpp b/node/Salsa20.hpp index 5e4c68be..c6af5700 100644 --- a/node/Salsa20.hpp +++ b/node/Salsa20.hpp @@ -15,77 +15,6 @@ #include "Constants.hpp" #include "Utils.hpp" -#ifdef ZT_USE_LIBSODIUM - -#include -#include - -namespace ZeroTier { - -/** - * Salsa20 stream cipher - */ -class Salsa20 -{ -public: - Salsa20() {} - ~Salsa20() { Utils::burn(_k,sizeof(_k)); } - - /** - * @param key 256-bit (32 byte) key - * @param iv 64-bit initialization vector - */ - Salsa20(const void *key,const void *iv) - { - memcpy(_k,key,32); - memcpy(&_iv,iv,8); - } - - /** - * Initialize cipher - * - * @param key Key bits - * @param iv 64-bit initialization vector - */ - inline void init(const void *key,const void *iv) - { - memcpy(_k,key,32); - memcpy(&_iv,iv,8); - } - - /** - * Encrypt/decrypt data using Salsa20/12 - * - * @param in Input data - * @param out Output buffer - * @param bytes Length of data - */ - inline void crypt12(const void *in,void *out,unsigned int bytes) - { - crypto_stream_salsa2012_xor(reinterpret_cast(out),reinterpret_cast(in),bytes,reinterpret_cast(&_iv),reinterpret_cast(_k)); - } - - /** - * Encrypt/decrypt data using Salsa20/20 - * - * @param in Input data - * @param out Output buffer - * @param bytes Length of data - */ - inline void crypt20(const void *in,void *out,unsigned int bytes) - { - crypto_stream_salsa20_xor(reinterpret_cast(out),reinterpret_cast(in),bytes,reinterpret_cast(&_iv),reinterpret_cast(_k)); - } - -private: - uint64_t _k[4]; - uint64_t _iv; -}; - -} // namespace ZeroTier - -#else // !ZT_USE_LIBSODIUM - #if (!defined(ZT_SALSA20_SSE)) && (defined(__SSE2__) || defined(__WINDOWS__)) #define ZT_SALSA20_SSE 1 #endif @@ -105,6 +34,11 @@ public: Salsa20() {} ~Salsa20() { Utils::burn(&_state,sizeof(_state)); } + /** + * If this returns true, crypt can only be done once + */ + static inline bool singleUseOnly() { return false; } + /** * @param key 256-bit (32 byte) key * @param iv 64-bit initialization vector @@ -151,6 +85,4 @@ private: } // namespace ZeroTier -#endif // ZT_USE_LIBSODIUM - #endif diff --git a/node/Utils.cpp b/node/Utils.cpp index 92d14d19..9ce1bf05 100644 --- a/node/Utils.cpp +++ b/node/Utils.cpp @@ -177,6 +177,7 @@ void Utils::getSecureRandom(void *buf,unsigned int bytes) } randomPtr = 0; s20.crypt12(randomBuf,randomBuf,sizeof(randomBuf)); + s20.init(randomBuf,randomBuf); } ((uint8_t *)buf)[i] = randomBuf[randomPtr++]; } @@ -209,6 +210,7 @@ void Utils::getSecureRandom(void *buf,unsigned int bytes) } randomPtr = 0; s20.crypt12(randomBuf,randomBuf,sizeof(randomBuf)); + s20.init(randomBuf,randomBuf); } ((uint8_t *)buf)[i] = randomBuf[randomPtr++]; } -- cgit v1.2.3 From a1e94154bebe17a24d2eed43be7d866e93c061fe Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Tue, 18 Apr 2017 08:45:37 -0700 Subject: Just incorporate the X64 ASM version of Salsa20/12 for X64 platforms. This gives us (for example) 1.5gb/sec encryption on a Core i5 2.8ghz. --- ext/x64-salsa2012-asm/README.md | 6 + ext/x64-salsa2012-asm/salsa2012.h | 13 + ext/x64-salsa2012-asm/salsa2012.s | 4488 +++++++++++++++++++++++++++++++++++++ make-mac.mk | 7 + node/Packet.cpp | 65 +- selftest.cpp | 22 + 6 files changed, 4597 insertions(+), 4 deletions(-) create mode 100644 ext/x64-salsa2012-asm/README.md create mode 100644 ext/x64-salsa2012-asm/salsa2012.h create mode 100644 ext/x64-salsa2012-asm/salsa2012.s (limited to 'make-mac.mk') diff --git a/ext/x64-salsa2012-asm/README.md b/ext/x64-salsa2012-asm/README.md new file mode 100644 index 00000000..a69a1a67 --- /dev/null +++ b/ext/x64-salsa2012-asm/README.md @@ -0,0 +1,6 @@ +Blazingly fast X64 ASM implementation of Salsa20/12 +====== + +This is ripped from the [cnacl](https://github.com/cjdelisle/cnacl) source. The actual code is by Danial J. Bernstein and is in the public domain. + +This is included on Linux and Mac 64-bit builds and is significantly faster than the SSE intrinsics or C versions. It's used for packet encode/decode only since its use differs a bit from the regular Salsa20 C++ class. Specifically it lacks the ability to be called on multiple blocks, preferring instead to take a key and a single stream to encrypt and that's it. diff --git a/ext/x64-salsa2012-asm/salsa2012.h b/ext/x64-salsa2012-asm/salsa2012.h new file mode 100644 index 00000000..d47059b4 --- /dev/null +++ b/ext/x64-salsa2012-asm/salsa2012.h @@ -0,0 +1,13 @@ +#ifdef __cplusplus +extern "C" { +#endif + +// output, outlen, nonce, key (256-bit / 32-byte) +extern int zt_salsa2012_amd64_xmm6(unsigned char *,unsigned long long,const unsigned char *,const unsigned char *); + +// ciphertext, message, mlen, nonce, key +extern int zt_salsa2012_amd64_xmm6_xor(unsigned char *,const unsigned char *,unsigned long long,const unsigned char *,const unsigned char *); + +#ifdef __cplusplus +} +#endif diff --git a/ext/x64-salsa2012-asm/salsa2012.s b/ext/x64-salsa2012-asm/salsa2012.s new file mode 100644 index 00000000..699c89ac --- /dev/null +++ b/ext/x64-salsa2012-asm/salsa2012.s @@ -0,0 +1,4488 @@ +# qhasm: enter zt_salsa2012_amd64_xmm6 +.text +.p2align 5 +.globl _zt_salsa2012_amd64_xmm6 +.globl zt_salsa2012_amd64_xmm6 +_zt_salsa2012_amd64_xmm6: +zt_salsa2012_amd64_xmm6: +mov %rsp,%r11 +and $31,%r11 +add $480,%r11 +sub %r11,%rsp + +# qhasm: r11_stack = r11_caller +# asm 1: movq r11_stack=stack64#1 +# asm 2: movq r11_stack=352(%rsp) +movq %r11,352(%rsp) + +# qhasm: r12_stack = r12_caller +# asm 1: movq r12_stack=stack64#2 +# asm 2: movq r12_stack=360(%rsp) +movq %r12,360(%rsp) + +# qhasm: r13_stack = r13_caller +# asm 1: movq r13_stack=stack64#3 +# asm 2: movq r13_stack=368(%rsp) +movq %r13,368(%rsp) + +# qhasm: r14_stack = r14_caller +# asm 1: movq r14_stack=stack64#4 +# asm 2: movq r14_stack=376(%rsp) +movq %r14,376(%rsp) + +# qhasm: r15_stack = r15_caller +# asm 1: movq r15_stack=stack64#5 +# asm 2: movq r15_stack=384(%rsp) +movq %r15,384(%rsp) + +# qhasm: rbx_stack = rbx_caller +# asm 1: movq rbx_stack=stack64#6 +# asm 2: movq rbx_stack=392(%rsp) +movq %rbx,392(%rsp) + +# qhasm: rbp_stack = rbp_caller +# asm 1: movq rbp_stack=stack64#7 +# asm 2: movq rbp_stack=400(%rsp) +movq %rbp,400(%rsp) + +# qhasm: bytes = arg2 +# asm 1: mov bytes=int64#6 +# asm 2: mov bytes=%r9 +mov %rsi,%r9 + +# qhasm: out = arg1 +# asm 1: mov out=int64#1 +# asm 2: mov out=%rdi +mov %rdi,%rdi + +# qhasm: m = out +# asm 1: mov m=int64#2 +# asm 2: mov m=%rsi +mov %rdi,%rsi + +# qhasm: iv = arg3 +# asm 1: mov iv=int64#3 +# asm 2: mov iv=%rdx +mov %rdx,%rdx + +# qhasm: k = arg4 +# asm 1: mov k=int64#8 +# asm 2: mov k=%r10 +mov %rcx,%r10 + +# qhasm: unsigned>? bytes - 0 +# asm 1: cmp $0, +jbe ._done + +# qhasm: a = 0 +# asm 1: mov $0,>a=int64#7 +# asm 2: mov $0,>a=%rax +mov $0,%rax + +# qhasm: i = bytes +# asm 1: mov i=int64#4 +# asm 2: mov i=%rcx +mov %r9,%rcx + +# qhasm: while (i) { *out++ = a; --i } +rep stosb + +# qhasm: out -= bytes +# asm 1: sub r11_stack=stack64#1 +# asm 2: movq r11_stack=352(%rsp) +movq %r11,352(%rsp) + +# qhasm: r12_stack = r12_caller +# asm 1: movq r12_stack=stack64#2 +# asm 2: movq r12_stack=360(%rsp) +movq %r12,360(%rsp) + +# qhasm: r13_stack = r13_caller +# asm 1: movq r13_stack=stack64#3 +# asm 2: movq r13_stack=368(%rsp) +movq %r13,368(%rsp) + +# qhasm: r14_stack = r14_caller +# asm 1: movq r14_stack=stack64#4 +# asm 2: movq r14_stack=376(%rsp) +movq %r14,376(%rsp) + +# qhasm: r15_stack = r15_caller +# asm 1: movq r15_stack=stack64#5 +# asm 2: movq r15_stack=384(%rsp) +movq %r15,384(%rsp) + +# qhasm: rbx_stack = rbx_caller +# asm 1: movq rbx_stack=stack64#6 +# asm 2: movq rbx_stack=392(%rsp) +movq %rbx,392(%rsp) + +# qhasm: rbp_stack = rbp_caller +# asm 1: movq rbp_stack=stack64#7 +# asm 2: movq rbp_stack=400(%rsp) +movq %rbp,400(%rsp) + +# qhasm: out = arg1 +# asm 1: mov out=int64#1 +# asm 2: mov out=%rdi +mov %rdi,%rdi + +# qhasm: m = arg2 +# asm 1: mov m=int64#2 +# asm 2: mov m=%rsi +mov %rsi,%rsi + +# qhasm: bytes = arg3 +# asm 1: mov bytes=int64#6 +# asm 2: mov bytes=%r9 +mov %rdx,%r9 + +# qhasm: iv = arg4 +# asm 1: mov iv=int64#3 +# asm 2: mov iv=%rdx +mov %rcx,%rdx + +# qhasm: k = arg5 +# asm 1: mov k=int64#8 +# asm 2: mov k=%r10 +mov %r8,%r10 + +# qhasm: unsigned>? bytes - 0 +# asm 1: cmp $0, +jbe ._done +# comment:fp stack unchanged by fallthrough + +# qhasm: start: +._start: + +# qhasm: in12 = *(uint32 *) (k + 20) +# asm 1: movl 20(in12=int64#4d +# asm 2: movl 20(in12=%ecx +movl 20(%r10),%ecx + +# qhasm: in1 = *(uint32 *) (k + 0) +# asm 1: movl 0(in1=int64#5d +# asm 2: movl 0(in1=%r8d +movl 0(%r10),%r8d + +# qhasm: in6 = *(uint32 *) (iv + 0) +# asm 1: movl 0(in6=int64#7d +# asm 2: movl 0(in6=%eax +movl 0(%rdx),%eax + +# qhasm: in11 = *(uint32 *) (k + 16) +# asm 1: movl 16(in11=int64#9d +# asm 2: movl 16(in11=%r11d +movl 16(%r10),%r11d + +# qhasm: ((uint32 *)&x1)[0] = in12 +# asm 1: movl x1=stack128#1 +# asm 2: movl x1=0(%rsp) +movl %ecx,0(%rsp) + +# qhasm: ((uint32 *)&x1)[1] = in1 +# asm 1: movl in8=int64#4 +# asm 2: mov $0,>in8=%rcx +mov $0,%rcx + +# qhasm: in13 = *(uint32 *) (k + 24) +# asm 1: movl 24(in13=int64#5d +# asm 2: movl 24(in13=%r8d +movl 24(%r10),%r8d + +# qhasm: in2 = *(uint32 *) (k + 4) +# asm 1: movl 4(in2=int64#7d +# asm 2: movl 4(in2=%eax +movl 4(%r10),%eax + +# qhasm: in7 = *(uint32 *) (iv + 4) +# asm 1: movl 4(in7=int64#3d +# asm 2: movl 4(in7=%edx +movl 4(%rdx),%edx + +# qhasm: ((uint32 *)&x2)[0] = in8 +# asm 1: movl x2=stack128#2 +# asm 2: movl x2=16(%rsp) +movl %ecx,16(%rsp) + +# qhasm: ((uint32 *)&x2)[1] = in13 +# asm 1: movl in4=int64#3d +# asm 2: movl 12(in4=%edx +movl 12(%r10),%edx + +# qhasm: in9 = 0 +# asm 1: mov $0,>in9=int64#4 +# asm 2: mov $0,>in9=%rcx +mov $0,%rcx + +# qhasm: in14 = *(uint32 *) (k + 28) +# asm 1: movl 28(in14=int64#5d +# asm 2: movl 28(in14=%r8d +movl 28(%r10),%r8d + +# qhasm: in3 = *(uint32 *) (k + 8) +# asm 1: movl 8(in3=int64#7d +# asm 2: movl 8(in3=%eax +movl 8(%r10),%eax + +# qhasm: ((uint32 *)&x3)[0] = in4 +# asm 1: movl x3=stack128#3 +# asm 2: movl x3=32(%rsp) +movl %edx,32(%rsp) + +# qhasm: ((uint32 *)&x3)[1] = in9 +# asm 1: movl in0=int64#3 +# asm 2: mov $1634760805,>in0=%rdx +mov $1634760805,%rdx + +# qhasm: in5 = 857760878 +# asm 1: mov $857760878,>in5=int64#4 +# asm 2: mov $857760878,>in5=%rcx +mov $857760878,%rcx + +# qhasm: in10 = 2036477234 +# asm 1: mov $2036477234,>in10=int64#5 +# asm 2: mov $2036477234,>in10=%r8 +mov $2036477234,%r8 + +# qhasm: in15 = 1797285236 +# asm 1: mov $1797285236,>in15=int64#7 +# asm 2: mov $1797285236,>in15=%rax +mov $1797285236,%rax + +# qhasm: ((uint32 *)&x0)[0] = in0 +# asm 1: movl x0=stack128#4 +# asm 2: movl x0=48(%rsp) +movl %edx,48(%rsp) + +# qhasm: ((uint32 *)&x0)[1] = in5 +# asm 1: movl z0=int6464#1 +# asm 2: movdqa z0=%xmm0 +movdqa 48(%rsp),%xmm0 + +# qhasm: z5 = z0[1,1,1,1] +# asm 1: pshufd $0x55,z5=int6464#2 +# asm 2: pshufd $0x55,z5=%xmm1 +pshufd $0x55,%xmm0,%xmm1 + +# qhasm: z10 = z0[2,2,2,2] +# asm 1: pshufd $0xaa,z10=int6464#3 +# asm 2: pshufd $0xaa,z10=%xmm2 +pshufd $0xaa,%xmm0,%xmm2 + +# qhasm: z15 = z0[3,3,3,3] +# asm 1: pshufd $0xff,z15=int6464#4 +# asm 2: pshufd $0xff,z15=%xmm3 +pshufd $0xff,%xmm0,%xmm3 + +# qhasm: z0 = z0[0,0,0,0] +# asm 1: pshufd $0x00,z0=int6464#1 +# asm 2: pshufd $0x00,z0=%xmm0 +pshufd $0x00,%xmm0,%xmm0 + +# qhasm: orig5 = z5 +# asm 1: movdqa orig5=stack128#5 +# asm 2: movdqa orig5=64(%rsp) +movdqa %xmm1,64(%rsp) + +# qhasm: orig10 = z10 +# asm 1: movdqa orig10=stack128#6 +# asm 2: movdqa orig10=80(%rsp) +movdqa %xmm2,80(%rsp) + +# qhasm: orig15 = z15 +# asm 1: movdqa orig15=stack128#7 +# asm 2: movdqa orig15=96(%rsp) +movdqa %xmm3,96(%rsp) + +# qhasm: orig0 = z0 +# asm 1: movdqa orig0=stack128#8 +# asm 2: movdqa orig0=112(%rsp) +movdqa %xmm0,112(%rsp) + +# qhasm: z1 = x1 +# asm 1: movdqa z1=int6464#1 +# asm 2: movdqa z1=%xmm0 +movdqa 0(%rsp),%xmm0 + +# qhasm: z6 = z1[2,2,2,2] +# asm 1: pshufd $0xaa,z6=int6464#2 +# asm 2: pshufd $0xaa,z6=%xmm1 +pshufd $0xaa,%xmm0,%xmm1 + +# qhasm: z11 = z1[3,3,3,3] +# asm 1: pshufd $0xff,z11=int6464#3 +# asm 2: pshufd $0xff,z11=%xmm2 +pshufd $0xff,%xmm0,%xmm2 + +# qhasm: z12 = z1[0,0,0,0] +# asm 1: pshufd $0x00,z12=int6464#4 +# asm 2: pshufd $0x00,z12=%xmm3 +pshufd $0x00,%xmm0,%xmm3 + +# qhasm: z1 = z1[1,1,1,1] +# asm 1: pshufd $0x55,z1=int6464#1 +# asm 2: pshufd $0x55,z1=%xmm0 +pshufd $0x55,%xmm0,%xmm0 + +# qhasm: orig6 = z6 +# asm 1: movdqa orig6=stack128#9 +# asm 2: movdqa orig6=128(%rsp) +movdqa %xmm1,128(%rsp) + +# qhasm: orig11 = z11 +# asm 1: movdqa orig11=stack128#10 +# asm 2: movdqa orig11=144(%rsp) +movdqa %xmm2,144(%rsp) + +# qhasm: orig12 = z12 +# asm 1: movdqa orig12=stack128#11 +# asm 2: movdqa orig12=160(%rsp) +movdqa %xmm3,160(%rsp) + +# qhasm: orig1 = z1 +# asm 1: movdqa orig1=stack128#12 +# asm 2: movdqa orig1=176(%rsp) +movdqa %xmm0,176(%rsp) + +# qhasm: z2 = x2 +# asm 1: movdqa z2=int6464#1 +# asm 2: movdqa z2=%xmm0 +movdqa 16(%rsp),%xmm0 + +# qhasm: z7 = z2[3,3,3,3] +# asm 1: pshufd $0xff,z7=int6464#2 +# asm 2: pshufd $0xff,z7=%xmm1 +pshufd $0xff,%xmm0,%xmm1 + +# qhasm: z13 = z2[1,1,1,1] +# asm 1: pshufd $0x55,z13=int6464#3 +# asm 2: pshufd $0x55,z13=%xmm2 +pshufd $0x55,%xmm0,%xmm2 + +# qhasm: z2 = z2[2,2,2,2] +# asm 1: pshufd $0xaa,z2=int6464#1 +# asm 2: pshufd $0xaa,z2=%xmm0 +pshufd $0xaa,%xmm0,%xmm0 + +# qhasm: orig7 = z7 +# asm 1: movdqa orig7=stack128#13 +# asm 2: movdqa orig7=192(%rsp) +movdqa %xmm1,192(%rsp) + +# qhasm: orig13 = z13 +# asm 1: movdqa orig13=stack128#14 +# asm 2: movdqa orig13=208(%rsp) +movdqa %xmm2,208(%rsp) + +# qhasm: orig2 = z2 +# asm 1: movdqa orig2=stack128#15 +# asm 2: movdqa orig2=224(%rsp) +movdqa %xmm0,224(%rsp) + +# qhasm: z3 = x3 +# asm 1: movdqa z3=int6464#1 +# asm 2: movdqa z3=%xmm0 +movdqa 32(%rsp),%xmm0 + +# qhasm: z4 = z3[0,0,0,0] +# asm 1: pshufd $0x00,z4=int6464#2 +# asm 2: pshufd $0x00,z4=%xmm1 +pshufd $0x00,%xmm0,%xmm1 + +# qhasm: z14 = z3[2,2,2,2] +# asm 1: pshufd $0xaa,z14=int6464#3 +# asm 2: pshufd $0xaa,z14=%xmm2 +pshufd $0xaa,%xmm0,%xmm2 + +# qhasm: z3 = z3[3,3,3,3] +# asm 1: pshufd $0xff,z3=int6464#1 +# asm 2: pshufd $0xff,z3=%xmm0 +pshufd $0xff,%xmm0,%xmm0 + +# qhasm: orig4 = z4 +# asm 1: movdqa orig4=stack128#16 +# asm 2: movdqa orig4=240(%rsp) +movdqa %xmm1,240(%rsp) + +# qhasm: orig14 = z14 +# asm 1: movdqa orig14=stack128#17 +# asm 2: movdqa orig14=256(%rsp) +movdqa %xmm2,256(%rsp) + +# qhasm: orig3 = z3 +# asm 1: movdqa orig3=stack128#18 +# asm 2: movdqa orig3=272(%rsp) +movdqa %xmm0,272(%rsp) + +# qhasm: bytesatleast256: +._bytesatleast256: + +# qhasm: in8 = ((uint32 *)&x2)[0] +# asm 1: movl in8=int64#3d +# asm 2: movl in8=%edx +movl 16(%rsp),%edx + +# qhasm: in9 = ((uint32 *)&x3)[1] +# asm 1: movl 4+in9=int64#4d +# asm 2: movl 4+in9=%ecx +movl 4+32(%rsp),%ecx + +# qhasm: ((uint32 *) &orig8)[0] = in8 +# asm 1: movl orig8=stack128#19 +# asm 2: movl orig8=288(%rsp) +movl %edx,288(%rsp) + +# qhasm: ((uint32 *) &orig9)[0] = in9 +# asm 1: movl orig9=stack128#20 +# asm 2: movl orig9=304(%rsp) +movl %ecx,304(%rsp) + +# qhasm: in8 += 1 +# asm 1: add $1,in9=int64#4 +# asm 2: mov in9=%rcx +mov %rdx,%rcx + +# qhasm: (uint64) in9 >>= 32 +# asm 1: shr $32,in9=int64#4 +# asm 2: mov in9=%rcx +mov %rdx,%rcx + +# qhasm: (uint64) in9 >>= 32 +# asm 1: shr $32,in9=int64#4 +# asm 2: mov in9=%rcx +mov %rdx,%rcx + +# qhasm: (uint64) in9 >>= 32 +# asm 1: shr $32,in9=int64#4 +# asm 2: mov in9=%rcx +mov %rdx,%rcx + +# qhasm: (uint64) in9 >>= 32 +# asm 1: shr $32,x2=stack128#2 +# asm 2: movl x2=16(%rsp) +movl %edx,16(%rsp) + +# qhasm: ((uint32 *)&x3)[1] = in9 +# asm 1: movl bytes_backup=stack64#8 +# asm 2: movq bytes_backup=408(%rsp) +movq %r9,408(%rsp) + +# qhasm: i = 12 +# asm 1: mov $12,>i=int64#3 +# asm 2: mov $12,>i=%rdx +mov $12,%rdx + +# qhasm: z5 = orig5 +# asm 1: movdqa z5=int6464#1 +# asm 2: movdqa z5=%xmm0 +movdqa 64(%rsp),%xmm0 + +# qhasm: z10 = orig10 +# asm 1: movdqa z10=int6464#2 +# asm 2: movdqa z10=%xmm1 +movdqa 80(%rsp),%xmm1 + +# qhasm: z15 = orig15 +# asm 1: movdqa z15=int6464#3 +# asm 2: movdqa z15=%xmm2 +movdqa 96(%rsp),%xmm2 + +# qhasm: z14 = orig14 +# asm 1: movdqa z14=int6464#4 +# asm 2: movdqa z14=%xmm3 +movdqa 256(%rsp),%xmm3 + +# qhasm: z3 = orig3 +# asm 1: movdqa z3=int6464#5 +# asm 2: movdqa z3=%xmm4 +movdqa 272(%rsp),%xmm4 + +# qhasm: z6 = orig6 +# asm 1: movdqa z6=int6464#6 +# asm 2: movdqa z6=%xmm5 +movdqa 128(%rsp),%xmm5 + +# qhasm: z11 = orig11 +# asm 1: movdqa z11=int6464#7 +# asm 2: movdqa z11=%xmm6 +movdqa 144(%rsp),%xmm6 + +# qhasm: z1 = orig1 +# asm 1: movdqa z1=int6464#8 +# asm 2: movdqa z1=%xmm7 +movdqa 176(%rsp),%xmm7 + +# qhasm: z7 = orig7 +# asm 1: movdqa z7=int6464#9 +# asm 2: movdqa z7=%xmm8 +movdqa 192(%rsp),%xmm8 + +# qhasm: z13 = orig13 +# asm 1: movdqa z13=int6464#10 +# asm 2: movdqa z13=%xmm9 +movdqa 208(%rsp),%xmm9 + +# qhasm: z2 = orig2 +# asm 1: movdqa z2=int6464#11 +# asm 2: movdqa z2=%xmm10 +movdqa 224(%rsp),%xmm10 + +# qhasm: z9 = orig9 +# asm 1: movdqa z9=int6464#12 +# asm 2: movdqa z9=%xmm11 +movdqa 304(%rsp),%xmm11 + +# qhasm: z0 = orig0 +# asm 1: movdqa z0=int6464#13 +# asm 2: movdqa z0=%xmm12 +movdqa 112(%rsp),%xmm12 + +# qhasm: z12 = orig12 +# asm 1: movdqa z12=int6464#14 +# asm 2: movdqa z12=%xmm13 +movdqa 160(%rsp),%xmm13 + +# qhasm: z4 = orig4 +# asm 1: movdqa z4=int6464#15 +# asm 2: movdqa z4=%xmm14 +movdqa 240(%rsp),%xmm14 + +# qhasm: z8 = orig8 +# asm 1: movdqa z8=int6464#16 +# asm 2: movdqa z8=%xmm15 +movdqa 288(%rsp),%xmm15 + +# qhasm: mainloop1: +._mainloop1: + +# qhasm: z10_stack = z10 +# asm 1: movdqa z10_stack=stack128#21 +# asm 2: movdqa z10_stack=320(%rsp) +movdqa %xmm1,320(%rsp) + +# qhasm: z15_stack = z15 +# asm 1: movdqa z15_stack=stack128#22 +# asm 2: movdqa z15_stack=336(%rsp) +movdqa %xmm2,336(%rsp) + +# qhasm: y4 = z12 +# asm 1: movdqa y4=int6464#2 +# asm 2: movdqa y4=%xmm1 +movdqa %xmm13,%xmm1 + +# qhasm: uint32323232 y4 += z0 +# asm 1: paddd r4=int6464#3 +# asm 2: movdqa r4=%xmm2 +movdqa %xmm1,%xmm2 + +# qhasm: uint32323232 y4 <<= 7 +# asm 1: pslld $7,>= 25 +# asm 1: psrld $25,y9=int6464#2 +# asm 2: movdqa y9=%xmm1 +movdqa %xmm7,%xmm1 + +# qhasm: uint32323232 y9 += z5 +# asm 1: paddd r9=int6464#3 +# asm 2: movdqa r9=%xmm2 +movdqa %xmm1,%xmm2 + +# qhasm: uint32323232 y9 <<= 7 +# asm 1: pslld $7,>= 25 +# asm 1: psrld $25,y8=int6464#2 +# asm 2: movdqa y8=%xmm1 +movdqa %xmm12,%xmm1 + +# qhasm: uint32323232 y8 += z4 +# asm 1: paddd r8=int6464#3 +# asm 2: movdqa r8=%xmm2 +movdqa %xmm1,%xmm2 + +# qhasm: uint32323232 y8 <<= 9 +# asm 1: pslld $9,>= 23 +# asm 1: psrld $23,y13=int6464#2 +# asm 2: movdqa y13=%xmm1 +movdqa %xmm0,%xmm1 + +# qhasm: uint32323232 y13 += z9 +# asm 1: paddd r13=int6464#3 +# asm 2: movdqa r13=%xmm2 +movdqa %xmm1,%xmm2 + +# qhasm: uint32323232 y13 <<= 9 +# asm 1: pslld $9,>= 23 +# asm 1: psrld $23,y12=int6464#2 +# asm 2: movdqa y12=%xmm1 +movdqa %xmm14,%xmm1 + +# qhasm: uint32323232 y12 += z8 +# asm 1: paddd r12=int6464#3 +# asm 2: movdqa r12=%xmm2 +movdqa %xmm1,%xmm2 + +# qhasm: uint32323232 y12 <<= 13 +# asm 1: pslld $13,>= 19 +# asm 1: psrld $19,y1=int6464#2 +# asm 2: movdqa y1=%xmm1 +movdqa %xmm11,%xmm1 + +# qhasm: uint32323232 y1 += z13 +# asm 1: paddd r1=int6464#3 +# asm 2: movdqa r1=%xmm2 +movdqa %xmm1,%xmm2 + +# qhasm: uint32323232 y1 <<= 13 +# asm 1: pslld $13,>= 19 +# asm 1: psrld $19,y0=int6464#2 +# asm 2: movdqa y0=%xmm1 +movdqa %xmm15,%xmm1 + +# qhasm: uint32323232 y0 += z12 +# asm 1: paddd r0=int6464#3 +# asm 2: movdqa r0=%xmm2 +movdqa %xmm1,%xmm2 + +# qhasm: uint32323232 y0 <<= 18 +# asm 1: pslld $18,>= 14 +# asm 1: psrld $14,z10=int6464#2 +# asm 2: movdqa z10=%xmm1 +movdqa 320(%rsp),%xmm1 + +# qhasm: z0_stack = z0 +# asm 1: movdqa z0_stack=stack128#21 +# asm 2: movdqa z0_stack=320(%rsp) +movdqa %xmm12,320(%rsp) + +# qhasm: y5 = z13 +# asm 1: movdqa y5=int6464#3 +# asm 2: movdqa y5=%xmm2 +movdqa %xmm9,%xmm2 + +# qhasm: uint32323232 y5 += z1 +# asm 1: paddd r5=int6464#13 +# asm 2: movdqa r5=%xmm12 +movdqa %xmm2,%xmm12 + +# qhasm: uint32323232 y5 <<= 18 +# asm 1: pslld $18,>= 14 +# asm 1: psrld $14,y14=int6464#3 +# asm 2: movdqa y14=%xmm2 +movdqa %xmm5,%xmm2 + +# qhasm: uint32323232 y14 += z10 +# asm 1: paddd r14=int6464#13 +# asm 2: movdqa r14=%xmm12 +movdqa %xmm2,%xmm12 + +# qhasm: uint32323232 y14 <<= 7 +# asm 1: pslld $7,>= 25 +# asm 1: psrld $25,z15=int6464#3 +# asm 2: movdqa z15=%xmm2 +movdqa 336(%rsp),%xmm2 + +# qhasm: z5_stack = z5 +# asm 1: movdqa z5_stack=stack128#22 +# asm 2: movdqa z5_stack=336(%rsp) +movdqa %xmm0,336(%rsp) + +# qhasm: y3 = z11 +# asm 1: movdqa y3=int6464#1 +# asm 2: movdqa y3=%xmm0 +movdqa %xmm6,%xmm0 + +# qhasm: uint32323232 y3 += z15 +# asm 1: paddd r3=int6464#13 +# asm 2: movdqa r3=%xmm12 +movdqa %xmm0,%xmm12 + +# qhasm: uint32323232 y3 <<= 7 +# asm 1: pslld $7,>= 25 +# asm 1: psrld $25,y2=int6464#1 +# asm 2: movdqa y2=%xmm0 +movdqa %xmm1,%xmm0 + +# qhasm: uint32323232 y2 += z14 +# asm 1: paddd r2=int6464#13 +# asm 2: movdqa r2=%xmm12 +movdqa %xmm0,%xmm12 + +# qhasm: uint32323232 y2 <<= 9 +# asm 1: pslld $9,>= 23 +# asm 1: psrld $23,y7=int6464#1 +# asm 2: movdqa y7=%xmm0 +movdqa %xmm2,%xmm0 + +# qhasm: uint32323232 y7 += z3 +# asm 1: paddd r7=int6464#13 +# asm 2: movdqa r7=%xmm12 +movdqa %xmm0,%xmm12 + +# qhasm: uint32323232 y7 <<= 9 +# asm 1: pslld $9,>= 23 +# asm 1: psrld $23,y6=int6464#1 +# asm 2: movdqa y6=%xmm0 +movdqa %xmm3,%xmm0 + +# qhasm: uint32323232 y6 += z2 +# asm 1: paddd r6=int6464#13 +# asm 2: movdqa r6=%xmm12 +movdqa %xmm0,%xmm12 + +# qhasm: uint32323232 y6 <<= 13 +# asm 1: pslld $13,>= 19 +# asm 1: psrld $19,y11=int6464#1 +# asm 2: movdqa y11=%xmm0 +movdqa %xmm4,%xmm0 + +# qhasm: uint32323232 y11 += z7 +# asm 1: paddd r11=int6464#13 +# asm 2: movdqa r11=%xmm12 +movdqa %xmm0,%xmm12 + +# qhasm: uint32323232 y11 <<= 13 +# asm 1: pslld $13,>= 19 +# asm 1: psrld $19,y10=int6464#1 +# asm 2: movdqa y10=%xmm0 +movdqa %xmm10,%xmm0 + +# qhasm: uint32323232 y10 += z6 +# asm 1: paddd r10=int6464#13 +# asm 2: movdqa r10=%xmm12 +movdqa %xmm0,%xmm12 + +# qhasm: uint32323232 y10 <<= 18 +# asm 1: pslld $18,>= 14 +# asm 1: psrld $14,z0=int6464#1 +# asm 2: movdqa z0=%xmm0 +movdqa 320(%rsp),%xmm0 + +# qhasm: z10_stack = z10 +# asm 1: movdqa z10_stack=stack128#21 +# asm 2: movdqa z10_stack=320(%rsp) +movdqa %xmm1,320(%rsp) + +# qhasm: y1 = z3 +# asm 1: movdqa y1=int6464#2 +# asm 2: movdqa y1=%xmm1 +movdqa %xmm4,%xmm1 + +# qhasm: uint32323232 y1 += z0 +# asm 1: paddd r1=int6464#13 +# asm 2: movdqa r1=%xmm12 +movdqa %xmm1,%xmm12 + +# qhasm: uint32323232 y1 <<= 7 +# asm 1: pslld $7,>= 25 +# asm 1: psrld $25,y15=int6464#2 +# asm 2: movdqa y15=%xmm1 +movdqa %xmm8,%xmm1 + +# qhasm: uint32323232 y15 += z11 +# asm 1: paddd r15=int6464#13 +# asm 2: movdqa r15=%xmm12 +movdqa %xmm1,%xmm12 + +# qhasm: uint32323232 y15 <<= 18 +# asm 1: pslld $18,>= 14 +# asm 1: psrld $14,z5=int6464#13 +# asm 2: movdqa z5=%xmm12 +movdqa 336(%rsp),%xmm12 + +# qhasm: z15_stack = z15 +# asm 1: movdqa z15_stack=stack128#22 +# asm 2: movdqa z15_stack=336(%rsp) +movdqa %xmm2,336(%rsp) + +# qhasm: y6 = z4 +# asm 1: movdqa y6=int6464#2 +# asm 2: movdqa y6=%xmm1 +movdqa %xmm14,%xmm1 + +# qhasm: uint32323232 y6 += z5 +# asm 1: paddd r6=int6464#3 +# asm 2: movdqa r6=%xmm2 +movdqa %xmm1,%xmm2 + +# qhasm: uint32323232 y6 <<= 7 +# asm 1: pslld $7,>= 25 +# asm 1: psrld $25,y2=int6464#2 +# asm 2: movdqa y2=%xmm1 +movdqa %xmm0,%xmm1 + +# qhasm: uint32323232 y2 += z1 +# asm 1: paddd r2=int6464#3 +# asm 2: movdqa r2=%xmm2 +movdqa %xmm1,%xmm2 + +# qhasm: uint32323232 y2 <<= 9 +# asm 1: pslld $9,>= 23 +# asm 1: psrld $23,y7=int6464#2 +# asm 2: movdqa y7=%xmm1 +movdqa %xmm12,%xmm1 + +# qhasm: uint32323232 y7 += z6 +# asm 1: paddd r7=int6464#3 +# asm 2: movdqa r7=%xmm2 +movdqa %xmm1,%xmm2 + +# qhasm: uint32323232 y7 <<= 9 +# asm 1: pslld $9,>= 23 +# asm 1: psrld $23,y3=int6464#2 +# asm 2: movdqa y3=%xmm1 +movdqa %xmm7,%xmm1 + +# qhasm: uint32323232 y3 += z2 +# asm 1: paddd r3=int6464#3 +# asm 2: movdqa r3=%xmm2 +movdqa %xmm1,%xmm2 + +# qhasm: uint32323232 y3 <<= 13 +# asm 1: pslld $13,>= 19 +# asm 1: psrld $19,y4=int6464#2 +# asm 2: movdqa y4=%xmm1 +movdqa %xmm5,%xmm1 + +# qhasm: uint32323232 y4 += z7 +# asm 1: paddd r4=int6464#3 +# asm 2: movdqa r4=%xmm2 +movdqa %xmm1,%xmm2 + +# qhasm: uint32323232 y4 <<= 13 +# asm 1: pslld $13,>= 19 +# asm 1: psrld $19,y0=int6464#2 +# asm 2: movdqa y0=%xmm1 +movdqa %xmm10,%xmm1 + +# qhasm: uint32323232 y0 += z3 +# asm 1: paddd r0=int6464#3 +# asm 2: movdqa r0=%xmm2 +movdqa %xmm1,%xmm2 + +# qhasm: uint32323232 y0 <<= 18 +# asm 1: pslld $18,>= 14 +# asm 1: psrld $14,z10=int6464#2 +# asm 2: movdqa z10=%xmm1 +movdqa 320(%rsp),%xmm1 + +# qhasm: z0_stack = z0 +# asm 1: movdqa z0_stack=stack128#21 +# asm 2: movdqa z0_stack=320(%rsp) +movdqa %xmm0,320(%rsp) + +# qhasm: y5 = z7 +# asm 1: movdqa y5=int6464#1 +# asm 2: movdqa y5=%xmm0 +movdqa %xmm8,%xmm0 + +# qhasm: uint32323232 y5 += z4 +# asm 1: paddd r5=int6464#3 +# asm 2: movdqa r5=%xmm2 +movdqa %xmm0,%xmm2 + +# qhasm: uint32323232 y5 <<= 18 +# asm 1: pslld $18,>= 14 +# asm 1: psrld $14,y11=int6464#1 +# asm 2: movdqa y11=%xmm0 +movdqa %xmm11,%xmm0 + +# qhasm: uint32323232 y11 += z10 +# asm 1: paddd r11=int6464#3 +# asm 2: movdqa r11=%xmm2 +movdqa %xmm0,%xmm2 + +# qhasm: uint32323232 y11 <<= 7 +# asm 1: pslld $7,>= 25 +# asm 1: psrld $25,z15=int6464#3 +# asm 2: movdqa z15=%xmm2 +movdqa 336(%rsp),%xmm2 + +# qhasm: z5_stack = z5 +# asm 1: movdqa z5_stack=stack128#22 +# asm 2: movdqa z5_stack=336(%rsp) +movdqa %xmm12,336(%rsp) + +# qhasm: y12 = z14 +# asm 1: movdqa y12=int6464#1 +# asm 2: movdqa y12=%xmm0 +movdqa %xmm3,%xmm0 + +# qhasm: uint32323232 y12 += z15 +# asm 1: paddd r12=int6464#13 +# asm 2: movdqa r12=%xmm12 +movdqa %xmm0,%xmm12 + +# qhasm: uint32323232 y12 <<= 7 +# asm 1: pslld $7,>= 25 +# asm 1: psrld $25,y8=int6464#1 +# asm 2: movdqa y8=%xmm0 +movdqa %xmm1,%xmm0 + +# qhasm: uint32323232 y8 += z11 +# asm 1: paddd r8=int6464#13 +# asm 2: movdqa r8=%xmm12 +movdqa %xmm0,%xmm12 + +# qhasm: uint32323232 y8 <<= 9 +# asm 1: pslld $9,>= 23 +# asm 1: psrld $23,y13=int6464#1 +# asm 2: movdqa y13=%xmm0 +movdqa %xmm2,%xmm0 + +# qhasm: uint32323232 y13 += z12 +# asm 1: paddd r13=int6464#13 +# asm 2: movdqa r13=%xmm12 +movdqa %xmm0,%xmm12 + +# qhasm: uint32323232 y13 <<= 9 +# asm 1: pslld $9,>= 23 +# asm 1: psrld $23,y9=int6464#1 +# asm 2: movdqa y9=%xmm0 +movdqa %xmm6,%xmm0 + +# qhasm: uint32323232 y9 += z8 +# asm 1: paddd r9=int6464#13 +# asm 2: movdqa r9=%xmm12 +movdqa %xmm0,%xmm12 + +# qhasm: uint32323232 y9 <<= 13 +# asm 1: pslld $13,>= 19 +# asm 1: psrld $19,y14=int6464#1 +# asm 2: movdqa y14=%xmm0 +movdqa %xmm13,%xmm0 + +# qhasm: uint32323232 y14 += z13 +# asm 1: paddd r14=int6464#13 +# asm 2: movdqa r14=%xmm12 +movdqa %xmm0,%xmm12 + +# qhasm: uint32323232 y14 <<= 13 +# asm 1: pslld $13,>= 19 +# asm 1: psrld $19,y10=int6464#1 +# asm 2: movdqa y10=%xmm0 +movdqa %xmm15,%xmm0 + +# qhasm: uint32323232 y10 += z9 +# asm 1: paddd r10=int6464#13 +# asm 2: movdqa r10=%xmm12 +movdqa %xmm0,%xmm12 + +# qhasm: uint32323232 y10 <<= 18 +# asm 1: pslld $18,>= 14 +# asm 1: psrld $14,y15=int6464#1 +# asm 2: movdqa y15=%xmm0 +movdqa %xmm9,%xmm0 + +# qhasm: uint32323232 y15 += z14 +# asm 1: paddd r15=int6464#13 +# asm 2: movdqa r15=%xmm12 +movdqa %xmm0,%xmm12 + +# qhasm: uint32323232 y15 <<= 18 +# asm 1: pslld $18,>= 14 +# asm 1: psrld $14,z0=int6464#13 +# asm 2: movdqa z0=%xmm12 +movdqa 320(%rsp),%xmm12 + +# qhasm: z5 = z5_stack +# asm 1: movdqa z5=int6464#1 +# asm 2: movdqa z5=%xmm0 +movdqa 336(%rsp),%xmm0 + +# qhasm: unsigned>? i -= 2 +# asm 1: sub $2, +ja ._mainloop1 + +# qhasm: uint32323232 z0 += orig0 +# asm 1: paddd in0=int64#3 +# asm 2: movd in0=%rdx +movd %xmm12,%rdx + +# qhasm: in1 = z1 +# asm 1: movd in1=int64#4 +# asm 2: movd in1=%rcx +movd %xmm7,%rcx + +# qhasm: in2 = z2 +# asm 1: movd in2=int64#5 +# asm 2: movd in2=%r8 +movd %xmm10,%r8 + +# qhasm: in3 = z3 +# asm 1: movd in3=int64#6 +# asm 2: movd in3=%r9 +movd %xmm4,%r9 + +# qhasm: z0 <<<= 96 +# asm 1: pshufd $0x39,in0=int64#3 +# asm 2: movd in0=%rdx +movd %xmm12,%rdx + +# qhasm: in1 = z1 +# asm 1: movd in1=int64#4 +# asm 2: movd in1=%rcx +movd %xmm7,%rcx + +# qhasm: in2 = z2 +# asm 1: movd in2=int64#5 +# asm 2: movd in2=%r8 +movd %xmm10,%r8 + +# qhasm: in3 = z3 +# asm 1: movd in3=int64#6 +# asm 2: movd in3=%r9 +movd %xmm4,%r9 + +# qhasm: z0 <<<= 96 +# asm 1: pshufd $0x39,in0=int64#3 +# asm 2: movd in0=%rdx +movd %xmm12,%rdx + +# qhasm: in1 = z1 +# asm 1: movd in1=int64#4 +# asm 2: movd in1=%rcx +movd %xmm7,%rcx + +# qhasm: in2 = z2 +# asm 1: movd in2=int64#5 +# asm 2: movd in2=%r8 +movd %xmm10,%r8 + +# qhasm: in3 = z3 +# asm 1: movd in3=int64#6 +# asm 2: movd in3=%r9 +movd %xmm4,%r9 + +# qhasm: z0 <<<= 96 +# asm 1: pshufd $0x39,in0=int64#3 +# asm 2: movd in0=%rdx +movd %xmm12,%rdx + +# qhasm: in1 = z1 +# asm 1: movd in1=int64#4 +# asm 2: movd in1=%rcx +movd %xmm7,%rcx + +# qhasm: in2 = z2 +# asm 1: movd in2=int64#5 +# asm 2: movd in2=%r8 +movd %xmm10,%r8 + +# qhasm: in3 = z3 +# asm 1: movd in3=int64#6 +# asm 2: movd in3=%r9 +movd %xmm4,%r9 + +# qhasm: (uint32) in0 ^= *(uint32 *) (m + 192) +# asm 1: xorl 192(in4=int64#3 +# asm 2: movd in4=%rdx +movd %xmm14,%rdx + +# qhasm: in5 = z5 +# asm 1: movd in5=int64#4 +# asm 2: movd in5=%rcx +movd %xmm0,%rcx + +# qhasm: in6 = z6 +# asm 1: movd in6=int64#5 +# asm 2: movd in6=%r8 +movd %xmm5,%r8 + +# qhasm: in7 = z7 +# asm 1: movd in7=int64#6 +# asm 2: movd in7=%r9 +movd %xmm8,%r9 + +# qhasm: z4 <<<= 96 +# asm 1: pshufd $0x39,in4=int64#3 +# asm 2: movd in4=%rdx +movd %xmm14,%rdx + +# qhasm: in5 = z5 +# asm 1: movd in5=int64#4 +# asm 2: movd in5=%rcx +movd %xmm0,%rcx + +# qhasm: in6 = z6 +# asm 1: movd in6=int64#5 +# asm 2: movd in6=%r8 +movd %xmm5,%r8 + +# qhasm: in7 = z7 +# asm 1: movd in7=int64#6 +# asm 2: movd in7=%r9 +movd %xmm8,%r9 + +# qhasm: z4 <<<= 96 +# asm 1: pshufd $0x39,in4=int64#3 +# asm 2: movd in4=%rdx +movd %xmm14,%rdx + +# qhasm: in5 = z5 +# asm 1: movd in5=int64#4 +# asm 2: movd in5=%rcx +movd %xmm0,%rcx + +# qhasm: in6 = z6 +# asm 1: movd in6=int64#5 +# asm 2: movd in6=%r8 +movd %xmm5,%r8 + +# qhasm: in7 = z7 +# asm 1: movd in7=int64#6 +# asm 2: movd in7=%r9 +movd %xmm8,%r9 + +# qhasm: z4 <<<= 96 +# asm 1: pshufd $0x39,in4=int64#3 +# asm 2: movd in4=%rdx +movd %xmm14,%rdx + +# qhasm: in5 = z5 +# asm 1: movd in5=int64#4 +# asm 2: movd in5=%rcx +movd %xmm0,%rcx + +# qhasm: in6 = z6 +# asm 1: movd in6=int64#5 +# asm 2: movd in6=%r8 +movd %xmm5,%r8 + +# qhasm: in7 = z7 +# asm 1: movd in7=int64#6 +# asm 2: movd in7=%r9 +movd %xmm8,%r9 + +# qhasm: (uint32) in4 ^= *(uint32 *) (m + 208) +# asm 1: xorl 208(in8=int64#3 +# asm 2: movd in8=%rdx +movd %xmm15,%rdx + +# qhasm: in9 = z9 +# asm 1: movd in9=int64#4 +# asm 2: movd in9=%rcx +movd %xmm11,%rcx + +# qhasm: in10 = z10 +# asm 1: movd in10=int64#5 +# asm 2: movd in10=%r8 +movd %xmm1,%r8 + +# qhasm: in11 = z11 +# asm 1: movd in11=int64#6 +# asm 2: movd in11=%r9 +movd %xmm6,%r9 + +# qhasm: z8 <<<= 96 +# asm 1: pshufd $0x39,in8=int64#3 +# asm 2: movd in8=%rdx +movd %xmm15,%rdx + +# qhasm: in9 = z9 +# asm 1: movd in9=int64#4 +# asm 2: movd in9=%rcx +movd %xmm11,%rcx + +# qhasm: in10 = z10 +# asm 1: movd in10=int64#5 +# asm 2: movd in10=%r8 +movd %xmm1,%r8 + +# qhasm: in11 = z11 +# asm 1: movd in11=int64#6 +# asm 2: movd in11=%r9 +movd %xmm6,%r9 + +# qhasm: z8 <<<= 96 +# asm 1: pshufd $0x39,in8=int64#3 +# asm 2: movd in8=%rdx +movd %xmm15,%rdx + +# qhasm: in9 = z9 +# asm 1: movd in9=int64#4 +# asm 2: movd in9=%rcx +movd %xmm11,%rcx + +# qhasm: in10 = z10 +# asm 1: movd in10=int64#5 +# asm 2: movd in10=%r8 +movd %xmm1,%r8 + +# qhasm: in11 = z11 +# asm 1: movd in11=int64#6 +# asm 2: movd in11=%r9 +movd %xmm6,%r9 + +# qhasm: z8 <<<= 96 +# asm 1: pshufd $0x39,in8=int64#3 +# asm 2: movd in8=%rdx +movd %xmm15,%rdx + +# qhasm: in9 = z9 +# asm 1: movd in9=int64#4 +# asm 2: movd in9=%rcx +movd %xmm11,%rcx + +# qhasm: in10 = z10 +# asm 1: movd in10=int64#5 +# asm 2: movd in10=%r8 +movd %xmm1,%r8 + +# qhasm: in11 = z11 +# asm 1: movd in11=int64#6 +# asm 2: movd in11=%r9 +movd %xmm6,%r9 + +# qhasm: (uint32) in8 ^= *(uint32 *) (m + 224) +# asm 1: xorl 224(in12=int64#3 +# asm 2: movd in12=%rdx +movd %xmm13,%rdx + +# qhasm: in13 = z13 +# asm 1: movd in13=int64#4 +# asm 2: movd in13=%rcx +movd %xmm9,%rcx + +# qhasm: in14 = z14 +# asm 1: movd in14=int64#5 +# asm 2: movd in14=%r8 +movd %xmm3,%r8 + +# qhasm: in15 = z15 +# asm 1: movd in15=int64#6 +# asm 2: movd in15=%r9 +movd %xmm2,%r9 + +# qhasm: z12 <<<= 96 +# asm 1: pshufd $0x39,in12=int64#3 +# asm 2: movd in12=%rdx +movd %xmm13,%rdx + +# qhasm: in13 = z13 +# asm 1: movd in13=int64#4 +# asm 2: movd in13=%rcx +movd %xmm9,%rcx + +# qhasm: in14 = z14 +# asm 1: movd in14=int64#5 +# asm 2: movd in14=%r8 +movd %xmm3,%r8 + +# qhasm: in15 = z15 +# asm 1: movd in15=int64#6 +# asm 2: movd in15=%r9 +movd %xmm2,%r9 + +# qhasm: z12 <<<= 96 +# asm 1: pshufd $0x39,in12=int64#3 +# asm 2: movd in12=%rdx +movd %xmm13,%rdx + +# qhasm: in13 = z13 +# asm 1: movd in13=int64#4 +# asm 2: movd in13=%rcx +movd %xmm9,%rcx + +# qhasm: in14 = z14 +# asm 1: movd in14=int64#5 +# asm 2: movd in14=%r8 +movd %xmm3,%r8 + +# qhasm: in15 = z15 +# asm 1: movd in15=int64#6 +# asm 2: movd in15=%r9 +movd %xmm2,%r9 + +# qhasm: z12 <<<= 96 +# asm 1: pshufd $0x39,in12=int64#3 +# asm 2: movd in12=%rdx +movd %xmm13,%rdx + +# qhasm: in13 = z13 +# asm 1: movd in13=int64#4 +# asm 2: movd in13=%rcx +movd %xmm9,%rcx + +# qhasm: in14 = z14 +# asm 1: movd in14=int64#5 +# asm 2: movd in14=%r8 +movd %xmm3,%r8 + +# qhasm: in15 = z15 +# asm 1: movd in15=int64#6 +# asm 2: movd in15=%r9 +movd %xmm2,%r9 + +# qhasm: (uint32) in12 ^= *(uint32 *) (m + 240) +# asm 1: xorl 240(bytes=int64#6 +# asm 2: movq bytes=%r9 +movq 408(%rsp),%r9 + +# qhasm: bytes -= 256 +# asm 1: sub $256,? bytes - 0 +# asm 1: cmp $0, +jbe ._done +# comment:fp stack unchanged by fallthrough + +# qhasm: bytesbetween1and255: +._bytesbetween1and255: + +# qhasm: unsignedctarget=int64#3 +# asm 2: mov ctarget=%rdx +mov %rdi,%rdx + +# qhasm: out = &tmp +# asm 1: leaq out=int64#1 +# asm 2: leaq out=%rdi +leaq 416(%rsp),%rdi + +# qhasm: i = bytes +# asm 1: mov i=int64#4 +# asm 2: mov i=%rcx +mov %r9,%rcx + +# qhasm: while (i) { *out++ = *m++; --i } +rep movsb + +# qhasm: out = &tmp +# asm 1: leaq out=int64#1 +# asm 2: leaq out=%rdi +leaq 416(%rsp),%rdi + +# qhasm: m = &tmp +# asm 1: leaq m=int64#2 +# asm 2: leaq m=%rsi +leaq 416(%rsp),%rsi +# comment:fp stack unchanged by fallthrough + +# qhasm: nocopy: +._nocopy: + +# qhasm: bytes_backup = bytes +# asm 1: movq bytes_backup=stack64#8 +# asm 2: movq bytes_backup=408(%rsp) +movq %r9,408(%rsp) + +# qhasm: diag0 = x0 +# asm 1: movdqa diag0=int6464#1 +# asm 2: movdqa diag0=%xmm0 +movdqa 48(%rsp),%xmm0 + +# qhasm: diag1 = x1 +# asm 1: movdqa diag1=int6464#2 +# asm 2: movdqa diag1=%xmm1 +movdqa 0(%rsp),%xmm1 + +# qhasm: diag2 = x2 +# asm 1: movdqa diag2=int6464#3 +# asm 2: movdqa diag2=%xmm2 +movdqa 16(%rsp),%xmm2 + +# qhasm: diag3 = x3 +# asm 1: movdqa diag3=int6464#4 +# asm 2: movdqa diag3=%xmm3 +movdqa 32(%rsp),%xmm3 + +# qhasm: a0 = diag1 +# asm 1: movdqa a0=int6464#5 +# asm 2: movdqa a0=%xmm4 +movdqa %xmm1,%xmm4 + +# qhasm: i = 12 +# asm 1: mov $12,>i=int64#4 +# asm 2: mov $12,>i=%rcx +mov $12,%rcx + +# qhasm: mainloop2: +._mainloop2: + +# qhasm: uint32323232 a0 += diag0 +# asm 1: paddd a1=int6464#6 +# asm 2: movdqa a1=%xmm5 +movdqa %xmm0,%xmm5 + +# qhasm: b0 = a0 +# asm 1: movdqa b0=int6464#7 +# asm 2: movdqa b0=%xmm6 +movdqa %xmm4,%xmm6 + +# qhasm: uint32323232 a0 <<= 7 +# asm 1: pslld $7,>= 25 +# asm 1: psrld $25,a2=int6464#5 +# asm 2: movdqa a2=%xmm4 +movdqa %xmm3,%xmm4 + +# qhasm: b1 = a1 +# asm 1: movdqa b1=int6464#7 +# asm 2: movdqa b1=%xmm6 +movdqa %xmm5,%xmm6 + +# qhasm: uint32323232 a1 <<= 9 +# asm 1: pslld $9,>= 23 +# asm 1: psrld $23,a3=int6464#6 +# asm 2: movdqa a3=%xmm5 +movdqa %xmm2,%xmm5 + +# qhasm: b2 = a2 +# asm 1: movdqa b2=int6464#7 +# asm 2: movdqa b2=%xmm6 +movdqa %xmm4,%xmm6 + +# qhasm: uint32323232 a2 <<= 13 +# asm 1: pslld $13,>= 19 +# asm 1: psrld $19,a4=int6464#5 +# asm 2: movdqa a4=%xmm4 +movdqa %xmm3,%xmm4 + +# qhasm: b3 = a3 +# asm 1: movdqa b3=int6464#7 +# asm 2: movdqa b3=%xmm6 +movdqa %xmm5,%xmm6 + +# qhasm: uint32323232 a3 <<= 18 +# asm 1: pslld $18,>= 14 +# asm 1: psrld $14,a5=int6464#6 +# asm 2: movdqa a5=%xmm5 +movdqa %xmm0,%xmm5 + +# qhasm: b4 = a4 +# asm 1: movdqa b4=int6464#7 +# asm 2: movdqa b4=%xmm6 +movdqa %xmm4,%xmm6 + +# qhasm: uint32323232 a4 <<= 7 +# asm 1: pslld $7,>= 25 +# asm 1: psrld $25,a6=int6464#5 +# asm 2: movdqa a6=%xmm4 +movdqa %xmm1,%xmm4 + +# qhasm: b5 = a5 +# asm 1: movdqa b5=int6464#7 +# asm 2: movdqa b5=%xmm6 +movdqa %xmm5,%xmm6 + +# qhasm: uint32323232 a5 <<= 9 +# asm 1: pslld $9,>= 23 +# asm 1: psrld $23,a7=int6464#6 +# asm 2: movdqa a7=%xmm5 +movdqa %xmm2,%xmm5 + +# qhasm: b6 = a6 +# asm 1: movdqa b6=int6464#7 +# asm 2: movdqa b6=%xmm6 +movdqa %xmm4,%xmm6 + +# qhasm: uint32323232 a6 <<= 13 +# asm 1: pslld $13,>= 19 +# asm 1: psrld $19,a0=int6464#5 +# asm 2: movdqa a0=%xmm4 +movdqa %xmm1,%xmm4 + +# qhasm: b7 = a7 +# asm 1: movdqa b7=int6464#7 +# asm 2: movdqa b7=%xmm6 +movdqa %xmm5,%xmm6 + +# qhasm: uint32323232 a7 <<= 18 +# asm 1: pslld $18,>= 14 +# asm 1: psrld $14,a1=int6464#6 +# asm 2: movdqa a1=%xmm5 +movdqa %xmm0,%xmm5 + +# qhasm: b0 = a0 +# asm 1: movdqa b0=int6464#7 +# asm 2: movdqa b0=%xmm6 +movdqa %xmm4,%xmm6 + +# qhasm: uint32323232 a0 <<= 7 +# asm 1: pslld $7,>= 25 +# asm 1: psrld $25,a2=int6464#5 +# asm 2: movdqa a2=%xmm4 +movdqa %xmm3,%xmm4 + +# qhasm: b1 = a1 +# asm 1: movdqa b1=int6464#7 +# asm 2: movdqa b1=%xmm6 +movdqa %xmm5,%xmm6 + +# qhasm: uint32323232 a1 <<= 9 +# asm 1: pslld $9,>= 23 +# asm 1: psrld $23,a3=int6464#6 +# asm 2: movdqa a3=%xmm5 +movdqa %xmm2,%xmm5 + +# qhasm: b2 = a2 +# asm 1: movdqa b2=int6464#7 +# asm 2: movdqa b2=%xmm6 +movdqa %xmm4,%xmm6 + +# qhasm: uint32323232 a2 <<= 13 +# asm 1: pslld $13,>= 19 +# asm 1: psrld $19,a4=int6464#5 +# asm 2: movdqa a4=%xmm4 +movdqa %xmm3,%xmm4 + +# qhasm: b3 = a3 +# asm 1: movdqa b3=int6464#7 +# asm 2: movdqa b3=%xmm6 +movdqa %xmm5,%xmm6 + +# qhasm: uint32323232 a3 <<= 18 +# asm 1: pslld $18,>= 14 +# asm 1: psrld $14,a5=int6464#6 +# asm 2: movdqa a5=%xmm5 +movdqa %xmm0,%xmm5 + +# qhasm: b4 = a4 +# asm 1: movdqa b4=int6464#7 +# asm 2: movdqa b4=%xmm6 +movdqa %xmm4,%xmm6 + +# qhasm: uint32323232 a4 <<= 7 +# asm 1: pslld $7,>= 25 +# asm 1: psrld $25,a6=int6464#5 +# asm 2: movdqa a6=%xmm4 +movdqa %xmm1,%xmm4 + +# qhasm: b5 = a5 +# asm 1: movdqa b5=int6464#7 +# asm 2: movdqa b5=%xmm6 +movdqa %xmm5,%xmm6 + +# qhasm: uint32323232 a5 <<= 9 +# asm 1: pslld $9,>= 23 +# asm 1: psrld $23,a7=int6464#6 +# asm 2: movdqa a7=%xmm5 +movdqa %xmm2,%xmm5 + +# qhasm: b6 = a6 +# asm 1: movdqa b6=int6464#7 +# asm 2: movdqa b6=%xmm6 +movdqa %xmm4,%xmm6 + +# qhasm: uint32323232 a6 <<= 13 +# asm 1: pslld $13,>= 19 +# asm 1: psrld $19,? i -= 4 +# asm 1: sub $4,a0=int6464#5 +# asm 2: movdqa a0=%xmm4 +movdqa %xmm1,%xmm4 + +# qhasm: b7 = a7 +# asm 1: movdqa b7=int6464#7 +# asm 2: movdqa b7=%xmm6 +movdqa %xmm5,%xmm6 + +# qhasm: uint32323232 a7 <<= 18 +# asm 1: pslld $18,b0=int6464#8,>b0=int6464#8 +# asm 2: pxor >b0=%xmm7,>b0=%xmm7 +pxor %xmm7,%xmm7 + +# qhasm: uint32323232 b7 >>= 14 +# asm 1: psrld $14, +ja ._mainloop2 + +# qhasm: uint32323232 diag0 += x0 +# asm 1: paddd in0=int64#4 +# asm 2: movd in0=%rcx +movd %xmm0,%rcx + +# qhasm: in12 = diag1 +# asm 1: movd in12=int64#5 +# asm 2: movd in12=%r8 +movd %xmm1,%r8 + +# qhasm: in8 = diag2 +# asm 1: movd in8=int64#6 +# asm 2: movd in8=%r9 +movd %xmm2,%r9 + +# qhasm: in4 = diag3 +# asm 1: movd in4=int64#7 +# asm 2: movd in4=%rax +movd %xmm3,%rax + +# qhasm: diag0 <<<= 96 +# asm 1: pshufd $0x39,in5=int64#4 +# asm 2: movd in5=%rcx +movd %xmm0,%rcx + +# qhasm: in1 = diag1 +# asm 1: movd in1=int64#5 +# asm 2: movd in1=%r8 +movd %xmm1,%r8 + +# qhasm: in13 = diag2 +# asm 1: movd in13=int64#6 +# asm 2: movd in13=%r9 +movd %xmm2,%r9 + +# qhasm: in9 = diag3 +# asm 1: movd in9=int64#7 +# asm 2: movd in9=%rax +movd %xmm3,%rax + +# qhasm: diag0 <<<= 96 +# asm 1: pshufd $0x39,in10=int64#4 +# asm 2: movd in10=%rcx +movd %xmm0,%rcx + +# qhasm: in6 = diag1 +# asm 1: movd in6=int64#5 +# asm 2: movd in6=%r8 +movd %xmm1,%r8 + +# qhasm: in2 = diag2 +# asm 1: movd in2=int64#6 +# asm 2: movd in2=%r9 +movd %xmm2,%r9 + +# qhasm: in14 = diag3 +# asm 1: movd in14=int64#7 +# asm 2: movd in14=%rax +movd %xmm3,%rax + +# qhasm: diag0 <<<= 96 +# asm 1: pshufd $0x39,in15=int64#4 +# asm 2: movd in15=%rcx +movd %xmm0,%rcx + +# qhasm: in11 = diag1 +# asm 1: movd in11=int64#5 +# asm 2: movd in11=%r8 +movd %xmm1,%r8 + +# qhasm: in7 = diag2 +# asm 1: movd in7=int64#6 +# asm 2: movd in7=%r9 +movd %xmm2,%r9 + +# qhasm: in3 = diag3 +# asm 1: movd in3=int64#7 +# asm 2: movd in3=%rax +movd %xmm3,%rax + +# qhasm: (uint32) in15 ^= *(uint32 *) (m + 60) +# asm 1: xorl 60(bytes=int64#6 +# asm 2: movq bytes=%r9 +movq 408(%rsp),%r9 + +# qhasm: in8 = ((uint32 *)&x2)[0] +# asm 1: movl in8=int64#4d +# asm 2: movl in8=%ecx +movl 16(%rsp),%ecx + +# qhasm: in9 = ((uint32 *)&x3)[1] +# asm 1: movl 4+in9=int64#5d +# asm 2: movl 4+in9=%r8d +movl 4+32(%rsp),%r8d + +# qhasm: in8 += 1 +# asm 1: add $1,in9=int64#5 +# asm 2: mov in9=%r8 +mov %rcx,%r8 + +# qhasm: (uint64) in9 >>= 32 +# asm 1: shr $32,x2=stack128#2 +# asm 2: movl x2=16(%rsp) +movl %ecx,16(%rsp) + +# qhasm: ((uint32 *)&x3)[1] = in9 +# asm 1: movl ? unsigned +ja ._bytesatleast65 +# comment:fp stack unchanged by jump + +# qhasm: goto bytesatleast64 if !unsigned< +jae ._bytesatleast64 + +# qhasm: m = out +# asm 1: mov m=int64#2 +# asm 2: mov m=%rsi +mov %rdi,%rsi + +# qhasm: out = ctarget +# asm 1: mov out=int64#1 +# asm 2: mov out=%rdi +mov %rdx,%rdi + +# qhasm: i = bytes +# asm 1: mov i=int64#4 +# asm 2: mov i=%rcx +mov %r9,%rcx + +# qhasm: while (i) { *out++ = *m++; --i } +rep movsb +# comment:fp stack unchanged by fallthrough + +# qhasm: bytesatleast64: +._bytesatleast64: +# comment:fp stack unchanged by fallthrough + +# qhasm: done: +._done: + +# qhasm: r11_caller = r11_stack +# asm 1: movq r11_caller=int64#9 +# asm 2: movq r11_caller=%r11 +movq 352(%rsp),%r11 + +# qhasm: r12_caller = r12_stack +# asm 1: movq r12_caller=int64#10 +# asm 2: movq r12_caller=%r12 +movq 360(%rsp),%r12 + +# qhasm: r13_caller = r13_stack +# asm 1: movq r13_caller=int64#11 +# asm 2: movq r13_caller=%r13 +movq 368(%rsp),%r13 + +# qhasm: r14_caller = r14_stack +# asm 1: movq r14_caller=int64#12 +# asm 2: movq r14_caller=%r14 +movq 376(%rsp),%r14 + +# qhasm: r15_caller = r15_stack +# asm 1: movq r15_caller=int64#13 +# asm 2: movq r15_caller=%r15 +movq 384(%rsp),%r15 + +# qhasm: rbx_caller = rbx_stack +# asm 1: movq rbx_caller=int64#14 +# asm 2: movq rbx_caller=%rbx +movq 392(%rsp),%rbx + +# qhasm: rbp_caller = rbp_stack +# asm 1: movq rbp_caller=int64#15 +# asm 2: movq rbp_caller=%rbp +movq 400(%rsp),%rbp + +# qhasm: leave +add %r11,%rsp +xor %rax,%rax +xor %rdx,%rdx +ret + +# qhasm: bytesatleast65: +._bytesatleast65: + +# qhasm: bytes -= 64 +# asm 1: sub $64, @@ -1064,7 +1068,7 @@ const char *Packet::errorString(ErrorCode e) void Packet::armor(const void *key,bool encryptPayload,unsigned int counter) { - uint8_t mangledKey[32],macKey[32],mac[16]; + uint8_t mangledKey[32]; uint8_t *const data = reinterpret_cast(unsafeData()); // Mask least significant 3 bits of packet ID with counter to embed packet send counter for QoS use @@ -1074,23 +1078,47 @@ void Packet::armor(const void *key,bool encryptPayload,unsigned int counter) setCipher(encryptPayload ? ZT_PROTO_CIPHER_SUITE__C25519_POLY1305_SALSA2012 : ZT_PROTO_CIPHER_SUITE__C25519_POLY1305_NONE); _salsa20MangleKey((const unsigned char *)key,mangledKey); + +#ifdef ZT_USE_X64_ASM_SALSA2012 + const unsigned int payloadLen = (encryptPayload) ? (size() - ZT_PACKET_IDX_VERB) : 0; + uint64_t keyStream[(ZT_PROTO_MAX_PACKET_LENGTH + 64 + 8) / 8]; + zt_salsa2012_amd64_xmm6(reinterpret_cast(keyStream),payloadLen + 64,reinterpret_cast(data + ZT_PACKET_IDX_IV),reinterpret_cast(mangledKey)); + + uint64_t *ksptr = keyStream + 8; // encryption starts after first Salsa20 block + uint8_t *dptr = data + ZT_PACKET_IDX_VERB; + unsigned int ksrem = payloadLen; + while (ksrem >= 8) { + ksrem -= 8; + *(reinterpret_cast(dptr)) ^= *(ksptr++); + dptr += 8; + } + for(unsigned int i=0;i(ksptr)[i]; + } + + uint64_t mac[2]; + Poly1305::compute(mac,data + ZT_PACKET_IDX_VERB,size() - ZT_PACKET_IDX_VERB,keyStream); + memcpy(data + ZT_PACKET_IDX_MAC,mac,8); +#else Salsa20 s20(mangledKey,data + ZT_PACKET_IDX_IV); - // MAC key is always the first 32 bytes of the Salsa20 key stream - // This is the same construction DJB's NaCl library uses + uint64_t macKey[4]; s20.crypt12(ZERO_KEY,macKey,sizeof(macKey)); uint8_t *const payload = data + ZT_PACKET_IDX_VERB; const unsigned int payloadLen = size() - ZT_PACKET_IDX_VERB; if (encryptPayload) s20.crypt12(payload,payload,payloadLen); + + uint64_t mac[2]; Poly1305::compute(mac,payload,payloadLen,macKey); memcpy(data + ZT_PACKET_IDX_MAC,mac,8); +#endif } bool Packet::dearmor(const void *key) { - uint8_t mangledKey[32],macKey[32],mac[16]; + uint8_t mangledKey[32]; uint8_t *const data = reinterpret_cast(unsafeData()); const unsigned int payloadLen = size() - ZT_PACKET_IDX_VERB; unsigned char *const payload = data + ZT_PACKET_IDX_VERB; @@ -1098,9 +1126,37 @@ bool Packet::dearmor(const void *key) if ((cs == ZT_PROTO_CIPHER_SUITE__C25519_POLY1305_NONE)||(cs == ZT_PROTO_CIPHER_SUITE__C25519_POLY1305_SALSA2012)) { _salsa20MangleKey((const unsigned char *)key,mangledKey); + +#ifdef ZT_USE_X64_ASM_SALSA2012 + uint64_t keyStream[(ZT_PROTO_MAX_PACKET_LENGTH + 64 + 8) / 8]; + zt_salsa2012_amd64_xmm6(reinterpret_cast(keyStream),((cs == ZT_PROTO_CIPHER_SUITE__C25519_POLY1305_SALSA2012) ? (payloadLen + 64) : 64),reinterpret_cast(data + ZT_PACKET_IDX_IV),reinterpret_cast(mangledKey)); + + uint64_t mac[2]; + Poly1305::compute(mac,payload,payloadLen,keyStream); + if (!Utils::secureEq(mac,data + ZT_PACKET_IDX_MAC,8)) + return false; // MAC failed, packet is corrupt, modified, or is not from the sender + + if (cs == ZT_PROTO_CIPHER_SUITE__C25519_POLY1305_SALSA2012) { + uint64_t *ksptr = keyStream + 8; // encryption starts after first Salsa20 block + uint8_t *dptr = data + ZT_PACKET_IDX_VERB; + unsigned int ksrem = payloadLen; + while (ksrem >= 8) { + ksrem -= 8; + *(reinterpret_cast(dptr)) ^= *(ksptr++); + dptr += 8; + } + for(unsigned int i=0;i(ksptr)[i]; + } + } + + return true; +#else Salsa20 s20(mangledKey,data + ZT_PACKET_IDX_IV); + uint64_t macKey[4]; s20.crypt12(ZERO_KEY,macKey,sizeof(macKey)); + uint64_t mac[2]; Poly1305::compute(mac,payload,payloadLen,macKey); if (!Utils::secureEq(mac,data + ZT_PACKET_IDX_MAC,8)) return false; // MAC failed, packet is corrupt, modified, or is not from the sender @@ -1109,6 +1165,7 @@ bool Packet::dearmor(const void *key) s20.crypt12(payload,payload,payloadLen); return true; +#endif } else { return false; // unrecognized cipher suite } diff --git a/selftest.cpp b/selftest.cpp index fe0aa933..b7a1cc4d 100644 --- a/selftest.cpp +++ b/selftest.cpp @@ -54,6 +54,10 @@ #include "controller/JSONDB.hpp" +#ifdef ZT_USE_X64_ASM_SALSA2012 +#include "ext/x64-salsa2012-asm/salsa2012.h" +#endif + #ifdef __WINDOWS__ #include #endif @@ -204,6 +208,24 @@ static int testCrypto() ::free((void *)bb); } +#ifdef ZT_USE_X64_ASM_SALSA2012 + std::cout << "[crypto] Benchmarking Salsa20/12 fast x64 ASM... "; std::cout.flush(); + { + unsigned char *bb = (unsigned char *)::malloc(1234567); + for(unsigned int i=0;i<1234567;++i) + bb[i] = (unsigned char)i; + double bytes = 0.0; + uint64_t start = OSUtils::now(); + for(unsigned int i=0;i<200;++i) { + zt_salsa2012_amd64_xmm6_xor(bb,bb,1234567,s20TV0Iv,s20TV0Key); + bytes += 1234567.0; + } + uint64_t end = OSUtils::now(); + std::cout << ((bytes / 1048576.0) / ((double)(end - start) / 1000.0)) << " MiB/second" << std::endl; + ::free((void *)bb); + } +#endif + std::cout << "[crypto] Benchmarking Salsa20/20... "; std::cout.flush(); { unsigned char *bb = (unsigned char *)::malloc(1234567); -- cgit v1.2.3 From f9ae5938d0f97e3020660bcbd04bc02f91e4b63d Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Thu, 20 Apr 2017 10:08:46 -0700 Subject: Makefile fix, only bifurcate routes for default, and release notes. --- RELEASE-NOTES.md | 11 +++++++++++ make-mac.mk | 4 ++-- osdep/ManagedRoute.cpp | 29 ++++------------------------- 3 files changed, 17 insertions(+), 27 deletions(-) (limited to 'make-mac.mk') diff --git a/RELEASE-NOTES.md b/RELEASE-NOTES.md index b54b7ea5..a08eda48 100644 --- a/RELEASE-NOTES.md +++ b/RELEASE-NOTES.md @@ -1,6 +1,17 @@ ZeroTier Release Notes ====== +# Version 1.2.4 + +Version 1.2.4 is a minor bug fix and performance improvement release. + + * Managed routes are now only bifurcated for the default route. This is a change in behavior, though few people will probably notice. Bifurcating all managed routes was causing more trouble than it was worth for most users. + * Up to 2X crypto speedup on x86-64 (except Windows, which will take some porting) and 32-bit ARM platforms due to integration of fast assembly language implementations of Salsa20/12 from the [supercop](http://bench.cr.yp.to/supercop.html) code base. These were written by Daniel J. Bernstein and are in the public domain. + * Refactored code that manages credentials to greatly reduce memory use in most cases. This may also result in a small performance improvement. + * Reworked and simplified path selection and priority logic to fix path instability and dead path persistence edge cases. There have been some sporadic reports of persistent path instabilities and dead paths hanging around that take minutes to resolve. These have proven difficult to reproduce in house, but hopefully this will fix them. In any case it seems to speed up path establishment in our tests and it makes the code simpler and more readable. + * Eliminated some unused cruft from the code around path management and in the peer class. + * Fixed an issue causing build problems on some MIPS architecture systems. + # 2017-03-17 -- Version 1.2.2 Version 1.2.2 fixes a few bugs discovered after the 1.2.0 release. These are: diff --git a/make-mac.mk b/make-mac.mk index fe832ae1..6676f457 100644 --- a/make-mac.mk +++ b/make-mac.mk @@ -61,11 +61,11 @@ endif CXXFLAGS=$(CFLAGS) -std=c++11 -stdlib=libc++ +all: one macui + ext/x64-salsa2012-asm/salsa2012.o: $(CC) $(CFLAGS) -c ext/x64-salsa2012-asm/salsa2012.s -o ext/x64-salsa2012-asm/salsa2012.o -all: one macui - one: $(OBJS) service/OneService.o one.o $(CXX) $(CXXFLAGS) -o zerotier-one $(OBJS) service/OneService.o one.o $(LIBS) $(STRIP) zerotier-one diff --git a/osdep/ManagedRoute.cpp b/osdep/ManagedRoute.cpp index 1fc6c78e..f24b6b34 100644 --- a/osdep/ManagedRoute.cpp +++ b/osdep/ManagedRoute.cpp @@ -57,8 +57,6 @@ #define ZT_LINUX_IP_COMMAND "/sbin/ip" #define ZT_LINUX_IP_COMMAND_2 "/usr/sbin/ip" -// NOTE: BSD is mostly tested on Apple/Mac but is likely to work on other BSD too - namespace ZeroTier { namespace { @@ -351,7 +349,7 @@ static bool _winRoute(bool del,const NET_LUID &interfaceLuid,const NET_IFINDEX & #endif // __WINDOWS__ -------------------------------------------------------- #ifndef ZT_ROUTING_SUPPORT_FOUND -#error "ManagedRoute.cpp has no support for managing routes on this platform! You'll need to check and see if one of the existing ones will work and make sure proper defines are set, or write one. Please do a Github pull request if you do this for a new OS." +#error "ManagedRoute.cpp has no support for managing routes on this platform! You'll need to check and see if one of the existing ones will work and make sure proper defines are set, or write one. Please do a GitHub pull request if you do this for a new OS." #endif } // anonymous namespace @@ -378,9 +376,10 @@ bool ManagedRoute::sync() return false; #endif - // Generate two more specific routes than target with one extra bit InetAddress leftt,rightt; - _forkTarget(_target,leftt,rightt); + if (_target.netmaskBits() == 0) // bifurcate only the default route + _forkTarget(_target,leftt,rightt); + else leftt = _target; #ifdef __BSD__ // ------------------------------------------------------------ @@ -447,26 +446,6 @@ bool ManagedRoute::sync() _routeCmd("change",rightt,_via,(const char *)0,(_via) ? (const char *)0 : _device); } - // Create a device-bound default target if there is none in the system. This - // is to allow e.g. IPv6 default route to work even if there is no native - // IPv6 on your LAN. - /* - if (_target.isDefaultRoute()) { - if (_systemVia) { - if (_applied.count(_target)) { - _applied.erase(_target); - _routeCmd("delete",_target,_via,_device,(_via) ? (const char *)0 : _device); - } - } else { - if (!_applied.count(_target)) { - _applied[_target] = true; // ifscoped - _routeCmd("add",_target,_via,_device,(_via) ? (const char *)0 : _device); - _routeCmd("change",_target,_via,_device,(_via) ? (const char *)0 : _device); - } - } - } - */ - #endif // __BSD__ ------------------------------------------------------------ #ifdef __LINUX__ // ---------------------------------------------------------- -- cgit v1.2.3 From 505b9c7f563ddc441db56489dc79c627971f32ea Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Mon, 8 May 2017 14:36:55 -0700 Subject: Makefile reorg to allow core to be built as a library. --- make-bsd.mk | 30 ++++++++++++++++++++++-------- make-linux.mk | 43 +++++++++++++++++++++++++++++-------------- make-mac.mk | 30 ++++++++++++++++++++++-------- objects.mk | 14 +++++++++----- 4 files changed, 82 insertions(+), 35 deletions(-) (limited to 'make-mac.mk') diff --git a/make-bsd.mk b/make-bsd.mk index 39c6cef1..6370c025 100644 --- a/make-bsd.mk +++ b/make-bsd.mk @@ -5,7 +5,7 @@ DEFS= LIBS= include objects.mk -OBJS+=osdep/BSDEthernetTap.o ext/http-parser/http_parser.o +ONE_OBJS+=osdep/BSDEthernetTap.o ext/http-parser/http_parser.o # Build with ZT_ENABLE_CLUSTER=1 to build with cluster support ifeq ($(ZT_ENABLE_CLUSTER),1) @@ -115,11 +115,11 @@ endif # Build faster crypto on some targets ifeq ($(ZT_USE_X64_ASM_SALSA2012),1) override DEFS+=-DZT_USE_X64_ASM_SALSA2012 - override OBJS+=ext/x64-salsa2012-asm/salsa2012.o + override CORE_OBJS+=ext/x64-salsa2012-asm/salsa2012.o endif ifeq ($(ZT_USE_ARM32_NEON_ASM_SALSA2012),1) override DEFS+=-DZT_USE_ARM32_NEON_ASM_SALSA2012 - override OBJS+=ext/arm32-neon-salsa2012-asm/salsa2012.o + override CORE_OBJS+=ext/arm32-neon-salsa2012-asm/salsa2012.o endif override DEFS+=-DZT_BUILD_PLATFORM=$(ZT_BUILD_PLATFORM) -DZT_BUILD_ARCHITECTURE=$(ZT_ARCHITECTURE) -DZT_SOFTWARE_UPDATE_DEFAULT="\"disable\"" @@ -128,18 +128,32 @@ CXXFLAGS+=$(CFLAGS) -fno-rtti -std=c++11 #-D_GLIBCXX_USE_C99 -D_GLIBCXX_USE_C99_ all: one -one: $(OBJS) service/OneService.o one.o - $(CXX) $(CXXFLAGS) $(LDFLAGS) -o zerotier-one $(OBJS) service/OneService.o one.o $(LIBS) +one: $(CORE_OBJS) $(ONE_OBJS) one.o + $(CXX) $(CXXFLAGS) $(LDFLAGS) -o zerotier-one $(CORE_OBJS) $(ONE_OBJS) one.o $(LIBS) $(STRIP) zerotier-one ln -sf zerotier-one zerotier-idtool ln -sf zerotier-one zerotier-cli -selftest: $(OBJS) selftest.o - $(CXX) $(CXXFLAGS) $(LDFLAGS) -o zerotier-selftest selftest.o $(OBJS) $(LIBS) +zerotier-one: one + +zerotier-idtool: one + +zerotier-cli: one + +libzerotiercore.a: $(CORE_OBJS) + ar rcs libzerotiercore.a $(CORE_OBJS) + ranlib libzerotiercore.a + +core: libzerotiercore.a + +selftest: $(CORE_OBJS) $(ONE_OBJS) selftest.o + $(CXX) $(CXXFLAGS) $(LDFLAGS) -o zerotier-selftest selftest.o $(CORE_OBJS) $(ONE_OBJS) $(LIBS) $(STRIP) zerotier-selftest +zerotier-selftest: selftest + clean: - rm -rf *.o node/*.o controller/*.o osdep/*.o service/*.o ext/http-parser/*.o build-* zerotier-one zerotier-idtool zerotier-selftest zerotier-cli ZeroTierOneInstaller-* $(OBJS) + rm -rf *.a *.o node/*.o controller/*.o osdep/*.o service/*.o ext/http-parser/*.o build-* zerotier-one zerotier-idtool zerotier-selftest zerotier-cli $(ONE_OBJS) $(CORE_OBJS) debug: FORCE make -j 4 ZT_DEBUG=1 diff --git a/make-linux.mk b/make-linux.mk index 9ec1a616..86309933 100644 --- a/make-linux.mk +++ b/make-linux.mk @@ -13,14 +13,11 @@ LDLIBS?= DESTDIR?= include objects.mk - -# Use bundled http-parser since distribution versions are NOT API-stable or compatible! -# Trying to use dynamically linked libhttp-parser causes tons of compatibility problems. -OBJS+=ext/http-parser/http_parser.o +ONE_OBJS+=osdep/LinuxEthernetTap.o # Auto-detect miniupnpc and nat-pmp as well and use system libs if present, # otherwise build into binary as done on Mac and Windows. -OBJS+=osdep/PortMapper.o +ONE_OBJS+=osdep/PortMapper.o DEFS+=-DZT_USE_MINIUPNPC MINIUPNPC_IS_NEW_ENOUGH=$(shell grep -sqr '.*define.*MINIUPNPC_VERSION.*"2.."' /usr/include/miniupnpc/miniupnpc.h && echo 1) ifeq ($(MINIUPNPC_IS_NEW_ENOUGH),1) @@ -28,15 +25,19 @@ ifeq ($(MINIUPNPC_IS_NEW_ENOUGH),1) LDLIBS+=-lminiupnpc else DEFS+=-DMINIUPNP_STATICLIB -DMINIUPNPC_SET_SOCKET_TIMEOUT -DMINIUPNPC_GET_SRC_ADDR -D_BSD_SOURCE -D_DEFAULT_SOURCE -D_XOPEN_SOURCE=600 -DOS_STRING=\"Linux\" -DMINIUPNPC_VERSION_STRING=\"2.0\" -DUPNP_VERSION_STRING=\"UPnP/1.1\" -DENABLE_STRNATPMPERR - OBJS+=ext/miniupnpc/connecthostport.o ext/miniupnpc/igd_desc_parse.o ext/miniupnpc/minisoap.o ext/miniupnpc/minissdpc.o ext/miniupnpc/miniupnpc.o ext/miniupnpc/miniwget.o ext/miniupnpc/minixml.o ext/miniupnpc/portlistingparse.o ext/miniupnpc/receivedata.o ext/miniupnpc/upnpcommands.o ext/miniupnpc/upnpdev.o ext/miniupnpc/upnperrors.o ext/miniupnpc/upnpreplyparse.o + ONE_OBJS+=ext/miniupnpc/connecthostport.o ext/miniupnpc/igd_desc_parse.o ext/miniupnpc/minisoap.o ext/miniupnpc/minissdpc.o ext/miniupnpc/miniupnpc.o ext/miniupnpc/miniwget.o ext/miniupnpc/minixml.o ext/miniupnpc/portlistingparse.o ext/miniupnpc/receivedata.o ext/miniupnpc/upnpcommands.o ext/miniupnpc/upnpdev.o ext/miniupnpc/upnperrors.o ext/miniupnpc/upnpreplyparse.o endif ifeq ($(wildcard /usr/include/natpmp.h),) - OBJS+=ext/libnatpmp/natpmp.o ext/libnatpmp/getgateway.o + ONE_OBJS+=ext/libnatpmp/natpmp.o ext/libnatpmp/getgateway.o else LDLIBS+=-lnatpmp DEFS+=-DZT_USE_SYSTEM_NATPMP endif +# Use bundled http-parser since distribution versions are NOT API-stable or compatible! +# Trying to use dynamically linked libhttp-parser causes tons of compatibility problems. +ONE_OBJS+=ext/http-parser/http_parser.o + ifeq ($(ZT_ENABLE_CLUSTER),1) DEFS+=-DZT_ENABLE_CLUSTER endif @@ -196,11 +197,11 @@ endif # Build faster crypto on some targets ifeq ($(ZT_USE_X64_ASM_SALSA2012),1) override DEFS+=-DZT_USE_X64_ASM_SALSA2012 - override OBJS+=ext/x64-salsa2012-asm/salsa2012.o + override CORE_OBJS+=ext/x64-salsa2012-asm/salsa2012.o endif ifeq ($(ZT_USE_ARM32_NEON_ASM_SALSA2012),1) override DEFS+=-DZT_USE_ARM32_NEON_ASM_SALSA2012 - override OBJS+=ext/arm32-neon-salsa2012-asm/salsa2012.o + override CORE_OBJS+=ext/arm32-neon-salsa2012-asm/salsa2012.o endif all: one @@ -211,23 +212,37 @@ ext/x64-salsa2012-asm/salsa2012.o: ext/arm32-neon-salsa2012-asm/salsa2012.o: $(CC) $(CFLAGS) -c ext/arm32-neon-salsa2012-asm/salsa2012.s -o ext/arm32-neon-salsa2012-asm/salsa2012.o -one: $(OBJS) service/OneService.o one.o osdep/LinuxEthernetTap.o - $(CXX) $(CXXFLAGS) $(LDFLAGS) -o zerotier-one $(OBJS) service/OneService.o one.o osdep/LinuxEthernetTap.o $(LDLIBS) +one: $(CORE_OBJS) $(ONE_OBJS) one.o + $(CXX) $(CXXFLAGS) $(LDFLAGS) -o zerotier-one $(CORE_OBJS) $(ONE_OBJS) one.o $(LDLIBS) $(STRIP) zerotier-one ln -sf zerotier-one zerotier-idtool ln -sf zerotier-one zerotier-cli -selftest: $(OBJS) selftest.o - $(CXX) $(CXXFLAGS) $(LDFLAGS) -o zerotier-selftest selftest.o $(OBJS) $(LDLIBS) +zerotier-one: one + +zerotier-idtool: one + +zerotier-cli: one + +libzerotiercore.a: $(CORE_OBJS) + ar rcs libzerotiercore.a $(CORE_OBJS) + ranlib libzerotiercore.a + +core: libzerotiercore.a + +selftest: $(CORE_OBJS) $(ONE_OBJS) selftest.o + $(CXX) $(CXXFLAGS) $(LDFLAGS) -o zerotier-selftest selftest.o $(CORE_OBJS) $(ONE_OBJS $(LDLIBS) $(STRIP) zerotier-selftest +zerotier-selftest: selftest + manpages: FORCE cd doc ; ./build.sh doc: manpages clean: FORCE - rm -rf *.so *.o node/*.o controller/*.o osdep/*.o service/*.o ext/http-parser/*.o ext/miniupnpc/*.o ext/libnatpmp/*.o $(OBJS) zerotier-one zerotier-idtool zerotier-cli zerotier-selftest build-* ZeroTierOneInstaller-* *.deb *.rpm .depend debian/files debian/zerotier-one*.debhelper debian/zerotier-one.substvars debian/*.log debian/zerotier-one doc/node_modules + rm -rf *.a *.so *.o node/*.o controller/*.o osdep/*.o service/*.o ext/http-parser/*.o ext/miniupnpc/*.o ext/libnatpmp/*.o $(CORE_OBJS) $(ONE_OBJS) zerotier-one zerotier-idtool zerotier-cli zerotier-selftest build-* ZeroTierOneInstaller-* *.deb *.rpm .depend debian/files debian/zerotier-one*.debhelper debian/zerotier-one.substvars debian/*.log debian/zerotier-one doc/node_modules distclean: clean diff --git a/make-mac.mk b/make-mac.mk index 6676f457..938bbf97 100644 --- a/make-mac.mk +++ b/make-mac.mk @@ -19,7 +19,7 @@ ZT_VERSION_BUILD=$(shell cat version.h | grep -F VERSION_BUILD | cut -d ' ' -f 3 DEFS+=-DZT_BUILD_PLATFORM=$(ZT_BUILD_PLATFORM) -DZT_BUILD_ARCHITECTURE=$(ZT_BUILD_ARCHITECTURE) include objects.mk -OBJS+=osdep/OSXEthernetTap.o ext/http-parser/http_parser.o +ONE_OBJS+=osdep/OSXEthernetTap.o ext/http-parser/http_parser.o # Official releases are signed with our Apple cert and apply software updates by default ifeq ($(ZT_OFFICIAL_RELEASE),1) @@ -39,11 +39,11 @@ endif # Use fast ASM Salsa20/12 for x64 processors DEFS+=-DZT_USE_X64_ASM_SALSA2012 -OBJS+=ext/x64-salsa2012-asm/salsa2012.o +CORE_OBJS+=ext/x64-salsa2012-asm/salsa2012.o # Build miniupnpc and nat-pmp as included libraries -- extra defs are required for these sources DEFS+=-DMACOSX -DZT_USE_MINIUPNPC -DMINIUPNP_STATICLIB -D_DARWIN_C_SOURCE -DMINIUPNPC_SET_SOCKET_TIMEOUT -DMINIUPNPC_GET_SRC_ADDR -D_BSD_SOURCE -D_DEFAULT_SOURCE -DOS_STRING=\"Darwin/15.0.0\" -DMINIUPNPC_VERSION_STRING=\"2.0\" -DUPNP_VERSION_STRING=\"UPnP/1.1\" -DENABLE_STRNATPMPERR -OBJS+=ext/libnatpmp/natpmp.o ext/libnatpmp/getgateway.o ext/miniupnpc/connecthostport.o ext/miniupnpc/igd_desc_parse.o ext/miniupnpc/minisoap.o ext/miniupnpc/minissdpc.o ext/miniupnpc/miniupnpc.o ext/miniupnpc/miniwget.o ext/miniupnpc/minixml.o ext/miniupnpc/portlistingparse.o ext/miniupnpc/receivedata.o ext/miniupnpc/upnpcommands.o ext/miniupnpc/upnpdev.o ext/miniupnpc/upnperrors.o ext/miniupnpc/upnpreplyparse.o osdep/PortMapper.o +ONE_OBJS+=ext/libnatpmp/natpmp.o ext/libnatpmp/getgateway.o ext/miniupnpc/connecthostport.o ext/miniupnpc/igd_desc_parse.o ext/miniupnpc/minisoap.o ext/miniupnpc/minissdpc.o ext/miniupnpc/miniupnpc.o ext/miniupnpc/miniwget.o ext/miniupnpc/minixml.o ext/miniupnpc/portlistingparse.o ext/miniupnpc/receivedata.o ext/miniupnpc/upnpcommands.o ext/miniupnpc/upnpdev.o ext/miniupnpc/upnperrors.o ext/miniupnpc/upnpreplyparse.o osdep/PortMapper.o # Debug mode -- dump trace output, build binary with -g ifeq ($(ZT_DEBUG),1) @@ -66,13 +66,25 @@ all: one macui ext/x64-salsa2012-asm/salsa2012.o: $(CC) $(CFLAGS) -c ext/x64-salsa2012-asm/salsa2012.s -o ext/x64-salsa2012-asm/salsa2012.o -one: $(OBJS) service/OneService.o one.o - $(CXX) $(CXXFLAGS) -o zerotier-one $(OBJS) service/OneService.o one.o $(LIBS) +one: $(CORE_OBJS) $(ONE_OBJS) one.o + $(CXX) $(CXXFLAGS) -o zerotier-one $(CORE_OBJS) $(ONE_OBJS) one.o $(LIBS) $(STRIP) zerotier-one ln -sf zerotier-one zerotier-idtool ln -sf zerotier-one zerotier-cli $(CODESIGN) -f -s $(CODESIGN_APP_CERT) zerotier-one +zerotier-one: one + +zerotier-idtool: one + +zerotier-cli: one + +libzerotiercore.a: $(CORE_OBJS) + ar rcs libzerotiercore.a $(CORE_OBJS) + ranlib libzerotiercore.a + +core: libzerotiercore.a + macui: FORCE cd macui && xcodebuild -target "ZeroTier One" -configuration Release $(CODESIGN) -f -s $(CODESIGN_APP_CERT) "macui/build/Release/ZeroTier One.app" @@ -81,10 +93,12 @@ macui: FORCE # $(CXX) $(CXXFLAGS) -o zerotier cli/zerotier.cpp osdep/OSUtils.cpp node/InetAddress.cpp node/Utils.cpp node/Salsa20.cpp node/Identity.cpp node/SHA512.cpp node/C25519.cpp -lcurl # $(STRIP) zerotier -selftest: $(OBJS) selftest.o - $(CXX) $(CXXFLAGS) -o zerotier-selftest selftest.o $(OBJS) $(LIBS) +selftest: $(CORE_OBJS) $(ONE_OBJS) selftest.o + $(CXX) $(CXXFLAGS) -o zerotier-selftest selftest.o $(CORE_OBJS) $(ONE_OBJS) $(LIBS) $(STRIP) zerotier-selftest +zerotier-selftest: selftest + # Requires Packages: http://s.sudre.free.fr/Software/Packages/about.html mac-dist-pkg: FORCE packagesbuild "ext/installfiles/mac/ZeroTier One.pkgproj" @@ -102,7 +116,7 @@ official: FORCE make ZT_OFFICIAL_RELEASE=1 mac-dist-pkg clean: - rm -rf *.dSYM build-* *.pkg *.dmg *.o node/*.o controller/*.o service/*.o osdep/*.o ext/http-parser/*.o $(OBJS) zerotier-one zerotier-idtool zerotier-selftest zerotier-cli zerotier mkworld doc/node_modules macui/build zt1_update_$(ZT_BUILD_PLATFORM)_$(ZT_BUILD_ARCHITECTURE)_* + rm -rf *.dSYM build-* *.a *.pkg *.dmg *.o node/*.o controller/*.o service/*.o osdep/*.o ext/http-parser/*.o $(CORE_OBJS) $(ONE_OBJS) zerotier-one zerotier-idtool zerotier-selftest zerotier-cli zerotier doc/node_modules macui/build zt1_update_$(ZT_BUILD_PLATFORM)_$(ZT_BUILD_ARCHITECTURE)_* distclean: clean diff --git a/objects.mk b/objects.mk index 74efc337..fd25a3d3 100644 --- a/objects.mk +++ b/objects.mk @@ -1,6 +1,4 @@ -OBJS=\ - controller/EmbeddedNetworkController.o \ - controller/JSONDB.o \ +CORE_OBJS=\ node/C25519.o \ node/Capability.o \ node/CertificateOfMembership.o \ @@ -26,9 +24,15 @@ OBJS=\ node/Switch.o \ node/Tag.o \ node/Topology.o \ - node/Utils.o \ + node/Utils.o + +ONE_OBJS=\ + controller/EmbeddedNetworkController.o \ + controller/JSONDB.o \ osdep/ManagedRoute.o \ osdep/Http.o \ osdep/OSUtils.o \ service/ClusterGeoIpService.o \ - service/SoftwareUpdater.o + service/SoftwareUpdater.o \ + service/OneService.o + -- cgit v1.2.3 From cf49e511a59911d36cea671d039599fcd7811032 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Mon, 8 May 2017 14:46:45 -0700 Subject: More makefile changes. --- make-bsd.mk | 42 +++++++++++++++++++++++------------------- make-linux.mk | 14 +++++++++----- make-mac.mk | 12 ++++++++---- 3 files changed, 40 insertions(+), 28 deletions(-) (limited to 'make-mac.mk') diff --git a/make-bsd.mk b/make-bsd.mk index 6370c025..cbab5810 100644 --- a/make-bsd.mk +++ b/make-bsd.mk @@ -14,10 +14,10 @@ endif # "make debug" is a shortcut for this ifeq ($(ZT_DEBUG),1) - DEFS+=-DZT_TRACE - CFLAGS+=-Wall -g -pthread $(INCLUDES) $(DEFS) + CFLAGS+=-Wall -Werror -g -pthread $(INCLUDES) $(DEFS) LDFLAGS+= STRIP=echo + ZT_TRACE=1 # The following line enables optimization for the crypto code, since # C25519 in particular is almost UNUSABLE in heavy testing without it. node/Salsa20.o node/SHA512.o node/C25519.o node/Poly1305.o: CFLAGS = -Wall -O2 -g -pthread $(INCLUDES) $(DEFS) @@ -28,80 +28,84 @@ else STRIP=strip --strip-all endif +ifeq ($(ZT_TRACE),1) + DEFS+=-DZT_TRACE +endif + # Determine system build architecture from compiler target CC_MACH=$(shell $(CC) -dumpmachine | cut -d '-' -f 1) ZT_ARCHITECTURE=999 ifeq ($(CC_MACH),x86_64) - ZT_ARCHITECTURE=2 + ZT_ARCHITECTURE=2 ZT_USE_X64_ASM_SALSA2012=1 endif ifeq ($(CC_MACH),amd64) - ZT_ARCHITECTURE=2 + ZT_ARCHITECTURE=2 ZT_USE_X64_ASM_SALSA2012=1 endif ifeq ($(CC_MACH),i386) - ZT_ARCHITECTURE=1 + ZT_ARCHITECTURE=1 endif ifeq ($(CC_MACH),i686) - ZT_ARCHITECTURE=1 + ZT_ARCHITECTURE=1 endif ifeq ($(CC_MACH),arm) - ZT_ARCHITECTURE=3 + ZT_ARCHITECTURE=3 override DEFS+=-DZT_NO_TYPE_PUNNING ZT_USE_ARM32_NEON_ASM_SALSA2012=1 endif ifeq ($(CC_MACH),armel) - ZT_ARCHITECTURE=3 + ZT_ARCHITECTURE=3 override DEFS+=-DZT_NO_TYPE_PUNNING ZT_USE_ARM32_NEON_ASM_SALSA2012=1 endif ifeq ($(CC_MACH),armhf) - ZT_ARCHITECTURE=3 + ZT_ARCHITECTURE=3 override DEFS+=-DZT_NO_TYPE_PUNNING ZT_USE_ARM32_NEON_ASM_SALSA2012=1 endif ifeq ($(CC_MACH),armv6) - ZT_ARCHITECTURE=3 + ZT_ARCHITECTURE=3 override DEFS+=-DZT_NO_TYPE_PUNNING ZT_USE_ARM32_NEON_ASM_SALSA2012=1 endif ifeq ($(CC_MACH),armv6zk) - ZT_ARCHITECTURE=3 + ZT_ARCHITECTURE=3 override DEFS+=-DZT_NO_TYPE_PUNNING ZT_USE_ARM32_NEON_ASM_SALSA2012=1 endif ifeq ($(CC_MACH),armv6kz) - ZT_ARCHITECTURE=3 + ZT_ARCHITECTURE=3 override DEFS+=-DZT_NO_TYPE_PUNNING ZT_USE_ARM32_NEON_ASM_SALSA2012=1 endif ifeq ($(CC_MACH),armv7) - ZT_ARCHITECTURE=3 + ZT_ARCHITECTURE=3 override DEFS+=-DZT_NO_TYPE_PUNNING ZT_USE_ARM32_NEON_ASM_SALSA2012=1 endif ifeq ($(CC_MACH),arm64) - ZT_ARCHITECTURE=4 + ZT_ARCHITECTURE=4 override DEFS+=-DZT_NO_TYPE_PUNNING endif ifeq ($(CC_MACH),aarch64) - ZT_ARCHITECTURE=4 + ZT_ARCHITECTURE=4 override DEFS+=-DZT_NO_TYPE_PUNNING endif ifeq ($(CC_MACH),mipsel) - ZT_ARCHITECTURE=5 + ZT_ARCHITECTURE=5 override DEFS+=-DZT_NO_TYPE_PUNNING endif ifeq ($(CC_MACH),mips) - ZT_ARCHITECTURE=5 + ZT_ARCHITECTURE=5 override DEFS+=-DZT_NO_TYPE_PUNNING endif ifeq ($(CC_MACH),mips64) - ZT_ARCHITECTURE=6 + ZT_ARCHITECTURE=6 override DEFS+=-DZT_NO_TYPE_PUNNING endif ifeq ($(CC_MACH),mips64el) - ZT_ARCHITECTURE=6 + ZT_ARCHITECTURE=6 override DEFS+=-DZT_NO_TYPE_PUNNING endif diff --git a/make-linux.mk b/make-linux.mk index 86309933..0deb8fcb 100644 --- a/make-linux.mk +++ b/make-linux.mk @@ -55,10 +55,10 @@ ifeq ($(ZT_RULES_ENGINE_DEBUGGING),1) endif ifeq ($(ZT_DEBUG),1) - override DEFS+=-DZT_TRACE - override CFLAGS+=-Wall -g -pthread $(INCLUDES) $(DEFS) - override CXXFLAGS+=-Wall -g -std=c++11 -pthread $(INCLUDES) $(DEFS) + override CFLAGS+=-Wall -Werror -g -pthread $(INCLUDES) $(DEFS) + override CXXFLAGS+=-Wall -Werror -g -std=c++11 -pthread $(INCLUDES) $(DEFS) override LDFLAGS+= + ZT_TRACE=1 STRIP?=echo # The following line enables optimization for the crypto code, since # C25519 in particular is almost UNUSABLE in -O0 even on a 3ghz box! @@ -66,14 +66,18 @@ node/Salsa20.o node/SHA512.o node/C25519.o node/Poly1305.o: CXXFLAGS=-Wall -O2 - else override DEFS+=-D_FORTIFY_SOURCE=2 CFLAGS?=-O3 -fstack-protector - override CFLAGS+=-Wall -fPIE -pthread $(INCLUDES) -DNDEBUG $(DEFS) + override CFLAGS+=-Wall -Werror -fPIE -pthread $(INCLUDES) -DNDEBUG $(DEFS) CXXFLAGS?=-O3 -fstack-protector - override CXXFLAGS+=-Wall -Wno-unused-result -Wreorder -fPIE -std=c++11 -pthread $(INCLUDES) -DNDEBUG $(DEFS) + override CXXFLAGS+=-Wall -Werror -Wno-unused-result -Wreorder -fPIE -std=c++11 -pthread $(INCLUDES) -DNDEBUG $(DEFS) override LDFLAGS+=-pie -Wl,-z,relro,-z,now STRIP?=strip STRIP+=--strip-all endif +ifeq ($(ZT_TRACE),1) + override DEFS+=-DZT_TRACE +endif + ifeq ($(ZT_USE_TEST_TAP),1) override DEFS+=-DZT_USE_TEST_TAP endif diff --git a/make-mac.mk b/make-mac.mk index 938bbf97..de66f49c 100644 --- a/make-mac.mk +++ b/make-mac.mk @@ -47,18 +47,22 @@ ONE_OBJS+=ext/libnatpmp/natpmp.o ext/libnatpmp/getgateway.o ext/miniupnpc/connec # Debug mode -- dump trace output, build binary with -g ifeq ($(ZT_DEBUG),1) - DEFS+=-DZT_TRACE - CFLAGS+=-Wall -g -pthread $(INCLUDES) $(DEFS) + ZT_TRACE=1 + CFLAGS+=-Wall -Werror -g $(INCLUDES) $(DEFS) STRIP=echo # The following line enables optimization for the crypto code, since # C25519 in particular is almost UNUSABLE in heavy testing without it. -node/Salsa20.o node/SHA512.o node/C25519.o node/Poly1305.o: CFLAGS = -Wall -O2 -g -pthread $(INCLUDES) $(DEFS) +node/Salsa20.o node/SHA512.o node/C25519.o node/Poly1305.o: CFLAGS = -Wall -O2 -g $(INCLUDES) $(DEFS) else CFLAGS?=-Ofast -fstack-protector-strong - CFLAGS+=$(ARCH_FLAGS) -Wall -flto -fPIE -pthread -mmacosx-version-min=10.7 -DNDEBUG -Wno-unused-private-field $(INCLUDES) $(DEFS) + CFLAGS+=$(ARCH_FLAGS) -Wall -Werror -flto -fPIE -mmacosx-version-min=10.7 -DNDEBUG -Wno-unused-private-field $(INCLUDES) $(DEFS) STRIP=strip endif +ifeq ($(ZT_TRACE),1) + DEFS+=-DZT_TRACE +endif + CXXFLAGS=$(CFLAGS) -std=c++11 -stdlib=libc++ all: one macui -- cgit v1.2.3 From 2a4a50b1daaec74d7a4d08869ead31ff1f966fa1 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Wed, 31 May 2017 08:36:09 -0700 Subject: Add some also-ZeroTier-written ext/ code for use in new clustering, delete some old code, and change Mac to use -Os which is just as fast as -Ofast and may be faster due to cache effects. --- ext/kissdb/Makefile | 7 + ext/kissdb/README.md | 69 +++ ext/kissdb/SPEC.txt | 62 ++ ext/kissdb/kissdb.c | 452 +++++++++++++++ ext/kissdb/kissdb.h | 173 ++++++ ext/vsdm/LICENSE.txt | 22 + ext/vsdm/Makefile | 5 + ext/vsdm/README.md | 40 ++ ext/vsdm/vsdm-test.cpp | 72 +++ ext/vsdm/vsdm.hpp | 1491 ++++++++++++++++++++++++++++++++++++++++++++++++ include/ZeroTierOne.h | 197 ------- make-mac.mk | 2 +- node/Node.cpp | 52 +- node/Node.hpp | 15 - 14 files changed, 2396 insertions(+), 263 deletions(-) create mode 100644 ext/kissdb/Makefile create mode 100644 ext/kissdb/README.md create mode 100644 ext/kissdb/SPEC.txt create mode 100644 ext/kissdb/kissdb.c create mode 100644 ext/kissdb/kissdb.h create mode 100644 ext/vsdm/LICENSE.txt create mode 100644 ext/vsdm/Makefile create mode 100644 ext/vsdm/README.md create mode 100644 ext/vsdm/vsdm-test.cpp create mode 100644 ext/vsdm/vsdm.hpp (limited to 'make-mac.mk') diff --git a/ext/kissdb/Makefile b/ext/kissdb/Makefile new file mode 100644 index 00000000..f47372c6 --- /dev/null +++ b/ext/kissdb/Makefile @@ -0,0 +1,7 @@ +# http://creativecommons.org/publicdomain/zero/1.0/ + +all: + gcc -Wall -O2 -DKISSDB_TEST -o kissdb-test kissdb.c + +clean: + rm -f kissdb-test *.o test.db diff --git a/ext/kissdb/README.md b/ext/kissdb/README.md new file mode 100644 index 00000000..ab8a6cff --- /dev/null +++ b/ext/kissdb/README.md @@ -0,0 +1,69 @@ +kissdb +====== + +(Keep It) Simple Stupid Database + +KISSDB is about the simplest key/value store you'll ever see, anywhere. +It's written in plain vanilla C using only the standard string and FILE +I/O functions, and should port to just about anything with a disk or +something that acts like one. + +It stores keys and values of fixed length in a stupid-simple file format +based on fixed-size hash tables. If a hash collision occurrs, a new "page" +of hash table is appended to the database. The format is append-only. +There is no delete. Puts that replace an existing value, however, will not +grow the file as they will overwrite the existing entry. + +Hash table size is a space/speed trade-off parameter. Larger hash tables +will reduce collisions and speed things up a bit, at the expense of memory +and disk space. A good size is usually about 1/2 the average number of +entries you expect. + +Features: + + * Tiny, compiles to ~4k on an x86_64 Linux system + * Small memory footprint (only caches hash tables) + * Very space-efficient (on disk) if small hash tables are used + * Makes a decent effort to be robust on power loss + * Pretty respectably fast, especially given its simplicity + * 64-bit, file size limit is 2^64 bytes + * Ports to anything with a C compiler and stdlib/stdio + * Public domain + +Limitations: + + * Fixed-size keys and values, must recreate and copy to change any init size parameter + * Add/update only, no delete + * Iteration is supported but key order is undefined + * No search for subsets of keys/values + * No indexes + * No transactions + * No special recovery features if a database gets corrupted + * No built-in thread-safety (guard it with a mutex in MT code) + * No built-in caching of data (only hash tables are cached for lookup speed) + * No endian-awareness (currently), so big-endian DBs won't read on little-endian machines + +Alternative key/value stores and embedded databases: + + * [MDB](http://symas.com/mdb/) uses mmap() and is very fast (not quite as tiny/simple/portable) + * [CDB](http://cr.yp.to/cdb.html) is also minimal and fast, probably the closest thing to this (but has a 4gb size limit) + * [Kyoto Cabinet](http://fallabs.com/kyotocabinet/) is very fast, full-featured, and modern (license required for commercial use) + * [SQLite](http://www.sqlite.org/) gives you a complete embedded SQL server (public domain, very mature, much larger) + * Others include GDBM, NDBM, Berkeley DB, etc. Use your Googles. :) + +KISSDB is good if you want space-efficient relatively fast write-once/read-many storage +of keys mapped to values. It's not a good choice if you need searches, indexes, delete, +structured storage, or widely varying key/value sizes. It's also probably not a good +choice if you need a long-lived database for critical data, since it lacks recovery +features and is brittle if its internals are modified. It would be better for a cache +of data that can be restored or "re-learned," such as keys, Bitcoin transactions, nodes +on a peer-to-peer network, log analysis results, rendered web pages, session cookies, +auth tokens, etc. + +KISSDB is in the public domain as according to the [Creative Commons Public Domain Dedication](http://creativecommons.org/publicdomain/zero/1.0/). +One reason it was written was the poverty of simple key/value databases with wide open licensing. Even old ones like GDBM have GPL, not LGPL, licenses. + +See comments in kissdb.h for documentation. Makefile can be used to build +a test program on systems with gcc. + +Author: Adam Ierymenko / ZeroTier Networks LLC diff --git a/ext/kissdb/SPEC.txt b/ext/kissdb/SPEC.txt new file mode 100644 index 00000000..732c4df5 --- /dev/null +++ b/ext/kissdb/SPEC.txt @@ -0,0 +1,62 @@ +----- + +KISSDB file format (version 2) +Author: Adam Ierymenko + +http://creativecommons.org/publicdomain/zero/1.0/ + +----- + +In keeping with the goal of minimalism the file format is very simple, the +sort of thing that would be given as an example in an introductory course in +data structures. It's a basic hash table that adds additional pages of hash +table entries on collision. + +It consists of a 28 byte header followed by a series of hash tables and data. +All integer values are stored in the native word order of the target +architecture (in the future the code might be fixed to make everything +little-endian if anyone cares about that). + +The header consists of the following fields: + +[0-3] magic numbers: (ASCII) 'K', 'd', 'B', KISSDB_VERSION (currently 2) +[4-11] 64-bit hash table size in entries +[12-19] 64-bit key size in bytes +[20-27] 64-bit value size in bytes + +Hash tables are arrays of [hash table size + 1] 64-bit integers. The extra +entry, if nonzero, is the offset in the file of the next hash table, forming +a linked list of hash tables across the file. + +Immediately following the header, the first hash table will be written when +the first key/value is added. The algorithm for adding new entries is as +follows: + +(1) The key is hashed using a 64-bit variant of the DJB2 hash function, and + this is taken modulo hash table size to get a bucket number. +(2) Hash tables are checked in order, starting with the first hash table, + until a zero (empty) bucket is found. If one is found, skip to step (4). +(3) If no empty buckets are found in any hash table, a new table is appended + to the file and the final pointer in the previous hash table is set to + its offset. (In the code the update of the next hash table pointer in + the previous hash table happens last, after the whole write is complete, + to avoid corruption on power loss.) +(4) The key and value are appended, in order with no additional meta-data, + to the database file. Before appending the offset in the file stream + where they will be stored is saved. After appending, this offset is + written to the empty hash table bucket we chose in steps 2/3. Hash table + updates happen last to avoid corruption if the write does not complete. + +Lookup of a key/value pair occurs as follows: + +(1) The key is hashed and taken modulo hash table size to get a bucket + number. +(2) If this bucket's entry in the hash table is nonzero, the key at the + offset specified by this bucket is compared to the key being looked up. + If they are equal, the value is read and returned. +(3) If the keys are not equal, the next hash table is checked and step (2) + is repeated. If an empty bucket is encountered or if we run out of hash + tables, the key was not found. + +To update an existing value, its location is looked up and the value portion +of the entry is rewritten. diff --git a/ext/kissdb/kissdb.c b/ext/kissdb/kissdb.c new file mode 100644 index 00000000..6b275686 --- /dev/null +++ b/ext/kissdb/kissdb.c @@ -0,0 +1,452 @@ +/* (Keep It) Simple Stupid Database + * + * Written by Adam Ierymenko + * KISSDB is in the public domain and is distributed with NO WARRANTY. + * + * http://creativecommons.org/publicdomain/zero/1.0/ */ + +/* Compile with KISSDB_TEST to build as a test program. */ + +/* Note: big-endian systems will need changes to implement byte swapping + * on hash table file I/O. Or you could just use it as-is if you don't care + * that your database files will be unreadable on little-endian systems. */ + +#define _FILE_OFFSET_BITS 64 + +#include "kissdb.h" + +#include +#include +#include + +#ifdef _WIN32 +#define fseeko _fseeki64 +#define ftello _ftelli64 +#endif + +#define KISSDB_HEADER_SIZE ((sizeof(uint64_t) * 3) + 4) + +/* djb2 hash function */ +static uint64_t KISSDB_hash(const void *b,unsigned long len) +{ + unsigned long i; + uint64_t hash = 5381; + for(i=0;if = (FILE *)0; + fopen_s(&db->f,path,((mode == KISSDB_OPEN_MODE_RWREPLACE) ? "w+b" : (((mode == KISSDB_OPEN_MODE_RDWR)||(mode == KISSDB_OPEN_MODE_RWCREAT)) ? "r+b" : "rb"))); +#else + db->f = fopen(path,((mode == KISSDB_OPEN_MODE_RWREPLACE) ? "w+b" : (((mode == KISSDB_OPEN_MODE_RDWR)||(mode == KISSDB_OPEN_MODE_RWCREAT)) ? "r+b" : "rb"))); +#endif + if (!db->f) { + if (mode == KISSDB_OPEN_MODE_RWCREAT) { +#ifdef _WIN32 + db->f = (FILE *)0; + fopen_s(&db->f,path,"w+b"); +#else + db->f = fopen(path,"w+b"); +#endif + } + if (!db->f) + return KISSDB_ERROR_IO; + } + + if (fseeko(db->f,0,SEEK_END)) { + fclose(db->f); + return KISSDB_ERROR_IO; + } + if (ftello(db->f) < KISSDB_HEADER_SIZE) { + /* write header if not already present */ + if ((hash_table_size)&&(key_size)&&(value_size)) { + if (fseeko(db->f,0,SEEK_SET)) { fclose(db->f); return KISSDB_ERROR_IO; } + tmp2[0] = 'K'; tmp2[1] = 'd'; tmp2[2] = 'B'; tmp2[3] = KISSDB_VERSION; + if (fwrite(tmp2,4,1,db->f) != 1) { fclose(db->f); return KISSDB_ERROR_IO; } + tmp = hash_table_size; + if (fwrite(&tmp,sizeof(uint64_t),1,db->f) != 1) { fclose(db->f); return KISSDB_ERROR_IO; } + tmp = key_size; + if (fwrite(&tmp,sizeof(uint64_t),1,db->f) != 1) { fclose(db->f); return KISSDB_ERROR_IO; } + tmp = value_size; + if (fwrite(&tmp,sizeof(uint64_t),1,db->f) != 1) { fclose(db->f); return KISSDB_ERROR_IO; } + fflush(db->f); + } else { + fclose(db->f); + return KISSDB_ERROR_INVALID_PARAMETERS; + } + } else { + if (fseeko(db->f,0,SEEK_SET)) { fclose(db->f); return KISSDB_ERROR_IO; } + if (fread(tmp2,4,1,db->f) != 1) { fclose(db->f); return KISSDB_ERROR_IO; } + if ((tmp2[0] != 'K')||(tmp2[1] != 'd')||(tmp2[2] != 'B')||(tmp2[3] != KISSDB_VERSION)) { + fclose(db->f); + return KISSDB_ERROR_CORRUPT_DBFILE; + } + if (fread(&tmp,sizeof(uint64_t),1,db->f) != 1) { fclose(db->f); return KISSDB_ERROR_IO; } + if (!tmp) { + fclose(db->f); + return KISSDB_ERROR_CORRUPT_DBFILE; + } + hash_table_size = (unsigned long)tmp; + if (fread(&tmp,sizeof(uint64_t),1,db->f) != 1) { fclose(db->f); return KISSDB_ERROR_IO; } + if (!tmp) { + fclose(db->f); + return KISSDB_ERROR_CORRUPT_DBFILE; + } + key_size = (unsigned long)tmp; + if (fread(&tmp,sizeof(uint64_t),1,db->f) != 1) { fclose(db->f); return KISSDB_ERROR_IO; } + if (!tmp) { + fclose(db->f); + return KISSDB_ERROR_CORRUPT_DBFILE; + } + value_size = (unsigned long)tmp; + } + + db->hash_table_size = hash_table_size; + db->key_size = key_size; + db->value_size = value_size; + db->hash_table_size_bytes = sizeof(uint64_t) * (hash_table_size + 1); /* [hash_table_size] == next table */ + + httmp = malloc(db->hash_table_size_bytes); + if (!httmp) { + fclose(db->f); + return KISSDB_ERROR_MALLOC; + } + db->num_hash_tables = 0; + db->hash_tables = (uint64_t *)0; + while (fread(httmp,db->hash_table_size_bytes,1,db->f) == 1) { + hash_tables_rea = realloc(db->hash_tables,db->hash_table_size_bytes * (db->num_hash_tables + 1)); + if (!hash_tables_rea) { + KISSDB_close(db); + free(httmp); + return KISSDB_ERROR_MALLOC; + } + db->hash_tables = hash_tables_rea; + + memcpy(((uint8_t *)db->hash_tables) + (db->hash_table_size_bytes * db->num_hash_tables),httmp,db->hash_table_size_bytes); + ++db->num_hash_tables; + if (httmp[db->hash_table_size]) { + if (fseeko(db->f,httmp[db->hash_table_size],SEEK_SET)) { + KISSDB_close(db); + free(httmp); + return KISSDB_ERROR_IO; + } + } else break; + } + free(httmp); + + return 0; +} + +void KISSDB_close(KISSDB *db) +{ + if (db->hash_tables) + free(db->hash_tables); + if (db->f) + fclose(db->f); + memset(db,0,sizeof(KISSDB)); +} + +int KISSDB_get(KISSDB *db,const void *key,void *vbuf) +{ + uint8_t tmp[4096]; + const uint8_t *kptr; + unsigned long klen,i; + uint64_t hash = KISSDB_hash(key,db->key_size) % (uint64_t)db->hash_table_size; + uint64_t offset; + uint64_t *cur_hash_table; + long n; + + cur_hash_table = db->hash_tables; + for(i=0;inum_hash_tables;++i) { + offset = cur_hash_table[hash]; + if (offset) { + if (fseeko(db->f,offset,SEEK_SET)) + return KISSDB_ERROR_IO; + + kptr = (const uint8_t *)key; + klen = db->key_size; + while (klen) { + n = (long)fread(tmp,1,(klen > sizeof(tmp)) ? sizeof(tmp) : klen,db->f); + if (n > 0) { + if (memcmp(kptr,tmp,n)) + goto get_no_match_next_hash_table; + kptr += n; + klen -= (unsigned long)n; + } else return 1; /* not found */ + } + + if (fread(vbuf,db->value_size,1,db->f) == 1) + return 0; /* success */ + else return KISSDB_ERROR_IO; + } else return 1; /* not found */ +get_no_match_next_hash_table: + cur_hash_table += db->hash_table_size + 1; + } + + return 1; /* not found */ +} + +int KISSDB_put(KISSDB *db,const void *key,const void *value) +{ + uint8_t tmp[4096]; + const uint8_t *kptr; + unsigned long klen,i; + uint64_t hash = KISSDB_hash(key,db->key_size) % (uint64_t)db->hash_table_size; + uint64_t offset; + uint64_t htoffset,lasthtoffset; + uint64_t endoffset; + uint64_t *cur_hash_table; + uint64_t *hash_tables_rea; + long n; + + lasthtoffset = htoffset = KISSDB_HEADER_SIZE; + cur_hash_table = db->hash_tables; + for(i=0;inum_hash_tables;++i) { + offset = cur_hash_table[hash]; + if (offset) { + /* rewrite if already exists */ + if (fseeko(db->f,offset,SEEK_SET)) + return KISSDB_ERROR_IO; + + kptr = (const uint8_t *)key; + klen = db->key_size; + while (klen) { + n = (long)fread(tmp,1,(klen > sizeof(tmp)) ? sizeof(tmp) : klen,db->f); + if (n > 0) { + if (memcmp(kptr,tmp,n)) + goto put_no_match_next_hash_table; + kptr += n; + klen -= (unsigned long)n; + } + } + + /* C99 spec demands seek after fread(), required for Windows */ + fseeko(db->f,0,SEEK_CUR); + + if (fwrite(value,db->value_size,1,db->f) == 1) { + fflush(db->f); + return 0; /* success */ + } else return KISSDB_ERROR_IO; + } else { + /* add if an empty hash table slot is discovered */ + if (fseeko(db->f,0,SEEK_END)) + return KISSDB_ERROR_IO; + endoffset = ftello(db->f); + + if (fwrite(key,db->key_size,1,db->f) != 1) + return KISSDB_ERROR_IO; + if (fwrite(value,db->value_size,1,db->f) != 1) + return KISSDB_ERROR_IO; + + if (fseeko(db->f,htoffset + (sizeof(uint64_t) * hash),SEEK_SET)) + return KISSDB_ERROR_IO; + if (fwrite(&endoffset,sizeof(uint64_t),1,db->f) != 1) + return KISSDB_ERROR_IO; + cur_hash_table[hash] = endoffset; + + fflush(db->f); + + return 0; /* success */ + } +put_no_match_next_hash_table: + lasthtoffset = htoffset; + htoffset = cur_hash_table[db->hash_table_size]; + cur_hash_table += (db->hash_table_size + 1); + } + + /* if no existing slots, add a new page of hash table entries */ + if (fseeko(db->f,0,SEEK_END)) + return KISSDB_ERROR_IO; + endoffset = ftello(db->f); + + hash_tables_rea = realloc(db->hash_tables,db->hash_table_size_bytes * (db->num_hash_tables + 1)); + if (!hash_tables_rea) + return KISSDB_ERROR_MALLOC; + db->hash_tables = hash_tables_rea; + cur_hash_table = &(db->hash_tables[(db->hash_table_size + 1) * db->num_hash_tables]); + memset(cur_hash_table,0,db->hash_table_size_bytes); + + cur_hash_table[hash] = endoffset + db->hash_table_size_bytes; /* where new entry will go */ + + if (fwrite(cur_hash_table,db->hash_table_size_bytes,1,db->f) != 1) + return KISSDB_ERROR_IO; + + if (fwrite(key,db->key_size,1,db->f) != 1) + return KISSDB_ERROR_IO; + if (fwrite(value,db->value_size,1,db->f) != 1) + return KISSDB_ERROR_IO; + + if (db->num_hash_tables) { + if (fseeko(db->f,lasthtoffset + (sizeof(uint64_t) * db->hash_table_size),SEEK_SET)) + return KISSDB_ERROR_IO; + if (fwrite(&endoffset,sizeof(uint64_t),1,db->f) != 1) + return KISSDB_ERROR_IO; + db->hash_tables[((db->hash_table_size + 1) * (db->num_hash_tables - 1)) + db->hash_table_size] = endoffset; + } + + ++db->num_hash_tables; + + fflush(db->f); + + return 0; /* success */ +} + +void KISSDB_Iterator_init(KISSDB *db,KISSDB_Iterator *dbi) +{ + dbi->db = db; + dbi->h_no = 0; + dbi->h_idx = 0; +} + +int KISSDB_Iterator_next(KISSDB_Iterator *dbi,void *kbuf,void *vbuf) +{ + uint64_t offset; + + if ((dbi->h_no < dbi->db->num_hash_tables)&&(dbi->h_idx < dbi->db->hash_table_size)) { + while (!(offset = dbi->db->hash_tables[((dbi->db->hash_table_size + 1) * dbi->h_no) + dbi->h_idx])) { + if (++dbi->h_idx >= dbi->db->hash_table_size) { + dbi->h_idx = 0; + if (++dbi->h_no >= dbi->db->num_hash_tables) + return 0; + } + } + if (fseeko(dbi->db->f,offset,SEEK_SET)) + return KISSDB_ERROR_IO; + if (fread(kbuf,dbi->db->key_size,1,dbi->db->f) != 1) + return KISSDB_ERROR_IO; + if (fread(vbuf,dbi->db->value_size,1,dbi->db->f) != 1) + return KISSDB_ERROR_IO; + if (++dbi->h_idx >= dbi->db->hash_table_size) { + dbi->h_idx = 0; + ++dbi->h_no; + } + return 1; + } + + return 0; +} + +#ifdef KISSDB_TEST + +#include + +int main(int argc,char **argv) +{ + uint64_t i,j; + uint64_t v[8]; + KISSDB db; + KISSDB_Iterator dbi; + char got_all_values[10000]; + int q; + + printf("Opening new empty database test.db...\n"); + + if (KISSDB_open(&db,"test.db",KISSDB_OPEN_MODE_RWREPLACE,1024,8,sizeof(v))) { + printf("KISSDB_open failed\n"); + return 1; + } + + printf("Adding and then re-getting 10000 64-byte values...\n"); + + for(i=0;i<10000;++i) { + for(j=0;j<8;++j) + v[j] = i; + if (KISSDB_put(&db,&i,v)) { + printf("KISSDB_put failed (%"PRIu64")\n",i); + return 1; + } + memset(v,0,sizeof(v)); + if ((q = KISSDB_get(&db,&i,v))) { + printf("KISSDB_get (1) failed (%"PRIu64") (%d)\n",i,q); + return 1; + } + for(j=0;j<8;++j) { + if (v[j] != i) { + printf("KISSDB_get (1) failed, bad data (%"PRIu64")\n",i); + return 1; + } + } + } + + printf("Getting 10000 64-byte values...\n"); + + for(i=0;i<10000;++i) { + if ((q = KISSDB_get(&db,&i,v))) { + printf("KISSDB_get (2) failed (%"PRIu64") (%d)\n",i,q); + return 1; + } + for(j=0;j<8;++j) { + if (v[j] != i) { + printf("KISSDB_get (2) failed, bad data (%"PRIu64")\n",i); + return 1; + } + } + } + + printf("Closing and re-opening database in read-only mode...\n"); + + KISSDB_close(&db); + + if (KISSDB_open(&db,"test.db",KISSDB_OPEN_MODE_RDONLY,1024,8,sizeof(v))) { + printf("KISSDB_open failed\n"); + return 1; + } + + printf("Getting 10000 64-byte values...\n"); + + for(i=0;i<10000;++i) { + if ((q = KISSDB_get(&db,&i,v))) { + printf("KISSDB_get (3) failed (%"PRIu64") (%d)\n",i,q); + return 1; + } + for(j=0;j<8;++j) { + if (v[j] != i) { + printf("KISSDB_get (3) failed, bad data (%"PRIu64")\n",i); + return 1; + } + } + } + + printf("Iterator test...\n"); + + KISSDB_Iterator_init(&db,&dbi); + i = 0xdeadbeef; + memset(got_all_values,0,sizeof(got_all_values)); + while (KISSDB_Iterator_next(&dbi,&i,&v) > 0) { + if (i < 10000) + got_all_values[i] = 1; + else { + printf("KISSDB_Iterator_next failed, bad data (%"PRIu64")\n",i); + return 1; + } + } + for(i=0;i<10000;++i) { + if (!got_all_values[i]) { + printf("KISSDB_Iterator failed, missing value index %"PRIu64"\n",i); + return 1; + } + } + + KISSDB_close(&db); + + printf("All tests OK!\n"); + + return 0; +} + +#endif diff --git a/ext/kissdb/kissdb.h b/ext/kissdb/kissdb.h new file mode 100644 index 00000000..926906b0 --- /dev/null +++ b/ext/kissdb/kissdb.h @@ -0,0 +1,173 @@ +/* (Keep It) Simple Stupid Database + * + * Written by Adam Ierymenko + * KISSDB is in the public domain and is distributed with NO WARRANTY. + * + * http://creativecommons.org/publicdomain/zero/1.0/ */ + +#ifndef ___KISSDB_H +#define ___KISSDB_H + +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +/** + * Version: 2 + * + * This is the file format identifier, and changes any time the file + * format changes. The code version will be this dot something, and can + * be seen in tags in the git repository. + */ +#define KISSDB_VERSION 2 + +/** + * KISSDB database state + * + * These fields can be read by a user, e.g. to look up key_size and + * value_size, but should never be changed. + */ +typedef struct { + unsigned long hash_table_size; + unsigned long key_size; + unsigned long value_size; + unsigned long hash_table_size_bytes; + unsigned long num_hash_tables; + uint64_t *hash_tables; + FILE *f; +} KISSDB; + +/** + * I/O error or file not found + */ +#define KISSDB_ERROR_IO -1 + +/** + * Out of memory + */ +#define KISSDB_ERROR_MALLOC -2 + +/** + * Invalid paramters (e.g. missing _size paramters on init to create database) + */ +#define KISSDB_ERROR_INVALID_PARAMETERS -3 + +/** + * Database file appears corrupt + */ +#define KISSDB_ERROR_CORRUPT_DBFILE -4 + +/** + * Open mode: read only + */ +#define KISSDB_OPEN_MODE_RDONLY 1 + +/** + * Open mode: read/write + */ +#define KISSDB_OPEN_MODE_RDWR 2 + +/** + * Open mode: read/write, create if doesn't exist + */ +#define KISSDB_OPEN_MODE_RWCREAT 3 + +/** + * Open mode: truncate database, open for reading and writing + */ +#define KISSDB_OPEN_MODE_RWREPLACE 4 + +/** + * Open database + * + * The three _size parameters must be specified if the database could + * be created or re-created. Otherwise an error will occur. If the + * database already exists, these parameters are ignored and are read + * from the database. You can check the struture afterwords to see what + * they were. + * + * @param db Database struct + * @param path Path to file + * @param mode One of the KISSDB_OPEN_MODE constants + * @param hash_table_size Size of hash table in 64-bit entries (must be >0) + * @param key_size Size of keys in bytes + * @param value_size Size of values in bytes + * @return 0 on success, nonzero on error + */ +extern int KISSDB_open( + KISSDB *db, + const char *path, + int mode, + unsigned long hash_table_size, + unsigned long key_size, + unsigned long value_size); + +/** + * Close database + * + * @param db Database struct + */ +extern void KISSDB_close(KISSDB *db); + +/** + * Get an entry + * + * @param db Database struct + * @param key Key (key_size bytes) + * @param vbuf Value buffer (value_size bytes capacity) + * @return -1 on I/O error, 0 on success, 1 on not found + */ +extern int KISSDB_get(KISSDB *db,const void *key,void *vbuf); + +/** + * Put an entry (overwriting it if it already exists) + * + * In the already-exists case the size of the database file does not + * change. + * + * @param db Database struct + * @param key Key (key_size bytes) + * @param value Value (value_size bytes) + * @return -1 on I/O error, 0 on success + */ +extern int KISSDB_put(KISSDB *db,const void *key,const void *value); + +/** + * Cursor used for iterating over all entries in database + */ +typedef struct { + KISSDB *db; + unsigned long h_no; + unsigned long h_idx; +} KISSDB_Iterator; + +/** + * Initialize an iterator + * + * @param db Database struct + * @param i Iterator to initialize + */ +extern void KISSDB_Iterator_init(KISSDB *db,KISSDB_Iterator *dbi); + +/** + * Get the next entry + * + * The order of entries returned by iterator is undefined. It depends on + * how keys hash. + * + * @param Database iterator + * @param kbuf Buffer to fill with next key (key_size bytes) + * @param vbuf Buffer to fill with next value (value_size bytes) + * @return 0 if there are no more entries, negative on error, positive if an kbuf/vbuf have been filled + */ +extern int KISSDB_Iterator_next(KISSDB_Iterator *dbi,void *kbuf,void *vbuf); + +#ifdef __cplusplus +} +#endif + +#endif + diff --git a/ext/vsdm/LICENSE.txt b/ext/vsdm/LICENSE.txt new file mode 100644 index 00000000..00b65473 --- /dev/null +++ b/ext/vsdm/LICENSE.txt @@ -0,0 +1,22 @@ +MIT LICENSE + +Copyright 2017 ZeroTier, Inc. +https://www.zerotier.com/ + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to +deal in the Software without restriction, including without limitation the +rights to use, copy, modify, merge, publish, distribute, sublicense, and/or +sell copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/ext/vsdm/Makefile b/ext/vsdm/Makefile new file mode 100644 index 00000000..828d6edc --- /dev/null +++ b/ext/vsdm/Makefile @@ -0,0 +1,5 @@ +all: + c++ -Os -std=c++11 -o vsdm-test vsdm-test.cpp + +clean: + rm -f vsdm-test *.o *.dSYM diff --git a/ext/vsdm/README.md b/ext/vsdm/README.md new file mode 100644 index 00000000..03354be6 --- /dev/null +++ b/ext/vsdm/README.md @@ -0,0 +1,40 @@ +VSDM: Very Simple Distributed Map +====== + +VSDM is a super-minimal replicated in-memory associative container. Its advantages are small code size, small footprint, simplicity, and lack of dependencies. + +VSDM uses a rumor mill replication algorithm that provides fast best-effort replication. If connectivity is stable this results in eventual consistency, but data loss or regression can occur under split brain conditions. This class is not recommended for data that is intolerant of loss or regression. Its ideal use case is a distributed cache for small data objects or a distributed database for ephemeral data. + +Transport is via TCP and can optionally be encrypted if one specifies a cryptor class (see below). The transport protocol does not implement any features for versioning or backward compatibility and changes to key/value type, cryptor, or other relevant parameters will render it incompatible. Again this is designed for simple app-embedded use cases. Use something more feature complete if you need to support different versions across the network. + +Each node maintains a 64-bit monotonically increasing revision counter that starts at zero. When a node connects to another node it sends this revision counter and the other party will send all updates with revision numbers greater than or equal to it. When a node receives an update it replaces the entry it has if the revision counter is higher and also sets its own revision counter to the new counter if it is higher. It then re-transmits the update if a replace event occurred (if the new update was newer). This is the "rumor mill" part: each node retransmits all changes to all other connected nodes (excluding the source). + +VSDM nodes can be connected according to any arbitrary connectivity graph. A more fully connected graph trades increased bandwidth consumption (due to redundant messages) for decreased likelihood of data loss or split brain conditions. + +The VSDM class is thread safe. It launches a single background thread to handle network I/O and periodic cleanup. Deleted keys are purged from memory after a period of time to allow time for propagation and possible re-propagation. + +## Template parameters + +VSDM supports a number of template parameters for customization. The only required parameters are K and V, the key and value types. The default serializer allows these to be one of the *uintX_t* stdint numeric types or *std::string*. Specify a different serializer for anything else. + +Here are the other parameters after K and V (in order): + + * **L**: Maximum message length, which also limits the max size of the combined key and value for a given entry. This imposes a sanity limit to prevent memory exhaustion. Default is 131072. Absolute max is UINT32_MAX - 4 (4294967291). + * **W**: Watcher function type. Default is `vsdm_watcher_noop` which does nothing. The watcher function (or function object) is passed into the constructor and receives notifications of remotely initiated changes to the map's contents. (Local changes via set() or del() do not trigger the watcher). The watcher must have the following methods: + * `void add(uint64_t remoteNodeId,const K &key,const V &value,uint64_t revision)` + * `void update(uint64_t remoteNodeId,const K &key,const V &value,uint64_t revision)` + * `void del(uint64_t remoteNodeId,const K &key)` + * **S**: Serializer class, which must contain *static* methods for serializing and deserializing keys and values. The following must be present for both key and value types: + * `unsigned long objectSize(const [K|V] &)` + * `const char *objectData(const [K|V] &)` + * `bool objectDeserialize(const char *,unsigned long,[K|V] &)` (false return means object was invalid and causes disconnect) + * **C**: Cryptor type to encrypt transport, default is `vsdm_cryptor_noop` (no encryption). It is also passed into the constructor for internal initialization. This must implement a static method `static unsigned long overhead()` that returns the *constant* per-message overhead for things like IV and MAC, and two methods to encrypt and decrypt the payload in place. The vsdm class will add space for overhead at the *end* of the message. Encrypt/decrypt mathods have these signatures: + * `void encrypt(void *,unsigned long)` + * `bool decrypt(void *,unsigned long)` (false return means invalid MAC and causes disconnect) + * **M**: Map type for underlying storage. Default is `std::unordered_map` with default STL hashers. Substitute a different container if you need to deal with non-hashable keys or need a sorted map. Containers supporting duplicate keys should not be used as the replication algorithm will not function properly. + +## License + +(c)2017 ZeroTier, Inc. (MIT license) + +Written by [Adam Ierymenko](https://github.com/adamierymenko) diff --git a/ext/vsdm/vsdm-test.cpp b/ext/vsdm/vsdm-test.cpp new file mode 100644 index 00000000..13c49afe --- /dev/null +++ b/ext/vsdm/vsdm-test.cpp @@ -0,0 +1,72 @@ +#include +#include +#include + +#include + +#define VSDM_DEBUG 1 + +#include "vsdm.hpp" + +int main(int argc,char **argv) +{ + if (argc < 4) { + printf("Usage: vsdm-test [//]\n"); + return 0; + } + + uint64_t id = (uint64_t)strtoull(argv[1],(char **)0,10); + uint64_t node = (uint64_t)strtoull(argv[2],(char **)0,10); + int port = (int)strtol(argv[3],(char **)0,10); + + struct sockaddr_in sa; + memset(&sa,0,sizeof(sa)); + sa.sin_family = AF_INET; + sa.sin_port = htons((uint16_t)port); + + vsdm m(id,node,false); + m.listen((const struct sockaddr *)&sa); + + for(int i=4;i + * License: MIT + */ + +#ifndef ZT_VSDM_HPP__ +#define ZT_VSDM_HPP__ + +#include +#include +#include + +#if defined(_WIN32) || defined(_WIN64) + +#include +#include +#include + +#define ZT_PHY_SOCKFD_TYPE SOCKET +#define ZT_PHY_SOCKFD_NULL (INVALID_SOCKET) +#define ZT_PHY_SOCKFD_VALID(s) ((s) != INVALID_SOCKET) +#define ZT_PHY_CLOSE_SOCKET(s) ::closesocket(s) +#define ZT_PHY_MAX_SOCKETS (FD_SETSIZE) +#define ZT_PHY_MAX_INTERCEPTS ZT_PHY_MAX_SOCKETS +#define ZT_PHY_SOCKADDR_STORAGE_TYPE struct sockaddr_storage + +#else // not Windows + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define ZT_PHY_SOCKFD_TYPE int +#define ZT_PHY_SOCKFD_NULL (-1) +#define ZT_PHY_SOCKFD_VALID(s) ((s) > -1) +#define ZT_PHY_CLOSE_SOCKET(s) ::close(s) +#define ZT_PHY_MAX_SOCKETS (FD_SETSIZE) +#define ZT_PHY_MAX_INTERCEPTS ZT_PHY_MAX_SOCKETS +#define ZT_PHY_SOCKADDR_STORAGE_TYPE struct sockaddr_storage + +#endif + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +/*********************************************************************************************************/ + +namespace ztVsdmInternal { + +/* This is the Phy<> adapter implementation for selected sockets from ZeroTier One. + * It should build and run out of the box on Windows and most *nix systems. Parts + * not used by VSDM have been removed. */ + +typedef void PhySocket; + +template +class Phy +{ +private: + HANDLER_PTR_TYPE _handler; + + enum PhySocketType + { + ZT_PHY_SOCKET_CLOSED = 0x00, // socket is closed, will be removed on next poll() + ZT_PHY_SOCKET_TCP_OUT_PENDING = 0x01, + ZT_PHY_SOCKET_TCP_OUT_CONNECTED = 0x02, + ZT_PHY_SOCKET_TCP_IN = 0x03, + ZT_PHY_SOCKET_TCP_LISTEN = 0x04, + ZT_PHY_SOCKET_UDP = 0x05, + ZT_PHY_SOCKET_FD = 0x06, + ZT_PHY_SOCKET_UNIX_IN = 0x07, + ZT_PHY_SOCKET_UNIX_LISTEN = 0x08 + }; + + struct PhySocketImpl + { + PhySocketType type; + ZT_PHY_SOCKFD_TYPE sock; + void *uptr; // user-settable pointer + ZT_PHY_SOCKADDR_STORAGE_TYPE saddr; // remote for TCP_OUT and TCP_IN, local for TCP_LISTEN, RAW, and UDP + }; + + std::list _socks; + fd_set _readfds; + fd_set _writefds; +#if defined(_WIN32) || defined(_WIN64) + fd_set _exceptfds; +#endif + long _nfds; + + ZT_PHY_SOCKFD_TYPE _whackReceiveSocket; + ZT_PHY_SOCKFD_TYPE _whackSendSocket; + + bool _noDelay; + bool _noCheck; + +public: + /** + * @param handler Pointer of type HANDLER_PTR_TYPE to handler + * @param noDelay If true, disable TCP NAGLE algorithm on TCP sockets + * @param noCheck If true, attempt to set UDP SO_NO_CHECK option to disable sending checksums + */ + Phy(HANDLER_PTR_TYPE handler,bool noDelay,bool noCheck) : + _handler(handler) + { + FD_ZERO(&_readfds); + FD_ZERO(&_writefds); + +#if defined(_WIN32) || defined(_WIN64) + FD_ZERO(&_exceptfds); + + SOCKET pipes[2]; + { // hack copied from StackOverflow, behaves a bit like pipe() on *nix systems + struct sockaddr_in inaddr; + struct sockaddr addr; + SOCKET lst=::socket(AF_INET, SOCK_STREAM,IPPROTO_TCP); + if (lst == INVALID_SOCKET) + throw std::runtime_error("unable to create pipes for select() abort"); + memset(&inaddr, 0, sizeof(inaddr)); + memset(&addr, 0, sizeof(addr)); + inaddr.sin_family = AF_INET; + inaddr.sin_addr.s_addr = htonl(INADDR_LOOPBACK); + inaddr.sin_port = 0; + int yes=1; + setsockopt(lst,SOL_SOCKET,SO_REUSEADDR,(char*)&yes,sizeof(yes)); + bind(lst,(struct sockaddr *)&inaddr,sizeof(inaddr)); + listen(lst,1); + int len=sizeof(inaddr); + getsockname(lst, &addr,&len); + pipes[0]=::socket(AF_INET, SOCK_STREAM,0); + if (pipes[0] == INVALID_SOCKET) + throw std::runtime_error("unable to create pipes for select() abort"); + connect(pipes[0],&addr,len); + pipes[1]=accept(lst,0,0); + closesocket(lst); + } +#else // not Windows + int pipes[2]; + if (::pipe(pipes)) + throw std::runtime_error("unable to create pipes for select() abort"); +#endif // Windows or not + + _nfds = (pipes[0] > pipes[1]) ? (long)pipes[0] : (long)pipes[1]; + _whackReceiveSocket = pipes[0]; + _whackSendSocket = pipes[1]; + _noDelay = noDelay; + _noCheck = noCheck; + } + + ~Phy() + { + for(typename std::list::const_iterator s(_socks.begin());s!=_socks.end();++s) { + if (s->type != ZT_PHY_SOCKET_CLOSED) + this->close((PhySocket *)&(*s),true); + } + ZT_PHY_CLOSE_SOCKET(_whackReceiveSocket); + ZT_PHY_CLOSE_SOCKET(_whackSendSocket); + } + + /** + * @param s Socket object + * @return Underlying OS-type (usually int or long) file descriptor associated with object + */ + static inline ZT_PHY_SOCKFD_TYPE getDescriptor(PhySocket *s) throw() { return reinterpret_cast(s)->sock; } + + /** + * @param s Socket object + * @return Pointer to user object + */ + static inline void** getuptr(PhySocket *s) throw() { return &(reinterpret_cast(s)->uptr); } + + /** + * Cause poll() to stop waiting immediately + * + * This can be used to reset the polling loop after changes that require + * attention, or to shut down a background thread that is waiting, etc. + */ + inline void whack() + { +#if defined(_WIN32) || defined(_WIN64) + ::send(_whackSendSocket,(const char *)this,1,0); +#else + (void)(::write(_whackSendSocket,(PhySocket *)this,1)); +#endif + } + + /** + * @return Number of open sockets + */ + inline unsigned long count() const throw() { return _socks.size(); } + + /** + * @return Maximum number of sockets allowed + */ + inline unsigned long maxCount() const throw() { return ZT_PHY_MAX_SOCKETS; } + + /** + * Bind a local listen socket to listen for new TCP connections + * + * @param localAddress Local address and port + * @param uptr Initial value of uptr for new socket (default: NULL) + * @return Socket or NULL on failure to bind + */ + inline PhySocket *tcpListen(const struct sockaddr *localAddress,void *uptr = (void *)0) + { + if (_socks.size() >= ZT_PHY_MAX_SOCKETS) + return (PhySocket *)0; + + ZT_PHY_SOCKFD_TYPE s = ::socket(localAddress->sa_family,SOCK_STREAM,0); + if (!ZT_PHY_SOCKFD_VALID(s)) + return (PhySocket *)0; + +#if defined(_WIN32) || defined(_WIN64) + { + BOOL f; + f = TRUE; ::setsockopt(s,IPPROTO_IPV6,IPV6_V6ONLY,(const char *)&f,sizeof(f)); + f = TRUE; ::setsockopt(s,SOL_SOCKET,SO_REUSEADDR,(const char *)&f,sizeof(f)); + f = (_noDelay ? TRUE : FALSE); setsockopt(s,IPPROTO_TCP,TCP_NODELAY,(char *)&f,sizeof(f)); + u_long iMode=1; + ioctlsocket(s,FIONBIO,&iMode); + } +#else + { + int f; + f = 1; ::setsockopt(s,IPPROTO_IPV6,IPV6_V6ONLY,(void *)&f,sizeof(f)); + f = 1; ::setsockopt(s,SOL_SOCKET,SO_REUSEADDR,(void *)&f,sizeof(f)); + f = (_noDelay ? 1 : 0); setsockopt(s,IPPROTO_TCP,TCP_NODELAY,(char *)&f,sizeof(f)); + fcntl(s,F_SETFL,O_NONBLOCK); + } +#endif + + if (::bind(s,localAddress,(localAddress->sa_family == AF_INET6) ? sizeof(struct sockaddr_in6) : sizeof(struct sockaddr_in))) { + ZT_PHY_CLOSE_SOCKET(s); + return (PhySocket *)0; + } + + if (::listen(s,1024)) { + ZT_PHY_CLOSE_SOCKET(s); + return (PhySocket *)0; + } + + try { + _socks.push_back(PhySocketImpl()); + } catch ( ... ) { + ZT_PHY_CLOSE_SOCKET(s); + return (PhySocket *)0; + } + PhySocketImpl &sws = _socks.back(); + + if ((long)s > _nfds) + _nfds = (long)s; + FD_SET(s,&_readfds); + sws.type = ZT_PHY_SOCKET_TCP_LISTEN; + sws.sock = s; + sws.uptr = uptr; + memset(&(sws.saddr),0,sizeof(struct sockaddr_storage)); + memcpy(&(sws.saddr),localAddress,(localAddress->sa_family == AF_INET6) ? sizeof(struct sockaddr_in6) : sizeof(struct sockaddr_in)); + + return (PhySocket *)&sws; + } + + /** + * Start a non-blocking connect; CONNECT handler is called on success or failure + * + * A return value of NULL indicates a synchronous failure such as a + * failure to open a socket. The TCP connection handler is not called + * in this case. + * + * It is possible on some platforms for an "instant connect" to occur, + * such as when connecting to a loopback address. In this case, the + * 'connected' result parameter will be set to 'true' and if the + * 'callConnectHandler' flag is true (the default) the TCP connect + * handler will be called before the function returns. + * + * These semantics can be a bit confusing, but they're less so than + * the underlying semantics of asynchronous TCP connect. + * + * @param remoteAddress Remote address + * @param connected Result parameter: set to whether an "instant connect" has occurred (true if yes) + * @param uptr Initial value of uptr for new socket (default: NULL) + * @param callConnectHandler If true, call TCP connect handler even if result is known before function exit (default: true) + * @return New socket or NULL on failure + */ + inline PhySocket *tcpConnect(const struct sockaddr *remoteAddress,bool &connected,void *uptr = (void *)0,bool callConnectHandler = true) + { + if (_socks.size() >= ZT_PHY_MAX_SOCKETS) + return (PhySocket *)0; + + ZT_PHY_SOCKFD_TYPE s = ::socket(remoteAddress->sa_family,SOCK_STREAM,0); + if (!ZT_PHY_SOCKFD_VALID(s)) { + connected = false; + return (PhySocket *)0; + } + +#if defined(_WIN32) || defined(_WIN64) + { + BOOL f; + if (remoteAddress->sa_family == AF_INET6) { f = TRUE; ::setsockopt(s,IPPROTO_IPV6,IPV6_V6ONLY,(const char *)&f,sizeof(f)); } + f = TRUE; ::setsockopt(s,SOL_SOCKET,SO_REUSEADDR,(const char *)&f,sizeof(f)); + f = (_noDelay ? TRUE : FALSE); setsockopt(s,IPPROTO_TCP,TCP_NODELAY,(char *)&f,sizeof(f)); + u_long iMode=1; + ioctlsocket(s,FIONBIO,&iMode); + } +#else + { + int f; + if (remoteAddress->sa_family == AF_INET6) { f = 1; ::setsockopt(s,IPPROTO_IPV6,IPV6_V6ONLY,(void *)&f,sizeof(f)); } + f = 1; ::setsockopt(s,SOL_SOCKET,SO_REUSEADDR,(void *)&f,sizeof(f)); + f = (_noDelay ? 1 : 0); setsockopt(s,IPPROTO_TCP,TCP_NODELAY,(char *)&f,sizeof(f)); + fcntl(s,F_SETFL,O_NONBLOCK); + } +#endif + + connected = true; + if (::connect(s,remoteAddress,(remoteAddress->sa_family == AF_INET6) ? sizeof(struct sockaddr_in6) : sizeof(struct sockaddr_in))) { + connected = false; +#if defined(_WIN32) || defined(_WIN64) + if (WSAGetLastError() != WSAEWOULDBLOCK) { +#else + if (errno != EINPROGRESS) { +#endif + ZT_PHY_CLOSE_SOCKET(s); + return (PhySocket *)0; + } // else connection is proceeding asynchronously... + } + + try { + _socks.push_back(PhySocketImpl()); + } catch ( ... ) { + ZT_PHY_CLOSE_SOCKET(s); + return (PhySocket *)0; + } + PhySocketImpl &sws = _socks.back(); + + if ((long)s > _nfds) + _nfds = (long)s; + if (connected) { + FD_SET(s,&_readfds); + sws.type = ZT_PHY_SOCKET_TCP_OUT_CONNECTED; + } else { + FD_SET(s,&_writefds); +#if defined(_WIN32) || defined(_WIN64) + FD_SET(s,&_exceptfds); +#endif + sws.type = ZT_PHY_SOCKET_TCP_OUT_PENDING; + } + sws.sock = s; + sws.uptr = uptr; + memset(&(sws.saddr),0,sizeof(struct sockaddr_storage)); + memcpy(&(sws.saddr),remoteAddress,(remoteAddress->sa_family == AF_INET6) ? sizeof(struct sockaddr_in6) : sizeof(struct sockaddr_in)); + + if ((callConnectHandler)&&(connected)) { + try { + _handler->phyOnTcpConnect((PhySocket *)&sws,&(sws.uptr),true); + } catch ( ... ) {} + } + + return (PhySocket *)&sws; + } + + /** + * Attempt to send data to a stream socket (non-blocking) + * + * If -1 is returned, the socket should no longer be used as it is now + * destroyed. If callCloseHandler is true, the close handler will be + * called before the function returns. + * + * This can be used with TCP, Unix, or socket pair sockets. + * + * @param sock An open stream socket (other socket types will fail) + * @param data Data to send + * @param len Length of data + * @param callCloseHandler If true, call close handler on socket closing failure condition (default: true) + * @return Number of bytes actually sent or -1 on fatal error (socket closure) + */ + inline long streamSend(PhySocket *sock,const void *data,unsigned long len,bool callCloseHandler = true) + { + PhySocketImpl &sws = *(reinterpret_cast(sock)); +#if defined(_WIN32) || defined(_WIN64) + long n = (long)::send(sws.sock,reinterpret_cast(data),len,0); + if (n == SOCKET_ERROR) { + switch(WSAGetLastError()) { + case WSAEINTR: + case WSAEWOULDBLOCK: + return 0; + default: + this->close(sock,callCloseHandler); + return -1; + } + } +#else // not Windows + long n = (long)::send(sws.sock,data,len,0); + if (n < 0) { + switch(errno) { +#ifdef EAGAIN + case EAGAIN: +#endif +#if defined(EWOULDBLOCK) && ( !defined(EAGAIN) || (EWOULDBLOCK != EAGAIN) ) + case EWOULDBLOCK: +#endif +#ifdef EINTR + case EINTR: +#endif + return 0; + default: + this->close(sock,callCloseHandler); + return -1; + } + } +#endif // Windows or not + return n; + } + + /** + * For streams, sets whether we want to be notified that the socket is writable + * + * This can be used with TCP, Unix, or socket pair sockets. + * + * Call whack() if this is being done from another thread and you want + * it to take effect immediately. Otherwise it is only guaranteed to + * take effect on the next poll(). + * + * @param sock Stream connection socket + * @param notifyWritable Want writable notifications? + */ + inline const void setNotifyWritable(PhySocket *sock,bool notifyWritable) + { + PhySocketImpl &sws = *(reinterpret_cast(sock)); + if (notifyWritable) { + FD_SET(sws.sock,&_writefds); + } else { + FD_CLR(sws.sock,&_writefds); + } + } + + /** + * Set whether we want to be notified that a socket is readable + * + * This is primarily for raw sockets added with wrapSocket(). It could be + * used with others, but doing so would essentially lock them and prevent + * data from being read from them until this is set to 'true' again. + * + * @param sock Socket to modify + * @param notifyReadable True if socket should be monitored for readability + */ + inline const void setNotifyReadable(PhySocket *sock,bool notifyReadable) + { + PhySocketImpl &sws = *(reinterpret_cast(sock)); + if (notifyReadable) { + FD_SET(sws.sock,&_readfds); + } else { + FD_CLR(sws.sock,&_readfds); + } + } + + /** + * Wait for activity and handle one or more events + * + * Note that this is not guaranteed to wait up to 'timeout' even + * if nothing happens, as whack() or other events such as signals + * may cause premature termination. + * + * @param timeout Timeout in milliseconds or 0 for none (forever) + */ + inline void poll(unsigned long timeout) + { + char buf[131072]; + struct sockaddr_storage ss; + struct timeval tv; + fd_set rfds,wfds,efds; + + memcpy(&rfds,&_readfds,sizeof(rfds)); + memcpy(&wfds,&_writefds,sizeof(wfds)); +#if defined(_WIN32) || defined(_WIN64) + memcpy(&efds,&_exceptfds,sizeof(efds)); +#else + FD_ZERO(&efds); +#endif + + tv.tv_sec = (long)(timeout / 1000); + tv.tv_usec = (long)((timeout % 1000) * 1000); + if (::select((int)_nfds + 1,&rfds,&wfds,&efds,(timeout > 0) ? &tv : (struct timeval *)0) <= 0) + return; + + if (FD_ISSET(_whackReceiveSocket,&rfds)) { + char tmp[16]; +#if defined(_WIN32) || defined(_WIN64) + ::recv(_whackReceiveSocket,tmp,16,0); +#else + ::read(_whackReceiveSocket,tmp,16); +#endif + } + + for(typename std::list::iterator s(_socks.begin());s!=_socks.end();) { + switch (s->type) { + + case ZT_PHY_SOCKET_TCP_OUT_PENDING: +#if defined(_WIN32) || defined(_WIN64) + if (FD_ISSET(s->sock,&efds)) { + this->close((PhySocket *)&(*s),true); + } else // ... if +#endif + if (FD_ISSET(s->sock,&wfds)) { + socklen_t slen = sizeof(ss); + if (::getpeername(s->sock,(struct sockaddr *)&ss,&slen) != 0) { + this->close((PhySocket *)&(*s),true); + } else { + s->type = ZT_PHY_SOCKET_TCP_OUT_CONNECTED; + FD_SET(s->sock,&_readfds); + FD_CLR(s->sock,&_writefds); +#if defined(_WIN32) || defined(_WIN64) + FD_CLR(s->sock,&_exceptfds); +#endif + try { + _handler->phyOnTcpConnect((PhySocket *)&(*s),&(s->uptr),true); + } catch ( ... ) {} + } + } + break; + + case ZT_PHY_SOCKET_TCP_OUT_CONNECTED: + case ZT_PHY_SOCKET_TCP_IN: { + ZT_PHY_SOCKFD_TYPE sock = s->sock; // if closed, s->sock becomes invalid as s is no longer dereferencable + if (FD_ISSET(sock,&rfds)) { + long n = (long)::recv(sock,buf,sizeof(buf),0); + if (n <= 0) { + this->close((PhySocket *)&(*s),true); + } else { + try { + _handler->phyOnTcpData((PhySocket *)&(*s),&(s->uptr),(void *)buf,(unsigned long)n); + } catch ( ... ) {} + } + } + if ((FD_ISSET(sock,&wfds))&&(FD_ISSET(sock,&_writefds))) { + try { + _handler->phyOnTcpWritable((PhySocket *)&(*s),&(s->uptr)); + } catch ( ... ) {} + } + } break; + + case ZT_PHY_SOCKET_TCP_LISTEN: + if (FD_ISSET(s->sock,&rfds)) { + memset(&ss,0,sizeof(ss)); + socklen_t slen = sizeof(ss); + ZT_PHY_SOCKFD_TYPE newSock = ::accept(s->sock,(struct sockaddr *)&ss,&slen); + if (ZT_PHY_SOCKFD_VALID(newSock)) { + if (_socks.size() >= ZT_PHY_MAX_SOCKETS) { + ZT_PHY_CLOSE_SOCKET(newSock); + } else { +#if defined(_WIN32) || defined(_WIN64) + { BOOL f = (_noDelay ? TRUE : FALSE); setsockopt(newSock,IPPROTO_TCP,TCP_NODELAY,(char *)&f,sizeof(f)); } + { u_long iMode=1; ioctlsocket(newSock,FIONBIO,&iMode); } +#else + { int f = (_noDelay ? 1 : 0); setsockopt(newSock,IPPROTO_TCP,TCP_NODELAY,(char *)&f,sizeof(f)); } + fcntl(newSock,F_SETFL,O_NONBLOCK); +#endif + _socks.push_back(PhySocketImpl()); + PhySocketImpl &sws = _socks.back(); + FD_SET(newSock,&_readfds); + if ((long)newSock > _nfds) + _nfds = (long)newSock; + sws.type = ZT_PHY_SOCKET_TCP_IN; + sws.sock = newSock; + sws.uptr = (void *)0; + memcpy(&(sws.saddr),&ss,sizeof(struct sockaddr_storage)); + try { + _handler->phyOnTcpAccept((PhySocket *)&(*s),(PhySocket *)&(_socks.back()),&(s->uptr),&(sws.uptr),(const struct sockaddr *)&(sws.saddr)); + } catch ( ... ) {} + } + } + } + break; + + case ZT_PHY_SOCKET_UDP: + if (FD_ISSET(s->sock,&rfds)) { + for(;;) { + memset(&ss,0,sizeof(ss)); + socklen_t slen = sizeof(ss); + long n = (long)::recvfrom(s->sock,buf,sizeof(buf),0,(struct sockaddr *)&ss,&slen); + if (n > 0) { + try { + _handler->phyOnDatagram((PhySocket *)&(*s),&(s->uptr),(const struct sockaddr *)&(s->saddr),(const struct sockaddr *)&ss,(void *)buf,(unsigned long)n); + } catch ( ... ) {} + } else if (n < 0) + break; + } + } + break; + + case ZT_PHY_SOCKET_UNIX_IN: { +#ifdef __UNIX_LIKE__ + ZT_PHY_SOCKFD_TYPE sock = s->sock; // if closed, s->sock becomes invalid as s is no longer dereferencable + if ((FD_ISSET(sock,&wfds))&&(FD_ISSET(sock,&_writefds))) { + try { + _handler->phyOnUnixWritable((PhySocket *)&(*s),&(s->uptr),false); + } catch ( ... ) {} + } + if (FD_ISSET(sock,&rfds)) { + long n = (long)::read(sock,buf,sizeof(buf)); + if (n <= 0) { + this->close((PhySocket *)&(*s),true); + } else { + try { + _handler->phyOnUnixData((PhySocket *)&(*s),&(s->uptr),(void *)buf,(unsigned long)n); + } catch ( ... ) {} + } + } +#endif // __UNIX_LIKE__ + } break; + + case ZT_PHY_SOCKET_UNIX_LISTEN: +#ifdef __UNIX_LIKE__ + if (FD_ISSET(s->sock,&rfds)) { + memset(&ss,0,sizeof(ss)); + socklen_t slen = sizeof(ss); + ZT_PHY_SOCKFD_TYPE newSock = ::accept(s->sock,(struct sockaddr *)&ss,&slen); + if (ZT_PHY_SOCKFD_VALID(newSock)) { + if (_socks.size() >= ZT_PHY_MAX_SOCKETS) { + ZT_PHY_CLOSE_SOCKET(newSock); + } else { + fcntl(newSock,F_SETFL,O_NONBLOCK); + _socks.push_back(PhySocketImpl()); + PhySocketImpl &sws = _socks.back(); + FD_SET(newSock,&_readfds); + if ((long)newSock > _nfds) + _nfds = (long)newSock; + sws.type = ZT_PHY_SOCKET_UNIX_IN; + sws.sock = newSock; + sws.uptr = (void *)0; + memcpy(&(sws.saddr),&ss,sizeof(struct sockaddr_storage)); + try { + //_handler->phyOnUnixAccept((PhySocket *)&(*s),(PhySocket *)&(_socks.back()),&(s->uptr),&(sws.uptr)); + } catch ( ... ) {} + } + } + } +#endif // __UNIX_LIKE__ + break; + + case ZT_PHY_SOCKET_FD: { + ZT_PHY_SOCKFD_TYPE sock = s->sock; + const bool readable = ((FD_ISSET(sock,&rfds))&&(FD_ISSET(sock,&_readfds))); + const bool writable = ((FD_ISSET(sock,&wfds))&&(FD_ISSET(sock,&_writefds))); + if ((readable)||(writable)) { + try { + //_handler->phyOnFileDescriptorActivity((PhySocket *)&(*s),&(s->uptr),readable,writable); + } catch ( ... ) {} + } + } break; + + default: + break; + + } + + if (s->type == ZT_PHY_SOCKET_CLOSED) + _socks.erase(s++); + else ++s; + } + } + + /** + * @param sock Socket to close + * @param callHandlers If true, call handlers for TCP connect (success: false) or close (default: true) + */ + inline void close(PhySocket *sock,bool callHandlers = true) + { + if (!sock) + return; + PhySocketImpl &sws = *(reinterpret_cast(sock)); + if (sws.type == ZT_PHY_SOCKET_CLOSED) + return; + + FD_CLR(sws.sock,&_readfds); + FD_CLR(sws.sock,&_writefds); +#if defined(_WIN32) || defined(_WIN64) + FD_CLR(sws.sock,&_exceptfds); +#endif + + if (sws.type != ZT_PHY_SOCKET_FD) + ZT_PHY_CLOSE_SOCKET(sws.sock); + +#ifdef __UNIX_LIKE__ + if (sws.type == ZT_PHY_SOCKET_UNIX_LISTEN) + ::unlink(((struct sockaddr_un *)(&(sws.saddr)))->sun_path); +#endif // __UNIX_LIKE__ + + if (callHandlers) { + switch(sws.type) { + case ZT_PHY_SOCKET_TCP_OUT_PENDING: + try { + _handler->phyOnTcpConnect(sock,&(sws.uptr),false); + } catch ( ... ) {} + break; + case ZT_PHY_SOCKET_TCP_OUT_CONNECTED: + case ZT_PHY_SOCKET_TCP_IN: + try { + _handler->phyOnTcpClose(sock,&(sws.uptr)); + } catch ( ... ) {} + break; + case ZT_PHY_SOCKET_UNIX_IN: +#ifdef __UNIX_LIKE__ + try { + _handler->phyOnUnixClose(sock,&(sws.uptr)); + } catch ( ... ) {} +#endif // __UNIX_LIKE__ + break; + default: + break; + } + } + + // Causes entry to be deleted from list in poll(), ignored elsewhere + sws.type = ZT_PHY_SOCKET_CLOSED; + + if ((long)sws.sock >= (long)_nfds) { + long nfds = (long)_whackSendSocket; + if ((long)_whackReceiveSocket > nfds) + nfds = (long)_whackReceiveSocket; + for(typename std::list::iterator s(_socks.begin());s!=_socks.end();++s) { + if ((s->type != ZT_PHY_SOCKET_CLOSED)&&((long)s->sock > nfds)) + nfds = (long)s->sock; + } + _nfds = nfds; + } + } +}; + +static inline uint64_t _swap64(const uint64_t n) +{ + return ( + ((n & 0x00000000000000FFULL) << 56) | + ((n & 0x000000000000FF00ULL) << 40) | + ((n & 0x0000000000FF0000ULL) << 24) | + ((n & 0x00000000FF000000ULL) << 8) | + ((n & 0x000000FF00000000ULL) >> 8) | + ((n & 0x0000FF0000000000ULL) >> 24) | + ((n & 0x00FF000000000000ULL) >> 40) | + ((n & 0xFF00000000000000ULL) >> 56) + ); +} + +} // namespace ztVsdmInternal + +/*********************************************************************************************************/ + +/** + * No-op update watcher + */ +class vsdm_watcher_noop +{ +public: + template + inline void add(uint64_t,const K &k,const V &v,uint64_t) {} + template + inline void update(uint64_t,const K &k,const V &v,uint64_t) {} + template + inline void del(uint64_t,const K &k) {} +}; + +/** + * No-op cryptor that adds no overhead and does no encryption + */ +class vsdm_cryptor_noop +{ +public: + static inline unsigned long overhead() { return 0; } + inline void encrypt(void *d,unsigned long l) {} + inline bool decrypt(void *d,unsigned long l) { return true; } +}; + +/** + * Default serializer supporting std::string and stdint.h types + */ +class vsdm_default_serializer +{ +public: + static inline unsigned long objectSize(const std::string &o) { return o.length(); } + static inline unsigned long objectSize(const uint8_t o) { return 1; } + static inline unsigned long objectSize(const int8_t o) { return 1; } + static inline unsigned long objectSize(const uint16_t o) { return 2; } + static inline unsigned long objectSize(const int16_t o) { return 2; } + static inline unsigned long objectSize(const uint32_t o) { return 4; } + static inline unsigned long objectSize(const int32_t o) { return 4; } + static inline unsigned long objectSize(const uint64_t o) { return 8; } + static inline unsigned long objectSize(const int64_t o) { return 8; } + + static inline const char *objectData(const std::string &o) { return o.data(); } + static inline const char *objectData(const uint8_t &o) { return reinterpret_cast(&o); } + static inline const char *objectData(const int8_t &o) { return reinterpret_cast(&o); } + static inline const char *objectData(const uint16_t &o) { return reinterpret_cast(&o); } + static inline const char *objectData(const int16_t &o) { return reinterpret_cast(&o); } + static inline const char *objectData(const uint32_t &o) { return reinterpret_cast(&o); } + static inline const char *objectData(const int32_t &o) { return reinterpret_cast(&o); } + static inline const char *objectData(const uint64_t &o) { return reinterpret_cast(&o); } + static inline const char *objectData(const int64_t &o) { return reinterpret_cast(&o); } + + static inline bool objectDeserialize(const char *d,unsigned long l,std::string &o) { o.assign(d,l); return true; } + static inline bool objectDeserialize(const char *d,unsigned long l,uint8_t &o) { if (l == 1) { memcpy(&o,d,1); return true; } else { return false; } } + static inline bool objectDeserialize(const char *d,unsigned long l,int8_t &o) { if (l == 1) { memcpy(&o,d,1); return true; } else { return false; } } + static inline bool objectDeserialize(const char *d,unsigned long l,uint16_t &o) { if (l == 2) { memcpy(&o,d,2); return true; } else { return false; } } + static inline bool objectDeserialize(const char *d,unsigned long l,int16_t &o) { if (l == 2) { memcpy(&o,d,2); return true; } else { return false; } } + static inline bool objectDeserialize(const char *d,unsigned long l,uint32_t &o) { if (l == 4) { memcpy(&o,d,4); return true; } else { return false; } } + static inline bool objectDeserialize(const char *d,unsigned long l,int32_t &o) { if (l == 4) { memcpy(&o,d,4); return true; } else { return false; } } + static inline bool objectDeserialize(const char *d,unsigned long l,uint64_t &o) { if (l == 8) { memcpy(&o,d,8); return true; } else { return false; } } + static inline bool objectDeserialize(const char *d,unsigned long l,int64_t &o) { if (l == 8) { memcpy(&o,d,8); return true; } else { return false; } } +}; + +/** + * VSDM: Very Simple Distributed Map + * + * See README.md for full docs. + * + * @tparam K Key type (must be supported by serializer) + * @tparam V Value type (must be supported by serializer) + * @tparam L Maximum message length (max allowed: UINT32_MAX - 1, default: 131072) + * @tparam W Watcher function (default: vsdm_watcher_noop) + * @tparam S Serializer class with static methods to serialize keys and values (default: vsdm_default_serializer) + * @tparam C Cryptor to encrypt/decrypt and authenticate network traffic (default: vsdm_cryptor_noop) + * @tparam M Map type for underlying data store (default: std::unordered_map) + */ +template< + typename K, + typename V, + unsigned long L = 131072, + typename W = vsdm_watcher_noop, + typename S = vsdm_default_serializer, + typename C = vsdm_cryptor_noop, + template class M = std::unordered_map +> +class vsdm +{ + friend void vsdm_thread_main(void *parent); + friend class ztVsdmInternal::Phy; + +private: + struct vsdm_entry + { + vsdm_entry() : rev(0),deletedAt(0),v() {} + uint64_t rev; + uint64_t deletedAt; + V v; + }; + + struct _connection + { + _connection() : outbuf(),inbuf(),gotHello(false),node(0),sock((ztVsdmInternal::PhySocket *)0) {} + std::string outbuf; + std::string inbuf; + bool gotHello; + uint64_t node; + ztVsdmInternal::PhySocket *sock; + }; + +public: + typedef K key_type; + typedef V value_type; + + /** + * @param id Cluster ID, must be the same on all nodes + * @param node Arbitrary unique node ID + * @param restrictInbound If true, restrict inbound connections to known peer IPs (added via link()) + * @param cryptor Encryptor/decryptor instance (default: C()) + * @param watcher Watcher function instance (default: W()) + */ + vsdm(uint64_t id,uint64_t node,bool restrictInbound,const C &cryptor = C(),const W &watcher = W()) : + _node(node), + _id(id), + _rev(0), + _connections(), + _m(), + _lock(), + _phy(this,false,false), + _cryptor(cryptor), + _watcher(watcher), + _run(true), + _restrictInbound(restrictInbound), + _t(_threadMain,reinterpret_cast(this)) + { + } + + ~vsdm() + { + _run = false; + _phy.whack(); + _t.join(); + } + + /** + * @param k Key to set + * @param v New value for key + * @return Revision of entry in map + */ + inline uint64_t set(const K &k,const V &v) + { + std::lock_guard l(_lock); + + vsdm_entry &e = _m[k]; + e.rev = ++_rev; + e.deletedAt = 0; + e.v = v; + + std::vector sentToNodes; + for(typename std::unordered_map::iterator c2(_connections.begin());c2!=_connections.end();++c2) { + if ((c2->second.gotHello)&&(std::find(sentToNodes.begin(),sentToNodes.end(),c2->second.node) == sentToNodes.end())) { + sendUpdate(c2->second,k,e); + sentToNodes.push_back(c2->second.node); +#ifdef VSDM_DEBUG + fprintf(stderr,">> %lu: %s=%s\n",(unsigned long)c2->second.node,k.c_str(),v.c_str()); fflush(stderr); +#endif + } + } + _phy.whack(); + + return _rev; + } + + /** + * @param k Key to check + * @return Revision of key that we have or 0 if not found + */ + inline uint64_t have(const K &k) const + { + std::lock_guard l(_lock); + typename std::unordered_map::const_iterator i(_m.find(k)); + if ((i == _m.end())||(i->second.deletedAt)) + return 0; + return i->second.rev; + } + + /** + * @param k Key to get + * @param dfl Default value if key is not found + * @param have If non-NULL, set to revision of this key or 0 if not found + * @return Key value or dfl if not found + */ + inline V get(const K &k,const V &dfl = V(),uint64_t *have = (uint64_t *)0) const + { + std::lock_guard l(_lock); + typename std::unordered_map::const_iterator i(_m.find(k)); + if ((i == _m.end())||(i->second.deletedAt)) { + if (have) + *have = 0; + return dfl; + } + if (have) + *have = i->second.rev; + return i->second.v; + } + + /** + * @param k Key to get + * @param have If non-NULL, set to revision of this key or 0 if not found + * @return Key's value or default/empty V() if not found + */ + inline V get(const K &k,uint64_t *have) const + { + return get(k,V(),have); + } + + /** + * Erase a key + * + * Erased entries are not wholly purged from memory immediately. They + * are marked as erased and purged after sufficient time for propagation. + * + * @param k Key to erase + * @return Previous revision of this key in map or 0 if not found + */ + inline bool del(const K &k) + { + uint64_t prev = 0; + std::lock_guard l(_lock); + + typename std::unordered_map::iterator i(_m.find(k)); + if (i == _m.end()) + return 0; + prev = i->second.rev; + i->second.rev = ++_rev; + i->second.deletedAt = _rev; + i->second.v.clear(); + + std::vector sentToNodes; + for(typename std::unordered_map::iterator c2(_connections.begin());c2!=_connections.end();++c2) { + if ((c2->second.gotHello)&&(std::find(sentToNodes.begin(),sentToNodes.end(),c2->second.node) == sentToNodes.end())) { + sendUpdate(c2->second,k,i->second); + sentToNodes.push_back(c2->second.node); +#ifdef VSDM_DEBUG + fprintf(stderr,">> %lu: %s=\n",(unsigned long)c2->second.node,k.c_str()); fflush(stderr); +#endif + } + } + _phy.whack(); + + return prev; + } + + /** + * Listen for incoming node connections on an address + * + * This can be called more than once to listen on more than one address and port. + * + * @param sa Socket address + * @return True if bind succeeded + */ + inline bool listen(const struct sockaddr *sa) + { + std::lock_guard l(_lock); + return (_phy.tcpListen(sa) != (ztVsdmInternal::PhySocket *)0); + } + + /** + * Add a remote node endpoint + * + * This can be called for an arbitrary number of other endpoints in the + * network to tell this node to attempt to maintain a link to them. + * + * @param node Node ID of remote + * @param sa Socket address of remote + * @param salen Length of socket address structure + */ + inline void link(uint64_t node,const struct sockaddr_in *sa,unsigned int salen) + { + std::lock_guard l(_lock); + if ((node != _node)&&(salen <= sizeof(struct sockaddr_storage))) + memcpy(&(_peers[node]),sa,salen); + } + + /** + * @return Node IDs of nodes that are currently connected + */ + inline std::vector who() const + { + std::vector w; + std::lock_guard l(_lock); + for(typename std::unordered_map::const_iterator i(_connections.begin());i!=_connections.end();++i) { + if ((i->gotHello)&&(std::find(w.begin(),w.end(),i->second.node) == w.end())) + w.push_back(i->second.node); + } + return w; + } + + /** + * @return True if we are currently connected to at least one other node + */ + inline bool connected() const + { + std::lock_guard l(_lock); + for(typename std::unordered_map::const_iterator i(_connections.begin());i!=_connections.end();++i) { + if (i->gotHello) + return true; + } + return false; + } + + /** + * Iterate through all members of this map, with optional deletion + * + * The function is executed against all key/value pairs and returns a signed integer. + * A negative return value causes the entry to be deleted, while a positive return + * value means the key's value (which is passed into the function as a reference) has + * been modified and should be replicated. A return value of zero means no change. + * + * Other methods should not be called since doing so can result in a deadlock. + * + * @param func Function to execute against all members of map, returns integer (see description) + */ + template + inline void each(F func) + { + std::vector sentToNodes; + bool whack = false; + std::lock_guard l(_lock); + for(typename M::iterator i(_m.begin());i!=_m.end();++i) { + if (!i->second.deletedAt) { + try { + const int result = func(i->first,i->second.v); + if (result < 0) { + i->second.rev = ++_rev; + i->second.deletedAt = _rev; + i->second.v.clear(); + } else if (result > 0) { + i->second.rev = ++_rev; + i->second.deletedAt = 0; + // v will have been modified in place + } + if (result != 0) { + sentToNodes.clear(); + for(typename std::unordered_map::iterator c2(_connections.begin());c2!=_connections.end();++c2) { + if ((c2->second.gotHello)&&(std::find(sentToNodes.begin(),sentToNodes.end(),c2->second.node) == sentToNodes.end())) { + sendUpdate(c2->second,i->first,i->second); + sentToNodes.push_back(c2->second.node); + } + } + whack = true; + } + } catch ( ... ) {} + } + } + if (whack) + _phy.whack(); + } + +private: + inline vsdm &operator=(const vsdm &v) { return *this; } + + static void _threadMain(void *p) { reinterpret_cast(p)->threadMain(); } + inline void threadMain() + { + std::vector haveNodes; + time_t lastcheck = 0; + time_t lastclean = 0; + while (_run) { + _phy.poll(1000); + + time_t now = time(0); + + // Check connections with other nodes and try to establish them + // if they're not present. + if ((now - lastcheck) >= 2) { + lastcheck = now; + haveNodes.clear(); + + std::lock_guard l(_lock); + + for(typename std::unordered_map::const_iterator c(_connections.begin());c!=_connections.end();++c) { + if (std::find(haveNodes.begin(),haveNodes.end(),c->second.node) == haveNodes.end()) + haveNodes.push_back(c->second.node); + } + + for(std::unordered_map::iterator p(_peers.begin());p!=_peers.end();++p) { + if (std::find(haveNodes.begin(),haveNodes.end(),p->first) == haveNodes.end()) { + bool connected = false; + ztVsdmInternal::PhySocket *ns = _phy.tcpConnect((const struct sockaddr *)&(p->second),connected,(void *)0,true); + if (ns) { + _connection &c = _connections[ns]; + c.gotHello = false; + c.node = p->first; + c.sock = ns; + } + } + } + } + + // Forget deleted entries if they've had ample time to propagate + if ((now - lastclean) >= 120) { + lastclean = now; + uint64_t delHorizon = _m.size() * (2 + _peers.size()); + if (_rev > delHorizon) { + delHorizon -= _rev; + std::lock_guard l(_lock); + for(typename M::iterator i(_m.begin());i!=_m.end();) { + if ((i->second.deletedAt > 0)&&(i->second.deletedAt < delHorizon)) + _m.erase(i++); + else ++i; + } + } + } + } + } + + inline void sendUpdate(_connection &c,const std::string &k,const vsdm_entry &e) + { + // assumes lock is locked + const uint32_t ks = (uint32_t)S::objectSize(k); + uint32_t vs = 0; + uint32_t hdr[4]; + hdr[0] = htonl((uint32_t)((e.rev >> 32) & 0xffffffff)); + hdr[1] = htonl((uint32_t)(e.rev & 0xffffffff)); + hdr[2] = htonl(ks); + if (e.deletedAt) { + hdr[3] = 0xffffffff; + } else { + vs = (uint32_t)S::objectSize(e.v); + hdr[3] = htonl(vs); + } + + const uint32_t s = htonl((uint32_t)(16 + C::overhead() + ks + vs)); + c.outbuf.append((const char *)&s,4); + + const unsigned long start = (unsigned long)c.outbuf.length(); + c.outbuf.append((const char *)hdr,16); + c.outbuf.append(S::objectData(k),ks); + if (!e.deletedAt) + c.outbuf.append(S::objectData(e.v),vs); + c.outbuf.append(C::overhead(),(char)0); + const unsigned long end = (unsigned long)c.outbuf.length(); + + _cryptor.encrypt(reinterpret_cast(const_cast(c.outbuf.data()) + start),end - start); + + _phy.setNotifyWritable(c.sock,true); + } + + inline void sendUpdateToAll(ztVsdmInternal::PhySocket *receivedOnSock,const uint64_t receivedFromNode,const std::string &k,const vsdm_entry &e) + { + // assumes lock is locked + std::vector sentToNodes; + for(typename std::unordered_map::iterator c2(_connections.begin());c2!=_connections.end();++c2) { + if ((c2->first != receivedOnSock)&&(c2->second.gotHello)&&(c2->second.node != receivedFromNode)&&(std::find(sentToNodes.begin(),sentToNodes.end(),c2->second.node) == sentToNodes.end())) { + sendUpdate(c2->second,k,e); + sentToNodes.push_back(c2->second.node); +#ifdef VSDM_DEBUG + fprintf(stderr,">> %lu: %s=%s\n",(unsigned long)c2->second.node,k.c_str(),(e.deletedAt) ? "" : e.v.c_str()); fflush(stderr); +#endif + } + } + _phy.whack(); + } + + inline void sendHello(ztVsdmInternal::PhySocket *sock) + { + uint64_t hdr[3]; + if (htonl(1) == 1) { + hdr[0] = _node; + hdr[1] = _id; + hdr[2] = _rev; + } else { + hdr[0] = ztVsdmInternal::_swap64(_node); + hdr[1] = ztVsdmInternal::_swap64(_id); + hdr[2] = ztVsdmInternal::_swap64(_rev); + } + uint8_t tmp[24 + C::overhead()]; + memcpy(tmp,hdr,24); + _cryptor.encrypt(reinterpret_cast(tmp),sizeof(tmp)); + _phy.streamSend(sock,reinterpret_cast(tmp),sizeof(tmp)); + } + + inline void phyOnTcpConnect(ztVsdmInternal::PhySocket *sock,void **uptr,bool success) + { + std::lock_guard l(_lock); + if (success) { + _connection &c = _connections[sock]; + c.gotHello = false; + c.sock = sock; + *uptr = (void *)&c; + sendHello(sock); + } else { + _connections.erase(sock); + } + } + + inline void phyOnTcpAccept(ztVsdmInternal::PhySocket *sockL,ztVsdmInternal::PhySocket *sockN,void **uptrL,void **uptrN,const struct sockaddr *from) + { + std::lock_guard l(_lock); + + if (_restrictInbound) { + bool ok = false; + for(typename std::unordered_map::const_iterator i(_peers.begin());i!=_peers.end();++i) { + if (from->sa_family == i->second.ss_family) { + if ( (from->sa_family == AF_INET) && (reinterpret_cast(from)->sin_addr.s_addr == reinterpret_cast(&(i->second))->sin_addr.s_addr) ) { + ok = true; + break; + } else if ( (from->sa_family == AF_INET6) && (memcmp(reinterpret_cast(from)->sin6_addr.s6_addr,reinterpret_cast(&(i->second))->sin6_addr.s6_addr,16) == 0) ) { + ok = true; + break; + } + } + } + if (!ok) { +#ifdef VSDM_DEBUG + fprintf(stderr," * dropped inbound connection: peer not from a known IP address\n"); fflush(stderr); +#endif + _phy.close(sockN,false); + return; + } + } + + _connection &c = _connections[sockN]; + c.gotHello = false; + c.node = _node; // impossible value for a remote + c.sock = sockN; + *uptrN = (void *)&c; + sendHello(sockN); + } + + inline void phyOnTcpClose(ztVsdmInternal::PhySocket *sock,void **uptr) + { + std::lock_guard l(_lock); + _connections.erase(sock); + } + + inline void phyOnTcpData(ztVsdmInternal::PhySocket *sock,void **uptr,void *data,unsigned long len) + { + _connection *const c = (_connection *)*uptr; + if (!c) return; + + std::unique_lock l(_lock); + c->inbuf.append(reinterpret_cast(data),len); + for(;;) { + if (c->gotHello) { + + if (c->inbuf.length() >= 20) { // got message size and header + uint32_t _totalLen; + memcpy(&_totalLen,c->inbuf.data(),4); + const unsigned long totalLen = ntohl(_totalLen); + if ((totalLen > L)||(totalLen < 16)) { // message too small or too large + _connections.erase(sock); + _phy.close(sock,false); + return; + } + + if (c->inbuf.length() >= (4 + totalLen)) { // got full message + + if (!_cryptor.decrypt(reinterpret_cast(const_cast(c->inbuf.data()) + 4),totalLen)) { + _connections.erase(sock); + _phy.close(sock,false); + return; + } + + uint32_t hdr[4]; + memcpy(hdr,c->inbuf.data() + 4,16); + + const uint64_t objectRev = ((uint64_t)ntohl(hdr[0]) << 32) | (uint64_t)ntohl(hdr[1]); + const unsigned long keyLen = (unsigned long)ntohl(hdr[2]); + unsigned long valueLen = (unsigned long)ntohl(hdr[3]); + + if (objectRev > _rev) + _rev = objectRev; + + uint64_t deletedAt = 0; + if (valueLen == 0xffffffff) { + valueLen = 0; + deletedAt = _rev; + } + if ((keyLen + valueLen + 16 + C::overhead()) > totalLen) { // key and/or value length invalid + _connections.erase(sock); + _phy.close(sock,false); + return; + } + + K k; + if (!S::objectDeserialize(c->inbuf.data() + 16 + 4,keyLen,k)) { + _connections.erase(sock); + _phy.close(sock,false); + return; + } + + vsdm_entry &e = _m[k]; + if (e.rev < objectRev) { + const bool added = (e.rev == 0); + e.rev = objectRev; + e.deletedAt = deletedAt; + if (e.deletedAt) { + e.v = V(); + } else { + if (!S::objectDeserialize(c->inbuf.data() + 16 + 4 + keyLen,valueLen,e.v)) { + _connections.erase(sock); + _phy.close(sock,false); + return; + } + } +#ifdef VSDM_DEBUG + fprintf(stderr,"<< %lu: %s=%s\n",(unsigned long)c->node,k.c_str(),(deletedAt) ? "" : e.v.c_str()); fflush(stderr); +#endif + sendUpdateToAll(sock,c->node,k,e); + + l.unlock(); + try { + if (added) { + _watcher.add(c->node,k,e.v,objectRev); + } else if (deletedAt) { + _watcher.del(c->node,k); + } else { + _watcher.update(c->node,k,e.v,objectRev); + } + } catch ( ... ) {} + l.lock(); + } + + c->inbuf.erase(c->inbuf.begin(),c->inbuf.begin() + totalLen + 4); + + // continue and process more messages in queue, if any + } else { // still waiting on full message + break; + } + } else { // still waiting on message size and header + break; + } + + } else if (c->inbuf.length() >= (24 + C::overhead())) { // got hello header + + if (!_cryptor.decrypt(reinterpret_cast(const_cast(c->inbuf.data())),24 + C::overhead())) { + _connections.erase(sock); + _phy.close(sock,false); + return; + } + + uint64_t hdr[3]; + memcpy(hdr,c->inbuf.data(),24); + c->inbuf.erase(c->inbuf.begin(),c->inbuf.begin() + 24 + C::overhead()); + + if (htonl(1) != 1) { + hdr[0] = ztVsdmInternal::_swap64(hdr[0]); + hdr[1] = ztVsdmInternal::_swap64(hdr[1]); + hdr[2] = ztVsdmInternal::_swap64(hdr[2]); + } + + if ((hdr[0] == _node)||(hdr[1] != _id)) { // don't connect to self, and don't connect to other map IDs + _connections.erase(sock); + _phy.close(sock,false); + break; + } else { + c->gotHello = true; + c->node = hdr[0]; + + if (hdr[2] > _rev) + _rev = hdr[2]; + + for(typename M::const_iterator i(_m.begin());i!=_m.end();++i) { + if (i->second.rev >= hdr[2]) { + sendUpdate(*c,i->first,i->second); +#ifdef VSDM_DEBUG + fprintf(stderr,">> %lu: %s=%s (new link)\n",(unsigned long)c->node,i->first.c_str(),i->second.v.c_str()); fflush(stderr); +#endif + } + } + _phy.whack(); + } + + // continue and process more messages in queue, if any + } else { // still waiting on hello header + break; + } + } + } + + inline void phyOnTcpWritable(ztVsdmInternal::PhySocket *sock,void **uptr) + { + std::lock_guard l(_lock); + _connection *c = (_connection *)*uptr; + if (c) { + if (c->outbuf.length() > 0) { + long n = _phy.streamSend(sock,c->outbuf.data(),c->outbuf.length()); + if (n <= 0) { + _connections.erase(sock); + _phy.close(sock,false); + return; + } else if (n == (long)c->outbuf.length()) { + c->outbuf.clear(); + } else { + c->outbuf.erase(c->outbuf.begin(),c->outbuf.begin() + n); + } + } + if (c->outbuf.length() == 0) { + _phy.setNotifyWritable(c->sock,false); + } + } + } + + inline void phyOnDatagram(ztVsdmInternal::PhySocket *sock,void **uptr,const struct sockaddr *localAddr,const struct sockaddr *from,void *data,unsigned long len) {} + inline void phyOnFileDescriptorActivity(ztVsdmInternal::PhySocket *sock,void **uptr,bool readable,bool writable) {} + inline void phyOnUnixAccept(ztVsdmInternal::PhySocket *sockL,ztVsdmInternal::PhySocket *sockN,void **uptrL,void **uptrN) {} + inline void phyOnUnixClose(ztVsdmInternal::PhySocket *sock,void **uptr) {} + inline void phyOnUnixData(ztVsdmInternal::PhySocket *sock,void **uptr,void *data,unsigned long len) {} + inline void phyOnUnixWritable(ztVsdmInternal::PhySocket *sock,void **uptr) {} + + const uint64_t _node; + const uint64_t _id; + uint64_t _rev; + std::unordered_map _peers; + std::unordered_map _connections; + M _m; + mutable std::mutex _lock; + ztVsdmInternal::Phy _phy; + C _cryptor; + W _watcher; + volatile bool _run; + bool _restrictInbound; + std::thread _t; +}; + +#endif diff --git a/include/ZeroTierOne.h b/include/ZeroTierOne.h index 5126c5a2..4709b116 100644 --- a/include/ZeroTierOne.h +++ b/include/ZeroTierOne.h @@ -157,21 +157,6 @@ extern "C" { */ #define ZT_CIRCUIT_TEST_REPORT_FLAGS_UPSTREAM_AUTHORIZED_IN_PATH 0x0000000000000001ULL -/** - * Maximum number of cluster members (and max member ID plus one) - */ -#define ZT_CLUSTER_MAX_MEMBERS 128 - -/** - * Maximum number of physical ZeroTier addresses a cluster member can report - */ -#define ZT_CLUSTER_MAX_ZT_PHYSICAL_ADDRESSES 16 - -/** - * Maximum allowed cluster message length in bytes - */ -#define ZT_CLUSTER_MAX_MESSAGE_LENGTH (1500 - 48) - /** * Maximum value for link quality (min is 0) */ @@ -1104,78 +1089,6 @@ typedef struct unsigned long peerCount; } ZT_PeerList; -/** - * A cluster member's status - */ -typedef struct { - /** - * This cluster member's ID (from 0 to 1-ZT_CLUSTER_MAX_MEMBERS) - */ - unsigned int id; - - /** - * Number of milliseconds since last 'alive' heartbeat message received via cluster backplane address - */ - unsigned int msSinceLastHeartbeat; - - /** - * Non-zero if cluster member is alive - */ - int alive; - - /** - * X, Y, and Z coordinates of this member (if specified, otherwise zero) - * - * What these mean depends on the location scheme being used for - * location-aware clustering. At present this is GeoIP and these - * will be the X, Y, and Z coordinates of the location on a spherical - * approximation of Earth where Earth's core is the origin (in km). - * They don't have to be perfect and need only be comparable with others - * to find shortest path via the standard vector distance formula. - */ - int x,y,z; - - /** - * Cluster member's last reported load - */ - uint64_t load; - - /** - * Number of peers - */ - uint64_t peers; - - /** - * Physical ZeroTier endpoints for this member (where peers are sent when directed here) - */ - struct sockaddr_storage zeroTierPhysicalEndpoints[ZT_CLUSTER_MAX_ZT_PHYSICAL_ADDRESSES]; - - /** - * Number of physical ZeroTier endpoints this member is announcing - */ - unsigned int numZeroTierPhysicalEndpoints; -} ZT_ClusterMemberStatus; - -/** - * ZeroTier cluster status - */ -typedef struct { - /** - * My cluster member ID (a record for 'self' is included in member[]) - */ - unsigned int myId; - - /** - * Number of cluster members - */ - unsigned int clusterSize; - - /** - * Cluster member statuses - */ - ZT_ClusterMemberStatus members[ZT_CLUSTER_MAX_MEMBERS]; -} ZT_ClusterStatus; - /** * An instance of a ZeroTier One node (opaque) */ @@ -1765,116 +1678,6 @@ int ZT_Node_sendUserMessage(ZT_Node *node,void *tptr,uint64_t dest,uint64_t type */ void ZT_Node_setNetconfMaster(ZT_Node *node,void *networkConfigMasterInstance); -/** - * Initialize cluster operation - * - * This initializes the internal structures and state for cluster operation. - * It takes two function pointers. The first is to a function that can be - * used to send data to cluster peers (mechanism is not defined by Node), - * and the second is to a function that can be used to get the location of - * a physical address in X,Y,Z coordinate space (e.g. as cartesian coordinates - * projected from the center of the Earth). - * - * Send function takes an arbitrary pointer followed by the cluster member ID - * to send data to, a pointer to the data, and the length of the data. The - * maximum message length is ZT_CLUSTER_MAX_MESSAGE_LENGTH (65535). Messages - * must be delivered whole and may be dropped or transposed, though high - * failure rates are undesirable and can cause problems. Validity checking or - * CRC is also not required since the Node validates the authenticity of - * cluster messages using cryptogrphic methods and will silently drop invalid - * messages. - * - * Address to location function is optional and if NULL geo-handoff is not - * enabled (in this case x, y, and z in clusterInit are also unused). It - * takes an arbitrary pointer followed by a physical address and three result - * parameters for x, y, and z. It returns zero on failure or nonzero if these - * three coordinates have been set. Coordinate space is arbitrary and can be - * e.g. coordinates on Earth relative to Earth's center. These can be obtained - * from latitutde and longitude with versions of the Haversine formula. - * - * See: http://stackoverflow.com/questions/1185408/converting-from-longitude-latitude-to-cartesian-coordinates - * - * Neither the send nor the address to location function should block. If the - * address to location function does not have a location for an address, it - * should return zero and then look up the address for future use since it - * will be called again in (typically) 1-3 minutes. - * - * Note that both functions can be called from any thread from which the - * various Node functions are called, and so must be thread safe if multiple - * threads are being used. - * - * @param node Node instance - * @param myId My cluster member ID (less than or equal to ZT_CLUSTER_MAX_MEMBERS) - * @param zeroTierPhysicalEndpoints Preferred physical address(es) for ZeroTier clients to contact this cluster member (for peer redirect) - * @param numZeroTierPhysicalEndpoints Number of physical endpoints in zeroTierPhysicalEndpoints[] (max allowed: 255) - * @param x My cluster member's X location - * @param y My cluster member's Y location - * @param z My cluster member's Z location - * @param sendFunction Function to be called to send data to other cluster members - * @param sendFunctionArg First argument to sendFunction() - * @param addressToLocationFunction Function to be called to get the location of a physical address or NULL to disable geo-handoff - * @param addressToLocationFunctionArg First argument to addressToLocationFunction() - * @return OK or UNSUPPORTED_OPERATION if this Node was not built with cluster support - */ -enum ZT_ResultCode ZT_Node_clusterInit( - ZT_Node *node, - unsigned int myId, - const struct sockaddr_storage *zeroTierPhysicalEndpoints, - unsigned int numZeroTierPhysicalEndpoints, - int x, - int y, - int z, - void (*sendFunction)(void *,unsigned int,const void *,unsigned int), - void *sendFunctionArg, - int (*addressToLocationFunction)(void *,const struct sockaddr_storage *,int *,int *,int *), - void *addressToLocationFunctionArg); - -/** - * Add a member to this cluster - * - * Calling this without having called clusterInit() will do nothing. - * - * @param node Node instance - * @param memberId Member ID (must be less than or equal to ZT_CLUSTER_MAX_MEMBERS) - * @return OK or error if clustering is disabled, ID invalid, etc. - */ -enum ZT_ResultCode ZT_Node_clusterAddMember(ZT_Node *node,unsigned int memberId); - -/** - * Remove a member from this cluster - * - * Calling this without having called clusterInit() will do nothing. - * - * @param node Node instance - * @param memberId Member ID to remove (nothing happens if not present) - */ -void ZT_Node_clusterRemoveMember(ZT_Node *node,unsigned int memberId); - -/** - * Handle an incoming cluster state message - * - * The message itself contains cluster member IDs, and invalid or badly - * addressed messages will be silently discarded. - * - * Calling this without having called clusterInit() will do nothing. - * - * @param node Node instance - * @param msg Cluster message - * @param len Length of cluster message - */ -void ZT_Node_clusterHandleIncomingMessage(ZT_Node *node,const void *msg,unsigned int len); - -/** - * Get the current status of the cluster from this node's point of view - * - * Calling this without clusterInit() or without cluster support will just - * zero out the structure and show a cluster size of zero. - * - * @param node Node instance - * @param cs Cluster status structure to fill with data - */ -void ZT_Node_clusterStatus(ZT_Node *node,ZT_ClusterStatus *cs); - /** * Set trusted paths * diff --git a/make-mac.mk b/make-mac.mk index de66f49c..5622a41b 100644 --- a/make-mac.mk +++ b/make-mac.mk @@ -54,7 +54,7 @@ ifeq ($(ZT_DEBUG),1) # C25519 in particular is almost UNUSABLE in heavy testing without it. node/Salsa20.o node/SHA512.o node/C25519.o node/Poly1305.o: CFLAGS = -Wall -O2 -g $(INCLUDES) $(DEFS) else - CFLAGS?=-Ofast -fstack-protector-strong + CFLAGS?=-Os -fstack-protector-strong CFLAGS+=$(ARCH_FLAGS) -Wall -Werror -flto -fPIE -mmacosx-version-min=10.7 -DNDEBUG -Wno-unused-private-field $(INCLUDES) $(DEFS) STRIP=strip endif diff --git a/node/Node.cpp b/node/Node.cpp index 911c9c4b..8849e0f4 100644 --- a/node/Node.cpp +++ b/node/Node.cpp @@ -503,6 +503,7 @@ void Node::setNetconfMaster(void *networkControllerInstance) RR->localNetworkController->init(RR->identity,this); } +/* ZT_ResultCode Node::clusterInit( unsigned int myId, const struct sockaddr_storage *zeroTierPhysicalEndpoints, @@ -570,6 +571,7 @@ void Node::clusterStatus(ZT_ClusterStatus *cs) #endif memset(cs,0,sizeof(ZT_ClusterStatus)); } +*/ /****************************************************************************/ /* Node methods used only within node/ */ @@ -998,56 +1000,6 @@ void ZT_Node_setNetconfMaster(ZT_Node *node,void *networkControllerInstance) } catch ( ... ) {} } -enum ZT_ResultCode ZT_Node_clusterInit( - ZT_Node *node, - unsigned int myId, - const struct sockaddr_storage *zeroTierPhysicalEndpoints, - unsigned int numZeroTierPhysicalEndpoints, - int x, - int y, - int z, - void (*sendFunction)(void *,unsigned int,const void *,unsigned int), - void *sendFunctionArg, - int (*addressToLocationFunction)(void *,const struct sockaddr_storage *,int *,int *,int *), - void *addressToLocationFunctionArg) -{ - try { - return reinterpret_cast(node)->clusterInit(myId,zeroTierPhysicalEndpoints,numZeroTierPhysicalEndpoints,x,y,z,sendFunction,sendFunctionArg,addressToLocationFunction,addressToLocationFunctionArg); - } catch ( ... ) { - return ZT_RESULT_FATAL_ERROR_INTERNAL; - } -} - -enum ZT_ResultCode ZT_Node_clusterAddMember(ZT_Node *node,unsigned int memberId) -{ - try { - return reinterpret_cast(node)->clusterAddMember(memberId); - } catch ( ... ) { - return ZT_RESULT_FATAL_ERROR_INTERNAL; - } -} - -void ZT_Node_clusterRemoveMember(ZT_Node *node,unsigned int memberId) -{ - try { - reinterpret_cast(node)->clusterRemoveMember(memberId); - } catch ( ... ) {} -} - -void ZT_Node_clusterHandleIncomingMessage(ZT_Node *node,const void *msg,unsigned int len) -{ - try { - reinterpret_cast(node)->clusterHandleIncomingMessage(msg,len); - } catch ( ... ) {} -} - -void ZT_Node_clusterStatus(ZT_Node *node,ZT_ClusterStatus *cs) -{ - try { - reinterpret_cast(node)->clusterStatus(cs); - } catch ( ... ) {} -} - void ZT_Node_setTrustedPaths(ZT_Node *node,const struct sockaddr_storage *networks,const uint64_t *ids,unsigned int count) { try { diff --git a/node/Node.hpp b/node/Node.hpp index 57b5489e..006551fa 100644 --- a/node/Node.hpp +++ b/node/Node.hpp @@ -117,21 +117,6 @@ public: void clearLocalInterfaceAddresses(); int sendUserMessage(void *tptr,uint64_t dest,uint64_t typeId,const void *data,unsigned int len); void setNetconfMaster(void *networkControllerInstance); - ZT_ResultCode clusterInit( - unsigned int myId, - const struct sockaddr_storage *zeroTierPhysicalEndpoints, - unsigned int numZeroTierPhysicalEndpoints, - int x, - int y, - int z, - void (*sendFunction)(void *,unsigned int,const void *,unsigned int), - void *sendFunctionArg, - int (*addressToLocationFunction)(void *,const struct sockaddr_storage *,int *,int *,int *), - void *addressToLocationFunctionArg); - ZT_ResultCode clusterAddMember(unsigned int memberId); - void clusterRemoveMember(unsigned int memberId); - void clusterHandleIncomingMessage(const void *msg,unsigned int len); - void clusterStatus(ZT_ClusterStatus *cs); // Internal functions ------------------------------------------------------ -- cgit v1.2.3 From 640ad577d1f52140adbe42e87b2da931bf15f430 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Thu, 6 Jul 2017 11:56:46 -0700 Subject: . --- attic/ClusterGeoIpService.cpp | 243 ++++++++++++++++++++++++++++++++++++++++ attic/ClusterGeoIpService.hpp | 151 +++++++++++++++++++++++++ make-bsd.mk | 5 - make-linux.mk | 4 - make-mac.mk | 4 - node/Constants.hpp | 5 - node/Path.hpp | 12 -- node/Peer.cpp | 157 -------------------------- node/Peer.hpp | 31 ----- node/Topology.cpp | 5 +- objects.mk | 1 - service/ClusterGeoIpService.cpp | 243 ---------------------------------------- service/ClusterGeoIpService.hpp | 151 ------------------------- 13 files changed, 395 insertions(+), 617 deletions(-) create mode 100644 attic/ClusterGeoIpService.cpp create mode 100644 attic/ClusterGeoIpService.hpp delete mode 100644 service/ClusterGeoIpService.cpp delete mode 100644 service/ClusterGeoIpService.hpp (limited to 'make-mac.mk') diff --git a/attic/ClusterGeoIpService.cpp b/attic/ClusterGeoIpService.cpp new file mode 100644 index 00000000..2dcc9179 --- /dev/null +++ b/attic/ClusterGeoIpService.cpp @@ -0,0 +1,243 @@ +/* + * ZeroTier One - Network Virtualization Everywhere + * Copyright (C) 2011-2017 ZeroTier, Inc. https://www.zerotier.com/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + * + * -- + * + * You can be released from the requirements of the license by purchasing + * a commercial license. Buying such a license is mandatory as soon as you + * develop commercial closed-source software that incorporates or links + * directly against ZeroTier software without disclosing the source code + * of your own application. + */ + +#ifdef ZT_ENABLE_CLUSTER + +#include + +#include + +#include "ClusterGeoIpService.hpp" + +#include "../node/Utils.hpp" +#include "../osdep/OSUtils.hpp" + +#define ZT_CLUSTERGEOIPSERVICE_FILE_MODIFICATION_CHECK_EVERY 10000 + +namespace ZeroTier { + +ClusterGeoIpService::ClusterGeoIpService() : + _pathToCsv(), + _ipStartColumn(-1), + _ipEndColumn(-1), + _latitudeColumn(-1), + _longitudeColumn(-1), + _lastFileCheckTime(0), + _csvModificationTime(0), + _csvFileSize(0) +{ +} + +ClusterGeoIpService::~ClusterGeoIpService() +{ +} + +bool ClusterGeoIpService::locate(const InetAddress &ip,int &x,int &y,int &z) +{ + Mutex::Lock _l(_lock); + + if ((_pathToCsv.length() > 0)&&((OSUtils::now() - _lastFileCheckTime) > ZT_CLUSTERGEOIPSERVICE_FILE_MODIFICATION_CHECK_EVERY)) { + _lastFileCheckTime = OSUtils::now(); + if ((_csvFileSize != OSUtils::getFileSize(_pathToCsv.c_str()))||(_csvModificationTime != OSUtils::getLastModified(_pathToCsv.c_str()))) + _load(_pathToCsv.c_str(),_ipStartColumn,_ipEndColumn,_latitudeColumn,_longitudeColumn); + } + + /* We search by looking up the upper bound of the sorted vXdb vectors + * and then iterating down for a matching IP range. We stop when we hit + * the beginning or an entry whose start and end are before the IP we + * are searching. */ + + if ((ip.ss_family == AF_INET)&&(_v4db.size() > 0)) { + _V4E key; + key.start = Utils::ntoh((uint32_t)(reinterpret_cast(&ip)->sin_addr.s_addr)); + std::vector<_V4E>::const_iterator i(std::upper_bound(_v4db.begin(),_v4db.end(),key)); + while (i != _v4db.begin()) { + --i; + if ((key.start >= i->start)&&(key.start <= i->end)) { + x = i->x; + y = i->y; + z = i->z; + //printf("%s : %f,%f %d,%d,%d\n",ip.toIpString().c_str(),i->lat,i->lon,x,y,z); + return true; + } else if ((key.start > i->start)&&(key.start > i->end)) + break; + } + } else if ((ip.ss_family == AF_INET6)&&(_v6db.size() > 0)) { + _V6E key; + memcpy(key.start,reinterpret_cast(&ip)->sin6_addr.s6_addr,16); + std::vector<_V6E>::const_iterator i(std::upper_bound(_v6db.begin(),_v6db.end(),key)); + while (i != _v6db.begin()) { + --i; + const int s_vs_s = memcmp(key.start,i->start,16); + const int s_vs_e = memcmp(key.start,i->end,16); + if ((s_vs_s >= 0)&&(s_vs_e <= 0)) { + x = i->x; + y = i->y; + z = i->z; + //printf("%s : %f,%f %d,%d,%d\n",ip.toIpString().c_str(),i->lat,i->lon,x,y,z); + return true; + } else if ((s_vs_s > 0)&&(s_vs_e > 0)) + break; + } + } + + return false; +} + +void ClusterGeoIpService::_parseLine(const char *line,std::vector<_V4E> &v4db,std::vector<_V6E> &v6db,int ipStartColumn,int ipEndColumn,int latitudeColumn,int longitudeColumn) +{ + std::vector ls(OSUtils::split(line,",\t","\\","\"'")); + if ( ((ipStartColumn >= 0)&&(ipStartColumn < (int)ls.size()))&& + ((ipEndColumn >= 0)&&(ipEndColumn < (int)ls.size()))&& + ((latitudeColumn >= 0)&&(latitudeColumn < (int)ls.size()))&& + ((longitudeColumn >= 0)&&(longitudeColumn < (int)ls.size())) ) { + InetAddress ipStart(ls[ipStartColumn].c_str(),0); + InetAddress ipEnd(ls[ipEndColumn].c_str(),0); + const double lat = strtod(ls[latitudeColumn].c_str(),(char **)0); + const double lon = strtod(ls[longitudeColumn].c_str(),(char **)0); + + if ((ipStart.ss_family == ipEnd.ss_family)&&(ipStart)&&(ipEnd)&&(std::isfinite(lat))&&(std::isfinite(lon))) { + const double latRadians = lat * 0.01745329251994; // PI / 180 + const double lonRadians = lon * 0.01745329251994; // PI / 180 + const double cosLat = cos(latRadians); + const int x = (int)round((-6371.0) * cosLat * cos(lonRadians)); // 6371 == Earth's approximate radius in kilometers + const int y = (int)round(6371.0 * sin(latRadians)); + const int z = (int)round(6371.0 * cosLat * sin(lonRadians)); + + if (ipStart.ss_family == AF_INET) { + v4db.push_back(_V4E()); + v4db.back().start = Utils::ntoh((uint32_t)(reinterpret_cast(&ipStart)->sin_addr.s_addr)); + v4db.back().end = Utils::ntoh((uint32_t)(reinterpret_cast(&ipEnd)->sin_addr.s_addr)); + v4db.back().lat = (float)lat; + v4db.back().lon = (float)lon; + v4db.back().x = x; + v4db.back().y = y; + v4db.back().z = z; + //printf("%s - %s : %d,%d,%d\n",ipStart.toIpString().c_str(),ipEnd.toIpString().c_str(),x,y,z); + } else if (ipStart.ss_family == AF_INET6) { + v6db.push_back(_V6E()); + memcpy(v6db.back().start,reinterpret_cast(&ipStart)->sin6_addr.s6_addr,16); + memcpy(v6db.back().end,reinterpret_cast(&ipEnd)->sin6_addr.s6_addr,16); + v6db.back().lat = (float)lat; + v6db.back().lon = (float)lon; + v6db.back().x = x; + v6db.back().y = y; + v6db.back().z = z; + //printf("%s - %s : %d,%d,%d\n",ipStart.toIpString().c_str(),ipEnd.toIpString().c_str(),x,y,z); + } + } + } +} + +long ClusterGeoIpService::_load(const char *pathToCsv,int ipStartColumn,int ipEndColumn,int latitudeColumn,int longitudeColumn) +{ + // assumes _lock is locked + + FILE *f = fopen(pathToCsv,"rb"); + if (!f) + return -1; + + std::vector<_V4E> v4db; + std::vector<_V6E> v6db; + v4db.reserve(16777216); + v6db.reserve(16777216); + + char buf[4096]; + char linebuf[1024]; + unsigned int lineptr = 0; + for(;;) { + int n = (int)fread(buf,1,sizeof(buf),f); + if (n <= 0) + break; + for(int i=0;i 0)||(v6db.size() > 0)) { + std::sort(v4db.begin(),v4db.end()); + std::sort(v6db.begin(),v6db.end()); + + _pathToCsv = pathToCsv; + _ipStartColumn = ipStartColumn; + _ipEndColumn = ipEndColumn; + _latitudeColumn = latitudeColumn; + _longitudeColumn = longitudeColumn; + + _lastFileCheckTime = OSUtils::now(); + _csvModificationTime = OSUtils::getLastModified(pathToCsv); + _csvFileSize = OSUtils::getFileSize(pathToCsv); + + _v4db.swap(v4db); + _v6db.swap(v6db); + + return (long)(_v4db.size() + _v6db.size()); + } else { + return 0; + } +} + +} // namespace ZeroTier + +#endif // ZT_ENABLE_CLUSTER + +/* +int main(int argc,char **argv) +{ + char buf[1024]; + + ZeroTier::ClusterGeoIpService gip; + printf("loading...\n"); + gip.load("/Users/api/Code/ZeroTier/Infrastructure/root-servers/zerotier-one/cluster-geoip.csv",0,1,5,6); + printf("... done!\n"); fflush(stdout); + + while (gets(buf)) { // unsafe, testing only + ZeroTier::InetAddress addr(buf,0); + printf("looking up: %s\n",addr.toString().c_str()); fflush(stdout); + int x = 0,y = 0,z = 0; + if (gip.locate(addr,x,y,z)) { + //printf("%s: %d,%d,%d\n",addr.toString().c_str(),x,y,z); fflush(stdout); + } else { + printf("%s: not found!\n",addr.toString().c_str()); fflush(stdout); + } + } + + return 0; +} +*/ diff --git a/attic/ClusterGeoIpService.hpp b/attic/ClusterGeoIpService.hpp new file mode 100644 index 00000000..380f944f --- /dev/null +++ b/attic/ClusterGeoIpService.hpp @@ -0,0 +1,151 @@ +/* + * ZeroTier One - Network Virtualization Everywhere + * Copyright (C) 2011-2017 ZeroTier, Inc. https://www.zerotier.com/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + * + * -- + * + * You can be released from the requirements of the license by purchasing + * a commercial license. Buying such a license is mandatory as soon as you + * develop commercial closed-source software that incorporates or links + * directly against ZeroTier software without disclosing the source code + * of your own application. + */ + +#ifndef ZT_CLUSTERGEOIPSERVICE_HPP +#define ZT_CLUSTERGEOIPSERVICE_HPP + +#ifdef ZT_ENABLE_CLUSTER + +#include +#include +#include +#include + +#include +#include +#include + +#include "../node/Constants.hpp" +#include "../node/Mutex.hpp" +#include "../node/NonCopyable.hpp" +#include "../node/InetAddress.hpp" + +namespace ZeroTier { + +/** + * Loads a GeoIP CSV into memory for fast lookup, reloading as needed + * + * This was designed around the CSV from https://db-ip.com but can be used + * with any similar GeoIP CSV database that is presented in the form of an + * IP range and lat/long coordinates. + * + * It loads the whole database into memory, which can be kind of large. If + * the CSV file changes, the changes are loaded automatically. + */ +class ClusterGeoIpService : NonCopyable +{ +public: + ClusterGeoIpService(); + ~ClusterGeoIpService(); + + /** + * Load or reload CSV file + * + * CSV column indexes start at zero. CSVs can be quoted with single or + * double quotes. Whitespace before or after commas is ignored. Backslash + * may be used for escaping whitespace as well. + * + * @param pathToCsv Path to (uncompressed) CSV file + * @param ipStartColumn Column with IP range start + * @param ipEndColumn Column with IP range end (inclusive) + * @param latitudeColumn Column with latitude + * @param longitudeColumn Column with longitude + * @return Number of valid records loaded or -1 on error (invalid file, not found, etc.) + */ + inline long load(const char *pathToCsv,int ipStartColumn,int ipEndColumn,int latitudeColumn,int longitudeColumn) + { + Mutex::Lock _l(_lock); + return _load(pathToCsv,ipStartColumn,ipEndColumn,latitudeColumn,longitudeColumn); + } + + /** + * Attempt to locate an IP + * + * This returns true if x, y, and z are set. If the return value is false + * the values of x, y, and z are undefined. + * + * @param ip IPv4 or IPv6 address + * @param x Reference to variable to receive X + * @param y Reference to variable to receive Y + * @param z Reference to variable to receive Z + * @return True if coordinates were set + */ + bool locate(const InetAddress &ip,int &x,int &y,int &z); + + /** + * @return True if IP database/service is available for queries (otherwise locate() will always be false) + */ + inline bool available() const + { + Mutex::Lock _l(_lock); + return ((_v4db.size() + _v6db.size()) > 0); + } + +private: + struct _V4E + { + uint32_t start; + uint32_t end; + float lat,lon; + int16_t x,y,z; + + inline bool operator<(const _V4E &e) const { return (start < e.start); } + }; + + struct _V6E + { + uint8_t start[16]; + uint8_t end[16]; + float lat,lon; + int16_t x,y,z; + + inline bool operator<(const _V6E &e) const { return (memcmp(start,e.start,16) < 0); } + }; + + static void _parseLine(const char *line,std::vector<_V4E> &v4db,std::vector<_V6E> &v6db,int ipStartColumn,int ipEndColumn,int latitudeColumn,int longitudeColumn); + long _load(const char *pathToCsv,int ipStartColumn,int ipEndColumn,int latitudeColumn,int longitudeColumn); + + std::string _pathToCsv; + int _ipStartColumn; + int _ipEndColumn; + int _latitudeColumn; + int _longitudeColumn; + + uint64_t _lastFileCheckTime; + uint64_t _csvModificationTime; + int64_t _csvFileSize; + + std::vector<_V4E> _v4db; + std::vector<_V6E> _v6db; + + Mutex _lock; +}; + +} // namespace ZeroTier + +#endif // ZT_ENABLE_CLUSTER + +#endif diff --git a/make-bsd.mk b/make-bsd.mk index cbab5810..c2fd6062 100644 --- a/make-bsd.mk +++ b/make-bsd.mk @@ -7,11 +7,6 @@ LIBS= include objects.mk ONE_OBJS+=osdep/BSDEthernetTap.o ext/http-parser/http_parser.o -# Build with ZT_ENABLE_CLUSTER=1 to build with cluster support -ifeq ($(ZT_ENABLE_CLUSTER),1) - DEFS+=-DZT_ENABLE_CLUSTER -endif - # "make debug" is a shortcut for this ifeq ($(ZT_DEBUG),1) CFLAGS+=-Wall -Werror -g -pthread $(INCLUDES) $(DEFS) diff --git a/make-linux.mk b/make-linux.mk index c27c600d..7017d31e 100644 --- a/make-linux.mk +++ b/make-linux.mk @@ -38,10 +38,6 @@ endif # Trying to use dynamically linked libhttp-parser causes tons of compatibility problems. ONE_OBJS+=ext/http-parser/http_parser.o -ifeq ($(ZT_ENABLE_CLUSTER),1) - DEFS+=-DZT_ENABLE_CLUSTER -endif - ifeq ($(ZT_SYNOLOGY), 1) DEFS+=-D__SYNOLOGY__ endif diff --git a/make-mac.mk b/make-mac.mk index 5622a41b..196b17cb 100644 --- a/make-mac.mk +++ b/make-mac.mk @@ -33,10 +33,6 @@ else DEFS+=-DZT_SOFTWARE_UPDATE_DEFAULT="\"download\"" endif -ifeq ($(ZT_ENABLE_CLUSTER),1) - DEFS+=-DZT_ENABLE_CLUSTER -endif - # Use fast ASM Salsa20/12 for x64 processors DEFS+=-DZT_USE_X64_ASM_SALSA2012 CORE_OBJS+=ext/x64-salsa2012-asm/salsa2012.o diff --git a/node/Constants.hpp b/node/Constants.hpp index 274b9564..12bf8d28 100644 --- a/node/Constants.hpp +++ b/node/Constants.hpp @@ -218,11 +218,6 @@ */ #define ZT_HOUSEKEEPING_PERIOD 60000 -/** - * How often in ms to write peer state to storage and/or cluster (approximate) - */ -#define ZT_PEER_STATE_WRITE_PERIOD 10000 - /** * How long to remember peer records in RAM if they haven't been used */ diff --git a/node/Path.hpp b/node/Path.hpp index 854b28e2..ac8e4c0e 100644 --- a/node/Path.hpp +++ b/node/Path.hpp @@ -283,18 +283,6 @@ public: */ inline uint64_t lastTrustEstablishedPacketReceived() const { return _lastTrustEstablishedPacketReceived; } - /** - * @param lo Last out send - * @param li Last in send - * @param lt Last trust established packet received - */ - inline void updateFromRemoteState(const uint64_t lo,const uint64_t li,const uint64_t lt) - { - _lastOut = lo; - _lastIn = li; - _lastTrustEstablishedPacketReceived = lt; - } - /** * Return and increment outgoing packet counter (used with Packet::armor()) * diff --git a/node/Peer.cpp b/node/Peer.cpp index 875d651e..fb9a72b1 100644 --- a/node/Peer.cpp +++ b/node/Peer.cpp @@ -38,8 +38,6 @@ namespace ZeroTier { Peer::Peer(const RuntimeEnvironment *renv,const Identity &myIdentity,const Identity &peerIdentity) : RR(renv), - _lastWroteState(0), - _lastReceivedStateTimestamp(0), _lastReceive(0), _lastNontrivialReceive(0), _lastTriedMemorizedPath(0), @@ -184,7 +182,6 @@ void Peer::received( if (verb == Packet::VERB_OK) { potentialNewPeerPath->lr = now; potentialNewPeerPath->p = path; - _lastWroteState = 0; // force state write now } else { TRACE("got %s via unknown path %s(%s), confirming...",Packet::verbString(verb),_id.address().toString().c_str(),path->address().toString().c_str()); attemptToContactAt(tPtr,path->localSocket(),path->address(),now,true,path->nextOutgoingCounter()); @@ -263,9 +260,6 @@ void Peer::received( } } } - - if ((now - _lastWroteState) > ZT_PEER_STATE_WRITE_PERIOD) - writeState(tPtr,now); } bool Peer::sendDirect(void *tPtr,const void *data,unsigned int len,uint64_t now,bool force) @@ -428,155 +422,4 @@ bool Peer::doPingAndKeepalive(void *tPtr,uint64_t now,int inetAddressFamily) return false; } -void Peer::writeState(void *tPtr,const uint64_t now) -{ - try { - Buffer b; - - b.append((uint8_t)1); // version - b.append(now); - - _id.serialize(b); - - { - Mutex::Lock _l(_paths_m); - unsigned int count = 0; - if (_v4Path.lr) - ++count; - if (_v6Path.lr) - ++count; - b.append((uint8_t)count); - if (_v4Path.lr) { - b.append(_v4Path.lr); - b.append(_v4Path.p->lastOut()); - b.append(_v4Path.p->lastIn()); - b.append(_v4Path.p->lastTrustEstablishedPacketReceived()); - _v4Path.p->address().serialize(b); - } - if (_v6Path.lr) { - b.append(_v6Path.lr); - b.append(_v6Path.p->lastOut()); - b.append(_v6Path.p->lastIn()); - b.append(_v6Path.p->lastTrustEstablishedPacketReceived()); - _v6Path.p->address().serialize(b); - } - } - - // Save space by sending these as time since now at 100ms resolution - b.append((uint16_t)(std::max(now - _lastReceive,(uint64_t)6553500) / 100)); - b.append((uint16_t)(std::max(now - _lastNontrivialReceive,(uint64_t)6553500) / 100)); - b.append((uint16_t)(std::max(now - _lastTriedMemorizedPath,(uint64_t)6553500) / 100)); - b.append((uint16_t)(std::max(now - _lastDirectPathPushSent,(uint64_t)6553500) / 100)); - b.append((uint16_t)(std::max(now - _lastDirectPathPushReceive,(uint64_t)6553500) / 100)); - b.append((uint16_t)(std::max(now - _lastCredentialRequestSent,(uint64_t)6553500) / 100)); - b.append((uint16_t)(std::max(now - _lastWhoisRequestReceived,(uint64_t)6553500) / 100)); - b.append((uint16_t)(std::max(now - _lastEchoRequestReceived,(uint64_t)6553500) / 100)); - b.append((uint16_t)(std::max(now - _lastComRequestReceived,(uint64_t)6553500) / 100)); - b.append((uint16_t)(std::max(now - _lastComRequestSent,(uint64_t)6553500) / 100)); - b.append((uint16_t)(std::max(now - _lastCredentialsReceived,(uint64_t)6553500) / 100)); - b.append((uint16_t)(std::max(now - _lastTrustEstablishedPacketReceived,(uint64_t)6553500) / 100)); - - b.append((uint8_t)_vProto); - b.append((uint8_t)_vMajor); - b.append((uint8_t)_vMinor); - b.append((uint16_t)_vRevision); - - b.append((uint16_t)0); // length of additional fields - - uint64_t tmp[2]; - tmp[0] = _id.address().toInt(); tmp[1] = 0; - //RR->node->stateObjectPut(tPtr,ZT_STATE_OBJECT_PEER_STATE,tmp,b.data(),b.size()); - - _lastWroteState = now; - } catch ( ... ) {} // sanity check, should not be possible -} - -bool Peer::applyStateUpdate(const void *data,unsigned int len) -{ - try { - Buffer b(data,len); - unsigned int ptr = 0; - - if (b[ptr++] != 1) - return false; - const uint64_t ts = b.at(ptr); ptr += 8; - if (ts <= _lastReceivedStateTimestamp) - return false; - - Identity id; - ptr += id.deserialize(b,ptr); - if (id != _id) // sanity check - return false; - - const unsigned int pathCount = (unsigned int)b[ptr++]; - { - Mutex::Lock _l(_paths_m); - for(unsigned int i=0;i(ptr); ptr += 8; - const uint64_t lastOut = b.at(ptr); ptr += 8; - const uint64_t lastIn = b.at(ptr); ptr += 8; - const uint64_t lastTrustEstablishedPacketReceived = b.at(ptr); ptr += 8; - InetAddress addr; - ptr += addr.deserialize(b,ptr); - _PeerPath *p = (_PeerPath *)0; - switch(addr.ss_family) { - case AF_INET: p = &_v4Path; break; - case AF_INET6: p = &_v6Path; break; - } - if (p) { - if ( (!p->p) || (p->p->address() != addr) ) { - p->p = RR->topology->getPath(-1,addr); - } - p->lr = lr; - p->p->updateFromRemoteState(lastOut,lastIn,lastTrustEstablishedPacketReceived); - } - } - } - - _lastReceive = std::max(_lastReceive,ts - ((uint64_t)b.at(ptr) * 100ULL)); ptr += 2; - _lastNontrivialReceive = std::max(_lastNontrivialReceive,ts - ((uint64_t)b.at(ptr) * 100ULL)); ptr += 2; - _lastTriedMemorizedPath = std::max(_lastTriedMemorizedPath,ts - ((uint64_t)b.at(ptr) * 100ULL)); ptr += 2; - _lastDirectPathPushSent = std::max(_lastDirectPathPushSent,ts - ((uint64_t)b.at(ptr) * 100ULL)); ptr += 2; - _lastDirectPathPushReceive = std::max(_lastDirectPathPushReceive,ts - ((uint64_t)b.at(ptr) * 100ULL)); ptr += 2; - _lastCredentialRequestSent = std::max(_lastCredentialRequestSent,ts - ((uint64_t)b.at(ptr) * 100ULL)); ptr += 2; - _lastWhoisRequestReceived = std::max(_lastWhoisRequestReceived,ts - ((uint64_t)b.at(ptr) * 100ULL)); ptr += 2; - _lastEchoRequestReceived = std::max(_lastEchoRequestReceived,ts - ((uint64_t)b.at(ptr) * 100ULL)); ptr += 2; - _lastComRequestReceived = std::max(_lastComRequestReceived,ts - ((uint64_t)b.at(ptr) * 100ULL)); ptr += 2; - _lastComRequestSent = std::max(_lastComRequestSent,ts - ((uint64_t)b.at(ptr) * 100ULL)); ptr += 2; - _lastCredentialsReceived = std::max(_lastCredentialsReceived,ts - ((uint64_t)b.at(ptr) * 100ULL)); ptr += 2; - _lastTrustEstablishedPacketReceived = std::max(_lastTrustEstablishedPacketReceived,ts - ((uint64_t)b.at(ptr) * 100ULL)); ptr += 2; - - _vProto = (uint16_t)b[ptr++]; - _vMajor = (uint16_t)b[ptr++]; - _vMinor = (uint16_t)b[ptr++]; - _vRevision = b.at(ptr); ptr += 2; - - _lastReceivedStateTimestamp = ts; - - return true; - } catch ( ... ) {} // ignore invalid state updates - return false; -} - -SharedPtr Peer::createFromStateUpdate(const RuntimeEnvironment *renv,void *tPtr,const void *data,unsigned int len) -{ - try { - Identity id; - { - Buffer b(data,len); - unsigned int ptr = 0; - if (b[ptr++] != 1) - return SharedPtr(); - ptr += 8; // skip TS, don't care - id.deserialize(b,ptr); - } - if (id) { - const SharedPtr p(new Peer(renv,renv->identity,id)); - if (p->applyStateUpdate(data,len)) - return renv->topology->addPeer(tPtr,p); - } - } catch ( ... ) {} - return SharedPtr(); -} - } // namespace ZeroTier diff --git a/node/Peer.hpp b/node/Peer.hpp index 478c7232..ad2d0ddc 100644 --- a/node/Peer.hpp +++ b/node/Peer.hpp @@ -195,23 +195,6 @@ public: */ bool doPingAndKeepalive(void *tPtr,uint64_t now,int inetAddressFamily); - /** - * Write object state to external storage and/or cluster network - * - * @param tPtr Thread pointer to be handed through to any callbacks called as a result of this call - * @param now Current time - */ - void writeState(void *tPtr,const uint64_t now); - - /** - * Apply a state update received from e.g. a remote cluster member - * - * @param data State update data - * @param len Length of state update - * @return True if state update was applied, false if ignored or invalid - */ - bool applyStateUpdate(const void *data,unsigned int len); - /** * Reset paths within a given IP scope and address family * @@ -440,17 +423,6 @@ public: return false; } - /** - * Create a peer from a remote state update - * - * @param renv Runtime environment - * @param tPtr Thread pointer to be handed through to any callbacks called as a result of this call - * @param data State update data - * @param len State update length - * @return Peer or NULL if data was invalid - */ - static SharedPtr createFromStateUpdate(const RuntimeEnvironment *renv,void *tPtr,const void *data,unsigned int len); - private: struct _PeerPath { @@ -463,9 +435,6 @@ private: const RuntimeEnvironment *RR; - uint64_t _lastWroteState; - uint64_t _lastReceivedStateTimestamp; - uint64_t _lastReceive; // direct or indirect uint64_t _lastNontrivialReceive; // frames, things like netconf, etc. uint64_t _lastTriedMemorizedPath; diff --git a/node/Topology.cpp b/node/Topology.cpp index d4632f43..809bc7e7 100644 --- a/node/Topology.cpp +++ b/node/Topology.cpp @@ -395,11 +395,8 @@ void Topology::doPeriodicTasks(void *tPtr,uint64_t now) Address *a = (Address *)0; SharedPtr *p = (SharedPtr *)0; while (i.next(a,p)) { - if ( (!(*p)->isAlive(now)) && (std::find(_upstreamAddresses.begin(),_upstreamAddresses.end(),*a) == _upstreamAddresses.end()) ) { + if ( (!(*p)->isAlive(now)) && (std::find(_upstreamAddresses.begin(),_upstreamAddresses.end(),*a) == _upstreamAddresses.end()) ) _peers.erase(*a); - } else { - (*p)->writeState(tPtr,now); - } } } diff --git a/objects.mk b/objects.mk index c8231f08..3a8bd645 100644 --- a/objects.mk +++ b/objects.mk @@ -31,7 +31,6 @@ ONE_OBJS=\ osdep/ManagedRoute.o \ osdep/Http.o \ osdep/OSUtils.o \ - service/ClusterGeoIpService.o \ service/SoftwareUpdater.o \ service/OneService.o diff --git a/service/ClusterGeoIpService.cpp b/service/ClusterGeoIpService.cpp deleted file mode 100644 index 2dcc9179..00000000 --- a/service/ClusterGeoIpService.cpp +++ /dev/null @@ -1,243 +0,0 @@ -/* - * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2017 ZeroTier, Inc. https://www.zerotier.com/ - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - * - * -- - * - * You can be released from the requirements of the license by purchasing - * a commercial license. Buying such a license is mandatory as soon as you - * develop commercial closed-source software that incorporates or links - * directly against ZeroTier software without disclosing the source code - * of your own application. - */ - -#ifdef ZT_ENABLE_CLUSTER - -#include - -#include - -#include "ClusterGeoIpService.hpp" - -#include "../node/Utils.hpp" -#include "../osdep/OSUtils.hpp" - -#define ZT_CLUSTERGEOIPSERVICE_FILE_MODIFICATION_CHECK_EVERY 10000 - -namespace ZeroTier { - -ClusterGeoIpService::ClusterGeoIpService() : - _pathToCsv(), - _ipStartColumn(-1), - _ipEndColumn(-1), - _latitudeColumn(-1), - _longitudeColumn(-1), - _lastFileCheckTime(0), - _csvModificationTime(0), - _csvFileSize(0) -{ -} - -ClusterGeoIpService::~ClusterGeoIpService() -{ -} - -bool ClusterGeoIpService::locate(const InetAddress &ip,int &x,int &y,int &z) -{ - Mutex::Lock _l(_lock); - - if ((_pathToCsv.length() > 0)&&((OSUtils::now() - _lastFileCheckTime) > ZT_CLUSTERGEOIPSERVICE_FILE_MODIFICATION_CHECK_EVERY)) { - _lastFileCheckTime = OSUtils::now(); - if ((_csvFileSize != OSUtils::getFileSize(_pathToCsv.c_str()))||(_csvModificationTime != OSUtils::getLastModified(_pathToCsv.c_str()))) - _load(_pathToCsv.c_str(),_ipStartColumn,_ipEndColumn,_latitudeColumn,_longitudeColumn); - } - - /* We search by looking up the upper bound of the sorted vXdb vectors - * and then iterating down for a matching IP range. We stop when we hit - * the beginning or an entry whose start and end are before the IP we - * are searching. */ - - if ((ip.ss_family == AF_INET)&&(_v4db.size() > 0)) { - _V4E key; - key.start = Utils::ntoh((uint32_t)(reinterpret_cast(&ip)->sin_addr.s_addr)); - std::vector<_V4E>::const_iterator i(std::upper_bound(_v4db.begin(),_v4db.end(),key)); - while (i != _v4db.begin()) { - --i; - if ((key.start >= i->start)&&(key.start <= i->end)) { - x = i->x; - y = i->y; - z = i->z; - //printf("%s : %f,%f %d,%d,%d\n",ip.toIpString().c_str(),i->lat,i->lon,x,y,z); - return true; - } else if ((key.start > i->start)&&(key.start > i->end)) - break; - } - } else if ((ip.ss_family == AF_INET6)&&(_v6db.size() > 0)) { - _V6E key; - memcpy(key.start,reinterpret_cast(&ip)->sin6_addr.s6_addr,16); - std::vector<_V6E>::const_iterator i(std::upper_bound(_v6db.begin(),_v6db.end(),key)); - while (i != _v6db.begin()) { - --i; - const int s_vs_s = memcmp(key.start,i->start,16); - const int s_vs_e = memcmp(key.start,i->end,16); - if ((s_vs_s >= 0)&&(s_vs_e <= 0)) { - x = i->x; - y = i->y; - z = i->z; - //printf("%s : %f,%f %d,%d,%d\n",ip.toIpString().c_str(),i->lat,i->lon,x,y,z); - return true; - } else if ((s_vs_s > 0)&&(s_vs_e > 0)) - break; - } - } - - return false; -} - -void ClusterGeoIpService::_parseLine(const char *line,std::vector<_V4E> &v4db,std::vector<_V6E> &v6db,int ipStartColumn,int ipEndColumn,int latitudeColumn,int longitudeColumn) -{ - std::vector ls(OSUtils::split(line,",\t","\\","\"'")); - if ( ((ipStartColumn >= 0)&&(ipStartColumn < (int)ls.size()))&& - ((ipEndColumn >= 0)&&(ipEndColumn < (int)ls.size()))&& - ((latitudeColumn >= 0)&&(latitudeColumn < (int)ls.size()))&& - ((longitudeColumn >= 0)&&(longitudeColumn < (int)ls.size())) ) { - InetAddress ipStart(ls[ipStartColumn].c_str(),0); - InetAddress ipEnd(ls[ipEndColumn].c_str(),0); - const double lat = strtod(ls[latitudeColumn].c_str(),(char **)0); - const double lon = strtod(ls[longitudeColumn].c_str(),(char **)0); - - if ((ipStart.ss_family == ipEnd.ss_family)&&(ipStart)&&(ipEnd)&&(std::isfinite(lat))&&(std::isfinite(lon))) { - const double latRadians = lat * 0.01745329251994; // PI / 180 - const double lonRadians = lon * 0.01745329251994; // PI / 180 - const double cosLat = cos(latRadians); - const int x = (int)round((-6371.0) * cosLat * cos(lonRadians)); // 6371 == Earth's approximate radius in kilometers - const int y = (int)round(6371.0 * sin(latRadians)); - const int z = (int)round(6371.0 * cosLat * sin(lonRadians)); - - if (ipStart.ss_family == AF_INET) { - v4db.push_back(_V4E()); - v4db.back().start = Utils::ntoh((uint32_t)(reinterpret_cast(&ipStart)->sin_addr.s_addr)); - v4db.back().end = Utils::ntoh((uint32_t)(reinterpret_cast(&ipEnd)->sin_addr.s_addr)); - v4db.back().lat = (float)lat; - v4db.back().lon = (float)lon; - v4db.back().x = x; - v4db.back().y = y; - v4db.back().z = z; - //printf("%s - %s : %d,%d,%d\n",ipStart.toIpString().c_str(),ipEnd.toIpString().c_str(),x,y,z); - } else if (ipStart.ss_family == AF_INET6) { - v6db.push_back(_V6E()); - memcpy(v6db.back().start,reinterpret_cast(&ipStart)->sin6_addr.s6_addr,16); - memcpy(v6db.back().end,reinterpret_cast(&ipEnd)->sin6_addr.s6_addr,16); - v6db.back().lat = (float)lat; - v6db.back().lon = (float)lon; - v6db.back().x = x; - v6db.back().y = y; - v6db.back().z = z; - //printf("%s - %s : %d,%d,%d\n",ipStart.toIpString().c_str(),ipEnd.toIpString().c_str(),x,y,z); - } - } - } -} - -long ClusterGeoIpService::_load(const char *pathToCsv,int ipStartColumn,int ipEndColumn,int latitudeColumn,int longitudeColumn) -{ - // assumes _lock is locked - - FILE *f = fopen(pathToCsv,"rb"); - if (!f) - return -1; - - std::vector<_V4E> v4db; - std::vector<_V6E> v6db; - v4db.reserve(16777216); - v6db.reserve(16777216); - - char buf[4096]; - char linebuf[1024]; - unsigned int lineptr = 0; - for(;;) { - int n = (int)fread(buf,1,sizeof(buf),f); - if (n <= 0) - break; - for(int i=0;i 0)||(v6db.size() > 0)) { - std::sort(v4db.begin(),v4db.end()); - std::sort(v6db.begin(),v6db.end()); - - _pathToCsv = pathToCsv; - _ipStartColumn = ipStartColumn; - _ipEndColumn = ipEndColumn; - _latitudeColumn = latitudeColumn; - _longitudeColumn = longitudeColumn; - - _lastFileCheckTime = OSUtils::now(); - _csvModificationTime = OSUtils::getLastModified(pathToCsv); - _csvFileSize = OSUtils::getFileSize(pathToCsv); - - _v4db.swap(v4db); - _v6db.swap(v6db); - - return (long)(_v4db.size() + _v6db.size()); - } else { - return 0; - } -} - -} // namespace ZeroTier - -#endif // ZT_ENABLE_CLUSTER - -/* -int main(int argc,char **argv) -{ - char buf[1024]; - - ZeroTier::ClusterGeoIpService gip; - printf("loading...\n"); - gip.load("/Users/api/Code/ZeroTier/Infrastructure/root-servers/zerotier-one/cluster-geoip.csv",0,1,5,6); - printf("... done!\n"); fflush(stdout); - - while (gets(buf)) { // unsafe, testing only - ZeroTier::InetAddress addr(buf,0); - printf("looking up: %s\n",addr.toString().c_str()); fflush(stdout); - int x = 0,y = 0,z = 0; - if (gip.locate(addr,x,y,z)) { - //printf("%s: %d,%d,%d\n",addr.toString().c_str(),x,y,z); fflush(stdout); - } else { - printf("%s: not found!\n",addr.toString().c_str()); fflush(stdout); - } - } - - return 0; -} -*/ diff --git a/service/ClusterGeoIpService.hpp b/service/ClusterGeoIpService.hpp deleted file mode 100644 index 380f944f..00000000 --- a/service/ClusterGeoIpService.hpp +++ /dev/null @@ -1,151 +0,0 @@ -/* - * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2017 ZeroTier, Inc. https://www.zerotier.com/ - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - * - * -- - * - * You can be released from the requirements of the license by purchasing - * a commercial license. Buying such a license is mandatory as soon as you - * develop commercial closed-source software that incorporates or links - * directly against ZeroTier software without disclosing the source code - * of your own application. - */ - -#ifndef ZT_CLUSTERGEOIPSERVICE_HPP -#define ZT_CLUSTERGEOIPSERVICE_HPP - -#ifdef ZT_ENABLE_CLUSTER - -#include -#include -#include -#include - -#include -#include -#include - -#include "../node/Constants.hpp" -#include "../node/Mutex.hpp" -#include "../node/NonCopyable.hpp" -#include "../node/InetAddress.hpp" - -namespace ZeroTier { - -/** - * Loads a GeoIP CSV into memory for fast lookup, reloading as needed - * - * This was designed around the CSV from https://db-ip.com but can be used - * with any similar GeoIP CSV database that is presented in the form of an - * IP range and lat/long coordinates. - * - * It loads the whole database into memory, which can be kind of large. If - * the CSV file changes, the changes are loaded automatically. - */ -class ClusterGeoIpService : NonCopyable -{ -public: - ClusterGeoIpService(); - ~ClusterGeoIpService(); - - /** - * Load or reload CSV file - * - * CSV column indexes start at zero. CSVs can be quoted with single or - * double quotes. Whitespace before or after commas is ignored. Backslash - * may be used for escaping whitespace as well. - * - * @param pathToCsv Path to (uncompressed) CSV file - * @param ipStartColumn Column with IP range start - * @param ipEndColumn Column with IP range end (inclusive) - * @param latitudeColumn Column with latitude - * @param longitudeColumn Column with longitude - * @return Number of valid records loaded or -1 on error (invalid file, not found, etc.) - */ - inline long load(const char *pathToCsv,int ipStartColumn,int ipEndColumn,int latitudeColumn,int longitudeColumn) - { - Mutex::Lock _l(_lock); - return _load(pathToCsv,ipStartColumn,ipEndColumn,latitudeColumn,longitudeColumn); - } - - /** - * Attempt to locate an IP - * - * This returns true if x, y, and z are set. If the return value is false - * the values of x, y, and z are undefined. - * - * @param ip IPv4 or IPv6 address - * @param x Reference to variable to receive X - * @param y Reference to variable to receive Y - * @param z Reference to variable to receive Z - * @return True if coordinates were set - */ - bool locate(const InetAddress &ip,int &x,int &y,int &z); - - /** - * @return True if IP database/service is available for queries (otherwise locate() will always be false) - */ - inline bool available() const - { - Mutex::Lock _l(_lock); - return ((_v4db.size() + _v6db.size()) > 0); - } - -private: - struct _V4E - { - uint32_t start; - uint32_t end; - float lat,lon; - int16_t x,y,z; - - inline bool operator<(const _V4E &e) const { return (start < e.start); } - }; - - struct _V6E - { - uint8_t start[16]; - uint8_t end[16]; - float lat,lon; - int16_t x,y,z; - - inline bool operator<(const _V6E &e) const { return (memcmp(start,e.start,16) < 0); } - }; - - static void _parseLine(const char *line,std::vector<_V4E> &v4db,std::vector<_V6E> &v6db,int ipStartColumn,int ipEndColumn,int latitudeColumn,int longitudeColumn); - long _load(const char *pathToCsv,int ipStartColumn,int ipEndColumn,int latitudeColumn,int longitudeColumn); - - std::string _pathToCsv; - int _ipStartColumn; - int _ipEndColumn; - int _latitudeColumn; - int _longitudeColumn; - - uint64_t _lastFileCheckTime; - uint64_t _csvModificationTime; - int64_t _csvFileSize; - - std::vector<_V4E> _v4db; - std::vector<_V6E> _v6db; - - Mutex _lock; -}; - -} // namespace ZeroTier - -#endif // ZT_ENABLE_CLUSTER - -#endif -- cgit v1.2.3 From ada611d597c935f95a807db8ac7bf01e386937fa Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Thu, 7 Dec 2017 08:56:15 -0800 Subject: Go ahead and load kext so MacOS High Sierra users will see kext auth dialog right away. --- ext/installfiles/mac/ZeroTier One.pkgproj | 22 +++++++++++++++++++++- ext/installfiles/mac/launch.sh | 6 +----- ext/installfiles/mac/postinst.sh | 14 ++++++++++---- make-mac.mk | 2 +- 4 files changed, 33 insertions(+), 11 deletions(-) (limited to 'make-mac.mk') diff --git a/ext/installfiles/mac/ZeroTier One.pkgproj b/ext/installfiles/mac/ZeroTier One.pkgproj index 96b1338a..59c22dde 100755 --- a/ext/installfiles/mac/ZeroTier One.pkgproj +++ b/ext/installfiles/mac/ZeroTier One.pkgproj @@ -613,6 +613,12 @@ PAYLOAD_TYPE 0 + SHOW_INVISIBLE + + SPLIT_FORKS + + TREAT_MISSING_FILES_AS_WARNING + VERSION 3 @@ -641,12 +647,24 @@ 1 CONCLUSION_ACTION 0 + FOLLOW_SYMBOLIC_LINKS + IDENTIFIER com.zerotier.pkg.ZeroTierOne + LOCATION + 0 + NAME + OVERWRITE_PERMISSIONS + PAYLOAD_SIZE + -1 + RELOCATABLE + + USE_HFS+_COMPRESSION + VERSION - 1.2.4 + 1.2.6 PROJECT_COMMENTS @@ -862,6 +880,8 @@ NAME ZeroTier One + PAYLOAD_ONLY + TYPE diff --git a/ext/installfiles/mac/launch.sh b/ext/installfiles/mac/launch.sh index 41c4b9c8..b02a6670 100755 --- a/ext/installfiles/mac/launch.sh +++ b/ext/installfiles/mac/launch.sh @@ -1,7 +1,3 @@ #!/bin/bash - -zthome="/Library/Application Support/ZeroTier/One" -export PATH="$zthome:/bin:/usr/bin:/sbin:/usr/sbin" - -# Launch ZeroTier One (not as daemon... launchd monitors it) +export PATH="/Library/Application Support/ZeroTier/One:/bin:/usr/bin:/sbin:/usr/sbin" exec zerotier-one diff --git a/ext/installfiles/mac/postinst.sh b/ext/installfiles/mac/postinst.sh index 4f5a50c2..b4ea2ee2 100755 --- a/ext/installfiles/mac/postinst.sh +++ b/ext/installfiles/mac/postinst.sh @@ -5,7 +5,7 @@ export PATH=/bin:/usr/bin:/sbin:/usr/sbin:/usr/local/bin OSX_RELEASE=`sw_vers -productVersion | cut -d . -f 1,2` launchctl unload /Library/LaunchDaemons/com.zerotier.one.plist >>/dev/null 2>&1 -sleep 1 +sleep 0.5 cd "/Library/Application Support/ZeroTier/One" @@ -23,29 +23,35 @@ if [ "$OSX_RELEASE" = "10.7" ]; then fi rm -rf node.log node.log.old root-topology shutdownIfUnreadable autoupdate.log updates.d ui peers.save + chown -R 0 tap.kext chgrp -R 0 tap.kext + if [ ! -f authtoken.secret ]; then - head -c 4096 /dev/urandom | md5 | head -c 24 >authtoken.secret + head -c 1024 /dev/urandom | md5 | head -c 24 >authtoken.secret chown 0 authtoken.secret chgrp 0 authtoken.secret chmod 0600 authtoken.secret fi + rm -f zerotier-cli zerotier-idtool ln -sf zerotier-one zerotier-cli ln -sf zerotier-one zerotier-idtool - mkdir -p /usr/local/bin cd /usr/local/bin rm -f zerotier-cli zerotier-idtool ln -sf "/Library/Application Support/ZeroTier/One/zerotier-one" zerotier-cli ln -sf "/Library/Application Support/ZeroTier/One/zerotier-one" zerotier-idtool +cd "/Library/Application Support/ZeroTier/One" +kextload -r . tap.kext >>/dev/null 2>&1 & +disown %1 + launchctl load /Library/LaunchDaemons/com.zerotier.one.plist >>/dev/null 2>&1 sleep 1 -if [ -f /tmp/zt1-gui-restart.tmp ]; +if [ -f /tmp/zt1-gui-restart.tmp ]; then for u in `cat /tmp/zt1-gui-restart.tmp`; do su $u -c '/Applications/ZeroTier\ One.app/Contents/MacOS/ZeroTier\ One &' >>/dev/null 2>&1 & done diff --git a/make-mac.mk b/make-mac.mk index 196b17cb..4184792f 100644 --- a/make-mac.mk +++ b/make-mac.mk @@ -111,7 +111,7 @@ mac-dist-pkg: FORCE # For ZeroTier, Inc. to build official signed packages official: FORCE make clean - make ZT_OFFICIAL_RELEASE=1 -j 4 one + make ZT_OFFICIAL_RELEASE=1 -j 8 one make ZT_OFFICIAL_RELEASE=1 macui make ZT_OFFICIAL_RELEASE=1 mac-dist-pkg -- cgit v1.2.3 From 346fa4d7f4506e9ddbf8dc8278210bc0c2b08251 Mon Sep 17 00:00:00 2001 From: Joseph Henry Date: Thu, 14 Dec 2017 13:27:49 -0800 Subject: Added ZT_SANITIZE option to makefiles. This your new memory debugging best friend --- make-bsd.mk | 4 ++++ make-linux.mk | 4 ++++ make-mac.mk | 4 ++++ 3 files changed, 12 insertions(+) (limited to 'make-mac.mk') diff --git a/make-bsd.mk b/make-bsd.mk index c3f2f3a1..713cafc0 100644 --- a/make-bsd.mk +++ b/make-bsd.mk @@ -7,6 +7,10 @@ LIBS= include objects.mk ONE_OBJS+=osdep/BSDEthernetTap.o ext/http-parser/http_parser.o +# Build with address sanitization library for advanced debugging (clang) +ifeq ($(ZT_SANITIZE),1) + SANFLAGS+=-fsanitize=address -DASAN_OPTIONS=symbolize=1 +endif # "make debug" is a shortcut for this ifeq ($(ZT_DEBUG),1) CFLAGS+=-Wall -Werror -g -pthread $(INCLUDES) $(DEFS) diff --git a/make-linux.mk b/make-linux.mk index 5b623247..5e0f5b98 100644 --- a/make-linux.mk +++ b/make-linux.mk @@ -55,6 +55,10 @@ ifeq ($(ZT_RULES_ENGINE_DEBUGGING),1) override DEFS+=-DZT_RULES_ENGINE_DEBUGGING endif +# Build with address sanitization library for advanced debugging (clang) +ifeq ($(ZT_SANITIZE),1) + SANFLAGS+=-fsanitize=address -DASAN_OPTIONS=symbolize=1 +endif ifeq ($(ZT_DEBUG),1) override CFLAGS+=-Wall -Wno-deprecated -Werror -g -pthread $(INCLUDES) $(DEFS) override CXXFLAGS+=-Wall -Wno-deprecated -Werror -g -std=c++11 -pthread $(INCLUDES) $(DEFS) diff --git a/make-mac.mk b/make-mac.mk index 4184792f..81159795 100644 --- a/make-mac.mk +++ b/make-mac.mk @@ -41,6 +41,10 @@ CORE_OBJS+=ext/x64-salsa2012-asm/salsa2012.o DEFS+=-DMACOSX -DZT_USE_MINIUPNPC -DMINIUPNP_STATICLIB -D_DARWIN_C_SOURCE -DMINIUPNPC_SET_SOCKET_TIMEOUT -DMINIUPNPC_GET_SRC_ADDR -D_BSD_SOURCE -D_DEFAULT_SOURCE -DOS_STRING=\"Darwin/15.0.0\" -DMINIUPNPC_VERSION_STRING=\"2.0\" -DUPNP_VERSION_STRING=\"UPnP/1.1\" -DENABLE_STRNATPMPERR ONE_OBJS+=ext/libnatpmp/natpmp.o ext/libnatpmp/getgateway.o ext/miniupnpc/connecthostport.o ext/miniupnpc/igd_desc_parse.o ext/miniupnpc/minisoap.o ext/miniupnpc/minissdpc.o ext/miniupnpc/miniupnpc.o ext/miniupnpc/miniwget.o ext/miniupnpc/minixml.o ext/miniupnpc/portlistingparse.o ext/miniupnpc/receivedata.o ext/miniupnpc/upnpcommands.o ext/miniupnpc/upnpdev.o ext/miniupnpc/upnperrors.o ext/miniupnpc/upnpreplyparse.o osdep/PortMapper.o +# Build with address sanitization library for advanced debugging (clang) +ifeq ($(ZT_SANITIZE),1) + SANFLAGS+=-fsanitize=address -DASAN_OPTIONS=symbolize=1 +endif # Debug mode -- dump trace output, build binary with -g ifeq ($(ZT_DEBUG),1) ZT_TRACE=1 -- cgit v1.2.3 From 6fbd15f8b22643a76c6b067335c5db6c406d209a Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Mon, 8 Jan 2018 15:56:53 -0800 Subject: cleanup --- make-mac.mk | 2 +- node/Identity.hpp | 1 - node/Switch.hpp | 1 - 3 files changed, 1 insertion(+), 3 deletions(-) (limited to 'make-mac.mk') diff --git a/make-mac.mk b/make-mac.mk index 81159795..60aa465a 100644 --- a/make-mac.mk +++ b/make-mac.mk @@ -54,7 +54,7 @@ ifeq ($(ZT_DEBUG),1) # C25519 in particular is almost UNUSABLE in heavy testing without it. node/Salsa20.o node/SHA512.o node/C25519.o node/Poly1305.o: CFLAGS = -Wall -O2 -g $(INCLUDES) $(DEFS) else - CFLAGS?=-Os -fstack-protector-strong + CFLAGS?=-Ofast -fstack-protector-strong CFLAGS+=$(ARCH_FLAGS) -Wall -Werror -flto -fPIE -mmacosx-version-min=10.7 -DNDEBUG -Wno-unused-private-field $(INCLUDES) $(DEFS) STRIP=strip endif diff --git a/node/Identity.hpp b/node/Identity.hpp index cd24757e..74514436 100644 --- a/node/Identity.hpp +++ b/node/Identity.hpp @@ -31,7 +31,6 @@ #include #include "Constants.hpp" -#include "Array.hpp" #include "Utils.hpp" #include "Address.hpp" #include "C25519.hpp" diff --git a/node/Switch.hpp b/node/Switch.hpp index 4fc5453d..08208735 100644 --- a/node/Switch.hpp +++ b/node/Switch.hpp @@ -40,7 +40,6 @@ #include "Utils.hpp" #include "InetAddress.hpp" #include "Topology.hpp" -#include "Array.hpp" #include "Network.hpp" #include "SharedPtr.hpp" #include "IncomingPacket.hpp" -- cgit v1.2.3