From e7a2c6ecefc95422145385fdfd3ff137e5d290ac Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Wed, 19 Apr 2017 17:11:56 -0700 Subject: Integrate ARM/NEON crypto. --- node/Packet.cpp | 161 +++++++++++++++++++++++++++++++++----------------------- 1 file changed, 95 insertions(+), 66 deletions(-) (limited to 'node') diff --git a/node/Packet.cpp b/node/Packet.cpp index 39a0a4d5..85b18cff 100644 --- a/node/Packet.cpp +++ b/node/Packet.cpp @@ -27,6 +27,9 @@ #ifdef ZT_USE_X64_ASM_SALSA2012 #include "../ext/x64-salsa2012-asm/salsa2012.h" #endif +#ifdef ZT_USE_ARM32_NEON_ASM_SALSA2012 +#include "../ext/arm32-neon-salsa2012-asm/salsa2012.h" +#endif #ifdef _MSC_VER #define FORCE_INLINE static __forceinline @@ -40,6 +43,34 @@ namespace ZeroTier { /************************************************************************** */ + +/* Set up macros for fast single-pass ASM Salsa20/12 crypto, if we have it */ + +// x64 SSE crypto +#ifdef ZT_USE_X64_ASM_SALSA2012 +#define ZT_HAS_FAST_CRYPTO() (true) +#define ZT_FAST_SINGLE_PASS_SALSA2012(b,l,n,k) zt_salsa2012_amd64_xmm6(reinterpret_cast(b),(l),reinterpret_cast(n),reinterpret_cast(k)) +#endif + +// ARM (32-bit) NEON crypto (must be detected) +#ifdef ZT_USE_ARM32_NEON_ASM_SALSA2012 +class _FastCryptoChecker +{ +public: + _FastCryptoChecker() : canHas(zt_arm_has_neon()) {} + bool canHas; +}; +static const _FastCryptoChecker _ZT_FAST_CRYPTO_CHECK; +#define ZT_HAS_FAST_CRYPTO() (_ZT_FAST_CRYPTO_CHECK.canHas) +#define ZT_FAST_SINGLE_PASS_SALSA2012(b,l,n,k) zt_salsa2012_armneon3_xor(reinterpret_cast(b),(const unsigned char *)0,(l),reinterpret_cast(n),reinterpret_cast(k)) +#endif + +// No fast crypto available +#ifndef ZT_HAS_FAST_CRYPTO +#define ZT_HAS_FAST_CRYPTO() (false) +#define ZT_FAST_SINGLE_PASS_SALSA2012(b,l,n,k) {} +#endif + /************************************************************************** */ /* LZ4 is shipped encapsulated into Packet in an anonymous namespace. @@ -1079,41 +1110,41 @@ void Packet::armor(const void *key,bool encryptPayload,unsigned int counter) _salsa20MangleKey((const unsigned char *)key,mangledKey); -#ifdef ZT_USE_X64_ASM_SALSA2012 - const unsigned int payloadLen = (encryptPayload) ? (size() - ZT_PACKET_IDX_VERB) : 0; - uint64_t keyStream[(ZT_PROTO_MAX_PACKET_LENGTH + 64 + 8) / 8]; - zt_salsa2012_amd64_xmm6(reinterpret_cast(keyStream),payloadLen + 64,reinterpret_cast(data + ZT_PACKET_IDX_IV),reinterpret_cast(mangledKey)); - - uint64_t *ksptr = keyStream + 8; // encryption starts after first Salsa20 block - uint8_t *dptr = data + ZT_PACKET_IDX_VERB; - unsigned int ksrem = payloadLen; - while (ksrem >= 8) { - ksrem -= 8; - *(reinterpret_cast(dptr)) ^= *(ksptr++); - dptr += 8; - } - for(unsigned int i=0;i(ksptr)[i]; - } + if (ZT_HAS_FAST_CRYPTO()) { + const unsigned int payloadLen = (encryptPayload) ? (size() - ZT_PACKET_IDX_VERB) : 0; + uint64_t keyStream[(ZT_PROTO_MAX_PACKET_LENGTH + 64 + 8) / 8]; + ZT_FAST_SINGLE_PASS_SALSA2012(keyStream,payloadLen + 64,(data + ZT_PACKET_IDX_IV),mangledKey); + + uint64_t *ksptr = keyStream + 8; // encryption starts after first Salsa20 block + uint8_t *dptr = data + ZT_PACKET_IDX_VERB; + unsigned int ksrem = payloadLen; + while (ksrem >= 8) { + ksrem -= 8; + *(reinterpret_cast(dptr)) ^= *(ksptr++); + dptr += 8; + } + for(unsigned int i=0;i(ksptr)[i]; + } - uint64_t mac[2]; - Poly1305::compute(mac,data + ZT_PACKET_IDX_VERB,size() - ZT_PACKET_IDX_VERB,keyStream); - memcpy(data + ZT_PACKET_IDX_MAC,mac,8); -#else - Salsa20 s20(mangledKey,data + ZT_PACKET_IDX_IV); + uint64_t mac[2]; + Poly1305::compute(mac,data + ZT_PACKET_IDX_VERB,size() - ZT_PACKET_IDX_VERB,keyStream); + memcpy(data + ZT_PACKET_IDX_MAC,mac,8); + } else { + Salsa20 s20(mangledKey,data + ZT_PACKET_IDX_IV); - uint64_t macKey[4]; - s20.crypt12(ZERO_KEY,macKey,sizeof(macKey)); + uint64_t macKey[4]; + s20.crypt12(ZERO_KEY,macKey,sizeof(macKey)); - uint8_t *const payload = data + ZT_PACKET_IDX_VERB; - const unsigned int payloadLen = size() - ZT_PACKET_IDX_VERB; - if (encryptPayload) - s20.crypt12(payload,payload,payloadLen); + uint8_t *const payload = data + ZT_PACKET_IDX_VERB; + const unsigned int payloadLen = size() - ZT_PACKET_IDX_VERB; + if (encryptPayload) + s20.crypt12(payload,payload,payloadLen); - uint64_t mac[2]; - Poly1305::compute(mac,payload,payloadLen,macKey); - memcpy(data + ZT_PACKET_IDX_MAC,mac,8); -#endif + uint64_t mac[2]; + Poly1305::compute(mac,payload,payloadLen,macKey); + memcpy(data + ZT_PACKET_IDX_MAC,mac,8); + } } bool Packet::dearmor(const void *key) @@ -1127,45 +1158,43 @@ bool Packet::dearmor(const void *key) if ((cs == ZT_PROTO_CIPHER_SUITE__C25519_POLY1305_NONE)||(cs == ZT_PROTO_CIPHER_SUITE__C25519_POLY1305_SALSA2012)) { _salsa20MangleKey((const unsigned char *)key,mangledKey); -#ifdef ZT_USE_X64_ASM_SALSA2012 - uint64_t keyStream[(ZT_PROTO_MAX_PACKET_LENGTH + 64 + 8) / 8]; - zt_salsa2012_amd64_xmm6(reinterpret_cast(keyStream),((cs == ZT_PROTO_CIPHER_SUITE__C25519_POLY1305_SALSA2012) ? (payloadLen + 64) : 64),reinterpret_cast(data + ZT_PACKET_IDX_IV),reinterpret_cast(mangledKey)); - - uint64_t mac[2]; - Poly1305::compute(mac,payload,payloadLen,keyStream); - if (!Utils::secureEq(mac,data + ZT_PACKET_IDX_MAC,8)) - return false; // MAC failed, packet is corrupt, modified, or is not from the sender - - if (cs == ZT_PROTO_CIPHER_SUITE__C25519_POLY1305_SALSA2012) { - uint64_t *ksptr = keyStream + 8; // encryption starts after first Salsa20 block - uint8_t *dptr = data + ZT_PACKET_IDX_VERB; - unsigned int ksrem = payloadLen; - while (ksrem >= 8) { - ksrem -= 8; - *(reinterpret_cast(dptr)) ^= *(ksptr++); - dptr += 8; - } - for(unsigned int i=0;i(ksptr)[i]; + if (ZT_HAS_FAST_CRYPTO()) { + uint64_t keyStream[(ZT_PROTO_MAX_PACKET_LENGTH + 64 + 8) / 8]; + ZT_FAST_SINGLE_PASS_SALSA2012(keyStream,((cs == ZT_PROTO_CIPHER_SUITE__C25519_POLY1305_SALSA2012) ? (payloadLen + 64) : 64),(data + ZT_PACKET_IDX_IV),mangledKey); + + uint64_t mac[2]; + Poly1305::compute(mac,payload,payloadLen,keyStream); + if (!Utils::secureEq(mac,data + ZT_PACKET_IDX_MAC,8)) + return false; // MAC failed, packet is corrupt, modified, or is not from the sender + + if (cs == ZT_PROTO_CIPHER_SUITE__C25519_POLY1305_SALSA2012) { + uint64_t *ksptr = keyStream + 8; // encryption starts after first Salsa20 block + uint8_t *dptr = data + ZT_PACKET_IDX_VERB; + unsigned int ksrem = payloadLen; + while (ksrem >= 8) { + ksrem -= 8; + *(reinterpret_cast(dptr)) ^= *(ksptr++); + dptr += 8; + } + for(unsigned int i=0;i(ksptr)[i]; + } } + } else { + Salsa20 s20(mangledKey,data + ZT_PACKET_IDX_IV); + + uint64_t macKey[4]; + s20.crypt12(ZERO_KEY,macKey,sizeof(macKey)); + uint64_t mac[2]; + Poly1305::compute(mac,payload,payloadLen,macKey); + if (!Utils::secureEq(mac,data + ZT_PACKET_IDX_MAC,8)) + return false; // MAC failed, packet is corrupt, modified, or is not from the sender + + if (cs == ZT_PROTO_CIPHER_SUITE__C25519_POLY1305_SALSA2012) + s20.crypt12(payload,payload,payloadLen); } return true; -#else - Salsa20 s20(mangledKey,data + ZT_PACKET_IDX_IV); - - uint64_t macKey[4]; - s20.crypt12(ZERO_KEY,macKey,sizeof(macKey)); - uint64_t mac[2]; - Poly1305::compute(mac,payload,payloadLen,macKey); - if (!Utils::secureEq(mac,data + ZT_PACKET_IDX_MAC,8)) - return false; // MAC failed, packet is corrupt, modified, or is not from the sender - - if (cs == ZT_PROTO_CIPHER_SUITE__C25519_POLY1305_SALSA2012) - s20.crypt12(payload,payload,payloadLen); - - return true; -#endif } else { return false; // unrecognized cipher suite } -- cgit v1.2.3