More refactoring... and update the API a bit... turns out my strategy for reducing indirect function calls also increased memcpy()s which are more expensive. This is simpler and faster.

4 files changed, 254 insertions, 246 deletions

diff --git a/include/ZeroTierOne.h b/include/ZeroTierOne.h
index 3ba98d28..63b3603a 100644
--- a/include/ZeroTierOne.h
+++ b/include/ZeroTierOne.h
@@ -35,6 +35,7 @@
 
 #include <stdint.h>
 
+// For the struct sockaddr_storage structure
 #if defined(_WIN32) || defined(_WIN64)
 #include <WinSock2.h>
 #include <WS2tcpip.h>
@@ -53,14 +54,44 @@ extern "C" {
 /****************************************************************************/
 
 /**
- * Maximum frame MTU
+ * Maximum MTU for ZeroTier virtual networks
+ *
+ * This is pretty much an unchangeable global constant. To make it change
+ * across nodes would require logic to send ICMP packet too big messages,
+ * which would complicate things. 1500 has been good enough on most LANs
+ * for ages, so a larger MTU should be fine for the forseeable future. This
+ * typically results in two UDP packets per single large frame. Experimental
+ * results seem to show that this is good. Larger MTUs resulting in more
+ * fragments seemed too brittle on slow/crummy links for no benefit.
+ *
+ * If this does change, also change it in tap.h in the tuntaposx code under
+ * mac-tap.
+ * 
+ * Overhead for a normal frame split into two packets:
+ *
+ * 1414 = 1444 (typical UDP MTU) - 28 (packet header) - 2 (ethertype)
+ * 1428 = 1444 (typical UDP MTU) - 16 (fragment header)
+ * SUM: 2842
+ *
+ * We use 2800, which leaves some room for other payload in other types of
+ * messages such as multicast propagation or future support for bridging.
  */
 #define ZT1_MAX_MTU 2800
 
 /**
- * Maximum length of a wire message packet in bytes
+ * Feature flag: this is an official ZeroTier, Inc. binary build (built with ZT_OFFICIAL_RELEASE)
+ */
+#define ZT1_FEATURE_FLAG_OFFICIAL 0x00000001
+
+/**
+ * Feature flag: ZeroTier One was built to be thread-safe -- concurrent processXXX() calls are okay
+ */
+#define ZT1_FEATURE_FLAG_THREAD_SAFE 0x00000002
+
+/**
+ * Feature flag: FIPS compliant build (not available yet, but reserved for future use if we ever do this)
  */
-#define ZT1_MAX_WIRE_MESSAGE_LENGTH 1500
+#define ZT1_FEATURE_FLAG_FIPS 0x00000004
 
 /****************************************************************************/
 /* Structures and other types                                               */
@@ -156,103 +187,6 @@ typedef struct
 } ZT1_NodeStatus;
 
 /**
- * A message to or from a physical address (e.g. IP or physical Ethernet)
- */
-typedef struct
-{
-	/**
-	 * Socket address
-	 */
-	struct sockaddr_storage address;
-
-	/**
-	 * Link desperation -- higher equals "worse" or "slower"
-	 *
-	 * This is very similar to an interface metric. Higher values indicate
-	 * worse links. For incoming wire messages, it should be sent to the
-	 * desperation metric for the originating socket. For outgoing wire
-	 * messages, ZeroTier will increment this from zero as it grows more
-	 * and more desperate to communicate.
-	 *
-	 * In other words, this value controls fallback to things like TCP
-	 * tunnels to relays. As desperation increases, ZeroTier becomes
-	 * more and more willing to use these links.
-	 *
-	 * Desperation values shouldn't be arbitrary. They should be tied to
-	 * specific transport types. For example: 0 might be UDP, 1 might be
-	 * TCP, and 2 might be HTTP relay via a ZeroTier relay server. There
-	 * should be no gaps. Negative values are permitted and may refer to
-	 * better-than-normal links such as direct raw Ethernet framing over
-	 * a trusted backplane.
-	 */
-	int desperation;
-
-	/**
-	 * If nonzero (true), spam this message across paths up to 'desperation'
-	 *
-	 * This works with 'desperation' to allow fall-forward to less desperate
-	 * paths. When this flag is set, this message should be sent across all
-	 * applicable transports up to and including the specified level of
-	 * desperation.
-	 *
-	 * For example, if spam==1 and desperation==2 the packet might be sent
-	 * via both UDP and HTTP tunneling.
-	 */
-	int spam;
-
-	/**
-	 * Packet data
-	 */
-	char packetData[ZT1_MAX_WIRE_MESSAGE_LENGTH];
-
-	/**
-	 * Length of packet
-	 */
-	unsigned int packetLength;
-} ZT1_WireMessage;
-
-/**
- * A message to or from a virtual LAN port
- */
-typedef struct
-{
-	/**
-	 * ZeroTier network ID of virtual LAN port
-	 */
-	uint64_t nwid;
-
-	/**
-	 * Source MAC address
-	 */
-	uint64_t sourceMac;
-
-	/**
-	 * Destination MAC address
-	 */
-	uint64_t destMac;
-
-	/**
-	 * 16-bit Ethernet frame type
-	 */
-	unsigned int etherType;
-
-	/**
-	 * 10-bit VLAN ID or 0 for none
-	 */
-	unsigned int vlanId;
-
-	/**
-	 * Ethernet frame data
-	 */
-	char frameData[ZT1_MAX_MTU];
-
-	/**
-	 * Ethernet frame length
-	 */
-	unsigned int frameLength;
-} ZT1_VirtualNetworkFrame;
-
-/**
  * Virtual network status codes
  */
 enum ZT1_VirtualNetworkStatus
@@ -295,6 +229,22 @@ enum ZT1_VirtualNetworkType
 };
 
 /**
+ * An Ethernet multicast group
+ */
+typedef struct
+{
+	/**
+	 * MAC address (least significant 48 bits)
+	 */
+	uint64_t mac;
+
+	/**
+	 * Additional distinguishing information (usually zero)
+	 */
+	unsigned long adi;
+} ZT1_MulticastGroup;
+
+/**
  * Virtual LAN configuration
  */
 typedef struct
@@ -342,6 +292,11 @@ typedef struct
 	int bridge;
 
 	/**
+	 * If nonzero, this network supports and allows broadcast (ff:ff:ff:ff:ff:ff) traffic
+	 */
+	int broadcastEnabled;
+
+	/**
 	 * Network config revision as reported by netconf master
 	 *
 	 * If this is zero, it means we're still waiting for our netconf.
@@ -366,6 +321,16 @@ typedef struct
 	unsigned int assignedAddressCount;
 
 	/**
+	 * Multicast group subscriptions
+	 */
+	ZT1_MulticastGroup *multicastSubscriptions;
+
+	/**
+	 * Number of multicast group subscriptions
+	 */
+	unsigned int multicastSubscriptionCount;
+
+	/**
 	 * Network name (from network configuration master)
 	 */
 	const char *networkName;
@@ -551,8 +516,8 @@ typedef long (*ZT1_DataStoreGetFunction)(ZT1_Node *,const char *,void *,unsigned
 /**
  * Function to store an object in the data store
  *
- * Parameters: (1) object name, (2) object data, (3) object size. Naming
- * semantics are the same as the get function. This must return zero on
+ * Parameters: (1) node, (2) object name, (3) object data, (4) object size.
+ * Name semantics are the same as the get function. This must return zero on
  * success. You can return any OS-specific error code on failure, as these
  * may be visible in logs or error messages and might aid in debugging.
  *
@@ -560,6 +525,30 @@ typedef long (*ZT1_DataStoreGetFunction)(ZT1_Node *,const char *,void *,unsigned
  */
 typedef int (*ZT1_DataStorePutFunction)(ZT1_Node *,const char *,const void *,unsigned long);
 
+/**
+ * Function to send a ZeroTier packet out over the wire
+ *
+ * Parameters: (1) node, (2) address, (3) desperation, (4) spam? (bool),
+ * (5) packet data, (6) packet data length.
+ *
+ * If spam is nonzero, the implementation should attempt to send the packet
+ * over all link types or protocols up to and including the stated level of
+ * desperation. Non-applicable link types can of course be skipped.
+ *
+ * The function must return zero on success and may return any error code
+ * on failure. Note that success does not (of course) guarantee packet
+ * delivery. It only means that the packet appears to have been sent.
+ */
+typedef int (*ZT1_WirePacketSendFunction)(ZT1_Node *,const struct sockaddr_storage *,int,int,const void *,unsigned int);
+
+/**
+ * Function to send a frame out to a virtual network port
+ *
+ * Parameters: (1) node, (2) network ID, (3) source MAC, (4) destination MAC,
+ * (5) ethertype, (6) VLAN ID, (7) frame data, (8) frame length.
+ */
+typedef void (*ZT1_VirtualNetworkFrameFunction)(ZT1_Node *,uint64_t,uint64_t,uint64_t,unsigned int,unsigned int,const void *,unsigned int);
+
 /****************************************************************************/
 /* C Node API                                                               */
 /****************************************************************************/
@@ -581,53 +570,74 @@ enum ZT1_ResultCode ZT1_Node_new(
 	ZT1_Node **node,
 	ZT1_DataStoreGetFunction *dataStoreGetFunction,
 	ZT1_DataStorePutFunction *dataStorePutFunction,
+	ZT1_WirePacketSendFunction *wirePacketSendFunction,
+	ZT1_VirtualNetworkFrameFunction *virtualNetworkFrameFunction,
 	ZT1_VirtualNetworkConfigCallback *networkConfigCallback,
 	ZT1_StatusCallback *statusCallback);
 
 /**
- * Process wire messages and/or LAN frames
+ * Process a packet received from the physical wire
  *
- * This runs the ZeroTier core loop once with input packets and frames and
- * returns zero or more resulting packets or frames. It also sets a max
- * interval value. The calling code must call run() again after no more
- * than this many milliseconds of inactivity. If no packets have been
- * received, it's fine to call run() with no inputs after the inactivity
- * timeout.
+ * @param node Node instance
+ * @param now Current clock in milliseconds
+ * @param remoteAddress Origin of packet
+ * @param linkDesperation Link desperation metric for link or protocol over which packet arrived
+ * @param packetData Packet data
+ * @param packetLength Packet length
+ * @param nextCallDeadline Result: set to deadline for next call to one of the three processXXX() methods
+ * @return OK (0) or error code if a fatal error condition has occurred
+ */
+enum ZT1_ResultCode ZT1_Node_processWirePacket(
+	ZT1_Node *node,
+	uint64_t now,
+	const struct sockaddr_storage *remoteAddress,
+	int linkDesperation,
+	const void *packetData,
+	unsigned int packetLength,
+	uint64_t *nextCallDeadline);
+
+/**
+ * Process a frame from a virtual network port (tap)
  *
- * In addition to normal inputs and outputs, any callbacks registered
- * with the ZeroTier One core may also be called such as virtual network
- * endpoint configuration update or diagnostic message handlers.
+ * @param node Node instance
+ * @param now Current clock in milliseconds
+ * @param nwid ZeroTier 64-bit virtual network ID
+ * @param sourceMac Source MAC address (least significant 48 bits)
+ * @param destMac Destination MAC address (least significant 48 bits)
+ * @param etherType 16-bit Ethernet frame type
+ * @param vlanId 10-bit VLAN ID or 0 if none
+ * @param frameData Frame payload data
+ * @param frameLength Frame payload length
+ * @param nextCallDeadline Result: set to deadline for next call to one of the three processXXX() methods
+ * @return OK (0) or error code if a fatal error condition has occurred
+ */
+enum ZT1_ResultCode ZT1_Node_processVirtualNetworkFrame(
+	ZT1_Node *node,
+	uint64_t now,
+	uint64_t nwid,
+	uint64_t sourceMac,
+	uint64_t destMac,
+	unsigned int etherType,
+	unsigned int vlanId,
+	const void *frameData,
+	unsigned int frameLength,
+	uint64_t *nextCallDeadline);
+
+/**
+ * Perform required periodic operations even if no new frames or packets have arrived
  *
- * The supplied time must be at millisecond resolution and must increment
- * monotonically from the time the Node is created. Other than that, there
- * are no other restrictions. On normal systems this is usually the system
- * clock measured in milliseconds since the epoch.
+ * If the nextCallDeadline arrives and nothing has happened, call this method
+ * to do required background tasks like pinging and cleanup.
  *
  * @param node Node instance
- * @param now Current time at millisecond resolution (typically since epoch)
- * @param inputWireMessages ZeroTier transport packets from the wire
- * @param inputWireMessageCount Number of packets received
- * @param inputLanFrames Frames read from virtual LAN tap device
- * @param inputLanFrameCount Number of frames read
- * @param outputWireMessages Result: set to array of wire messages to be sent
- * @param outputWireMessageCount Result: set to size of *outputWireMessages[]
- * @param outputLanFrames Result: set to array of LAN frames to post to tap device
- * @param outputLanFrameCount Result: set to size of outputLanFrames[]
- * @param maxNextInterval Result: maximum number of milliseconds before next call to run() is needed
+ * @param now Current clock in milliseconds
+ * @param nextCallDeadline Result: set to deadline for next call to one of the three processXXX() methods
  * @return OK (0) or error code if a fatal error condition has occurred
  */
-enum ZT1_ResultCode ZT1_Node_run(
+enum ZT1_Resultcode ZT1_Node_processNothing(
 	ZT1_Node *node,
 	uint64_t now,
-	const ZT1_WireMessage *inputWireMessages,
-	unsigned int inputWireMessageCount,
-	const ZT1_VirtualNetworkFrame *inputFrames,
-	unsigned int inputFrameCount,
-	const ZT1_WireMessage **outputWireMessages,
-	unsigned int *outputWireMessageCount,
-	const ZT1_VirtualNetworkFrame **outputFrames,
-	unsigned int *outputLanFrameCount,
-	unsigned long *maxNextInterval);
+	uint64_t *nextCallDeadline);
 
 /**
  * Join a network
@@ -636,7 +646,7 @@ enum ZT1_ResultCode ZT1_Node_run(
  * or these may be deffered if a netconf is not available yet.
  *
  * @param node Node instance
- * @param nwid 64-bit ZeroTIer network ID
+ * @param nwid 64-bit ZeroTier network ID
  * @return OK (0) or error code if a fatal error condition has occurred
  */
 enum ZT1_ResultCode ZT1_Node_join(ZT1_Node *node,uint64_t nwid);
@@ -655,6 +665,49 @@ enum ZT1_ResultCode ZT1_Node_join(ZT1_Node *node,uint64_t nwid);
 enum ZT1_ResultCode ZT1_Node_leave(ZT1_Node *node,uint64_t nwid);
 
 /**
+ * Subscribe to an Ethernet multicast group
+ *
+ * ADI stands for additional distinguishing information. This defaults to zero
+ * and is rarely used. Right now its only use is to enable IPv4 ARP to scale,
+ * and this must be done.
+ *
+ * For IPv4 ARP, the implementation must subscribe to 0xffffffffffff (the
+ * broadcast address) but with an ADI equal to each IPv4 address in host
+ * byte order. This converts ARP from a non-scalable broadcast protocol to
+ * a scalable multicast protocol with perfect address specificity.
+ *
+ * If this is not done, ARP will not work reliably.
+ *
+ * Multiple calls to subscribe to the same multicast address will have no
+ * effect.
+ *
+ * This does not generate an update call to networkConfigCallback().
+ *
+ * @param node Node instance
+ * @param nwid 64-bit network ID
+ * @param multicastGroup Ethernet multicast or broadcast MAC (least significant 48 bits)
+ * @param multicastAdi Multicast ADI (least significant 32 bits only, default: 0)
+ * @return OK (0) or error code if a fatal error condition has occurred
+ */
+enum ZT1_ResultCode ZT1_Node_multicastSubscribe(ZT1_Node *node,uint64_t nwid,uint64_t multicastGroup,unsigned long multicastAdi = 0);
+
+/**
+ * Unsubscribe from an Ethernet multicast group (or all groups)
+ *
+ * If multicastGroup is zero (0), this will unsubscribe from all groups. If
+ * you are not subscribed to a group this has no effect.
+ *
+ * This does not generate an update call to networkConfigCallback().
+ *
+ * @param node Node instance
+ * @param nwid 64-bit network ID
+ * @param multicastGroup Ethernet multicast or broadcast MAC (least significant 48 bits)
+ * @param multicastAdi Multicast ADI (least significant 32 bits only, default: 0)
+ * @return OK (0) or error code if a fatal error condition has occurred
+ */
+enum ZT1_ResultCode ZT1_Node_multicastUnsubscribe(ZT1_Node *node,uint64_t nwid,uint64_t multicastGroup,unsigned long multicastAdi = 0);
+
+/**
  * Get the status of this node
  *
  * @param node Node instance
@@ -725,8 +778,9 @@ void ZT1_Node_setNetconfMaster(ZT1_Node *node,void *networkConfigMasterInstance)
  * @param major Result: major version
  * @param minor Result: minor version
  * @param revision Result: revision
+ * @param featureFlags: Result: feature flag bitmap
  */
-void ZT1_version(int *major,int *minor,int *revision);
+void ZT1_version(int *major,int *minor,int *revision,unsigned long *featureFlags);
 
 #ifdef __cplusplus
 }
diff --git a/node/Constants.hpp b/node/Constants.hpp
index ff91417c..21227682 100644
--- a/node/Constants.hpp
+++ b/node/Constants.hpp
@@ -28,6 +28,8 @@
 #ifndef ZT_CONSTANTS_HPP
 #define ZT_CONSTANTS_HPP
 
+#include "../include/ZeroTierOne.h"
+
 //
 // This include file also auto-detects and canonicalizes some environment
 // information defines:
@@ -154,28 +156,8 @@
 
 /**
  * Default MTU used for Ethernet tap device
- *
- * This is pretty much an unchangeable global constant. To make it change
- * across nodes would require logic to send ICMP packet too big messages,
- * which would complicate things. 1500 has been good enough on most LANs
- * for ages, so a larger MTU should be fine for the forseeable future. This
- * typically results in two UDP packets per single large frame. Experimental
- * results seem to show that this is good. Larger MTUs resulting in more
- * fragments seemed too brittle on slow/crummy links for no benefit.
- *
- * If this does change, also change it in tap.h in the tuntaposx code under
- * mac-tap.
- * 
- * Overhead for a normal frame split into two packets:
- *
- * 1414 = 1444 (typical UDP MTU) - 28 (packet header) - 2 (ethertype)
- * 1428 = 1444 (typical UDP MTU) - 16 (fragment header)
- * SUM: 2842
- *
- * We use 2800, which leaves some room for other payload in other types of
- * messages such as multicast propagation or future support for bridging.
  */
-#define ZT_IF_MTU 2800
+#define ZT_IF_MTU ZT1_MAX_MTU
 
 /**
  * Default interface metric for ZeroTier taps -- should be higher than physical ports
diff --git a/node/Node.cpp b/node/Node.cpp
index 7b51d3bd..0808062f 100644
--- a/node/Node.cpp
+++ b/node/Node.cpp
@@ -44,19 +44,15 @@ namespace ZeroTier {
 Node::Node(
 	ZT1_DataStoreGetFunction *dataStoreGetFunction,
 	ZT1_DataStorePutFunction *dataStorePutFunction,
+	ZT1_WirePacketSendFunction *wirePacketSendFunction,
+	ZT1_VirtualNetworkFrameFunction *virtualNetworkFrameFunction,
 	ZT1_VirtualNetworkConfigCallback *networkConfigCallback,
 	ZT1_StatusCallback *statusCallback) :
 	RR(new RuntimeEnvironment(this)),
-	_outputWireMessages((ZT1_WireMessage *)0),
-	_outputWireMessageCount(0),
-	_outputWireMessageCapacity(8),
-	_outputWireMessages_m(),
-	_outputFrames((ZT1_VirtualNetworkFrame *)0),
-	_outputFrameCount(0),
-	_outputFrameCapacity(8),
-	_outputFrames_m(),
 	_dataStoreGetFunction(dataStoreGetFunction),
 	_dataStorePutFunction(dataStorePutFunction),
+	_wirePacketSendFunction(wirePacketSendFunction),
+	_virtualNetworkFrameFunction(virtualNetworkFrameFunction),
 	_networkConfigCallback(networkConfigCallback),
 	_statusCallback(statusCallback),
 	_networks(),
@@ -67,16 +63,12 @@ Node::Node(
 	_spamCounter(0)
 {
 	try {
-		_outputWireMessages = new ZT1_WireMessage[_outputWireMessageCapacity];
-		_outputFrames = new ZT1_VirtualNetworkFrame[_outputFrameCapacity];
 		RR->prng = new CMWC4096();
 		RR->sw = new Switch(RR);
 		RR->mc = new Multicaster(RR);
 		RR->antiRec = new AntiRecursion(RR);
 		RR->topology = new Topology(RR);
 	} catch ( ... ) {
-		delete [] _outputFrames;
-		delete [] _outputWireMessages;
 		delete RR->topology;
 		delete RR->antiRec;
 		delete RR->mc;
@@ -90,8 +82,6 @@ Node::Node(
 
 Node::~Node()
 {
-	delete [] _outputFrames;
-	delete [] _outputWireMessages;
 	delete RR->topology;
 	delete RR->antiRec;
 	delete RR->mc;
@@ -101,25 +91,19 @@ Node::~Node()
 	delete RR;
 }
 
-ZT1_ResultCode Node::run(
-	uint64_t now,
-	const ZT1_WireMessage *inputWireMessages,
-	unsigned int inputWireMessageCount,
-	const ZT1_VirtualNetworkFrame *inputFrames,
-	unsigned int inputFrameCount,
-	const ZT1_WireMessage **outputWireMessages,
-	unsigned int *outputWireMessageCount,
-	const ZT1_VirtualNetworkFrame **outputFrames,
-	unsigned int *outputLanFrameCount,
-	unsigned long *maxNextInterval)
+ZT1_ResultCode Node::join(uint64_t nwid)
 {
 }
 
-ZT1_ResultCode Node::join(uint64_t nwid)
+ZT1_ResultCode Node::leave(uint64_t nwid)
 {
 }
 
-ZT1_ResultCode Node::leave(uint64_t nwid)
+ZT1_ResultCode Node::multicastSubscribe(ZT1_Node *node,uint64_t nwid,uint64_t multicastGroup,unsigned long multicastAdi)
+{
+}
+
+ZT1_ResultCode Node::multicastUnsubscribe(ZT1_Node *node,uint64_t nwid,uint64_t multicastGroup,unsigned long multicastAdi)
 {
 }
 
diff --git a/node/Node.hpp b/node/Node.hpp
index 6986cc5f..0f11f3b7 100644
--- a/node/Node.hpp
+++ b/node/Node.hpp
@@ -58,6 +58,8 @@ public:
 	Node(
 		ZT1_DataStoreGetFunction *dataStoreGetFunction,
 		ZT1_DataStorePutFunction *dataStorePutFunction,
+		ZT1_WirePacketSendFunction *wirePacketSendFunction,
+		ZT1_VirtualNetworkFrameFunction *virtualNetworkFrameFunction,
 		ZT1_VirtualNetworkConfigCallback *networkConfigCallback,
 		ZT1_StatusCallback *statusCallback);
 
@@ -65,32 +67,38 @@ public:
 
 	// Public API Functions ----------------------------------------------------
 
-	ZT1_ResultCode run(
+	ZT1_ResultCode processWirePacket(
+		ZT1_Node *node,
 		uint64_t now,
-		const ZT1_WireMessage *inputWireMessages,
-		unsigned int inputWireMessageCount,
-		const ZT1_VirtualNetworkFrame *inputFrames,
-		unsigned int inputFrameCount,
-		const ZT1_WireMessage **outputWireMessages,
-		unsigned int *outputWireMessageCount,
-		const ZT1_VirtualNetworkFrame **outputFrames,
-		unsigned int *outputLanFrameCount,
-		unsigned long *maxNextInterval);
-
+		const struct sockaddr_storage *remoteAddress,
+		int linkDesperation,
+		const void *packetData,
+		unsigned int packetLength,
+		uint64_t *nextCallDeadline);
+	ZT1_ResultCode processVirtualNetworkFrame(
+		ZT1_Node *node,
+		uint64_t now,
+		uint64_t nwid,
+		uint64_t sourceMac,
+		uint64_t destMac,
+		unsigned int etherType,
+		unsigned int vlanId,
+		const void *frameData,
+		unsigned int frameLength,
+		uint64_t *nextCallDeadline);
+	ZT1_Resultcode processNothing(
+		ZT1_Node *node,
+		uint64_t now,
+		uint64_t *nextCallDeadline);
 	ZT1_ResultCode join(uint64_t nwid);
-
 	ZT1_ResultCode leave(uint64_t nwid);
-
+	ZT1_ResultCode multicastSubscribe(ZT1_Node *node,uint64_t nwid,uint64_t multicastGroup,unsigned long multicastAdi = 0);
+	ZT1_ResultCode multicastUnsubscribe(ZT1_Node *node,uint64_t nwid,uint64_t multicastGroup,unsigned long multicastAdi = 0);
 	void status(ZT1_NodeStatus *status);
-
 	ZT1_PeerList *peers();
-
 	ZT1_VirtualNetworkConfig *networkConfig(uint64_t nwid);
-
 	ZT1_VirtualNetworkList *listNetworks();
-
 	void freeQueryResult(void *qr);
-
 	void setNetconfMaster(void *networkConfigMasterInstance);
 
 	// Internal functions ------------------------------------------------------
@@ -114,19 +122,13 @@ public:
 	 */
 	inline void putPacket(const InetAddress &addr,const void *data,unsigned int len)
 	{
-		Mutex::Lock _l(_outputWireMessages_m);
-		if (_outputWireMessageCount >= _outputWireMessageCapacity) {
-			ZT1_WireMessage *old = _outputWireMessages;
-			_outputWireMessages = new ZT1_WireMessage[_outputWireMessageCapacity *= 2];
-			memcpy(_outputWireMessages,old,sizeof(ZT1_WireMessage) * _outputWireMessageCount);
-			delete [] old;
-		}
-		ZT1_WireMessage &wm = _outputWireMessages[_outputWireMessageCount++];
-		memcpy(&(wm.address),&addr,sizeof(struct sockaddr_storage));
-		wm.desperation = this->desperation();
-		wm.spam = (int)((++_spamCounter % ZT_DESPERATION_SPAM_EVERY) == 0);
-		memcpy(wm.packetData,data,len);
-		wm.packetLength = len;
+		_wirePacketSendFunction(
+			reinterpret_cast<ZT1_Node *>(this),
+			reinterpret_cast<const struct sockaddr_storage *>(&addr),
+			this->desperation(),
+			(int)((++_spamCounter % ZT_DESPERATION_SPAM_EVERY) == 0),
+			data,
+			len);
 	}
 
 	/**
@@ -142,21 +144,15 @@ public:
 	 */
 	inline void putFrame(uint64_t nwid,const MAC &source,const MAC &dest,unsigned int etherType,unsigned int vlanId,const void *data,unsigned int len)
 	{
-		Mutex::Lock _l(_outputFrames_m);
-		if (_outputFrameCount >= _outputFrameCapacity) {
-			ZT1_VirtualNetworkFrame *old = _outputFrames;
-			_outputFrames = new ZT1_VirtualNetworkFrame[_outputFrameCapacity *= 2];
-			memcpy(_outputFrames,old,sizeof(ZT1_VirtualNetworkFrame) * _outputFrameCount);
-			delete [] old;
-		}
-		ZT1_VirtualNetworkFrame &f = _outputFrames[_outputFrameCount++];
-		f.nwid = nwid;
-		f.sourceMac = source.toInt();
-		f.destMac = dest.toInt();
-		f.etherType = etherType;
-		f.vlanId = vlanId;
-		memcpy(f.frameData,data,len);
-		f.frameLength = len;
+		_virtualNetworkFrameFunction(
+			reinterpret_cast<ZT1_Node *>(this),
+			nwid,
+			source.toInt(),
+			dest.toInt(),
+			etherType,
+			vlanId,
+			data,
+			len);
 	}
 
 	/**
@@ -173,18 +169,10 @@ public:
 private:
 	RuntimeEnvironment *RR;
 
-	ZT1_WireMessage *_outputWireMessages;
-	unsigned long _outputWireMessageCount;
-	unsigned long _outputWireMessageCapacity;
-	Mutex _outputWireMessages_m;
-
-	ZT1_VirtualNetworkFrame *_outputFrames;
-	unsigned long _outputFrameCount;
-	unsigned long _outputFrameCapacity;
-	Mutex _outputFrames_m;
-
 	ZT1_DataStoreGetFunction *_dataStoreGetFunction;
 	ZT1_DataStorePutFunction *_dataStorePutFunction;
+	ZT1_WirePacketSendFunction *_wirePacketSendFunction;
+	ZT1_VirtualNetworkFrameFunction *_virtualNetworkFrameFunction;
 	ZT1_VirtualNetworkConfigCallback *_networkConfigCallback;
 	ZT1_StatusCallback *_statusCallback;
 
@@ -194,10 +182,10 @@ private:
 	std::map< uint64_t,Network * > _networks;
 	Mutex _networks_m;
 
-	uint64_t _now; // time of last run()
-	uint64_t _timeOfLastPacketReceived;
-	uint64_t _timeOfLastPrivilgedPacket;
-	unsigned int _spamCounter; // used to "spam" every Nth packet
+	volatile uint64_t _now; // time of last run()
+	volatile uint64_t _timeOfLastPacketReceived;
+	volatile _timeOfLastPrivilgedPacket;
+	volatile unsigned int _spamCounter; // used to "spam" every Nth packet
 };
 
 } // namespace ZeroTier