Work on defining new direct broadcast multicast algorithm.

13 files changed, 352 insertions, 441 deletions

diff --git a/node/Constants.hpp b/node/Constants.hpp
index 9aa16ddf..b085d446 100644
--- a/node/Constants.hpp
+++ b/node/Constants.hpp
@@ -228,36 +228,19 @@
 #define ZT_RELAY_MAX_HOPS 3
 
 /**
- * Size of multicast deduplication ring buffer in 64-bit ints
+ * Expire time for multicast 'likes' and indirect multicast memberships in ms
  */
-#define ZT_MULTICAST_DEDUP_HISTORY_LENGTH 512
+#define ZT_MULTICAST_LIKE_EXPIRE 600000
 
 /**
- * Default number of bits in multicast propagation prefix
- */
-#define ZT_DEFAULT_MULTICAST_PREFIX_BITS 2
-
-/**
- * Default max depth (TTL) for multicast propagation
- */
-#define ZT_DEFAULT_MULTICAST_DEPTH 32
-
-/**
- * Global maximum for multicast propagation depth
- *
- * This is kind of an insane value, meant as a sanity check.
- */
-#define ZT_MULTICAST_GLOBAL_MAX_DEPTH 500
-
-/**
- * Expire time for multicast 'likes' in ms
+ * Time between polls of local tap devices for multicast membership changes
  */
-#define ZT_MULTICAST_LIKE_EXPIRE 120000
+#define ZT_MULTICAST_LOCAL_POLL_PERIOD 10000
 
 /**
- * Time between polls of local tap devices for multicast membership changes
+ * Minimum delay between attempts to gather multicast topology info if members > 0
  */
-#define ZT_MULTICAST_LOCAL_POLL_PERIOD 10000
+#define ZT_MULTICAST_TOPOLOGY_RESEARCH_RATE_THROTTLE 120000
 
 /**
  * Delay between scans of the topology active peer DB for peers that need ping
diff --git a/node/MulticastTopology.cpp b/node/MulticastTopology.cpp
new file mode 100644
index 00000000..a594d33e
--- /dev/null
+++ b/node/MulticastTopology.cpp
@@ -0,0 +1,88 @@
+/*
+ * ZeroTier One - Global Peer to Peer Ethernet
+ * Copyright (C) 2011-2014  ZeroTier Networks LLC
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ * --
+ *
+ * ZeroTier may be used and distributed under the terms of the GPLv3, which
+ * are available at: http://www.gnu.org/licenses/gpl-3.0.html
+ *
+ * If you would like to embed ZeroTier into a commercial application or
+ * redistribute it in a modified binary form, please contact ZeroTier Networks
+ * LLC. Start here: http://www.zerotier.com/
+ */
+
+#include <algorithm>
+
+#include "Constants.hpp"
+#include "MulticastTopology.hpp"
+#include "Topology.hpp"
+
+namespace ZeroTier {
+
+MulticastTopology::MulticastTopology()
+{
+}
+
+MulticastTopology::~MulticastTopology()
+{
+}
+
+void MulticastTopology::clean(const Topology &topology)
+{
+	uint64_t now = Utils::now();
+
+	for(std::map< MulticastGroup,std::vector<MulticastGroupMember> >::iterator mm(_members.begin());mm!=_members.end();) {
+		std::vector<MulticastGroupMember>::iterator reader(mm->second.begin());
+		std::vector<MulticastGroupMember>::iterator writer(mm->second.begin());
+		unsigned long count = 0;
+		while (reader != mm->second.end()) {
+			if ((now - reader->timestamp) < ZT_MULTICAST_LIKE_EXPIRE) {
+				*writer = *reader;
+
+				/* We sort in ascending order of most recent relevant activity. For peers we've learned
+				 * about by direct LIKEs, we do this in order of their own activity. For indirectly
+				 * acquired peers we do this minus a constant to place these categorically below directly
+				 * learned peers. For peers with no active Peer record, we use the time we last learned
+				 * about them minus one day (a large constant) to put these at the bottom of the list.
+				 * List is sorted in ascending order of rank and multicasts are sent last-to-first. */
+				if (writer->learnedFrom) {
+					SharedPtr<Peer> p(topology.getPeer(writer->learnedFrom));
+					if (p)
+						writer->rank = p->lastUnicastFrame() - ZT_MULTICAST_LIKE_EXPIRE;
+					else writer->rank = writer->timestamp - 86400000;
+				} else {
+					SharedPtr<Peer> p(topology.getPeer(writer->address));
+					if (p)
+						writer->rank = p->lastUnicastFrame();
+					else writer->rank = writer->timestamp - 86400000;
+				}
+
+				++writer;
+				++count;
+			}
+			++reader;
+		}
+
+		if (count) {
+			mm->second.resize(count);
+			std::sort(mm->second.begin(),mm->second.end()); // sorts in ascending order of rank
+			++mm;
+		} else _members.erase(mm++);
+	}
+}
+
+} // namespace ZeroTier
diff --git a/node/MulticastTopology.hpp b/node/MulticastTopology.hpp
new file mode 100644
index 00000000..fc49dbd3
--- /dev/null
+++ b/node/MulticastTopology.hpp
@@ -0,0 +1,162 @@
+/*
+ * ZeroTier One - Global Peer to Peer Ethernet
+ * Copyright (C) 2011-2014  ZeroTier Networks LLC
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ * --
+ *
+ * ZeroTier may be used and distributed under the terms of the GPLv3, which
+ * are available at: http://www.gnu.org/licenses/gpl-3.0.html
+ *
+ * If you would like to embed ZeroTier into a commercial application or
+ * redistribute it in a modified binary form, please contact ZeroTier Networks
+ * LLC. Start here: http://www.zerotier.com/
+ */
+
+#ifndef ZT_MULTICASTTOPOLOGY_HPP
+#define ZT_MULTICASTTOPOLOGY_HPP
+
+#include <stdint.h>
+#include <string.h>
+
+#include <map>
+#include <vector>
+
+#include "Constants.hpp"
+#include "Address.hpp"
+#include "MulticastGroup.hpp"
+#include "Mutex.hpp"
+#include "Utils.hpp"
+
+namespace ZeroTier {
+
+class Topology;
+
+/**
+ * Database of known multicast peers within a network
+ */
+class MulticastTopology
+{
+private:
+	struct MulticastGroupMember
+	{
+		MulticastGroupMember() {}
+		MulticastGroupMember(const Address &a,const Address &lf,uint64_t ts) : address(a),learnedFrom(lf),timestamp(ts) {}
+
+		Address address;
+		Address learnedFrom; // NULL/0 for addresses directly learned from LIKE
+		uint64_t timestamp; // time of last LIKE or OK response to MULTICAST_LONELY
+		uint64_t rank; // used by sorting algorithm in clean()
+
+		// for sorting in ascending order of rank
+		inline bool operator<(const MulticastGroupMember &m) const throw() { return (rank < m.rank); }
+	};
+
+public:
+	MulticastTopology();
+	~MulticastTopology();
+
+	/**
+	 * Add or update a member in a multicast group
+	 *
+	 * @param mg Multicast group
+	 * @param member Member to add/update
+	 * @param learnedFrom Address from which we learned this member or NULL/0 Address if direct
+	 */
+	inline void add(const MulticastGroup &mg,const Address &member,const Address &learnedFrom)
+	{
+		Mutex::Lock _l(_members_m);
+		std::vector<MulticastGroupMember> &mv = _members[mg];
+		for(std::vector<MulticastGroupMember>::iterator m(mv.begin());m!=mv.end();++m) {
+			if (m->address == member) {
+				if (m->learnedFrom) // once a member has been seen directly, we keep its status as direct
+					m->learnedFrom = learnedFrom;
+				m->timestamp = Utils::now();
+				return;
+			}
+		}
+		mv.push_back(MulticastGroupMember(member,learnedFrom,Utils::now()));
+	}
+
+	/**
+	 * Erase a member from a multicast group (if present)
+	 *
+	 * @param mg Multicast group
+	 * @param member Member to erase
+	 */
+	inline void erase(const MulticastGroup &mg,const Address &member)
+	{
+		Mutex::Lock _l(_members_m);
+		std::map< MulticastGroup,std::vector<MulticastGroupMember> >::iterator r(_members.find(mg));
+		if (r != _members.end()) {
+			for(std::vector<MulticastGroupMember>::iterator m(r->second.begin());m!=r->second.end();++m) {
+				if (m->address == member) {
+					r->second.erase(m);
+					return;
+				}
+			}
+		}
+	}
+
+	/**
+	 * @param mg Multicast group
+	 * @return Number of known peers in group
+	 */
+	inline unsigned int memberCount(const MulticastGroup &mg) const
+	{
+		Mutex::Lock _l(_members_m);
+		std::map< MulticastGroup,std::vector<MulticastGroupMember> >::const_iterator r(_members.find(mg));
+		return ((r != _members.end()) ? (unsigned int)r->second.size() : (unsigned int)0);
+	}
+
+	/**
+	 * Iterate over the known members of a multicast group
+	 *
+	 * @param mg Multicast group
+	 * @param func Function to be called with multicast group and address of member
+	 * @tparam F Function type (explicitly template on "FuncObj &" if reference instead of copy should be passed)
+	 * @return Number of members in multicast group for which function was called
+	 */
+	template<typename F>
+	inline unsigned int eachMember(const MulticastGroup &mg,F func) const
+	{
+		Mutex::Lock _l(_members_m);
+		std::map< MulticastGroup,std::vector<MulticastGroupMember> >::const_iterator r(_members.find(mg));
+		if (r != _members.end()) {
+			// We go in reverse order because most recently learned members are pushed to the end
+			// of the vector. The priority resort algorithm in clean() sorts in ascending order
+			// of propagation priority too.
+			for(std::vector<MulticastGroupMember>::const_reverse_iterator m(r->second.rbegin());m!=r->second.rend();++m) {
+				func(mg,m->address);
+			}
+			return (unsigned int)r->second.size();
+		} else return 0;
+	}
+
+	/**
+	 * Clean up and resort database
+	 *
+	 * @param topology Global peer topology
+	 */
+	void clean(const Topology &topology);
+
+private:
+	std::map< MulticastGroup,std::vector<MulticastGroupMember> > _members;
+	Mutex _members_m;
+};
+
+} // namespace ZeroTier
+
+#endif
diff --git a/node/Multicaster.cpp b/node/Multicaster.cpp
deleted file mode 100644
index d75c3c93..00000000
--- a/node/Multicaster.cpp
+++ /dev/null
@@ -1,105 +0,0 @@
-/*
- * ZeroTier One - Global Peer to Peer Ethernet
- * Copyright (C) 2011-2014  ZeroTier Networks LLC
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program.  If not, see <http://www.gnu.org/licenses/>.
- *
- * --
- *
- * ZeroTier may be used and distributed under the terms of the GPLv3, which
- * are available at: http://www.gnu.org/licenses/gpl-3.0.html
- *
- * If you would like to embed ZeroTier into a commercial application or
- * redistribute it in a modified binary form, please contact ZeroTier Networks
- * LLC. Start here: http://www.zerotier.com/
- */
-
-#include <stdio.h>
-
-#include "Constants.hpp"
-#include "Multicaster.hpp"
-#include "Utils.hpp"
-
-namespace ZeroTier {
-
-Multicaster::Multicaster()
-{
-}
-
-Multicaster::~Multicaster()
-{
-}
-
-void Multicaster::likesGroup(uint64_t nwid,const Address &a,const MulticastGroup &mg,uint64_t now)
-{
-	Mutex::Lock _l(_lock);
-	_NetInfo &n = _nets[nwid];
-	_SubInfo &si = n.subscriptions[_Subscription(a,mg)];
-	if (!si.lastLike) { // on first LIKE, we must add to _proximity[mg]
-		std::list< Address > &p = n.proximity[mg];
-		p.push_front(a);
-		si.proximitySlot = p.begin(); // list's iterators remain valid until erase()
-	}
-	si.lastLike = now;
-}
-
-void Multicaster::bringCloser(uint64_t nwid,const Address &a)
-{
-	Mutex::Lock _l(_lock);
-
-	std::map< uint64_t,_NetInfo >::iterator n(_nets.find(nwid));
-	if (n == _nets.end())
-		return;
-
-	/* _subscriptions contains pairs of <Address,MulticastGroup>, so we can
-	 * easily iterate through all subscriptions for a given address by
-	 * starting with the default all-zero MulticastGroup() as lower bound
-	 * and stopping when we're not looking at the right address anymore.
-	 * Then we can look up _proximity and rapidly splice() the list using
-	 * the saved iterator in _SubInfo. */
-
-	std::map< _Subscription,_SubInfo >::iterator s(n->second.subscriptions.lower_bound(_Subscription(a,MulticastGroup())));
-	while ((s != n->second.subscriptions.end())&&(s->first.first == a)) {
-		std::map< MulticastGroup,std::list< Address > >::iterator p(n->second.proximity.find(s->first.second));
-		if (s->second.proximitySlot != p->second.begin())
-			p->second.splice(p->second.begin(),p->second,s->second.proximitySlot);
-		++s;
-	}
-}
-
-void Multicaster::clean()
-{
-	Mutex::Lock _l(_lock);
-
-	uint64_t now = Utils::now();
-
-	for(std::map< uint64_t,_NetInfo >::iterator n(_nets.begin());n!=_nets.end();) {
-		for(std::map< _Subscription,_SubInfo >::iterator s(n->second.subscriptions.begin());s!=n->second.subscriptions.end();) {
-			if ((now - s->second.lastLike) >= ZT_MULTICAST_LIKE_EXPIRE) {
-				std::map< MulticastGroup,std::list< Address > >::iterator p(n->second.proximity.find(s->first.second));
-				p->second.erase(s->second.proximitySlot);
-				if (p->second.empty())
-					n->second.proximity.erase(p);
-				n->second.subscriptions.erase(s++);
-			} else ++s;
-		}
-
-		if (n->second.proximity.empty()&&n->second.subscriptions.empty())
-			_nets.erase(n++);
-		else ++n;
-	}
-}
-
-} // namespace ZeroTier
-
diff --git a/node/Multicaster.hpp b/node/Multicaster.hpp
deleted file mode 100644
index 262f0304..00000000
--- a/node/Multicaster.hpp
+++ /dev/null
@@ -1,275 +0,0 @@
-/*
- * ZeroTier One - Global Peer to Peer Ethernet
- * Copyright (C) 2011-2014  ZeroTier Networks LLC
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program.  If not, see <http://www.gnu.org/licenses/>.
- *
- * --
- *
- * ZeroTier may be used and distributed under the terms of the GPLv3, which
- * are available at: http://www.gnu.org/licenses/gpl-3.0.html
- *
- * If you would like to embed ZeroTier into a commercial application or
- * redistribute it in a modified binary form, please contact ZeroTier Networks
- * LLC. Start here: http://www.zerotier.com/
- */
-
-#ifndef ZT_MULTICASTER_HPP
-#define ZT_MULTICASTER_HPP
-
-#include <stdint.h>
-#include <string.h>
-
-#include <stdexcept>
-#include <map>
-#include <set>
-#include <list>
-#include <algorithm>
-
-#include "Constants.hpp"
-#include "Mutex.hpp"
-#include "MulticastGroup.hpp"
-#include "Topology.hpp"
-#include "Address.hpp"
-#include "Buffer.hpp"
-
-namespace ZeroTier {
-
-/**
- * Multicast propagation algorithm core and database
- */
-class Multicaster
-{
-public:
-	Multicaster();
-	~Multicaster();
-
-	/**
-	 * Add or renew a peer's subscription to a multicast group
-	 *
-	 * @param nwid Network ID
-	 * @param a Address that LIKEd
-	 * @param mg Multicast group
-	 * @param now Current time
-	 */
-	void likesGroup(uint64_t nwid,const Address &a,const MulticastGroup &mg,uint64_t now);
-
-	/**
-	 * Bring a peer closer in terms of propagation priority
-	 *
-	 * This gets called from PacketDecoder when a unicast frame is received.
-	 *
-	 * @param nwid Network ID
-	 * @param a Address to bring closer (e.g. due to unicast message)
-	 * @param now Current time
-	 */
-	void bringCloser(uint64_t nwid,const Address &a);
-
-	/**
-	 * Erase entries for expired LIKEs and GOT records
-	 */
-	void clean();
-
-	/**
-	 * Multicast deduplicator
-	 *
-	 * This checks to see if a multicast GUID has been seen before. If not, it
-	 * adds it to the history and returns false.
-	 *
-	 * @param nwid Network ID
-	 * @param mcGuid Multicast GUID (sender address + sender unique ID)
-	 * @return True if multicast IS a duplicate, false otherwise
-	 */
-	inline bool deduplicate(uint64_t nwid,uint64_t mcGuid)
-		throw()
-	{
-		Mutex::Lock _l(_lock);
-		_NetInfo &n = _nets[nwid];
-		for(unsigned int i=0;i<ZT_MULTICAST_DEDUP_HISTORY_LENGTH;++i) {
-			if (n.multicastHistory[i] == mcGuid)
-				return true;
-		}
-		n.multicastHistory[n.multicastHistoryPtr++ % ZT_MULTICAST_DEDUP_HISTORY_LENGTH] = mcGuid;
-		return false;
-	}
-
-	/**
-	 * Pick next hops for a multicast by proximity
-	 *
-	 * The function or function object must return true if more hops are desired
-	 * or false to stop finding new hops and return.
-	 *
-	 * @param nwid Network ID
-	 * @param mg Multicast group
-	 * @param nextHopFunc Function to call for each address, search stops if it returns false
-	 * @tparam F Function to receive each next hop address
-	 */
-	template<typename F>
-	inline void getNextHops(uint64_t nwid,const MulticastGroup &mg,F nextHopFunc)
-	{
-		Mutex::Lock _l(_lock);
-
-		std::map< uint64_t,_NetInfo >::iterator n(_nets.find(nwid));
-		if (n == _nets.end())
-			return;
-		std::map< MulticastGroup,std::list< Address > >::iterator p(n->second.proximity.find(mg));
-		if (p == n->second.proximity.end())
-			return;
-
-		for(std::list< Address >::iterator a(p->second.begin());a!=p->second.end();++a) {
-			if (!nextHopFunc(*a))
-				break;
-		}
-	}
-
-	/**
-	 * Functor to add addresses to multicast frame propagation queues
-	 *
-	 * This function object checks the origin, bloom filter, and restriction
-	 * prefix for each address and if all these pass it adds the address and
-	 * increments the pointer pointed to by ptr. It stops (returns false) when
-	 * *ptr reaches end. It's used in PacketDecoder and Switch with getNextHops()
-	 * to compose multicast frame headers.
-	 */
-	class AddToPropagationQueue
-	{
-	public:
-		/**
-		 * @param ptr Pointer to pointer to current position in queue
-		 * @param end End of queue
-		 * @param bloom Bloom filter field (must be 1024 bytes in length)
-		 * @param bloomNonce Random nonce for bloom filter randomization
-		 * @param origin Originating address
-		 * @param prefixBits Number of bits in propagation restriction prefix
-		 * @param prefix Propagation restrition prefix
-		 * @param topology Topology database
-		 * @param now Current time
-		 */
-		AddToPropagationQueue(
-			unsigned char **ptr,
-			unsigned char *end,
-			unsigned char *bloom,
-			uint16_t bloomNonce,
-			const Address &origin,
-			unsigned int prefixBits,
-			uint64_t prefix,
-			const Topology *topology,
-			uint64_t now)
-		throw() :
-			_origin(origin),
-			_bloomNonce((uint64_t)bloomNonce),
-			_prefix(prefix),
-			_now(now),
-			_ptr(ptr),
-			_end(end),
-			_bloom(bloom),
-			_topology(topology),
-			_prefixBits(prefixBits) {}
-
-		/**
-		 * @param a Address to (possibly) add
-		 * @return True if FIFO still contains room for more possible addresses
-		 */
-		inline bool operator()(const Address &a)
-			throw()
-		{
-			if (*_ptr >= _end)
-				return false;
-
-			// Exclude original sender -- obviously they've already seen it
-			if (a == _origin)
-				return true;
-
-			// Exclude addresses not in this prefix domain
-			if (!a.withinMulticastPropagationPrefix(_prefix,_prefixBits))
-				return true;
-
-			// Exclude addresses remembered in bloom filter
-			uint64_t aint = a.toInt() + _bloomNonce;
-			const unsigned int bit = (unsigned int)(aint ^ (aint >> 13) ^ (aint >> 26) ^ (aint >> 39)) & 0x1fff;
-			unsigned char *const bbyte = _bloom + (bit >> 3); // note: bloom filter size == 1024 is hard-coded here
-			const unsigned char bmask = 1 << (bit & 7);
-			if ((*bbyte & bmask))
-				return true; // address already visited
-
-			// Exclude peers that don't appear to be online
-			SharedPtr<Peer> p(_topology->getPeer(a));
-			if ((!p)||(!p->alive(_now)))
-				return true;
-
-			// Remember address in bloom filter
-			*bbyte |= bmask;
-
-			a.copyTo(*_ptr,ZT_ADDRESS_LENGTH);
-			return ((*_ptr += ZT_ADDRESS_LENGTH) < _end);
-		}
-
-	private:
-		const Address _origin;
-		const uint64_t _bloomNonce;
-		const uint64_t _prefix;
-		const uint64_t _now;
-		unsigned char **const _ptr;
-		unsigned char *const _end;
-		unsigned char *const _bloom;
-		const Topology *const _topology;
-		const unsigned int _prefixBits;
-	};
-
-private:
-	// Information about a subscription
-	struct _SubInfo
-	{
-		_SubInfo() :
-			lastLike(0),
-			proximitySlot() {}
-
-		// Time of last MULTICAST_LIKE for this group
-		uint64_t lastLike;
-
-		// Slot in corresponding list in _proximity
-		std::list< Address >::iterator proximitySlot;
-	};
-
-	// An address and multicast group tuple
-	typedef std::pair< Address,MulticastGroup > _Subscription;
-
-	// Multicast info for a given network
-	struct _NetInfo
-	{
-		_NetInfo()
-			throw()
-		{
-			memset(multicastHistory,0,sizeof(multicastHistory));
-			multicastHistoryPtr = 0;
-		}
-
-		// Ring buffer of most recently injected multicast packet GUIDs
-		uint64_t multicastHistory[ZT_MULTICAST_DEDUP_HISTORY_LENGTH];
-		unsigned int multicastHistoryPtr;
-
-		// Peer proximity ordering for peers subscribed to each group
-		std::map< MulticastGroup,std::list< Address > > proximity;
-
-		// Peer subscriptions to multicast groups
-		std::map< _Subscription,_SubInfo > subscriptions;
-	};
-
-	std::map< uint64_t,_NetInfo > _nets;
-	Mutex _lock;
-};
-
-} // namespace ZeroTier
-
-#endif
diff --git a/node/Network.cpp b/node/Network.cpp
index 0eb3b359..8bcf185c 100644
--- a/node/Network.cpp
+++ b/node/Network.cpp
@@ -117,23 +117,23 @@ bool Network::updateMulticastGroups()
 	EthernetTap *t = _tap;
 	if (t) {
 		// Grab current groups from the local tap
-		bool updated = t->updateMulticastGroups(_multicastGroups);
+		bool updated = t->updateMulticastGroups(_myMulticastGroups);
 
 		// Merge in learned groups from any hosts bridged in behind us
-		for(std::map<MulticastGroup,uint64_t>::const_iterator mg(_bridgedMulticastGroups.begin());mg!=_bridgedMulticastGroups.end();++mg)
-			_multicastGroups.insert(mg->first);
+		for(std::map<MulticastGroup,uint64_t>::const_iterator mg(_multicastGroupsBehindMe.begin());mg!=_multicastGroupsBehindMe.end();++mg)
+			_myMulticastGroups.insert(mg->first);
 
 		// Add or remove BROADCAST group based on broadcast enabled netconf flag
 		if ((_config)&&(_config->enableBroadcast())) {
-			if (_multicastGroups.count(BROADCAST))
+			if (_myMulticastGroups.count(BROADCAST))
 				return updated;
 			else {
-				_multicastGroups.insert(BROADCAST);
+				_myMulticastGroups.insert(BROADCAST);
 				return true;
 			}
 		} else {
-			if (_multicastGroups.count(BROADCAST)) {
-				_multicastGroups.erase(BROADCAST);
+			if (_myMulticastGroups.count(BROADCAST)) {
+				_myMulticastGroups.erase(BROADCAST);
 				return true;
 			} else return updated;
 		}
@@ -311,9 +311,9 @@ void Network::clean()
 	}
 
 	// Clean learned multicast groups if we haven't heard from them in a while
-	for(std::map<MulticastGroup,uint64_t>::iterator mg(_bridgedMulticastGroups.begin());mg!=_bridgedMulticastGroups.end();) {
+	for(std::map<MulticastGroup,uint64_t>::iterator mg(_multicastGroupsBehindMe.begin());mg!=_multicastGroupsBehindMe.end();) {
 		if ((now - mg->second) > (ZT_MULTICAST_LIKE_EXPIRE * 2))
-			_bridgedMulticastGroups.erase(mg++);
+			_multicastGroupsBehindMe.erase(mg++);
 		else ++mg;
 	}
 }
@@ -419,23 +419,23 @@ void Network::threadMain()
 void Network::learnBridgeRoute(const MAC &mac,const Address &addr)
 {
 	Mutex::Lock _l(_lock);
-	_bridgeRoutes[mac] = addr;
+	_remoteBridgeRoutes[mac] = addr;
 
-	// If _bridgeRoutes exceeds sanity limit, trim worst offenders until below -- denial of service circuit breaker
-	while (_bridgeRoutes.size() > ZT_MAX_BRIDGE_ROUTES) {
+	// If _remoteBridgeRoutes exceeds sanity limit, trim worst offenders until below -- denial of service circuit breaker
+	while (_remoteBridgeRoutes.size() > ZT_MAX_BRIDGE_ROUTES) {
 		std::map<Address,unsigned long> counts;
 		Address maxAddr;
 		unsigned long maxCount = 0;
-		for(std::map<MAC,Address>::iterator br(_bridgeRoutes.begin());br!=_bridgeRoutes.end();++br) {
+		for(std::map<MAC,Address>::iterator br(_remoteBridgeRoutes.begin());br!=_remoteBridgeRoutes.end();++br) {
 			unsigned long c = ++counts[br->second];
 			if (c > maxCount) {
 				maxCount = c;
 				maxAddr = br->second;
 			}
 		}
-		for(std::map<MAC,Address>::iterator br(_bridgeRoutes.begin());br!=_bridgeRoutes.end();) {
+		for(std::map<MAC,Address>::iterator br(_remoteBridgeRoutes.begin());br!=_remoteBridgeRoutes.end();) {
 			if (br->second == maxAddr)
-				_bridgeRoutes.erase(br++);
+				_remoteBridgeRoutes.erase(br++);
 			else ++br;
 		}
 	}
diff --git a/node/Network.hpp b/node/Network.hpp
index a24f161c..33c3226c 100644
--- a/node/Network.hpp
+++ b/node/Network.hpp
@@ -159,7 +159,7 @@ public:
 	inline std::set<MulticastGroup> multicastGroups() const
 	{
 		Mutex::Lock _l(_lock);
-		return _multicastGroups;
+		return _myMulticastGroups;
 	}
 
 	/**
@@ -378,8 +378,8 @@ public:
 	inline Address findBridgeTo(const MAC &mac) const
 	{
 		Mutex::Lock _l(_lock);
-		std::map<MAC,Address>::const_iterator br(_bridgeRoutes.find(mac));
-		if (br == _bridgeRoutes.end())
+		std::map<MAC,Address>::const_iterator br(_remoteBridgeRoutes.find(mac));
+		if (br == _remoteBridgeRoutes.end())
 			return Address();
 		return br->second;
 	}
@@ -401,7 +401,7 @@ public:
 	inline void learnBridgedMulticastGroup(const MulticastGroup &mg,uint64_t now)
 	{
 		Mutex::Lock _l(_lock);
-		_bridgedMulticastGroups[mg] = now;
+		_multicastGroupsBehindMe[mg] = now;
 	}
 
 	/**
@@ -445,16 +445,20 @@ private:
 	EthernetTap *volatile _tap; // tap device or NULL if not initialized yet
 	volatile bool _enabled;
 
-	std::set<MulticastGroup> _multicastGroups;
-	std::map< std::pair<Address,MulticastGroup>,BandwidthAccount > _multicastRateAccounts;
+	std::set<MulticastGroup> _myMulticastGroups; // multicast groups that we belong to including those behind us (updated periodically)
+	std::map<MulticastGroup,uint64_t> _multicastGroupsBehindMe; // multicast groups bridged to us and when we last saw activity on each
+
+	std::map<MAC,Address> _remoteBridgeRoutes; // remote addresses where given MACs are reachable
 
-	std::map<Address,CertificateOfMembership> _membershipCertificates;
-	std::map<Address,uint64_t> _lastPushedMembershipCertificate;
+	// Deprecated, but will be kept around until P5_MULTICAST_FRAME is gone -- but the
+	// entry for us is still used by both. Eventually there will only be one BandwidthAccount,
+	// namely ours.
+	std::map< std::pair<Address,MulticastGroup>,BandwidthAccount > _multicastRateAccounts;
 
-	std::map<MAC,Address> _bridgeRoutes; // remote addresses where given MACs are reachable
-	std::map<MulticastGroup,uint64_t> _bridgedMulticastGroups; // multicast groups of interest on our side of the bridge
+	std::map<Address,CertificateOfMembership> _membershipCertificates; // Other members' certificates of membership
+	std::map<Address,uint64_t> _lastPushedMembershipCertificate; // When did we last push our certificate to each remote member?
 
-	SharedPtr<NetworkConfig> _config;
+	SharedPtr<NetworkConfig> _config; // Most recent network configuration, which is an immutable value-object
 	volatile uint64_t _lastConfigUpdate;
 
 	volatile bool _destroyed;
diff --git a/node/Node.cpp b/node/Node.cpp
index 03749474..8301bd88 100644
--- a/node/Node.cpp
+++ b/node/Node.cpp
@@ -70,7 +70,6 @@
 #include "Network.hpp"
 #include "MulticastGroup.hpp"
 #include "Mutex.hpp"
-#include "Multicaster.hpp"
 #include "Service.hpp"
 #include "SoftwareUpdater.hpp"
 #include "Buffer.hpp"
@@ -113,7 +112,6 @@ struct _NodeImpl
 		delete renv.topology; renv.topology = (Topology *)0;        // now we no longer need routing info
 		delete renv.sm;       renv.sm = (SocketManager *)0;         // close all sockets
 		delete renv.sw;       renv.sw = (Switch *)0;                // order matters less from here down
-		delete renv.mc;       renv.mc = (Multicaster *)0;
 		delete renv.antiRec;  renv.antiRec = (AntiRecursion *)0;
 		delete renv.http;     renv.http = (HttpClient *)0;
 		delete renv.prng;     renv.prng = (CMWC4096 *)0;
@@ -382,7 +380,6 @@ Node::ReasonForTermination Node::run()
 
 		_r->http = new HttpClient();
 		_r->antiRec = new AntiRecursion();
-		_r->mc = new Multicaster();
 		_r->sw = new Switch(_r);
 		_r->sm = new SocketManager(impl->udpPort,impl->tcpPort,&_CBztTraffic,_r);
 		_r->topology = new Topology(_r,Utils::fileExists((_r->homePath + ZT_PATH_SEPARATOR_S + "iddb.d").c_str()));
diff --git a/node/Packet.cpp b/node/Packet.cpp
index 07d40b71..b43f3cad 100644
--- a/node/Packet.cpp
+++ b/node/Packet.cpp
@@ -43,11 +43,13 @@ const char *Packet::verbString(Verb v)
 		case VERB_RENDEZVOUS: return "RENDEZVOUS";
 		case VERB_FRAME: return "FRAME";
 		case VERB_EXT_FRAME: return "EXT_FRAME";
-		case VERB_MULTICAST_FRAME: return "MULTICAST_FRAME";
+		case VERB_P5_MULTICAST_FRAME: return "P5_MULTICAST_FRAME";
 		case VERB_MULTICAST_LIKE: return "MULTICAST_LIKE";
 		case VERB_NETWORK_MEMBERSHIP_CERTIFICATE: return "NETWORK_MEMBERSHIP_CERTIFICATE";
 		case VERB_NETWORK_CONFIG_REQUEST: return "NETWORK_CONFIG_REQUEST";
 		case VERB_NETWORK_CONFIG_REFRESH: return "NETWORK_CONFIG_REFRESH";
+		case VERB_MULTICAST_LONELY: return "MULTICAST_LONELY";
+		case VERB_MULTICAST_FRAME: return "MULTICAST_FRAME";
 	}
 	return "(unknown)";
 }
@@ -64,6 +66,7 @@ const char *Packet::errorString(ErrorCode e)
 		case ERROR_UNSUPPORTED_OPERATION: return "UNSUPPORTED_OPERATION";
 		case ERROR_NEED_MEMBERSHIP_CERTIFICATE: return "NEED_MEMBERSHIP_CERTIFICATE";
 		case ERROR_NETWORK_ACCESS_DENIED_: return "NETWORK_ACCESS_DENIED";
+		case ERROR_UNWANTED_MULTICAST: return "UNWANTED_MULTICAST";
 	}
 	return "(unknown)";
 }
diff --git a/node/Packet.hpp b/node/Packet.hpp
index ddafe4ee..7bcbb821 100644
--- a/node/Packet.hpp
+++ b/node/Packet.hpp
@@ -527,7 +527,7 @@ public:
 		 */
 		VERB_EXT_FRAME = 7,
 
-		/* A multicast frame:
+		/* A multicast frame [old multicast protocol, deprecated]:
 		 *   <[2] 16-bit propagation depth or 0xffff for "do not forward">
 		 *   <[320] propagation FIFO>
 		 *   <[1024] propagation bloom filter>
@@ -593,7 +593,7 @@ public:
 		 * ERROR may be generated if a membership certificate is needed for a
 		 * closed network. Payload will be network ID.
 		 */
-		VERB_MULTICAST_FRAME = 8,
+		VERB_P5_MULTICAST_FRAME = 8,
 
 		/* Announce interest in multicast group(s):
 		 *   <[8] 64-bit network ID>
@@ -657,7 +657,60 @@ public:
 		 * It does not generate an OK or ERROR message, and is treated only as
 		 * a hint to refresh now.
 		 */
-		VERB_NETWORK_CONFIG_REFRESH = 12
+		VERB_NETWORK_CONFIG_REFRESH = 12,
+
+		/* Request endpoints for multicast distribution:
+		 *   <[1] flags>
+		 *   <[8] 64-bit network ID>
+		 *   <[6] MAC address of multicast group being queried>
+		 *   <[4] 32-bit ADI for multicast group being queried>
+		 *   <[2] 16-bit (suggested) max number of multicast peers desired>
+		 *  [<[...] network membership certificate (optional)>]
+		 *
+		 * Flags are:
+		 *   0x01 - network membership certificate is included
+		 *
+		 * This message asks a peer for additional known endpoints that have
+		 * LIKEd a given multicast group. It's sent when the sender wishes
+		 * to send multicast but does not have the desired number of recipient
+		 * peers. (Hence it is "lonely." :)
+		 *
+		 * OK response payload:
+		 *   <[8] 64-bit network ID>
+		 *   <[6] MAC address of multicast group being queried>
+		 *   <[4] 32-bit ADI for multicast group being queried>
+		 *   <[2] 16-bit total number of known members in this multicast group>
+		 *   <[2] 16-bit number of members enumerated in this packet>
+		 *   <[...] series of 5-byte ZeroTier addresses of enumerated members>
+		 *
+		 * ERROR response payload:
+		 *   <[8] 64-bit network ID>
+		 *   <[6] MAC address of multicast group being queried>
+		 *   <[4] 32-bit ADI for multicast group being queried>
+		 *
+		 * ERRORs are optional and are only generated if permission is denied,
+		 * certificate of membership is out of date, etc.
+		 */
+		VERB_MULTICAST_LONELY = 13,
+
+		/* Multicast frame:
+		 *   <[1] flags (currently unused, must be 0)>
+		 *   <[4] 32-bit multicast ADI (note that this is out of order here -- it precedes MAC)>
+		 *   <[6] destination MAC or all zero for destination node>
+		 *   <[6] source MAC or all zero for node of origin>
+		 *   <[2] 16-bit ethertype>
+		 *   <[...] ethernet payload>
+		 *
+		 * This is similar to EXT_FRAME but carries a multicast, and is sent
+		 * out to recipients on a multicast list.
+		 *
+		 * OK is not generated.
+		 *
+		 * ERROR response payload:
+		 *   <[6] multicast group MAC>
+		 *   <[4] 32-bit multicast group ADI>
+		 */
+		VERB_MULTICAST_FRAME = 14
 	};
 
 	/**
@@ -687,7 +740,10 @@ public:
 		ERROR_NEED_MEMBERSHIP_CERTIFICATE = 6,
 
 		/* Tried to join network, but you're not a member */
-		ERROR_NETWORK_ACCESS_DENIED_ = 7 /* extra _ to avoid Windows name conflict */
+		ERROR_NETWORK_ACCESS_DENIED_ = 7, /* extra _ to avoid Windows name conflict */
+
+		/* Multicasts to this group are not wanted */
+		ERROR_UNWANTED_MULTICAST = 8
 	};
 
 	/**
diff --git a/node/RuntimeEnvironment.hpp b/node/RuntimeEnvironment.hpp
index 767dd564..83932bc6 100644
--- a/node/RuntimeEnvironment.hpp
+++ b/node/RuntimeEnvironment.hpp
@@ -42,7 +42,6 @@ class Topology;
 class CMWC4096;
 class Service;
 class Node;
-class Multicaster;
 class SoftwareUpdater;
 class SocketManager;
 class AntiRecursion;
@@ -79,7 +78,6 @@ public:
 		prng((CMWC4096 *)0),
 		http((HttpClient *)0),
 		antiRec((AntiRecursion *)0),
-		mc((Multicaster *)0),
 		sw((Switch *)0),
 		sm((SocketManager *)0),
 		topology((Topology *)0),
@@ -129,7 +127,6 @@ public:
 	CMWC4096 *prng;
 	HttpClient *http;
 	AntiRecursion *antiRec;
-	Multicaster *mc;
 	Switch *sw;
 	SocketManager *sm;
 	Topology *topology;
diff --git a/node/Switch.cpp b/node/Switch.cpp
index dd4aec21..22b698bf 100644
--- a/node/Switch.cpp
+++ b/node/Switch.cpp
@@ -151,6 +151,7 @@ void Switch::onLocalEthernet(const SharedPtr<Network> &network,const MAC &from,c
 
 		TRACE("%s: MULTICAST %s -> %s %s %d",network->tapDeviceName().c_str(),from.toString().c_str(),mg.toString().c_str(),etherTypeName(etherType),(int)data.size());
 
+		/* old P5 multicast algorithm
 		const unsigned int mcid = ++_multicastIdCounter & 0xffffff;
 		const uint16_t bloomNonce = (uint16_t)(_r->prng->next32() & 0xffff); // doesn't need to be cryptographically strong
 		unsigned char bloom[ZT_PROTO_VERB_MULTICAST_FRAME_LEN_PROPAGATION_BLOOM];
@@ -226,6 +227,7 @@ void Switch::onLocalEthernet(const SharedPtr<Network> &network,const MAC &from,c
 			outp.compress();
 			send(outp,true);
 		}
+		*/
 
 		return;
 	}
diff --git a/node/Switch.hpp b/node/Switch.hpp
index 64457109..4cc34d46 100644
--- a/node/Switch.hpp
+++ b/node/Switch.hpp
@@ -44,7 +44,6 @@
 #include "Array.hpp"
 #include "Network.hpp"
 #include "SharedPtr.hpp"
-#include "Multicaster.hpp"
 #include "PacketDecoder.hpp"
 #include "Socket.hpp"