From c6a918d9962dcf2354483b709b8bf0fffbbc3983 Mon Sep 17 00:00:00 2001
From: Adam Ierymenko <adam.ierymenko@gmail.com>
Date: Wed, 28 Oct 2015 12:50:48 -0700
Subject: HTTP test code.

---
 tests/http/README.md    |   5 ++
 tests/http/agent.js     | 224 ++++++++++++++++++++++++++++++++++++++++++++++++
 tests/http/package.json |  16 ++++
 tests/http/server.js    |  48 +++++++++++
 4 files changed, 293 insertions(+)
 create mode 100644 tests/http/README.md
 create mode 100644 tests/http/agent.js
 create mode 100644 tests/http/package.json
 create mode 100644 tests/http/server.js

(limited to 'tests')

diff --git a/tests/http/README.md b/tests/http/README.md
new file mode 100644
index 00000000..ae7f08f1
--- /dev/null
+++ b/tests/http/README.md
@@ -0,0 +1,5 @@
+HTTP one-to-all test
+======
+
+This code can be deployed across a large number of VMs or containers to test and benchmark HTTP traffic within a virtual network at scale. The agent acts as a server and can query other agents, while the server collects agent data and tells agents about each other. It's designed to use RFC4193-based ZeroTier IPv6 addresses within the cluster, which allows the easy provisioning of a large cluster without IP conflicts.
+
diff --git a/tests/http/agent.js b/tests/http/agent.js
new file mode 100644
index 00000000..14964d87
--- /dev/null
+++ b/tests/http/agent.js
@@ -0,0 +1,224 @@
+// ---------------------------------------------------------------------------
+// Customizable parameters:
+
+// How frequently in ms to run tests
+//var RUN_TEST_EVERY = (60 * 5 * 1000);
+var RUN_TEST_EVERY = 1000;
+
+// Maximum test duration in milliseconds (must be less than RUN_TEST_EVERY)
+var TEST_DURATION = (60 * 1000);
+
+// Where should I contact to register and query a list of other nodes?
+var SERVER_HOST = '127.0.0.1';
+var SERVER_PORT = 18080;
+
+// Which port should agents use for their HTTP?
+var AGENT_PORT = 18888;
+
+// Payload size in bytes
+var PAYLOAD_SIZE = 4096;
+
+// ---------------------------------------------------------------------------
+
+var ipaddr = require('ipaddr.js');
+var os = require('os');
+var http = require('http');
+var async = require('async');
+
+var express = require('express');
+var app = express();
+
+// Find our ZeroTier-assigned RFC4193 IPv6 address
+var thisAgentId = null;
+var interfaces = os.networkInterfaces();
+if (!interfaces) {
+	console.error('FATAL: os.networkInterfaces() failed.');
+	process.exit(1);
+}
+for(var ifname in interfaces) {
+	var ifaddrs = interfaces[ifname];
+	if (Array.isArray(ifaddrs)) {
+		for(var i=0;i<ifaddrs.length;++i) {
+			if (ifaddrs[i].family == 'IPv6') {
+				try {
+					var ipbytes = ipaddr.parse(ifaddrs[i].address).toByteArray();
+					if ((ipbytes.length === 16)&&(ipbytes[0] == 0xfd)&&(ipbytes[9] == 0x99)&&(ipbytes[10] == 0x93)) {
+						thisAgentId = '';
+						for(var j=0;j<16;++j) {
+							var tmp = ipbytes[j].toString(16);
+							if (tmp.length === 1)
+								thisAgentId += '0';
+							thisAgentId += tmp;
+						}
+					}
+				} catch (e) {
+					console.error(e);
+				}
+			}
+		}
+	}
+}
+if (thisAgentId === null) {
+	console.error('FATAL: no ZeroTier-assigned RFC4193 IPv6 addresses found on any local interface!');
+	process.exit(1);
+}
+
+//console.log(thisAgentId);
+
+// Create a random (and therefore not very compressable) payload
+var payload = '';
+while (payload.length < PAYLOAD_SIZE) {
+	payload += String.fromCharCode(Math.round(Math.random() * 255.0));
+}
+
+// Incremented for each test
+var testCounter = 0;
+
+function registerAndGetPeers(callback)
+{
+	http.get({
+		host: SERVER_HOST,
+		port: SERVER_PORT,
+		path: '/'+thisAgentId
+	},function(res) {
+		var body = '';
+		res.on('data',function(chunk) { body += chunk.toString(); });
+		res.on('end',function() {
+			try {
+				var peers = JSON.parse(body);
+				if (Array.isArray(peers))
+					return callback(null,peers);
+				else return callback(new Error('invalid JSON response from server'),null);
+			} catch (e) {
+				return callback(new Error('invalid JSON response from server'),null);
+			}
+		});
+	}).on('error',function(e) {
+		return callback(e,null);
+	});
+};
+
+function performTestOnAllPeers(peers,callback)
+{
+	var allResults = {};
+	var timedOut = false;
+	var endOfTestTimer = setTimeout(function() {
+		timedOut = true;
+		return callback(allResults);
+	},TEST_DURATION);
+	var testStartTime = Date.now();
+
+	async.each(peers,function(peer,next) {
+		if (timedOut)
+			return next(null);
+		if (peer.length !== 32)
+			return next(null);
+
+		var connectionStartTime = Date.now();
+		allResults[peer] = {
+			testStart: testStartTime,
+			start: connectionStartTime,
+			end: null,
+			error: null,
+			bytes: 0,
+			test: testCounter
+		};
+
+		var peerHost = '';
+		peerHost += peer.substr(0,4);
+		peerHost += ':';
+		peerHost += peer.substr(4,4);
+		peerHost += ':';
+		peerHost += peer.substr(8,4);
+		peerHost += ':';
+		peerHost += peer.substr(12,4);
+		peerHost += ':';
+		peerHost += peer.substr(16,4);
+		peerHost += ':';
+		peerHost += peer.substr(20,4);
+		peerHost += ':';
+		peerHost += peer.substr(24,4);
+		peerHost += ':';
+		peerHost += peer.substr(28,4);
+
+		http.get({
+			host: peerHost,
+			port: AGENT_PORT,
+			path: '/'
+		},function(res) {
+			var bytes = 0;
+			res.on('data',function(chunk) { bytes += chunk.length; });
+			res.on('end',function() {
+				if (timedOut)
+					return next(null);
+				allResults[peer] = {
+					testStart: testStartTime,
+					start: connectionStartTime,
+					end: Date.now(),
+					error: null,
+					bytes: bytes,
+					test: testCounter
+				};
+				return next(null);
+			});
+		}).on('error',function(e) {
+			if (timedOut)
+				return next(null);
+			allResults[peer] = {
+				testStart: testStartTime,
+				start: connectionStartTime,
+				end: Date.now(),
+				error: e.toString(),
+				bytes: 0,
+				test: testCounter
+			};
+			return next(null);
+		});
+	},function(err) {
+		if (!timedOut) {
+			clearTimeout(endOfTestTimer);
+			return callback(allResults);
+		}
+	});
+};
+
+// Agents just serve up a test payload
+app.get('/',function(req,res) {
+	return res.status(200).send(payload);
+});
+
+var expressServer = app.listen(AGENT_PORT,function () {
+	registerAndGetPeers(function(err,peers) {
+		if (err) {
+			console.error('FATAL: unable to contact or query server: '+err.toString());
+			process.exit(1);
+		}
+
+		setInterval(function() {
+			++testCounter;
+
+			registerAndGetPeers(function(err,peers) {
+				if (err) {
+					console.error('WARNING: unable to contact or query server, test aborted: '+err.toString());
+					return;
+				}
+
+				performTestOnAllPeers(peers,function(results) {
+					console.log(results);
+
+					var submit = http.request({
+						host: SERVER_HOST,
+						port: SERVER_PORT,
+						path: '/'+thisAgentId,
+						method: 'POST'
+					},function(res) {
+					}).on('error',function(e) {
+						console.error('WARNING: unable to submit results to server: '+err.toString());
+					});
+					submit.write(JSON.stringify(results));
+					submit.end();
+				});
+			});
+		},RUN_TEST_EVERY);
+	});
+});
diff --git a/tests/http/package.json b/tests/http/package.json
new file mode 100644
index 00000000..173a6f99
--- /dev/null
+++ b/tests/http/package.json
@@ -0,0 +1,16 @@
+{
+  "name": "zerotier-test-http",
+  "version": "1.0.0",
+  "description": "ZeroTier in-network HTTP test",
+  "main": "agent.js",
+  "scripts": {
+    "test": "echo \"Error: no test specified\" && exit 1"
+  },
+  "author": "ZeroTier, Inc.",
+  "license": "GPL-3.0",
+  "dependencies": {
+    "async": "^1.5.0",
+    "express": "^4.13.3",
+    "ipaddr.js": "^1.0.3"
+  }
+}
diff --git a/tests/http/server.js b/tests/http/server.js
new file mode 100644
index 00000000..221dcda9
--- /dev/null
+++ b/tests/http/server.js
@@ -0,0 +1,48 @@
+// ---------------------------------------------------------------------------
+// Customizable parameters:
+
+var SERVER_PORT = 18080;
+
+// ---------------------------------------------------------------------------
+
+var express = require('express');
+var app = express();
+
+app.use(function(req,res,next) {
+	req.rawBody = '';
+	req.on('data', function(chunk) { req.rawBody += chunk.toString(); });
+	req.on('end', function() { return next(); });
+});
+
+var knownAgents = {};
+
+app.get('/:agentId',function(req,res) {
+	var agentId = req.params.agentId;
+	if ((!agentId)||(agentId.length !== 32))
+		return res.status(404).send('');
+	knownAgents[agentId] = Date.now();
+	return res.status(200).send(JSON.stringify(Object.keys(knownAgents)));
+});
+
+app.post('/:agentId',function(req,res) {
+	var agentId = req.params.agentId;
+	if ((!agentId)||(agentId.length !== 32))
+		return res.status(404).send('');
+	var resultData = null;
+	try {
+		resultData = JSON.parse(req.rawBody);
+	} catch (e) {
+		resultData = req.rawBody;
+	}
+	result = {
+		agentId: agentId,
+		result: resultData
+	};
+	console.log(result);
+	return res.status(200).send('');
+});
+
+var expressServer = app.listen(SERVER_PORT,function () {
+	console.log('LISTENING ON '+SERVER_PORT);
+	console.log('');
+});
-- 
cgit v1.2.3


From c03550de3598e3c55ea6c181148286b8673b6df1 Mon Sep 17 00:00:00 2001
From: Adam Ierymenko <adam.ierymenko@gmail.com>
Date: Wed, 28 Oct 2015 13:14:53 -0700
Subject: HTTP test works!

---
 tests/http/agent.js  | 16 +++++++++-------
 tests/http/server.js | 14 +++++++++++++-
 2 files changed, 22 insertions(+), 8 deletions(-)

(limited to 'tests')

diff --git a/tests/http/agent.js b/tests/http/agent.js
index 14964d87..34837411 100644
--- a/tests/http/agent.js
+++ b/tests/http/agent.js
@@ -9,14 +9,14 @@ var RUN_TEST_EVERY = 1000;
 var TEST_DURATION = (60 * 1000);
 
 // Where should I contact to register and query a list of other nodes?
-var SERVER_HOST = '127.0.0.1';
+var SERVER_HOST = '174.136.102.178';
 var SERVER_PORT = 18080;
 
 // Which port should agents use for their HTTP?
 var AGENT_PORT = 18888;
 
 // Payload size in bytes
-var PAYLOAD_SIZE = 4096;
+var PAYLOAD_SIZE = 100000;
 
 // ---------------------------------------------------------------------------
 
@@ -66,9 +66,9 @@ if (thisAgentId === null) {
 //console.log(thisAgentId);
 
 // Create a random (and therefore not very compressable) payload
-var payload = '';
-while (payload.length < PAYLOAD_SIZE) {
-	payload += String.fromCharCode(Math.round(Math.random() * 255.0));
+var payload = new Buffer(PAYLOAD_SIZE);
+for(var xx=0;xx<PAYLOAD_SIZE;++xx) {
+	payload.writeUInt8(Math.round(Math.random() * 255.0),xx);
 }
 
 // Incremented for each test
@@ -147,7 +147,9 @@ function performTestOnAllPeers(peers,callback)
 			path: '/'
 		},function(res) {
 			var bytes = 0;
-			res.on('data',function(chunk) { bytes += chunk.length; });
+			res.on('data',function(chunk) {
+				bytes += chunk.length;
+			});
 			res.on('end',function() {
 				if (timedOut)
 					return next(null);
@@ -204,7 +206,7 @@ var expressServer = app.listen(AGENT_PORT,function () {
 				}
 
 				performTestOnAllPeers(peers,function(results) {
-					console.log(results);
+					//console.log(results);
 
 					var submit = http.request({
 						host: SERVER_HOST,
diff --git a/tests/http/server.js b/tests/http/server.js
index 221dcda9..a58756bc 100644
--- a/tests/http/server.js
+++ b/tests/http/server.js
@@ -5,6 +5,8 @@ var SERVER_PORT = 18080;
 
 // ---------------------------------------------------------------------------
 
+var fs = require('fs');
+
 var express = require('express');
 var app = express();
 
@@ -28,6 +30,8 @@ app.post('/:agentId',function(req,res) {
 	var agentId = req.params.agentId;
 	if ((!agentId)||(agentId.length !== 32))
 		return res.status(404).send('');
+
+	var receiveTime = Date.now();
 	var resultData = null;
 	try {
 		resultData = JSON.parse(req.rawBody);
@@ -36,9 +40,17 @@ app.post('/:agentId',function(req,res) {
 	}
 	result = {
 		agentId: agentId,
+		receiveTime: receiveTime,
 		result: resultData
 	};
-	console.log(result);
+
+	var nows = receiveTime.toString(16);
+	while (nows.length < 16)
+		nows = '0' + nows;
+	fs.writeFile('result_'+agentId+'_'+nows,JSON.stringify(result),function(err) {
+		console.log(result);
+	});
+
 	return res.status(200).send('');
 });
 
-- 
cgit v1.2.3


From 1cae7327aeabeb4546a302c00985fb9413b63443 Mon Sep 17 00:00:00 2001
From: Adam Ierymenko <adam.ierymenko@gmail.com>
Date: Wed, 28 Oct 2015 13:35:52 -0700
Subject: Basic Dockerfile for building test agents.

---
 .gitignore                |  5 ++---
 tests/http/Dockerfile     | 23 +++++++++++++++++++++++
 tests/http/docker-main.sh |  6 ++++++
 3 files changed, 31 insertions(+), 3 deletions(-)
 create mode 100644 tests/http/Dockerfile
 create mode 100644 tests/http/docker-main.sh

(limited to 'tests')

diff --git a/.gitignore b/.gitignore
index 06b06b7d..89ab049f 100755
--- a/.gitignore
+++ b/.gitignore
@@ -34,7 +34,7 @@ Thumbs.db
 /world/mkworld
 /world/*.c25519
 
-# Miscellaneous file types that we don't want to check in
+# Miscellaneous temporaries, build files, etc.
 *.log
 *.opensdf
 *.user
@@ -50,10 +50,9 @@ Thumbs.db
 *.autosave
 *.tmp
 node_modules
-
-# cluster-geo stuff
 cluster-geo/cluster-geo/config.js
 cluster-geo/cluster-geo/cache.*
+tests/http/zerotier-one
 
 # MacGap wrapper build files
 /ext/mac-ui-macgap1-wrapper/src/MacGap.xcodeproj/project.xcworkspace/xcuserdata/*
diff --git a/tests/http/Dockerfile b/tests/http/Dockerfile
new file mode 100644
index 00000000..02578cd5
--- /dev/null
+++ b/tests/http/Dockerfile
@@ -0,0 +1,23 @@
+FROM centos:latest
+
+MAINTAINER https://www.zerotier.com/
+
+EXPOSE 9993/udp
+
+RUN yum -y update && yum -y install epel-release && yum -y install nodejs npm && yum clean all
+
+RUN mkdir -p /var/lib/zerotier-one
+RUN mkdir -p /var/lib/zerotier-one/networks.d
+RUN touch /var/lib/zerotier-one/networks.d/ffffffffffffffff.conf
+
+ADD package.json /
+RUN npm install
+
+ADD zerotier-one /
+RUN chmod a+x /zerotier-one
+
+ADD agent.js /
+ADD main.sh /
+RUN chmod a+x /docker-main.sh
+
+CMD ["./docker-main.sh"]
diff --git a/tests/http/docker-main.sh b/tests/http/docker-main.sh
new file mode 100644
index 00000000..947ccf47
--- /dev/null
+++ b/tests/http/docker-main.sh
@@ -0,0 +1,6 @@
+#!/bin/bash
+
+export PATH=/bin:/usr/bin:/usr/local/bin:/sbin:/usr/sbin
+
+/zerotier-one -d
+exec node --harmony /agent.js
-- 
cgit v1.2.3


From 07c1b4ddee1b6ba1a4399dc62c8e3c6f5367afa0 Mon Sep 17 00:00:00 2001
From: Adam Ierymenko <adam.ierymenko@gmail.com>
Date: Wed, 28 Oct 2015 14:16:58 -0700
Subject: test stuff

---
 tests/http/Dockerfile     |   2 +-
 tests/http/agent.js       | 148 ++++++++++++++++++++++------------------------
 tests/http/docker-main.sh |   7 ++-
 tests/http/server.js      |   6 +-
 4 files changed, 83 insertions(+), 80 deletions(-)
 mode change 100644 => 100755 tests/http/docker-main.sh

(limited to 'tests')

diff --git a/tests/http/Dockerfile b/tests/http/Dockerfile
index 02578cd5..7bba2fc0 100644
--- a/tests/http/Dockerfile
+++ b/tests/http/Dockerfile
@@ -17,7 +17,7 @@ ADD zerotier-one /
 RUN chmod a+x /zerotier-one
 
 ADD agent.js /
-ADD main.sh /
+ADD docker-main.sh /
 RUN chmod a+x /docker-main.sh
 
 CMD ["./docker-main.sh"]
diff --git a/tests/http/agent.js b/tests/http/agent.js
index 34837411..465a2e28 100644
--- a/tests/http/agent.js
+++ b/tests/http/agent.js
@@ -1,14 +1,13 @@
 // ---------------------------------------------------------------------------
 // Customizable parameters:
 
-// How frequently in ms to run tests
-//var RUN_TEST_EVERY = (60 * 5 * 1000);
-var RUN_TEST_EVERY = 1000;
+// Maximum test duration in milliseconds
+var TEST_DURATION = (30 * 1000);
 
-// Maximum test duration in milliseconds (must be less than RUN_TEST_EVERY)
-var TEST_DURATION = (60 * 1000);
+// Interval between tests (should be several times longer than TEST_DURATION)
+var TEST_INTERVAL = (60 * 2 * 1000);
 
-// Where should I contact to register and query a list of other nodes?
+// Where should I contact to register and query a list of other test agents?
 var SERVER_HOST = '174.136.102.178';
 var SERVER_PORT = 18080;
 
@@ -71,8 +70,29 @@ for(var xx=0;xx<PAYLOAD_SIZE;++xx) {
 	payload.writeUInt8(Math.round(Math.random() * 255.0),xx);
 }
 
-// Incremented for each test
-var testCounter = 0;
+// Incremented with each test
+var testNumber = 0;
+
+function agentIdToIp(agentId)
+{
+	var ip = '';
+	ip += agentId.substr(0,4);
+	ip += ':';
+	ip += agentId.substr(4,4);
+	ip += ':';
+	ip += agentId.substr(8,4);
+	ip += ':';
+	ip += agentId.substr(12,4);
+	ip += ':';
+	ip += agentId.substr(16,4);
+	ip += ':';
+	ip += agentId.substr(20,4);
+	ip += ':';
+	ip += agentId.substr(24,4);
+	ip += ':';
+	ip += agentId.substr(28,4);
+	return ip;
+};
 
 function registerAndGetPeers(callback)
 {
@@ -84,13 +104,15 @@ function registerAndGetPeers(callback)
 		var body = '';
 		res.on('data',function(chunk) { body += chunk.toString(); });
 		res.on('end',function() {
+			if (!body)
+				return callback(null,[]);
 			try {
 				var peers = JSON.parse(body);
 				if (Array.isArray(peers))
 					return callback(null,peers);
 				else return callback(new Error('invalid JSON response from server'),null);
 			} catch (e) {
-				return callback(new Error('invalid JSON response from server'),null);
+				return callback(new Error('invalid JSON response from server: '+e.toString()),null);
 			}
 		});
 	}).on('error',function(e) {
@@ -101,12 +123,13 @@ function registerAndGetPeers(callback)
 function performTestOnAllPeers(peers,callback)
 {
 	var allResults = {};
+	var allRequests = [];
 	var timedOut = false;
 	var endOfTestTimer = setTimeout(function() {
 		timedOut = true;
-		return callback(allResults);
+		for(var x=0;x<allRequests.length;++x)
+			allRequests[x].abort();
 	},TEST_DURATION);
-	var testStartTime = Date.now();
 
 	async.each(peers,function(peer,next) {
 		if (timedOut)
@@ -116,33 +139,15 @@ function performTestOnAllPeers(peers,callback)
 
 		var connectionStartTime = Date.now();
 		allResults[peer] = {
-			testStart: testStartTime,
 			start: connectionStartTime,
-			end: null,
+			end: 0,
 			error: null,
-			bytes: 0,
-			test: testCounter
+			timedOut: false,
+			bytes: 0
 		};
 
-		var peerHost = '';
-		peerHost += peer.substr(0,4);
-		peerHost += ':';
-		peerHost += peer.substr(4,4);
-		peerHost += ':';
-		peerHost += peer.substr(8,4);
-		peerHost += ':';
-		peerHost += peer.substr(12,4);
-		peerHost += ':';
-		peerHost += peer.substr(16,4);
-		peerHost += ':';
-		peerHost += peer.substr(20,4);
-		peerHost += ':';
-		peerHost += peer.substr(24,4);
-		peerHost += ':';
-		peerHost += peer.substr(28,4);
-
-		http.get({
-			host: peerHost,
+		allRequests.push(http.get({
+			host: agentIdToIp(peer),
 			port: AGENT_PORT,
 			path: '/'
 		},function(res) {
@@ -151,76 +156,67 @@ function performTestOnAllPeers(peers,callback)
 				bytes += chunk.length;
 			});
 			res.on('end',function() {
-				if (timedOut)
-					return next(null);
 				allResults[peer] = {
-					testStart: testStartTime,
 					start: connectionStartTime,
 					end: Date.now(),
 					error: null,
-					bytes: bytes,
-					test: testCounter
+					timedOut: timedOut,
+					bytes: bytes
 				};
 				return next(null);
 			});
 		}).on('error',function(e) {
-			if (timedOut)
-				return next(null);
 			allResults[peer] = {
-				testStart: testStartTime,
 				start: connectionStartTime,
 				end: Date.now(),
 				error: e.toString(),
-				bytes: 0,
-				test: testCounter
+				timedOut: timedOut,
+				bytes: 0
 			};
 			return next(null);
-		});
+		}));
 	},function(err) {
-		if (!timedOut) {
+		if (!timedOut)
 			clearTimeout(endOfTestTimer);
-			return callback(allResults);
-		}
+		return callback(allResults);
 	});
 };
 
+function doTestsAndReport()
+{
+	registerAndGetPeers(function(err,peers) {
+		if (err) {
+			console.error('WARNING: skipping test: unable to contact or query server: '+err.toString());
+		} else {
+			performTestOnAllPeers(peers,function(results) {
+				++testNumber;
+				var submit = http.request({
+					host: SERVER_HOST,
+					port: SERVER_PORT,
+					path: '/'+testNumber+'/'+thisAgentId,
+					method: 'POST'
+				},function(res) {
+				}).on('error',function(e) {
+					console.error('WARNING: unable to submit results to server: '+err.toString());
+				});
+				submit.write(JSON.stringify(results));
+				submit.end();
+			});
+		}
+	});
+}
+
 // Agents just serve up a test payload
 app.get('/',function(req,res) {
 	return res.status(200).send(payload);
 });
 
-var expressServer = app.listen(AGENT_PORT,function () {
+var expressServer = app.listen(AGENT_PORT,agentIdToIp(thisAgentId),function () {
 	registerAndGetPeers(function(err,peers) {
 		if (err) {
 			console.error('FATAL: unable to contact or query server: '+err.toString());
 			process.exit(1);
 		}
-
-		setInterval(function() {
-			++testCounter;
-
-			registerAndGetPeers(function(err,peers) {
-				if (err) {
-					console.error('WARNING: unable to contact or query server, test aborted: '+err.toString());
-					return;
-				}
-
-				performTestOnAllPeers(peers,function(results) {
-					//console.log(results);
-
-					var submit = http.request({
-						host: SERVER_HOST,
-						port: SERVER_PORT,
-						path: '/'+thisAgentId,
-						method: 'POST'
-					},function(res) {
-					}).on('error',function(e) {
-						console.error('WARNING: unable to submit results to server: '+err.toString());
-					});
-					submit.write(JSON.stringify(results));
-					submit.end();
-				});
-			});
-		},RUN_TEST_EVERY);
+		setInterval(doTestsAndReport,TEST_INTERVAL);
 	});
 });
diff --git a/tests/http/docker-main.sh b/tests/http/docker-main.sh
old mode 100644
new mode 100755
index 947ccf47..ad80af0c
--- a/tests/http/docker-main.sh
+++ b/tests/http/docker-main.sh
@@ -3,4 +3,9 @@
 export PATH=/bin:/usr/bin:/usr/local/bin:/sbin:/usr/sbin
 
 /zerotier-one -d
-exec node --harmony /agent.js
+
+while [ ! -d "/proc/sys/net/ipv6/conf/zt0" ]; do
+	sleep 0.25
+done
+
+exec node --harmony /agent.js >>agent.out 2>&1
diff --git a/tests/http/server.js b/tests/http/server.js
index a58756bc..6211b4ee 100644
--- a/tests/http/server.js
+++ b/tests/http/server.js
@@ -26,7 +26,8 @@ app.get('/:agentId',function(req,res) {
 	return res.status(200).send(JSON.stringify(Object.keys(knownAgents)));
 });
 
-app.post('/:agentId',function(req,res) {
+app.post('/:testNumber/:agentId',function(req,res) {
+	var testNumber = req.params.testNumber;
 	var agentId = req.params.agentId;
 	if ((!agentId)||(agentId.length !== 32))
 		return res.status(404).send('');
@@ -40,8 +41,9 @@ app.post('/:agentId',function(req,res) {
 	}
 	result = {
 		agentId: agentId,
+		testNumber: testNumber,
 		receiveTime: receiveTime,
-		result: resultData
+		results: resultData
 	};
 
 	var nows = receiveTime.toString(16);
-- 
cgit v1.2.3


From 9653531242dd5f66e331fc716c4aacd1aece30c5 Mon Sep 17 00:00:00 2001
From: Adam Ierymenko <adam.ierymenko@gmail.com>
Date: Wed, 28 Oct 2015 14:18:58 -0700
Subject: .

---
 tests/http/agent.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'tests')

diff --git a/tests/http/agent.js b/tests/http/agent.js
index 465a2e28..53b2e9f9 100644
--- a/tests/http/agent.js
+++ b/tests/http/agent.js
@@ -204,7 +204,7 @@ function doTestsAndReport()
 			});
 		}
 	});
-}
+};
 
 // Agents just serve up a test payload
 app.get('/',function(req,res) {
-- 
cgit v1.2.3


From 4c24e0cfb0bb9b61a4e19ac81b89dc1cbce6ea99 Mon Sep 17 00:00:00 2001
From: Adam Ierymenko <adam.ierymenko@gmail.com>
Date: Wed, 28 Oct 2015 14:24:54 -0700
Subject: More tweaks to tests... just about ready to run at scale.

---
 .gitignore           |  1 +
 tests/http/agent.js  |  1 +
 tests/http/server.js | 10 +++++-----
 3 files changed, 7 insertions(+), 5 deletions(-)

(limited to 'tests')

diff --git a/.gitignore b/.gitignore
index 89ab049f..87e387a6 100755
--- a/.gitignore
+++ b/.gitignore
@@ -53,6 +53,7 @@ node_modules
 cluster-geo/cluster-geo/config.js
 cluster-geo/cluster-geo/cache.*
 tests/http/zerotier-one
+tests/http/result_*
 
 # MacGap wrapper build files
 /ext/mac-ui-macgap1-wrapper/src/MacGap.xcodeproj/project.xcworkspace/xcuserdata/*
diff --git a/tests/http/agent.js b/tests/http/agent.js
index 53b2e9f9..061c7ba7 100644
--- a/tests/http/agent.js
+++ b/tests/http/agent.js
@@ -217,6 +217,7 @@ var expressServer = app.listen(AGENT_PORT,agentIdToIp(thisAgentId),function () {
 			console.error('FATAL: unable to contact or query server: '+err.toString());
 			process.exit(1);
 		}
+		doTestsAndReport();
 		setInterval(doTestsAndReport,TEST_INTERVAL);
 	});
 });
diff --git a/tests/http/server.js b/tests/http/server.js
index 6211b4ee..69ccf527 100644
--- a/tests/http/server.js
+++ b/tests/http/server.js
@@ -41,15 +41,15 @@ app.post('/:testNumber/:agentId',function(req,res) {
 	}
 	result = {
 		agentId: agentId,
-		testNumber: testNumber,
+		testNumber: parseInt(testNumber),
 		receiveTime: receiveTime,
 		results: resultData
 	};
 
-	var nows = receiveTime.toString(16);
-	while (nows.length < 16)
-		nows = '0' + nows;
-	fs.writeFile('result_'+agentId+'_'+nows,JSON.stringify(result),function(err) {
+	testNumber = testNumber.toString();
+	while (testNumber.length < 10)
+		testNumber = '0' + testNumber;
+	fs.writeFile('result_'+testNumber+'_'+agentId,JSON.stringify(result),function(err) {
 		console.log(result);
 	});
 
-- 
cgit v1.2.3


From 1f5ef968cff51721d97587cb1c5402c988c92d5e Mon Sep 17 00:00:00 2001
From: Adam Ierymenko <adam.ierymenko@gmail.com>
Date: Wed, 28 Oct 2015 15:08:00 -0700
Subject: Test need a more recent version of NodeJS so update Dockerfile.

---
 tests/http/Dockerfile         | 3 ++-
 tests/http/agent.js           | 2 +-
 tests/http/docker-main.sh     | 3 +++
 tests/http/nodesource-el.repo | 6 ++++++
 4 files changed, 12 insertions(+), 2 deletions(-)
 create mode 100644 tests/http/nodesource-el.repo

(limited to 'tests')

diff --git a/tests/http/Dockerfile b/tests/http/Dockerfile
index 7bba2fc0..e19b3fee 100644
--- a/tests/http/Dockerfile
+++ b/tests/http/Dockerfile
@@ -4,7 +4,8 @@ MAINTAINER https://www.zerotier.com/
 
 EXPOSE 9993/udp
 
-RUN yum -y update && yum -y install epel-release && yum -y install nodejs npm && yum clean all
+ADD nodesource-el.repo /etc/yum.repos.d/nodesource-el.repo
+RUN yum -y update && yum install -y nodejs && yum clean all
 
 RUN mkdir -p /var/lib/zerotier-one
 RUN mkdir -p /var/lib/zerotier-one/networks.d
diff --git a/tests/http/agent.js b/tests/http/agent.js
index 061c7ba7..8b1cf512 100644
--- a/tests/http/agent.js
+++ b/tests/http/agent.js
@@ -211,7 +211,7 @@ app.get('/',function(req,res) {
 	return res.status(200).send(payload);
 });
 
-var expressServer = app.listen(AGENT_PORT,agentIdToIp(thisAgentId),function () {
+var expressServer = app.listen(AGENT_PORT,function () {
 	registerAndGetPeers(function(err,peers) {
 		if (err) {
 			console.error('FATAL: unable to contact or query server: '+err.toString());
diff --git a/tests/http/docker-main.sh b/tests/http/docker-main.sh
index ad80af0c..72023668 100755
--- a/tests/http/docker-main.sh
+++ b/tests/http/docker-main.sh
@@ -8,4 +8,7 @@ while [ ! -d "/proc/sys/net/ipv6/conf/zt0" ]; do
 	sleep 0.25
 done
 
+sleep 2
+
 exec node --harmony /agent.js >>agent.out 2>&1
+#exec node --harmony /agent.js
diff --git a/tests/http/nodesource-el.repo b/tests/http/nodesource-el.repo
new file mode 100644
index 00000000..b785d3d0
--- /dev/null
+++ b/tests/http/nodesource-el.repo
@@ -0,0 +1,6 @@
+[nodesource]
+name=Node.js Packages for Enterprise Linux 7 - $basearch
+baseurl=https://rpm.nodesource.com/pub_4.x/el/7/$basearch
+failovermethod=priority
+enabled=1
+gpgcheck=0
-- 
cgit v1.2.3


From cabb8752cb9d9c85dc2aa2cac22a4ad101614577 Mon Sep 17 00:00:00 2001
From: Adam Ierymenko <adam.ierymenko@gmail.com>
Date: Wed, 28 Oct 2015 15:28:05 -0700
Subject: docs

---
 tests/http/README.md | 3 +++
 1 file changed, 3 insertions(+)

(limited to 'tests')

diff --git a/tests/http/README.md b/tests/http/README.md
index ae7f08f1..7852ac16 100644
--- a/tests/http/README.md
+++ b/tests/http/README.md
@@ -3,3 +3,6 @@ HTTP one-to-all test
 
 This code can be deployed across a large number of VMs or containers to test and benchmark HTTP traffic within a virtual network at scale. The agent acts as a server and can query other agents, while the server collects agent data and tells agents about each other. It's designed to use RFC4193-based ZeroTier IPv6 addresses within the cluster, which allows the easy provisioning of a large cluster without IP conflicts.
 
+A Dockerfile is also included which will build a simple Docker image that runs the agent. The image must be launched with "--device=/dev/net/tun --privileged" to permit it to open a tun/tap device within the container.
+
+Before using this code you will want to edit agent.js to change SERVER_HOST to the IP address of where you will run server.js. This should typically be an open Internet IP, since this makes reporting not dependent upon the thing being tested. Also note that this thing does no security of any kind. It's designed for one-off tests run over a short period of time, not to be anything that runs permanently.
-- 
cgit v1.2.3


From e3d811b04b7fb04981d65a85d9042e2bd31798b7 Mon Sep 17 00:00:00 2001
From: Adam Ierymenko <adam.ierymenko@gmail.com>
Date: Wed, 28 Oct 2015 15:55:40 -0700
Subject: docs

---
 tests/http/README.md | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

(limited to 'tests')

diff --git a/tests/http/README.md b/tests/http/README.md
index 7852ac16..ab4827ec 100644
--- a/tests/http/README.md
+++ b/tests/http/README.md
@@ -3,6 +3,8 @@ HTTP one-to-all test
 
 This code can be deployed across a large number of VMs or containers to test and benchmark HTTP traffic within a virtual network at scale. The agent acts as a server and can query other agents, while the server collects agent data and tells agents about each other. It's designed to use RFC4193-based ZeroTier IPv6 addresses within the cluster, which allows the easy provisioning of a large cluster without IP conflicts.
 
-A Dockerfile is also included which will build a simple Docker image that runs the agent. The image must be launched with "--device=/dev/net/tun --privileged" to permit it to open a tun/tap device within the container.
-
 Before using this code you will want to edit agent.js to change SERVER_HOST to the IP address of where you will run server.js. This should typically be an open Internet IP, since this makes reporting not dependent upon the thing being tested. Also note that this thing does no security of any kind. It's designed for one-off tests run over a short period of time, not to be anything that runs permanently.
+
+A Dockerfile is also included which will build a simple Docker image that runs the agent. The image must be launched with "--device=/dev/net/tun --privileged" to permit it to open a tun/tap device within the container. You can run a bunch with a command like:
+
+    for ((n=0;n<10;n++)); do docker run --device=/dev/net/tun --privileged -d zerotier/http-test; done
-- 
cgit v1.2.3


From 80e62ad29180800fd1669aa68515876f0e8add54 Mon Sep 17 00:00:00 2001
From: Adam Ierymenko <adam.ierymenko@gmail.com>
Date: Fri, 30 Oct 2015 10:55:05 -0700
Subject: docs

---
 tests/http/README.md | 4 ++--
 tests/http/agent.js  | 2 ++
 tests/http/server.js | 2 ++
 3 files changed, 6 insertions(+), 2 deletions(-)

(limited to 'tests')

diff --git a/tests/http/README.md b/tests/http/README.md
index ab4827ec..58d4a989 100644
--- a/tests/http/README.md
+++ b/tests/http/README.md
@@ -3,8 +3,8 @@ HTTP one-to-all test
 
 This code can be deployed across a large number of VMs or containers to test and benchmark HTTP traffic within a virtual network at scale. The agent acts as a server and can query other agents, while the server collects agent data and tells agents about each other. It's designed to use RFC4193-based ZeroTier IPv6 addresses within the cluster, which allows the easy provisioning of a large cluster without IP conflicts.
 
-Before using this code you will want to edit agent.js to change SERVER_HOST to the IP address of where you will run server.js. This should typically be an open Internet IP, since this makes reporting not dependent upon the thing being tested. Also note that this thing does no security of any kind. It's designed for one-off tests run over a short period of time, not to be anything that runs permanently.
+Before using this code you will want to edit agent.js to change SERVER_HOST to the IP address of where you will run server.js. This should typically be an open Internet IP, since this makes reporting not dependent upon the thing being tested. Also note that this thing does no security of any kind. It's designed for one-off tests run over a short period of time, not to be anything that runs permanently. You will also want to edit the Dockerfile if you want to build containers and change the network ID to the network you want to run tests over.
 
-A Dockerfile is also included which will build a simple Docker image that runs the agent. The image must be launched with "--device=/dev/net/tun --privileged" to permit it to open a tun/tap device within the container. You can run a bunch with a command like:
+The Dockerfile builds an image that launches the agent. The image must be "docker run" with "--device=/dev/net/tun --privileged" to permit it to open a tun/tap device within the container. (Unfortunately CAP_NET_ADMIN may not work due to a bug in Docker and/or Linux.) You can run a bunch with a command like:
 
     for ((n=0;n<10;n++)); do docker run --device=/dev/net/tun --privileged -d zerotier/http-test; done
diff --git a/tests/http/agent.js b/tests/http/agent.js
index 8b1cf512..8a8b785e 100644
--- a/tests/http/agent.js
+++ b/tests/http/agent.js
@@ -1,3 +1,5 @@
+// ZeroTier distributed HTTP test agent
+
 // ---------------------------------------------------------------------------
 // Customizable parameters:
 
diff --git a/tests/http/server.js b/tests/http/server.js
index 69ccf527..30d8339a 100644
--- a/tests/http/server.js
+++ b/tests/http/server.js
@@ -1,3 +1,5 @@
+// ZeroTier distributed HTTP test coordinator and result-reporting server
+
 // ---------------------------------------------------------------------------
 // Customizable parameters:
 
-- 
cgit v1.2.3


From b845dd1b88b966c7931524721d8369e6db240ed7 Mon Sep 17 00:00:00 2001
From: Adam Ierymenko <adam.ierymenko@gmail.com>
Date: Fri, 30 Oct 2015 12:38:12 -0700
Subject: Set contact IP for real test.

---
 tests/http/agent.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'tests')

diff --git a/tests/http/agent.js b/tests/http/agent.js
index 8a8b785e..d0c33917 100644
--- a/tests/http/agent.js
+++ b/tests/http/agent.js
@@ -10,7 +10,7 @@ var TEST_DURATION = (30 * 1000);
 var TEST_INTERVAL = (60 * 2 * 1000);
 
 // Where should I contact to register and query a list of other test agents?
-var SERVER_HOST = '174.136.102.178';
+var SERVER_HOST = '104.238.141.145';
 var SERVER_PORT = 18080;
 
 // Which port should agents use for their HTTP?
-- 
cgit v1.2.3


From f808138a942893188537bf82a064932a2523d34d Mon Sep 17 00:00:00 2001
From: Adam Ierymenko <adam.ierymenko@gmail.com>
Date: Fri, 30 Oct 2015 13:05:34 -0700
Subject: docs and stuff

---
 tests/http/README.md         |  4 +++-
 tests/http/run-a-big-test.sh | 28 ++++++++++++++++++++++++++++
 2 files changed, 31 insertions(+), 1 deletion(-)
 create mode 100755 tests/http/run-a-big-test.sh

(limited to 'tests')

diff --git a/tests/http/README.md b/tests/http/README.md
index 58d4a989..23a95605 100644
--- a/tests/http/README.md
+++ b/tests/http/README.md
@@ -1,10 +1,12 @@
 HTTP one-to-all test
 ======
 
-This code can be deployed across a large number of VMs or containers to test and benchmark HTTP traffic within a virtual network at scale. The agent acts as a server and can query other agents, while the server collects agent data and tells agents about each other. It's designed to use RFC4193-based ZeroTier IPv6 addresses within the cluster, which allows the easy provisioning of a large cluster without IP conflicts.
+*This is really internal use code. You're free to test it out but expect to do some editing/tweaking to make it work. We used this to run some massive scale tests of our new geo-cluster-based root server infrastructure prior to taking it live.*
 
 Before using this code you will want to edit agent.js to change SERVER_HOST to the IP address of where you will run server.js. This should typically be an open Internet IP, since this makes reporting not dependent upon the thing being tested. Also note that this thing does no security of any kind. It's designed for one-off tests run over a short period of time, not to be anything that runs permanently. You will also want to edit the Dockerfile if you want to build containers and change the network ID to the network you want to run tests over.
 
+This code can be deployed across a large number of VMs or containers to test and benchmark HTTP traffic within a virtual network at scale. The agent acts as a server and can query other agents, while the server collects agent data and tells agents about each other. It's designed to use RFC4193-based ZeroTier IPv6 addresses within the cluster, which allows the easy provisioning of a large cluster without IP conflicts.
+
 The Dockerfile builds an image that launches the agent. The image must be "docker run" with "--device=/dev/net/tun --privileged" to permit it to open a tun/tap device within the container. (Unfortunately CAP_NET_ADMIN may not work due to a bug in Docker and/or Linux.) You can run a bunch with a command like:
 
     for ((n=0;n<10;n++)); do docker run --device=/dev/net/tun --privileged -d zerotier/http-test; done
diff --git a/tests/http/run-a-big-test.sh b/tests/http/run-a-big-test.sh
new file mode 100755
index 00000000..1c125345
--- /dev/null
+++ b/tests/http/run-a-big-test.sh
@@ -0,0 +1,28 @@
+#!/bin/bash
+
+# Edit as needed -- note that >1000 per host is likely problematic due to Linux kernel limits
+NUM_CONTAINERS=100
+CONTAINER_IMAGE=zerotier/http-test
+
+#
+# This script is designed to be run on Docker hosts to run NUM_CONTAINERS
+#
+# It can then be run on each Docker host via pssh or similar to run very
+# large scale tests.
+#
+
+export PATH=/bin:/usr/bin:/usr/local/bin:/usr/sbin:/sbin
+
+# Kill and clean up old test containers if any -- note that this kills all containers on the system!
+docker ps -q | xargs -n 1 docker kill
+docker ps -aq | xargs -n 1 docker rm
+
+# Pull latest if needed -- change this to your image name and/or where to pull it from
+docker pull $CONTAINER_IMAGE
+
+# Run NUM_CONTAINERS
+for ((n=0;n<$NUM_CONTAINERS;n++)); do
+	docker run --device=/dev/net/tun --privileged -d $CONTAINER_IMAGE
+done
+
+exit 0
-- 
cgit v1.2.3


From f974517f64aac6b527fefa8f3b30088b804d2ae2 Mon Sep 17 00:00:00 2001
From: Adam Ierymenko <adam.ierymenko@gmail.com>
Date: Fri, 30 Oct 2015 13:06:30 -0700
Subject: Save zerotier output in containers.

---
 tests/http/docker-main.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'tests')

diff --git a/tests/http/docker-main.sh b/tests/http/docker-main.sh
index 72023668..f9e11de5 100755
--- a/tests/http/docker-main.sh
+++ b/tests/http/docker-main.sh
@@ -2,7 +2,7 @@
 
 export PATH=/bin:/usr/bin:/usr/local/bin:/sbin:/usr/sbin
 
-/zerotier-one -d
+/zerotier-one -d >>zerotier-one.out 2>&1
 
 while [ ! -d "/proc/sys/net/ipv6/conf/zt0" ]; do
 	sleep 0.25
-- 
cgit v1.2.3


From 29249db5d295b72cd28f73550ff7727b34fd5c9a Mon Sep 17 00:00:00 2001
From: Adam Ierymenko <adam.ierymenko@gmail.com>
Date: Mon, 2 Nov 2015 11:37:32 -0800
Subject: Big test stuff.

---
 tests/http/big-test-hosts    |  4 ++++
 tests/http/big-test-kill.sh  | 30 ++++++++++++++++++++++++++++++
 tests/http/big-test-start.sh | 30 ++++++++++++++++++++++++++++++
 tests/http/run-a-big-test.sh | 28 ----------------------------
 4 files changed, 64 insertions(+), 28 deletions(-)
 create mode 100644 tests/http/big-test-hosts
 create mode 100755 tests/http/big-test-kill.sh
 create mode 100755 tests/http/big-test-start.sh
 delete mode 100755 tests/http/run-a-big-test.sh

(limited to 'tests')

diff --git a/tests/http/big-test-hosts b/tests/http/big-test-hosts
new file mode 100644
index 00000000..27c0c656
--- /dev/null
+++ b/tests/http/big-test-hosts
@@ -0,0 +1,4 @@
+root@104.156.246.48
+root@104.156.252.136
+root@46.101.72.130
+root@188.166.240.16
diff --git a/tests/http/big-test-kill.sh b/tests/http/big-test-kill.sh
new file mode 100755
index 00000000..fbb34c10
--- /dev/null
+++ b/tests/http/big-test-kill.sh
@@ -0,0 +1,30 @@
+#!/bin/bash
+
+# Edit as needed -- note that >1000 per host is likely problematic due to Linux kernel limits
+NUM_CONTAINERS=100
+CONTAINER_IMAGE=zerotier/http-test
+
+#
+# This script is designed to be run on Docker hosts to run NUM_CONTAINERS
+#
+# It can then be run on each Docker host via pssh or similar to run very
+# large scale tests.
+#
+
+export PATH=/bin:/usr/bin:/usr/local/bin:/usr/sbin:/sbin
+
+# Kill and clean up old test containers if any -- note that this kills all containers on the system!
+#docker ps -q | xargs -n 1 docker kill
+#docker ps -aq | xargs -n 1 docker rm
+
+# Pull latest if needed -- change this to your image name and/or where to pull it from
+#docker pull $CONTAINER_IMAGE
+
+# Run NUM_CONTAINERS
+#for ((n=0;n<$NUM_CONTAINERS;n++)); do
+#	docker run --device=/dev/net/tun --privileged -d $CONTAINER_IMAGE
+#done
+
+pssh -h big-test-hosts -i -p 256 "docker ps -q | xargs -r -n 128 docker kill && docker ps -aq | xargs -r -P 16 -n 1 docker rm"
+
+exit 0
diff --git a/tests/http/big-test-start.sh b/tests/http/big-test-start.sh
new file mode 100755
index 00000000..79b6f93a
--- /dev/null
+++ b/tests/http/big-test-start.sh
@@ -0,0 +1,30 @@
+#!/bin/bash
+
+# Edit as needed -- note that >1000 per host is likely problematic due to Linux kernel limits
+NUM_CONTAINERS=100
+CONTAINER_IMAGE=zerotier/http-test
+
+#
+# This script is designed to be run on Docker hosts to run NUM_CONTAINERS
+#
+# It can then be run on each Docker host via pssh or similar to run very
+# large scale tests.
+#
+
+export PATH=/bin:/usr/bin:/usr/local/bin:/usr/sbin:/sbin
+
+# Kill and clean up old test containers if any -- note that this kills all containers on the system!
+#docker ps -q | xargs -n 1 docker kill
+#docker ps -aq | xargs -n 1 docker rm
+
+# Pull latest if needed -- change this to your image name and/or where to pull it from
+#docker pull $CONTAINER_IMAGE
+
+# Run NUM_CONTAINERS
+#for ((n=0;n<$NUM_CONTAINERS;n++)); do
+#	docker run --device=/dev/net/tun --privileged -d $CONTAINER_IMAGE
+#done
+
+pssh -o big-test-out -h big-test-hosts -i -p 256 "docker pull $CONTAINER_IMAGE && for ((n=0;n<$NUM_CONTAINERS;n++)); do docker run --device=/dev/net/tun --privileged -d $CONTAINER_IMAGE; done"
+
+exit 0
diff --git a/tests/http/run-a-big-test.sh b/tests/http/run-a-big-test.sh
deleted file mode 100755
index 1c125345..00000000
--- a/tests/http/run-a-big-test.sh
+++ /dev/null
@@ -1,28 +0,0 @@
-#!/bin/bash
-
-# Edit as needed -- note that >1000 per host is likely problematic due to Linux kernel limits
-NUM_CONTAINERS=100
-CONTAINER_IMAGE=zerotier/http-test
-
-#
-# This script is designed to be run on Docker hosts to run NUM_CONTAINERS
-#
-# It can then be run on each Docker host via pssh or similar to run very
-# large scale tests.
-#
-
-export PATH=/bin:/usr/bin:/usr/local/bin:/usr/sbin:/sbin
-
-# Kill and clean up old test containers if any -- note that this kills all containers on the system!
-docker ps -q | xargs -n 1 docker kill
-docker ps -aq | xargs -n 1 docker rm
-
-# Pull latest if needed -- change this to your image name and/or where to pull it from
-docker pull $CONTAINER_IMAGE
-
-# Run NUM_CONTAINERS
-for ((n=0;n<$NUM_CONTAINERS;n++)); do
-	docker run --device=/dev/net/tun --privileged -d $CONTAINER_IMAGE
-done
-
-exit 0
-- 
cgit v1.2.3


From e53ef9642e0201880672699ca12edd50d103be9e Mon Sep 17 00:00:00 2001
From: Adam Ierymenko <adam.ierymenko@gmail.com>
Date: Mon, 2 Nov 2015 12:31:34 -0800
Subject: test stuff.

---
 tests/http/agent.js                          | 26 +++++++---
 tests/http/big-test-hosts                    |  2 -
 tests/http/big-test-kill.sh                  | 14 +-----
 tests/http/big-test-out/root@104.156.246.48  | 67 +++++++++++++++++++++++++
 tests/http/big-test-out/root@104.156.252.136 | 75 ++++++++++++++++++++++++++++
 tests/http/big-test-out/root@188.166.240.16  |  0
 tests/http/big-test-out/root@46.101.72.130   |  0
 tests/http/big-test-ready.sh                 | 30 +++++++++++
 tests/http/big-test-start.sh                 |  4 +-
 9 files changed, 193 insertions(+), 25 deletions(-)
 create mode 100644 tests/http/big-test-out/root@104.156.246.48
 create mode 100644 tests/http/big-test-out/root@104.156.252.136
 create mode 100644 tests/http/big-test-out/root@188.166.240.16
 create mode 100644 tests/http/big-test-out/root@46.101.72.130
 create mode 100755 tests/http/big-test-ready.sh

(limited to 'tests')

diff --git a/tests/http/agent.js b/tests/http/agent.js
index d0c33917..e11fed60 100644
--- a/tests/http/agent.js
+++ b/tests/http/agent.js
@@ -214,12 +214,22 @@ app.get('/',function(req,res) {
 });
 
 var expressServer = app.listen(AGENT_PORT,function () {
-	registerAndGetPeers(function(err,peers) {
-		if (err) {
-			console.error('FATAL: unable to contact or query server: '+err.toString());
-			process.exit(1);
-		}
-		doTestsAndReport();
-		setInterval(doTestsAndReport,TEST_INTERVAL);
-	});
+	var serverUp = false;
+	async.whilst(
+		function() { return (!serverUp); },
+		function(nextTry) {
+			registerAndGetPeers(function(err,peers) {
+				if ((err)||(!peers)) {
+					setTimeout(nextTry,1000);
+				} else {
+					serverUp = true;
+					return nextTry(null);
+				}
+			});
+		},
+		function(err) {
+			console.log('Server up, starting!');
+			doTestsAndReport();
+			setInterval(doTestsAndReport,TEST_INTERVAL);
+		});
 });
diff --git a/tests/http/big-test-hosts b/tests/http/big-test-hosts
index 27c0c656..93b6f23f 100644
--- a/tests/http/big-test-hosts
+++ b/tests/http/big-test-hosts
@@ -1,4 +1,2 @@
 root@104.156.246.48
 root@104.156.252.136
-root@46.101.72.130
-root@188.166.240.16
diff --git a/tests/http/big-test-kill.sh b/tests/http/big-test-kill.sh
index fbb34c10..917a7791 100755
--- a/tests/http/big-test-kill.sh
+++ b/tests/http/big-test-kill.sh
@@ -13,18 +13,6 @@ CONTAINER_IMAGE=zerotier/http-test
 
 export PATH=/bin:/usr/bin:/usr/local/bin:/usr/sbin:/sbin
 
-# Kill and clean up old test containers if any -- note that this kills all containers on the system!
-#docker ps -q | xargs -n 1 docker kill
-#docker ps -aq | xargs -n 1 docker rm
-
-# Pull latest if needed -- change this to your image name and/or where to pull it from
-#docker pull $CONTAINER_IMAGE
-
-# Run NUM_CONTAINERS
-#for ((n=0;n<$NUM_CONTAINERS;n++)); do
-#	docker run --device=/dev/net/tun --privileged -d $CONTAINER_IMAGE
-#done
-
-pssh -h big-test-hosts -i -p 256 "docker ps -q | xargs -r -n 128 docker kill && docker ps -aq | xargs -r -P 16 -n 1 docker rm"
+pssh -h big-test-hosts -i -t 128 -p 256 "docker ps -q | xargs -r docker kill && docker ps -aq | xargs -r docker rm"
 
 exit 0
diff --git a/tests/http/big-test-out/root@104.156.246.48 b/tests/http/big-test-out/root@104.156.246.48
new file mode 100644
index 00000000..afcda19f
--- /dev/null
+++ b/tests/http/big-test-out/root@104.156.246.48
@@ -0,0 +1,67 @@
+5fc1676570b10196c5e261dac6ff28998c4f66c706b131d6423532a38b8b7a15
+894bbfa612f772aa0170d5c5ddb9362a9edb0f5fbe22c81591b50dfa0a0eadeb
+bae9bf7c37806adbbdfe9e01b4e62034953d01ecae55e0cd450a21b1415f5c00
+26e83eccbd6083f2c7a8532786fd1ec5c96d6a51bb3508ec4fd1919a1883630a
+90e95ac0cd3b4ba6ffa33fe946a1f12a2cdba61168770af9fca7b42df45c9530
+88a11bbc89c0b3579de7faf4db4dbe3f0a5a073aa49fd7a5482eff35c93fab4e
+50be80765f09000085d6c0895958e6d3794f672f562ff5bc27634b15c77b7583
+99630786ef067f4c053f7bca96e2869f14c9ceb9f1d07b7635945c143700b950
+8422d4cb09ed44e34488163a96910ece8f1dd39c280daf3d369d17ce2ac70766
+e0a0af014418076adc351827fabbd9ef6da7404f2d6184f20d601736c4685154
+387cd6dbbd8e8b17d9ccf943fd13dcaddf27c178834a9630e357407a42733fb8
+b06b75cb0c5748321ab4501e1055dd5ff8457e81502548954702ead843011388
+59d874f5e36f15b62e8540b86d552950c72f086d90667780b6d82b5595326fce
+69dc1347f8671ba4bbe1da12f26e8f67b0980f1e2ad73bc0b77cc06e5cbf1b06
+68ef4b43d3f5559b5b0c82ba2f396a5b6dcb6001a67efd3a3b3b2a415c2b7b61
+3ecc9f45151f95a194d8274a88f433f83540f5397523de7a86db714cd9155bb2
+3b24b66b1dcd5e8ce1fa24d33cb2eb3cc55f3a157602a09bca4942089e25790f
+ba34465f6cedd2f306022cc9259ef4e43819959f51f980a8cb94ea33a29c7e34
+4e9a701f18a0ac42ee24f3bf2dd6dc442706a0bf4b6288336447b03752640852
+5865df2182165576a0825644d9b7537314c9fd4323cb2023260382acc2f9d7fd
+f1498fda2dd6d1f1fdfc95d539d6ee511ca8baad65b1a1b44d76309e84015550
+7d9232ecda50856096523dc2d0c1fae46481053067d8bfe024e1dfccf8f9c0fc
+173b47f327a3c21187e25d2d02dbde49760182a40e00cf4a64746c00537a3ff6
+9221fdb92693a29f720a5c41f09791c35f2b220459a2e4bdcc24fd5e3af4ad85
+ef8def0260ba3ff21aa292bb482bedf94b17fd9306e529f6feae5cab04e1bdc2
+7392179b9d51cad626b6cf78b00c355b280ddccccd7013e11a8cecbdd2db1c15
+fc31766af9265f2c496ee3847971ee0f5249aca8ad0cd214620643c7347436b2
+7f8bd6d1fde6948a09b132b4b2c3919ca11c932bfdf8a279c4cbe140daf24287
+ce9596c1d8dd42929820e4d3a56e8b1f8eb2d8b67474c525bbed1b888143cc14
+67c9de4f3d88daa89796e8a03306d2c5f2bcc4c409eef89e67c6ffc6a6282060
+705c1973e4a9ef77ffe6319375e585aec1d762a031c8b93ba9883af98e377590
+c7fdcc10eed007a30da7c7bbb8767d0b6a7287fadd67d935651f0bb265a71e1b
+5102e854fe72cdcd91b228d520a9380a36a273df7949de054355e8f99bd14e95
+77c37e503ccaa1350bf2fe16daaef486ec223348bc8f678b6f6d6bb477ae10c1
+fd97792d7bd61449163fa4e953212446e11cf02b27a45a2073a1de5148da63e8
+7d5b84b290f727713ad02f0817856c9b891996bdcc6ad4d0994005608cf9bbb7
+104c41c9cce7934f0e205cfc90b65f5ba89ca696cab41771a12b4384af7f6805
+75e1751a7e290a34faddea54c98e870b4a1cd5bb37c810cafabda8d3ba1faa5f
+2293a17e82ff54e04218c5aafd904079d15d71e47496249dc125f90f0960039f
+3f3563782349fddb61cdc638cf5f54030f726d9759cb104253f5b8b04ae8e2b0
+2328da71617e1abc0e2335e5974a70f45f8abaedba641292ad4d87e2f27a6b83
+00e79b478925c7b866f7d669ee73af1a7b377fb8bf22a04f2bd5356f256f59ba
+2960f757cca294c32abd51f072798f5457a1552de52bd42ca9233cd075da086f
+efbc248c9a3ce7b7a52724b67f14e27b02f5f37da295bc905d4f8fabe847cd00
+a068d735478e065236e840e50697a078d77aad9b82f906555279e5dee074db5d
+c4a366870dd1f3d1fb776b25f009d9079a6f7d0d83b03cd178b237b412d8dce6
+20340c7bd3bd9d32d6ecb7a451c377fb239bfe2d2c976886e2bd59746ed180ee
+8c522ecbabb9580528794d94f82bdef2982f0458b677598f236acc14e819e480
+8dfcc3407b050a39d82af263eec6e332bd69cb848ffa660344644ea10e3c2221
+74e06d9deba29982f9dccdb841163715dd419dee6a54eaf3ac987cd3458a1a2a
+725d990ca2ea34dc3e9acc02480cc8009a9c2016414c9c9cca3c7b135bd384f4
+120d7eac6a5bd761ee6acfb751f48bf7075c0316123326ce6bbb6ce3fe05c3a7
+91aa44d2650ff308877d9c81357619c51c0c0d05dc9f4c899df9ce460673b2aa
+3c203cd73c6be2606397357154789a94fbfe8670271825f75bbfc6c97fa0e048
+f9018c8390a472798c7a39bfe834cd01bb62bb4b0882dfe1108bf43334a3bd0e
+cdf8afae641e0423b2a7a1ff92cca80c7db478ff8deb1d81032808ac84415921
+9a5364df1df5460f6a9ed15e020b8bd283c47464556a7c5896707cee00c01a14
+df9365b8816e9d67f484adf94eef53aab236a92a588a1f3f650fd36b8073f7bb
+bd0172f67fa20716502e2bee82387de7f426e3cdb19d5ac6ce9dcc177f919cde
+90dd259c03b11625c09b8db614f45759e67edf07fe350681d273bfd988b45443
+9f69d376248b6851aa7837a7d09a1b9eda917601049e5942796815069d09a80f
+1a065bee20e8f4c6a91bb92ec9cca6ca16e8eb434798ed433a5248c48d91f596
+fb5a6a9397546a97ccdc4252603d5e774d8430195b06ec74926c48cf372b9906
+51cf0867773bb298140eb09c88d69587aae3a6da3e63275d32f3d32d98a737d7
+70574f3e616413ce90b045e0e9fd92353766e216eabf8139556fa61efea9c3b4
+ccd21257994f9eaf309d06b2fc5652b14ea80796a79face304fcc8fd1da53423
+7d8417be656f17fbfb779a9803b2de045e2f496f75ea5f1ea69e223572bde2a5
diff --git a/tests/http/big-test-out/root@104.156.252.136 b/tests/http/big-test-out/root@104.156.252.136
new file mode 100644
index 00000000..c11311b4
--- /dev/null
+++ b/tests/http/big-test-out/root@104.156.252.136
@@ -0,0 +1,75 @@
+5ea6049a0b92070494f40a5ccecccaf788a5aafccee7c2eada9b9eb8731bc002
+798f8beecd2e3fbb50df49b7ef57cdd1e8e00c0680046b3c2d53a3554f956fda
+dd20da6340210e1b7612d8922aaa4b045e84da32f264add073a65a15f676a9e4
+0479f14d0aa68e835c07dc5ca413febd9da19b6554fd8bdba7e319e5f4661f80
+df6747868f90cfa069f5f9f954626b7392cd99026e43e6d6c83ab7c16d5cbdc6
+c24799f74233b1bc7d7d936d57699b955000c640531f3db38be8196a87eb262c
+46b00a65d527738c0bfad924051bd2117563e0c6ad74b803b662e74720d8d085
+dc14b9428032771388d30c6002bb5cba05131972cab53360f088c51769786c47
+e7dc364aeccf60bafa5a42787cc6de231612782252f57b9f03ebae3a309b2352
+4e8e578a8948ab384525646a17c2e0cb9f2b9ef67fd0c489ad6aa2bffbfabddc
+cc626d978e32dfe14782011218ba265ae4e69886f44335a2c402001dc0c4c3c7
+36a148254d34f954906a810ba4a8644a4433e8847d3cc30e091b1f63723f0590
+1bdd06b691fa06b3af77d2868b78f2a01b91026b4f483fb278cb8872a9150987
+d6d5709039708a515b295519a4007c3b49361ef67465ddca2dfba9a473b9c37c
+f8a3a0b3dd5ce42bd87cdba28df7469071f948fb28151955cfa75abe0455a000
+f7eb37fecb17571091f05f1bfc66f6fba731ff934988a529813f4751451401af
+1c2f2c77c429095b1dad53032684df672f351489ed6b7e00e1097f7dc1c0ba97
+361dd9cc6764facd2aca0b462e64a2400f6b41e124c4d9be71466d801763270a
+3f5f49ce854439bc672c4b0cf4c1e1b9a978e8f2db14709977ddcfd39b7d6bb3
+5221bec1aec2c9f08ddec548a24b0700e3d0c0568d10caa753564c914b35c95f
+ebdcc27c264d326619262f82b5d7dfcbf102d720ad9bab4428b9725118bf627a
+bc01c863316a29c8f119da7cce1c891185c43d385521d46b06186a89cb6cff9d
+47fb8b119e3d098f22cc6f4a7867f2016f244cc8b114aa66630dadbd4bfb2a0c
+4ca2402d762adbd7ce860c3f3d072e948bb33afb7c2830ad51ae9d3fb2c714df
+328e4fda6dd1befcfda64d5c89c458fcc9386d88375218cb5197d479d3d292d0
+099636c67de66af06ee8493ccd55b588cf8bbacf67352877f37a077a2166117a
+624e0c032e4b8a78af1387c0199c5b02da68e0795e9b6397ba8bdd5ebcf7daa4
+265e038f17e4bc3c99737bf4ac98364c98a12d9d22a28b6a9302ce5a83c7d4f8
+de198984870126801aa20f25c459ea8f89bb7a4782614e91659b820c42a33c93
+ad48788c3f91a1292fb28dde84750f94e27ef150b2ebb52b7807f3d0c7986c4b
+34ab2f912eae5e75bbde977b8bb2952e552e36c83f29c2cd64d7ac3165aa9726
+951efa9c57da7eb2a1f6578927f809ab0c9feac2aa4326ea42f182e3ad74e600
+8b8193d9ebd89c36a728a3fb89282854bfb89b27172d93952747126c22ea6a97
+79a78a9f96f2c961b2d06dfd484a495ddc3809228243c608ecd51715a228e528
+4301dfc330137391585d36ddc4b54971999a1da96b1c12aa43221dd92cc32e79
+f956660447a893adbc0fc4382fe67ad2a7bb8591a647a130faa9e17aa380328a
+0cee22f18107559eff4f4eba20320c349d70d82b72197ae380fc514e6241730c
+d74601767fec3cff13062ad1393fad9a88277bbc6710c4fce6d78c21d001bfa4
+c1c46fd958b7806bf0c22c77c997317b7e4dffd7243d0a5919d4922fafd841a2
+a07e0f4e1cac4a84e3ca922b8c59a37ed8049096d9154be76e6d2d9094ba3714
+6861c27e7584e83a792ce710d004f7dc22213cc732b23f0a2025606cc5e9e325
+ee334626da5143ddf49561522483eb689663d031ba6cf9891204b709b279b28b
+3be351d25381f07b85c7f5c2bb08b80b5bbed80c348515187d727ccbf293b13b
+268da41127aa3b7768bbf6075baad5397f0af4f0ff16f5b8dff2cad9c3019750
+74cc47af92d6cf6b2315f62e0261d4461c6297c5ffc19b50a97784cd6271acfc
+8e52beef1bd61be4d223c460b589d32d1a64d5525406ab179d1962fccd734309
+9471871b2f1ec2331c1f9855b408d212ba868965f99aeee91ddd0b7dd76b2985
+16eb9e7e24b61ade0939345118d5d14032ae496de3b5fc702c0c1356662b8a80
+f503549eb9c03c8dc7e54559bf076e1f7eaf7c8599c84722062851597a4c91c1
+d9dce304b504003c97ffdf9d076ff348343d7d0ce50070038d049b71239bc70e
+2ed388e1730860b7605527213d7d61bbc8f29703a2f586d127b37e7ed8eca708
+d20988195a901d19597bf4ed2b136c3d2aed2d169093409a5d3ea8daf1f983af
+a9be22365634b68f0ff5ba9228550eba9b3923319eff07f5cf5785eec85fa11e
+99b512fe14b21569c81d358b161976c9dce45c608cd1a03da6f6063dbabb6c41
+88f76143cb289e8daf8edbc183a4b760c0d86efd1cc9da3933e59603bf92c539
+62054c1a23221461f1c06b94148614754f7da0cd7145eb85290bb11bdf7b9af8
+8dffffd5b63672f8749160dbff55c489f1f1f6c41ce01e086f2d9aa8bc6150ad
+f86a0512be3ec599065dec982545ecb96bdb0d82dda7285c58b3c3b7666988c9
+773786ea8094e7f013702ef721a2dde7657dcb6c44927f41f5acb32161957d83
+ef981b569d0c98e79bfc5bb387cf00457c4254e13f1ec625d0149b06ef92d5e1
+77841624d9fbba73cf722327f531ea36cfa42be59b4709c3b4ca4e7e453ac7fa
+f90ada865c766ef7e0e47d0a677b64183aa1e612f14f3f1f996d411b7197ea3e
+11c278763c1f2ff8d9b2ee5e3100a40538adcc1b74f6ca7d9668fcc8c1ab2f9a
+4a72c59eda21b43136cb5f308298aba39235bbd227668c2c9f3830fbab7b4a34
+f12a3d9308eb7800a1436b3258d899aae3b643f2c78648258f126266f9707032
+ce651585240975fc5c954c526203a048c6e4326e2d16feb083ca3a3ff4ad682a
+43aa6c22cb636983ff1b6e169057b0d7a70ad75754ef1b3a3bc7f49461c84cdb
+7e71c166a6f285c4c501b2125713e698575d1987f1819b76d4fabbbf246eae6c
+d38478eec0109a5b76da1a6e3de982382cf01bbc871b651d3e258330642cbe27
+c4c6ce60a0b1c4f2553ecb0f551de2c7df2e2d2cf2101e80af48b29e03cfefd5
+38905fb0f59281f1d4aa80894654e56df76653f3d3545a883e37c80053e72977
+1a6ee044aa753748035975b12885e73504ffde1bf129a7ff992f012d9cff111b
+f0d37e0a5ccb7871a30b143fad68a4e07624aa2a153e295022868e68e34ff770
+ee9d0ef6b557bd5e7c8fba9e087a428a98ee5350ec86785205db6ea10493b21c
+53ca280c12ba4d5be4ea78144fb2d411ecd9910f5105d04537d4bec362865c40
diff --git a/tests/http/big-test-out/root@188.166.240.16 b/tests/http/big-test-out/root@188.166.240.16
new file mode 100644
index 00000000..e69de29b
diff --git a/tests/http/big-test-out/root@46.101.72.130 b/tests/http/big-test-out/root@46.101.72.130
new file mode 100644
index 00000000..e69de29b
diff --git a/tests/http/big-test-ready.sh b/tests/http/big-test-ready.sh
new file mode 100755
index 00000000..391ca2a1
--- /dev/null
+++ b/tests/http/big-test-ready.sh
@@ -0,0 +1,30 @@
+#!/bin/bash
+
+# Edit as needed -- note that >1000 per host is likely problematic due to Linux kernel limits
+NUM_CONTAINERS=100
+CONTAINER_IMAGE=zerotier/http-test
+
+#
+# This script is designed to be run on Docker hosts to run NUM_CONTAINERS
+#
+# It can then be run on each Docker host via pssh or similar to run very
+# large scale tests.
+#
+
+export PATH=/bin:/usr/bin:/usr/local/bin:/usr/sbin:/sbin
+
+# Kill and clean up old test containers if any -- note that this kills all containers on the system!
+#docker ps -q | xargs -n 1 docker kill
+#docker ps -aq | xargs -n 1 docker rm
+
+# Pull latest if needed -- change this to your image name and/or where to pull it from
+#docker pull $CONTAINER_IMAGE
+
+# Run NUM_CONTAINERS
+#for ((n=0;n<$NUM_CONTAINERS;n++)); do
+#	docker run --device=/dev/net/tun --privileged -d $CONTAINER_IMAGE
+#done
+
+pssh -h big-test-hosts -i -t 128 -p 256 "docker pull $CONTAINER_IMAGE"
+
+exit 0
diff --git a/tests/http/big-test-start.sh b/tests/http/big-test-start.sh
index 79b6f93a..a4c7e6c1 100755
--- a/tests/http/big-test-start.sh
+++ b/tests/http/big-test-start.sh
@@ -1,7 +1,7 @@
 #!/bin/bash
 
 # Edit as needed -- note that >1000 per host is likely problematic due to Linux kernel limits
-NUM_CONTAINERS=100
+NUM_CONTAINERS=64
 CONTAINER_IMAGE=zerotier/http-test
 
 #
@@ -25,6 +25,6 @@ export PATH=/bin:/usr/bin:/usr/local/bin:/usr/sbin:/sbin
 #	docker run --device=/dev/net/tun --privileged -d $CONTAINER_IMAGE
 #done
 
-pssh -o big-test-out -h big-test-hosts -i -p 256 "docker pull $CONTAINER_IMAGE && for ((n=0;n<$NUM_CONTAINERS;n++)); do docker run --device=/dev/net/tun --privileged -d $CONTAINER_IMAGE; done"
+pssh -o big-test-out -h big-test-hosts -i -t 128 -p 256 "for ((n=0;n<$NUM_CONTAINERS;n++)); do docker run --device=/dev/net/tun --privileged -d $CONTAINER_IMAGE; done"
 
 exit 0
-- 
cgit v1.2.3


From fd3916a49e7923e95c47c70afb8696f110b79951 Mon Sep 17 00:00:00 2001
From: Adam Ierymenko <adam.ierymenko@gmail.com>
Date: Mon, 2 Nov 2015 13:17:11 -0800
Subject: More test stuff... make it more granular and less batch based.

---
 tests/http/agent.js                          | 132 ++++++++++++++++++---------
 tests/http/big-test-kill.sh                  |   2 +-
 tests/http/big-test-out/root@104.156.246.48  |  67 --------------
 tests/http/big-test-out/root@104.156.252.136 |  75 ---------------
 tests/http/big-test-out/root@188.166.240.16  |   0
 tests/http/big-test-out/root@46.101.72.130   |   0
 tests/http/big-test-start.sh                 |   4 +-
 tests/http/server.js                         |  40 ++------
 8 files changed, 100 insertions(+), 220 deletions(-)
 delete mode 100644 tests/http/big-test-out/root@104.156.246.48
 delete mode 100644 tests/http/big-test-out/root@104.156.252.136
 delete mode 100644 tests/http/big-test-out/root@188.166.240.16
 delete mode 100644 tests/http/big-test-out/root@46.101.72.130

(limited to 'tests')

diff --git a/tests/http/agent.js b/tests/http/agent.js
index e11fed60..1d4a4320 100644
--- a/tests/http/agent.js
+++ b/tests/http/agent.js
@@ -3,21 +3,23 @@
 // ---------------------------------------------------------------------------
 // Customizable parameters:
 
-// Maximum test duration in milliseconds
-var TEST_DURATION = (30 * 1000);
+// Maximum interval between test attempts
+//var TEST_INTERVAL_MAX = (60 * 1 * 1000);
+var TEST_INTERVAL_MAX = 1000;
 
-// Interval between tests (should be several times longer than TEST_DURATION)
-var TEST_INTERVAL = (60 * 2 * 1000);
+// Test timeout in ms
+var TEST_TIMEOUT = 30000;
 
 // Where should I contact to register and query a list of other test agents?
-var SERVER_HOST = '104.238.141.145';
+var SERVER_HOST = '127.0.0.1';
+//var SERVER_HOST = '104.238.141.145';
 var SERVER_PORT = 18080;
 
 // Which port should agents use for their HTTP?
 var AGENT_PORT = 18888;
 
 // Payload size in bytes
-var PAYLOAD_SIZE = 100000;
+var PAYLOAD_SIZE = 10000;
 
 // ---------------------------------------------------------------------------
 
@@ -72,9 +74,6 @@ for(var xx=0;xx<PAYLOAD_SIZE;++xx) {
 	payload.writeUInt8(Math.round(Math.random() * 255.0),xx);
 }
 
-// Incremented with each test
-var testNumber = 0;
-
 function agentIdToIp(agentId)
 {
 	var ip = '';
@@ -96,32 +95,93 @@ function agentIdToIp(agentId)
 	return ip;
 };
 
-function registerAndGetPeers(callback)
+var lastTestResult = null;
+var allOtherAgents = [];
+
+function doTest()
 {
-	http.get({
+	var submit = http.request({
 		host: SERVER_HOST,
 		port: SERVER_PORT,
-		path: '/'+thisAgentId
+		path: '/'+thisAgentId,
+		method: 'POST'
 	},function(res) {
 		var body = '';
 		res.on('data',function(chunk) { body += chunk.toString(); });
 		res.on('end',function() {
-			if (!body)
-				return callback(null,[]);
-			try {
-				var peers = JSON.parse(body);
-				if (Array.isArray(peers))
-					return callback(null,peers);
-				else return callback(new Error('invalid JSON response from server'),null);
-			} catch (e) {
-				return callback(new Error('invalid JSON response from server: '+e.toString()),null);
+
+			if (body) {
+				try {
+					var peers = JSON.parse(body);
+					if (Array.isArray(peers))
+						allOtherAgents = peers;
+				} catch (e) {}
+			}
+
+			if (allOtherAgents.length > 0) {
+
+				var target = allOtherAgents[Math.floor(Math.random() * allOtherAgents.length)];
+
+				var testRequest = null;
+				var timeoutId = null;
+				timeoutId = setTimeout(function() {
+					if (testRequest !== null)
+						testRequest.abort();
+					timeoutId = null;
+				});
+				var startTime = Date.now();
+
+				testRequest = http.get({
+					host: agentIdToIp(target),
+					port: AGENT_PORT,
+					path: '/'
+				},function(res) {
+					var bytes = 0;
+					res.on('data',function(chunk) { bytes += chunk.length; });
+					res.on('end',function() {
+						lastTestResult = {
+							source: thisAgentId,
+							target: target,
+							time: (Date.now() - startTime),
+							bytes: bytes,
+							timedOut: (timeoutId === null),
+							error: null
+						};
+						if (timeoutId !== null)
+							clearTimeout(timeoutId);
+						return setTimeout(doTest,Math.round(Math.random() * TEST_INTERVAL_MAX) + 1);
+					});
+				}).on('error',function(e) {
+					lastTestResult = {
+						source: thisAgentId,
+						target: target,
+						time: (Date.now() - startTime),
+						bytes: 0,
+						timedOut: (timeoutId === null),
+						error: e.toString()
+					};
+					if (timeoutId !== null)
+						clearTimeout(timeoutId);
+					return setTimeout(doTest,Math.round(Math.random() * TEST_INTERVAL_MAX) + 1);
+				});
+
+			} else {
+				return setTimeout(doTest,Math.round(Math.random() * TEST_INTERVAL_MAX) + 1);
 			}
+
 		});
 	}).on('error',function(e) {
-		return callback(e,null);
+		console.log('POST failed: '+e.toString());
+		return setTimeout(doTest,1000);
 	});
+	if (lastTestResult !== null) {
+		submit.write(JSON.stringify(lastTestResult));
+		lastTestResult = null;
+	}
+	submit.end();
 };
 
+/*
 function performTestOnAllPeers(peers,callback)
 {
 	var allResults = {};
@@ -191,11 +251,10 @@ function doTestsAndReport()
 			console.error('WARNING: skipping test: unable to contact or query server: '+err.toString());
 		} else {
 			performTestOnAllPeers(peers,function(results) {
-				++testNumber;
 				var submit = http.request({
 					host: SERVER_HOST,
 					port: SERVER_PORT,
-					path: '/'+testNumber+'/'+thisAgentId,
+					path: '/'+thisAgentId,
 					method: 'POST'
 				},function(res) {
 				}).on('error',function(e) {
@@ -207,29 +266,12 @@ function doTestsAndReport()
 		}
 	});
 };
+*/
 
 // Agents just serve up a test payload
-app.get('/',function(req,res) {
-	return res.status(200).send(payload);
-});
+app.get('/',function(req,res) { return res.status(200).send(payload); });
 
 var expressServer = app.listen(AGENT_PORT,function () {
-	var serverUp = false;
-	async.whilst(
-		function() { return (!serverUp); },
-		function(nextTry) {
-			registerAndGetPeers(function(err,peers) {
-				if ((err)||(!peers)) {
-					setTimeout(nextTry,1000);
-				} else {
-					serverUp = true;
-					return nextTry(null);
-				}
-			});
-		},
-		function(err) {
-			console.log('Server up, starting!');
-			doTestsAndReport();
-			setInterval(doTestsAndReport,TEST_INTERVAL);
-		});
+	// Start timeout-based loop
+	doTest();
 });
diff --git a/tests/http/big-test-kill.sh b/tests/http/big-test-kill.sh
index 917a7791..4a764d1f 100755
--- a/tests/http/big-test-kill.sh
+++ b/tests/http/big-test-kill.sh
@@ -13,6 +13,6 @@ CONTAINER_IMAGE=zerotier/http-test
 
 export PATH=/bin:/usr/bin:/usr/local/bin:/usr/sbin:/sbin
 
-pssh -h big-test-hosts -i -t 128 -p 256 "docker ps -q | xargs -r docker kill && docker ps -aq | xargs -r docker rm"
+pssh -h big-test-hosts -i -t 128 -p 256 "docker ps -aq | xargs -r docker rm -f"
 
 exit 0
diff --git a/tests/http/big-test-out/root@104.156.246.48 b/tests/http/big-test-out/root@104.156.246.48
deleted file mode 100644
index afcda19f..00000000
--- a/tests/http/big-test-out/root@104.156.246.48
+++ /dev/null
@@ -1,67 +0,0 @@
-5fc1676570b10196c5e261dac6ff28998c4f66c706b131d6423532a38b8b7a15
-894bbfa612f772aa0170d5c5ddb9362a9edb0f5fbe22c81591b50dfa0a0eadeb
-bae9bf7c37806adbbdfe9e01b4e62034953d01ecae55e0cd450a21b1415f5c00
-26e83eccbd6083f2c7a8532786fd1ec5c96d6a51bb3508ec4fd1919a1883630a
-90e95ac0cd3b4ba6ffa33fe946a1f12a2cdba61168770af9fca7b42df45c9530
-88a11bbc89c0b3579de7faf4db4dbe3f0a5a073aa49fd7a5482eff35c93fab4e
-50be80765f09000085d6c0895958e6d3794f672f562ff5bc27634b15c77b7583
-99630786ef067f4c053f7bca96e2869f14c9ceb9f1d07b7635945c143700b950
-8422d4cb09ed44e34488163a96910ece8f1dd39c280daf3d369d17ce2ac70766
-e0a0af014418076adc351827fabbd9ef6da7404f2d6184f20d601736c4685154
-387cd6dbbd8e8b17d9ccf943fd13dcaddf27c178834a9630e357407a42733fb8
-b06b75cb0c5748321ab4501e1055dd5ff8457e81502548954702ead843011388
-59d874f5e36f15b62e8540b86d552950c72f086d90667780b6d82b5595326fce
-69dc1347f8671ba4bbe1da12f26e8f67b0980f1e2ad73bc0b77cc06e5cbf1b06
-68ef4b43d3f5559b5b0c82ba2f396a5b6dcb6001a67efd3a3b3b2a415c2b7b61
-3ecc9f45151f95a194d8274a88f433f83540f5397523de7a86db714cd9155bb2
-3b24b66b1dcd5e8ce1fa24d33cb2eb3cc55f3a157602a09bca4942089e25790f
-ba34465f6cedd2f306022cc9259ef4e43819959f51f980a8cb94ea33a29c7e34
-4e9a701f18a0ac42ee24f3bf2dd6dc442706a0bf4b6288336447b03752640852
-5865df2182165576a0825644d9b7537314c9fd4323cb2023260382acc2f9d7fd
-f1498fda2dd6d1f1fdfc95d539d6ee511ca8baad65b1a1b44d76309e84015550
-7d9232ecda50856096523dc2d0c1fae46481053067d8bfe024e1dfccf8f9c0fc
-173b47f327a3c21187e25d2d02dbde49760182a40e00cf4a64746c00537a3ff6
-9221fdb92693a29f720a5c41f09791c35f2b220459a2e4bdcc24fd5e3af4ad85
-ef8def0260ba3ff21aa292bb482bedf94b17fd9306e529f6feae5cab04e1bdc2
-7392179b9d51cad626b6cf78b00c355b280ddccccd7013e11a8cecbdd2db1c15
-fc31766af9265f2c496ee3847971ee0f5249aca8ad0cd214620643c7347436b2
-7f8bd6d1fde6948a09b132b4b2c3919ca11c932bfdf8a279c4cbe140daf24287
-ce9596c1d8dd42929820e4d3a56e8b1f8eb2d8b67474c525bbed1b888143cc14
-67c9de4f3d88daa89796e8a03306d2c5f2bcc4c409eef89e67c6ffc6a6282060
-705c1973e4a9ef77ffe6319375e585aec1d762a031c8b93ba9883af98e377590
-c7fdcc10eed007a30da7c7bbb8767d0b6a7287fadd67d935651f0bb265a71e1b
-5102e854fe72cdcd91b228d520a9380a36a273df7949de054355e8f99bd14e95
-77c37e503ccaa1350bf2fe16daaef486ec223348bc8f678b6f6d6bb477ae10c1
-fd97792d7bd61449163fa4e953212446e11cf02b27a45a2073a1de5148da63e8
-7d5b84b290f727713ad02f0817856c9b891996bdcc6ad4d0994005608cf9bbb7
-104c41c9cce7934f0e205cfc90b65f5ba89ca696cab41771a12b4384af7f6805
-75e1751a7e290a34faddea54c98e870b4a1cd5bb37c810cafabda8d3ba1faa5f
-2293a17e82ff54e04218c5aafd904079d15d71e47496249dc125f90f0960039f
-3f3563782349fddb61cdc638cf5f54030f726d9759cb104253f5b8b04ae8e2b0
-2328da71617e1abc0e2335e5974a70f45f8abaedba641292ad4d87e2f27a6b83
-00e79b478925c7b866f7d669ee73af1a7b377fb8bf22a04f2bd5356f256f59ba
-2960f757cca294c32abd51f072798f5457a1552de52bd42ca9233cd075da086f
-efbc248c9a3ce7b7a52724b67f14e27b02f5f37da295bc905d4f8fabe847cd00
-a068d735478e065236e840e50697a078d77aad9b82f906555279e5dee074db5d
-c4a366870dd1f3d1fb776b25f009d9079a6f7d0d83b03cd178b237b412d8dce6
-20340c7bd3bd9d32d6ecb7a451c377fb239bfe2d2c976886e2bd59746ed180ee
-8c522ecbabb9580528794d94f82bdef2982f0458b677598f236acc14e819e480
-8dfcc3407b050a39d82af263eec6e332bd69cb848ffa660344644ea10e3c2221
-74e06d9deba29982f9dccdb841163715dd419dee6a54eaf3ac987cd3458a1a2a
-725d990ca2ea34dc3e9acc02480cc8009a9c2016414c9c9cca3c7b135bd384f4
-120d7eac6a5bd761ee6acfb751f48bf7075c0316123326ce6bbb6ce3fe05c3a7
-91aa44d2650ff308877d9c81357619c51c0c0d05dc9f4c899df9ce460673b2aa
-3c203cd73c6be2606397357154789a94fbfe8670271825f75bbfc6c97fa0e048
-f9018c8390a472798c7a39bfe834cd01bb62bb4b0882dfe1108bf43334a3bd0e
-cdf8afae641e0423b2a7a1ff92cca80c7db478ff8deb1d81032808ac84415921
-9a5364df1df5460f6a9ed15e020b8bd283c47464556a7c5896707cee00c01a14
-df9365b8816e9d67f484adf94eef53aab236a92a588a1f3f650fd36b8073f7bb
-bd0172f67fa20716502e2bee82387de7f426e3cdb19d5ac6ce9dcc177f919cde
-90dd259c03b11625c09b8db614f45759e67edf07fe350681d273bfd988b45443
-9f69d376248b6851aa7837a7d09a1b9eda917601049e5942796815069d09a80f
-1a065bee20e8f4c6a91bb92ec9cca6ca16e8eb434798ed433a5248c48d91f596
-fb5a6a9397546a97ccdc4252603d5e774d8430195b06ec74926c48cf372b9906
-51cf0867773bb298140eb09c88d69587aae3a6da3e63275d32f3d32d98a737d7
-70574f3e616413ce90b045e0e9fd92353766e216eabf8139556fa61efea9c3b4
-ccd21257994f9eaf309d06b2fc5652b14ea80796a79face304fcc8fd1da53423
-7d8417be656f17fbfb779a9803b2de045e2f496f75ea5f1ea69e223572bde2a5
diff --git a/tests/http/big-test-out/root@104.156.252.136 b/tests/http/big-test-out/root@104.156.252.136
deleted file mode 100644
index c11311b4..00000000
--- a/tests/http/big-test-out/root@104.156.252.136
+++ /dev/null
@@ -1,75 +0,0 @@
-5ea6049a0b92070494f40a5ccecccaf788a5aafccee7c2eada9b9eb8731bc002
-798f8beecd2e3fbb50df49b7ef57cdd1e8e00c0680046b3c2d53a3554f956fda
-dd20da6340210e1b7612d8922aaa4b045e84da32f264add073a65a15f676a9e4
-0479f14d0aa68e835c07dc5ca413febd9da19b6554fd8bdba7e319e5f4661f80
-df6747868f90cfa069f5f9f954626b7392cd99026e43e6d6c83ab7c16d5cbdc6
-c24799f74233b1bc7d7d936d57699b955000c640531f3db38be8196a87eb262c
-46b00a65d527738c0bfad924051bd2117563e0c6ad74b803b662e74720d8d085
-dc14b9428032771388d30c6002bb5cba05131972cab53360f088c51769786c47
-e7dc364aeccf60bafa5a42787cc6de231612782252f57b9f03ebae3a309b2352
-4e8e578a8948ab384525646a17c2e0cb9f2b9ef67fd0c489ad6aa2bffbfabddc
-cc626d978e32dfe14782011218ba265ae4e69886f44335a2c402001dc0c4c3c7
-36a148254d34f954906a810ba4a8644a4433e8847d3cc30e091b1f63723f0590
-1bdd06b691fa06b3af77d2868b78f2a01b91026b4f483fb278cb8872a9150987
-d6d5709039708a515b295519a4007c3b49361ef67465ddca2dfba9a473b9c37c
-f8a3a0b3dd5ce42bd87cdba28df7469071f948fb28151955cfa75abe0455a000
-f7eb37fecb17571091f05f1bfc66f6fba731ff934988a529813f4751451401af
-1c2f2c77c429095b1dad53032684df672f351489ed6b7e00e1097f7dc1c0ba97
-361dd9cc6764facd2aca0b462e64a2400f6b41e124c4d9be71466d801763270a
-3f5f49ce854439bc672c4b0cf4c1e1b9a978e8f2db14709977ddcfd39b7d6bb3
-5221bec1aec2c9f08ddec548a24b0700e3d0c0568d10caa753564c914b35c95f
-ebdcc27c264d326619262f82b5d7dfcbf102d720ad9bab4428b9725118bf627a
-bc01c863316a29c8f119da7cce1c891185c43d385521d46b06186a89cb6cff9d
-47fb8b119e3d098f22cc6f4a7867f2016f244cc8b114aa66630dadbd4bfb2a0c
-4ca2402d762adbd7ce860c3f3d072e948bb33afb7c2830ad51ae9d3fb2c714df
-328e4fda6dd1befcfda64d5c89c458fcc9386d88375218cb5197d479d3d292d0
-099636c67de66af06ee8493ccd55b588cf8bbacf67352877f37a077a2166117a
-624e0c032e4b8a78af1387c0199c5b02da68e0795e9b6397ba8bdd5ebcf7daa4
-265e038f17e4bc3c99737bf4ac98364c98a12d9d22a28b6a9302ce5a83c7d4f8
-de198984870126801aa20f25c459ea8f89bb7a4782614e91659b820c42a33c93
-ad48788c3f91a1292fb28dde84750f94e27ef150b2ebb52b7807f3d0c7986c4b
-34ab2f912eae5e75bbde977b8bb2952e552e36c83f29c2cd64d7ac3165aa9726
-951efa9c57da7eb2a1f6578927f809ab0c9feac2aa4326ea42f182e3ad74e600
-8b8193d9ebd89c36a728a3fb89282854bfb89b27172d93952747126c22ea6a97
-79a78a9f96f2c961b2d06dfd484a495ddc3809228243c608ecd51715a228e528
-4301dfc330137391585d36ddc4b54971999a1da96b1c12aa43221dd92cc32e79
-f956660447a893adbc0fc4382fe67ad2a7bb8591a647a130faa9e17aa380328a
-0cee22f18107559eff4f4eba20320c349d70d82b72197ae380fc514e6241730c
-d74601767fec3cff13062ad1393fad9a88277bbc6710c4fce6d78c21d001bfa4
-c1c46fd958b7806bf0c22c77c997317b7e4dffd7243d0a5919d4922fafd841a2
-a07e0f4e1cac4a84e3ca922b8c59a37ed8049096d9154be76e6d2d9094ba3714
-6861c27e7584e83a792ce710d004f7dc22213cc732b23f0a2025606cc5e9e325
-ee334626da5143ddf49561522483eb689663d031ba6cf9891204b709b279b28b
-3be351d25381f07b85c7f5c2bb08b80b5bbed80c348515187d727ccbf293b13b
-268da41127aa3b7768bbf6075baad5397f0af4f0ff16f5b8dff2cad9c3019750
-74cc47af92d6cf6b2315f62e0261d4461c6297c5ffc19b50a97784cd6271acfc
-8e52beef1bd61be4d223c460b589d32d1a64d5525406ab179d1962fccd734309
-9471871b2f1ec2331c1f9855b408d212ba868965f99aeee91ddd0b7dd76b2985
-16eb9e7e24b61ade0939345118d5d14032ae496de3b5fc702c0c1356662b8a80
-f503549eb9c03c8dc7e54559bf076e1f7eaf7c8599c84722062851597a4c91c1
-d9dce304b504003c97ffdf9d076ff348343d7d0ce50070038d049b71239bc70e
-2ed388e1730860b7605527213d7d61bbc8f29703a2f586d127b37e7ed8eca708
-d20988195a901d19597bf4ed2b136c3d2aed2d169093409a5d3ea8daf1f983af
-a9be22365634b68f0ff5ba9228550eba9b3923319eff07f5cf5785eec85fa11e
-99b512fe14b21569c81d358b161976c9dce45c608cd1a03da6f6063dbabb6c41
-88f76143cb289e8daf8edbc183a4b760c0d86efd1cc9da3933e59603bf92c539
-62054c1a23221461f1c06b94148614754f7da0cd7145eb85290bb11bdf7b9af8
-8dffffd5b63672f8749160dbff55c489f1f1f6c41ce01e086f2d9aa8bc6150ad
-f86a0512be3ec599065dec982545ecb96bdb0d82dda7285c58b3c3b7666988c9
-773786ea8094e7f013702ef721a2dde7657dcb6c44927f41f5acb32161957d83
-ef981b569d0c98e79bfc5bb387cf00457c4254e13f1ec625d0149b06ef92d5e1
-77841624d9fbba73cf722327f531ea36cfa42be59b4709c3b4ca4e7e453ac7fa
-f90ada865c766ef7e0e47d0a677b64183aa1e612f14f3f1f996d411b7197ea3e
-11c278763c1f2ff8d9b2ee5e3100a40538adcc1b74f6ca7d9668fcc8c1ab2f9a
-4a72c59eda21b43136cb5f308298aba39235bbd227668c2c9f3830fbab7b4a34
-f12a3d9308eb7800a1436b3258d899aae3b643f2c78648258f126266f9707032
-ce651585240975fc5c954c526203a048c6e4326e2d16feb083ca3a3ff4ad682a
-43aa6c22cb636983ff1b6e169057b0d7a70ad75754ef1b3a3bc7f49461c84cdb
-7e71c166a6f285c4c501b2125713e698575d1987f1819b76d4fabbbf246eae6c
-d38478eec0109a5b76da1a6e3de982382cf01bbc871b651d3e258330642cbe27
-c4c6ce60a0b1c4f2553ecb0f551de2c7df2e2d2cf2101e80af48b29e03cfefd5
-38905fb0f59281f1d4aa80894654e56df76653f3d3545a883e37c80053e72977
-1a6ee044aa753748035975b12885e73504ffde1bf129a7ff992f012d9cff111b
-f0d37e0a5ccb7871a30b143fad68a4e07624aa2a153e295022868e68e34ff770
-ee9d0ef6b557bd5e7c8fba9e087a428a98ee5350ec86785205db6ea10493b21c
-53ca280c12ba4d5be4ea78144fb2d411ecd9910f5105d04537d4bec362865c40
diff --git a/tests/http/big-test-out/root@188.166.240.16 b/tests/http/big-test-out/root@188.166.240.16
deleted file mode 100644
index e69de29b..00000000
diff --git a/tests/http/big-test-out/root@46.101.72.130 b/tests/http/big-test-out/root@46.101.72.130
deleted file mode 100644
index e69de29b..00000000
diff --git a/tests/http/big-test-start.sh b/tests/http/big-test-start.sh
index a4c7e6c1..a5e71ef1 100755
--- a/tests/http/big-test-start.sh
+++ b/tests/http/big-test-start.sh
@@ -1,7 +1,7 @@
 #!/bin/bash
 
 # Edit as needed -- note that >1000 per host is likely problematic due to Linux kernel limits
-NUM_CONTAINERS=64
+NUM_CONTAINERS=100
 CONTAINER_IMAGE=zerotier/http-test
 
 #
@@ -25,6 +25,6 @@ export PATH=/bin:/usr/bin:/usr/local/bin:/usr/sbin:/sbin
 #	docker run --device=/dev/net/tun --privileged -d $CONTAINER_IMAGE
 #done
 
-pssh -o big-test-out -h big-test-hosts -i -t 128 -p 256 "for ((n=0;n<$NUM_CONTAINERS;n++)); do docker run --device=/dev/net/tun --privileged -d $CONTAINER_IMAGE; done"
+pssh -h big-test-hosts -i -t 128 -p 256 "for ((n=0;n<$NUM_CONTAINERS;n++)); do docker run --device=/dev/net/tun --privileged -d $CONTAINER_IMAGE; sleep 0.25; done"
 
 exit 0
diff --git a/tests/http/server.js b/tests/http/server.js
index 30d8339a..57109392 100644
--- a/tests/http/server.js
+++ b/tests/http/server.js
@@ -20,42 +20,22 @@ app.use(function(req,res,next) {
 
 var knownAgents = {};
 
-app.get('/:agentId',function(req,res) {
-	var agentId = req.params.agentId;
-	if ((!agentId)||(agentId.length !== 32))
-		return res.status(404).send('');
-	knownAgents[agentId] = Date.now();
-	return res.status(200).send(JSON.stringify(Object.keys(knownAgents)));
-});
-
-app.post('/:testNumber/:agentId',function(req,res) {
-	var testNumber = req.params.testNumber;
+app.post('/:agentId',function(req,res) {
 	var agentId = req.params.agentId;
 	if ((!agentId)||(agentId.length !== 32))
 		return res.status(404).send('');
 
-	var receiveTime = Date.now();
-	var resultData = null;
-	try {
-		resultData = JSON.parse(req.rawBody);
-	} catch (e) {
-		resultData = req.rawBody;
+	if (req.rawBody) {
+		var receiveTime = Date.now();
+		var resultData = null;
+		try {
+			resultData = JSON.parse(req.rawBody);
+			console.log(resultData.source+','+resultData.target+','+resultData.time+','+resultData.bytes+','+resultData.timedOut+',"'+((resultData.error) ? resultData.error : '')+'"');
+		} catch (e) {}
 	}
-	result = {
-		agentId: agentId,
-		testNumber: parseInt(testNumber),
-		receiveTime: receiveTime,
-		results: resultData
-	};
-
-	testNumber = testNumber.toString();
-	while (testNumber.length < 10)
-		testNumber = '0' + testNumber;
-	fs.writeFile('result_'+testNumber+'_'+agentId,JSON.stringify(result),function(err) {
-		console.log(result);
-	});
 
-	return res.status(200).send('');
+	knownAgents[agentId] = Date.now();
+	return res.status(200).send(JSON.stringify(Object.keys(knownAgents)));
 });
 
 var expressServer = app.listen(SERVER_PORT,function () {
-- 
cgit v1.2.3


From ab27a91b07278146975087e873577bed43793554 Mon Sep 17 00:00:00 2001
From: Adam Ierymenko <adam.ierymenko@gmail.com>
Date: Mon, 2 Nov 2015 13:53:27 -0800
Subject: .

---
 tests/http/agent.js | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

(limited to 'tests')

diff --git a/tests/http/agent.js b/tests/http/agent.js
index 1d4a4320..bc7c475e 100644
--- a/tests/http/agent.js
+++ b/tests/http/agent.js
@@ -4,15 +4,13 @@
 // Customizable parameters:
 
 // Maximum interval between test attempts
-//var TEST_INTERVAL_MAX = (60 * 1 * 1000);
-var TEST_INTERVAL_MAX = 1000;
+var TEST_INTERVAL_MAX = 60000;
 
 // Test timeout in ms
 var TEST_TIMEOUT = 30000;
 
 // Where should I contact to register and query a list of other test agents?
-var SERVER_HOST = '127.0.0.1';
-//var SERVER_HOST = '104.238.141.145';
+var SERVER_HOST = '104.238.141.145';
 var SERVER_PORT = 18080;
 
 // Which port should agents use for their HTTP?
@@ -118,9 +116,11 @@ function doTest()
 				} catch (e) {}
 			}
 
-			if (allOtherAgents.length > 0) {
+			if (allOtherAgents.length > 1) {
 
 				var target = allOtherAgents[Math.floor(Math.random() * allOtherAgents.length)];
+				while (target === thisAgentId)
+					target = allOtherAgents[Math.floor(Math.random() * allOtherAgents.length)];
 
 				var testRequest = null;
 				var timeoutId = null;
@@ -128,7 +128,7 @@ function doTest()
 					if (testRequest !== null)
 						testRequest.abort();
 					timeoutId = null;
-				});
+				},TEST_TIMEOUT);
 				var startTime = Date.now();
 
 				testRequest = http.get({
@@ -166,7 +166,7 @@ function doTest()
 				});
 
 			} else {
-				return setTimeout(doTest,Math.round(Math.random() * TEST_INTERVAL_MAX) + 1);
+				return setTimeout(doTest,1000);
 			}
 
 		});
-- 
cgit v1.2.3


From 60ce886605c0298fc22dbce48beb106a96bd35e2 Mon Sep 17 00:00:00 2001
From: Adam Ierymenko <adam.ierymenko@gmail.com>
Date: Mon, 2 Nov 2015 15:15:20 -0800
Subject: Tweak some timings for better reliability.

---
 node/Cluster.cpp             |   6 +-
 node/Cluster.hpp             |   8 +-
 node/Constants.hpp           |  24 +----
 node/Multicaster.cpp         | 220 ++++++++++++++++++++++---------------------
 node/Node.cpp                |  13 +--
 tests/http/big-test-kill.sh  |   2 +-
 tests/http/big-test-ready.sh |   2 +-
 tests/http/big-test-start.sh |   4 +-
 8 files changed, 131 insertions(+), 148 deletions(-)

(limited to 'tests')

diff --git a/node/Cluster.cpp b/node/Cluster.cpp
index d0daae43..e9e31ede 100644
--- a/node/Cluster.cpp
+++ b/node/Cluster.cpp
@@ -85,7 +85,8 @@ Cluster::Cluster(
 	_members(new _Member[ZT_CLUSTER_MAX_MEMBERS]),
 	_peerAffinities(65536),
 	_lastCleanedPeerAffinities(0),
-	_lastCheckedPeersForAnnounce(0)
+	_lastCheckedPeersForAnnounce(0),
+	_lastFlushed(0)
 {
 	uint16_t stmp[ZT_SHA512_DIGEST_LEN / sizeof(uint16_t)];
 
@@ -510,7 +511,8 @@ void Cluster::doPeriodicTasks()
 	}
 
 	// Flush outgoing packet send queue every doPeriodicTasks()
-	{
+	if ((now - _lastFlushed) >= ZT_CLUSTER_FLUSH_PERIOD) {
+		_lastFlushed = now;
 		Mutex::Lock _l(_memberIds_m);
 		for(std::vector<uint16_t>::const_iterator mid(_memberIds.begin());mid!=_memberIds.end();++mid) {
 			Mutex::Lock _l2(_members[*mid].lock);
diff --git a/node/Cluster.hpp b/node/Cluster.hpp
index 7d7a1ced..f1caa436 100644
--- a/node/Cluster.hpp
+++ b/node/Cluster.hpp
@@ -55,13 +55,18 @@
 /**
  * How often should we announce that we have a peer?
  */
-#define ZT_CLUSTER_HAVE_PEER_ANNOUNCE_PERIOD ((ZT_PEER_ACTIVITY_TIMEOUT / 2) - 1000)
+#define ZT_CLUSTER_HAVE_PEER_ANNOUNCE_PERIOD (ZT_PEER_DIRECT_PING_DELAY / 2)
 
 /**
  * Desired period between doPeriodicTasks() in milliseconds
  */
 #define ZT_CLUSTER_PERIODIC_TASK_PERIOD 250
 
+/**
+ * How often to flush outgoing message queues (maximum interval)
+ */
+#define ZT_CLUSTER_FLUSH_PERIOD 500
+
 namespace ZeroTier {
 
 class RuntimeEnvironment;
@@ -355,6 +360,7 @@ private:
 
 	uint64_t _lastCleanedPeerAffinities;
 	uint64_t _lastCheckedPeersForAnnounce;
+	uint64_t _lastFlushed;
 };
 
 } // namespace ZeroTier
diff --git a/node/Constants.hpp b/node/Constants.hpp
index 1d5fa6f4..bb62484d 100644
--- a/node/Constants.hpp
+++ b/node/Constants.hpp
@@ -173,13 +173,8 @@
 
 /**
  * Timeout for receipt of fragmented packets in ms
- *
- * Since there's no retransmits, this is just a really bad case scenario for
- * transit time. It's short enough that a DOS attack from exhausing buffers is
- * very unlikely, as the transfer rate would have to be fast enough to fill
- * system memory in this time.
  */
-#define ZT_FRAGMENTED_PACKET_RECEIVE_TIMEOUT 1000
+#define ZT_FRAGMENTED_PACKET_RECEIVE_TIMEOUT 500
 
 /**
  * Length of secret key in bytes -- 256-bit -- do not change
@@ -194,7 +189,7 @@
 /**
  * Overriding granularity for timer tasks to prevent CPU-intensive thrashing on every packet
  */
-#define ZT_CORE_TIMER_TASK_GRANULARITY 1000
+#define ZT_CORE_TIMER_TASK_GRANULARITY 500
 
 /**
  * How long to remember peer records in RAM if they haven't been used
@@ -269,7 +264,7 @@
 /**
  * Delay between ordinary case pings of direct links
  */
-#define ZT_PEER_DIRECT_PING_DELAY 120000
+#define ZT_PEER_DIRECT_PING_DELAY 60000
 
 /**
  * Delay between requests for updated network autoconf information
@@ -279,18 +274,7 @@
 /**
  * Timeout for overall peer activity (measured from last receive)
  */
-#define ZT_PEER_ACTIVITY_TIMEOUT (ZT_PEER_DIRECT_PING_DELAY + (ZT_PING_CHECK_INVERVAL * 3))
-
-/**
- * Stop relaying via peers that have not responded to direct sends
- *
- * When we send something (including frames), we generally expect a response.
- * Switching relays if no response in a short period of time causes more
- * rapid failover if a root server goes down or becomes unreachable. In the
- * mistaken case, little harm is done as it'll pick the next-fastest
- * root server and will switch back eventually.
- */
-#define ZT_PEER_RELAY_CONVERSATION_LATENCY_THRESHOLD 10000
+#define ZT_PEER_ACTIVITY_TIMEOUT ((ZT_PEER_DIRECT_PING_DELAY * 3) + (ZT_PING_CHECK_INVERVAL * 2))
 
 /**
  * Minimum interval between attempts by relays to unite peers
diff --git a/node/Multicaster.cpp b/node/Multicaster.cpp
index e43d7d88..01e6b799 100644
--- a/node/Multicaster.cpp
+++ b/node/Multicaster.cpp
@@ -175,128 +175,130 @@ void Multicaster::send(
 	unsigned long idxbuf[8194];
 	unsigned long *indexes = idxbuf;
 
-	Mutex::Lock _l(_groups_m);
-	MulticastGroupStatus &gs = _groups[Multicaster::Key(nwid,mg)];
-
-	if (!gs.members.empty()) {
-		// Allocate a memory buffer if group is monstrous
-		if (gs.members.size() > (sizeof(idxbuf) / sizeof(unsigned long)))
-			indexes = new unsigned long[gs.members.size()];
-
-		// Generate a random permutation of member indexes
-		for(unsigned long i=0;i<gs.members.size();++i)
-			indexes[i] = i;
-		for(unsigned long i=(unsigned long)gs.members.size()-1;i>0;--i) {
-			unsigned long j = (unsigned long)RR->node->prng() % (i + 1);
-			unsigned long tmp = indexes[j];
-			indexes[j] = indexes[i];
-			indexes[i] = tmp;
+	try {
+		Mutex::Lock _l(_groups_m);
+		MulticastGroupStatus &gs = _groups[Multicaster::Key(nwid,mg)];
+
+		if (!gs.members.empty()) {
+			// Allocate a memory buffer if group is monstrous
+			if (gs.members.size() > (sizeof(idxbuf) / sizeof(unsigned long)))
+				indexes = new unsigned long[gs.members.size()];
+
+			// Generate a random permutation of member indexes
+			for(unsigned long i=0;i<gs.members.size();++i)
+				indexes[i] = i;
+			for(unsigned long i=(unsigned long)gs.members.size()-1;i>0;--i) {
+				unsigned long j = (unsigned long)RR->node->prng() % (i + 1);
+				unsigned long tmp = indexes[j];
+				indexes[j] = indexes[i];
+				indexes[i] = tmp;
+			}
 		}
-	}
 
-	if (gs.members.size() >= limit) {
-		// Skip queue if we already have enough members to complete the send operation
-		OutboundMulticast out;
-
-		out.init(
-			RR,
-			now,
-			nwid,
-			com,
-			limit,
-			1, // we'll still gather a little from peers to keep multicast list fresh
-			src,
-			mg,
-			etherType,
-			data,
-			len);
-
-		unsigned int count = 0;
-
-		for(std::vector<Address>::const_iterator ast(alwaysSendTo.begin());ast!=alwaysSendTo.end();++ast) {
-			if (*ast != RR->identity.address()) {
-				out.sendOnly(RR,*ast);
-				if (++count >= limit)
-					break;
+		if (gs.members.size() >= limit) {
+			// Skip queue if we already have enough members to complete the send operation
+			OutboundMulticast out;
+
+			out.init(
+				RR,
+				now,
+				nwid,
+				com,
+				limit,
+				1, // we'll still gather a little from peers to keep multicast list fresh
+				src,
+				mg,
+				etherType,
+				data,
+				len);
+
+			unsigned int count = 0;
+
+			for(std::vector<Address>::const_iterator ast(alwaysSendTo.begin());ast!=alwaysSendTo.end();++ast) {
+				if (*ast != RR->identity.address()) {
+					out.sendOnly(RR,*ast); // optimization: don't use dedup log if it's a one-pass send
+					if (++count >= limit)
+						break;
+				}
 			}
-		}
 
-		unsigned long idx = 0;
-		while ((count < limit)&&(idx < gs.members.size())) {
-			Address ma(gs.members[indexes[idx++]].address);
-			if (std::find(alwaysSendTo.begin(),alwaysSendTo.end(),ma) == alwaysSendTo.end()) {
-				out.sendOnly(RR,ma);
-				++count;
+			unsigned long idx = 0;
+			while ((count < limit)&&(idx < gs.members.size())) {
+				Address ma(gs.members[indexes[idx++]].address);
+				if (std::find(alwaysSendTo.begin(),alwaysSendTo.end(),ma) == alwaysSendTo.end()) {
+					out.sendOnly(RR,ma); // optimization: don't use dedup log if it's a one-pass send
+					++count;
+				}
 			}
-		}
-	} else {
-		unsigned int gatherLimit = (limit - (unsigned int)gs.members.size()) + 1;
-
-		if ((now - gs.lastExplicitGather) >= ZT_MULTICAST_EXPLICIT_GATHER_DELAY) {
-			gs.lastExplicitGather = now;
-			SharedPtr<Peer> r(RR->topology->getBestRoot());
-			if (r) {
-				TRACE(">>MC upstream GATHER up to %u for group %.16llx/%s",gatherLimit,nwid,mg.toString().c_str());
-
-				const CertificateOfMembership *com = (CertificateOfMembership *)0;
-				{
-					SharedPtr<Network> nw(RR->node->network(nwid));
-					if (nw) {
-						SharedPtr<NetworkConfig> nconf(nw->config2());
-						if ((nconf)&&(nconf->com())&&(nconf->isPrivate())&&(r->needsOurNetworkMembershipCertificate(nwid,now,true)))
-							com = &(nconf->com());
+		} else {
+			unsigned int gatherLimit = (limit - (unsigned int)gs.members.size()) + 1;
+
+			if ((gs.members.empty())||((now - gs.lastExplicitGather) >= ZT_MULTICAST_EXPLICIT_GATHER_DELAY)) {
+				gs.lastExplicitGather = now;
+				SharedPtr<Peer> r(RR->topology->getBestRoot());
+				if (r) {
+					TRACE(">>MC upstream GATHER up to %u for group %.16llx/%s",gatherLimit,nwid,mg.toString().c_str());
+
+					const CertificateOfMembership *com = (CertificateOfMembership *)0;
+					{
+						SharedPtr<Network> nw(RR->node->network(nwid));
+						if (nw) {
+							SharedPtr<NetworkConfig> nconf(nw->config2());
+							if ((nconf)&&(nconf->com())&&(nconf->isPrivate())&&(r->needsOurNetworkMembershipCertificate(nwid,now,true)))
+								com = &(nconf->com());
+						}
 					}
-				}
 
-				Packet outp(r->address(),RR->identity.address(),Packet::VERB_MULTICAST_GATHER);
-				outp.append(nwid);
-				outp.append((uint8_t)(com ? 0x01 : 0x00));
-				mg.mac().appendTo(outp);
-				outp.append((uint32_t)mg.adi());
-				outp.append((uint32_t)gatherLimit);
-				if (com)
-					com->serialize(outp);
-				outp.armor(r->key(),true);
-				r->send(RR,outp.data(),outp.size(),now);
+					Packet outp(r->address(),RR->identity.address(),Packet::VERB_MULTICAST_GATHER);
+					outp.append(nwid);
+					outp.append((uint8_t)(com ? 0x01 : 0x00));
+					mg.mac().appendTo(outp);
+					outp.append((uint32_t)mg.adi());
+					outp.append((uint32_t)gatherLimit);
+					if (com)
+						com->serialize(outp);
+					outp.armor(r->key(),true);
+					r->send(RR,outp.data(),outp.size(),now);
+				}
+				gatherLimit = 0;
 			}
-			gatherLimit = 0;
-		}
 
-		gs.txQueue.push_back(OutboundMulticast());
-		OutboundMulticast &out = gs.txQueue.back();
-
-		out.init(
-			RR,
-			now,
-			nwid,
-			com,
-			limit,
-			gatherLimit,
-			src,
-			mg,
-			etherType,
-			data,
-			len);
-
-		unsigned int count = 0;
-
-		for(std::vector<Address>::const_iterator ast(alwaysSendTo.begin());ast!=alwaysSendTo.end();++ast) {
-			if (*ast != RR->identity.address()) {
-				out.sendAndLog(RR,*ast);
-				if (++count >= limit)
-					break;
+			gs.txQueue.push_back(OutboundMulticast());
+			OutboundMulticast &out = gs.txQueue.back();
+
+			out.init(
+				RR,
+				now,
+				nwid,
+				com,
+				limit,
+				gatherLimit,
+				src,
+				mg,
+				etherType,
+				data,
+				len);
+
+			unsigned int count = 0;
+
+			for(std::vector<Address>::const_iterator ast(alwaysSendTo.begin());ast!=alwaysSendTo.end();++ast) {
+				if (*ast != RR->identity.address()) {
+					out.sendAndLog(RR,*ast);
+					if (++count >= limit)
+						break;
+				}
 			}
-		}
 
-		unsigned long idx = 0;
-		while ((count < limit)&&(idx < gs.members.size())) {
-			Address ma(gs.members[indexes[idx++]].address);
-			if (std::find(alwaysSendTo.begin(),alwaysSendTo.end(),ma) == alwaysSendTo.end()) {
-				out.sendAndLog(RR,ma);
-				++count;
+			unsigned long idx = 0;
+			while ((count < limit)&&(idx < gs.members.size())) {
+				Address ma(gs.members[indexes[idx++]].address);
+				if (std::find(alwaysSendTo.begin(),alwaysSendTo.end(),ma) == alwaysSendTo.end()) {
+					out.sendAndLog(RR,ma);
+					++count;
+				}
 			}
 		}
-	}
+	} catch ( ... ) {} // this is a sanity check to catch any failures and make sure indexes[] still gets deleted
 
 	// Free allocated memory buffer if any
 	if (indexes != idxbuf)
diff --git a/node/Node.cpp b/node/Node.cpp
index 42180e99..74acc869 100644
--- a/node/Node.cpp
+++ b/node/Node.cpp
@@ -305,18 +305,7 @@ ZT_ResultCode Node::processBackgroundTasks(uint64_t now,volatile uint64_t *nextB
 			for(std::vector< SharedPtr<Network> >::const_iterator n(needConfig.begin());n!=needConfig.end();++n)
 				(*n)->requestConfiguration();
 
-			// Attempt to contact network preferred relays that we don't have direct links to
-			std::sort(networkRelays.begin(),networkRelays.end());
-			networkRelays.erase(std::unique(networkRelays.begin(),networkRelays.end()),networkRelays.end());
-			for(std::vector< std::pair<Address,InetAddress> >::const_iterator nr(networkRelays.begin());nr!=networkRelays.end();++nr) {
-				if (nr->second) {
-					SharedPtr<Peer> rp(RR->topology->getPeer(nr->first));
-					if ((rp)&&(!rp->hasActiveDirectPath(now)))
-						rp->attemptToContactAt(RR,InetAddress(),nr->second,now);
-				}
-			}
-
-			// Ping living or root server/relay peers
+			// Do pings and keepalives
 			_PingPeersThatNeedPing pfunc(RR,now,networkRelays);
 			RR->topology->eachPeer<_PingPeersThatNeedPing &>(pfunc);
 
diff --git a/tests/http/big-test-kill.sh b/tests/http/big-test-kill.sh
index 4a764d1f..59f36788 100755
--- a/tests/http/big-test-kill.sh
+++ b/tests/http/big-test-kill.sh
@@ -13,6 +13,6 @@ CONTAINER_IMAGE=zerotier/http-test
 
 export PATH=/bin:/usr/bin:/usr/local/bin:/usr/sbin:/sbin
 
-pssh -h big-test-hosts -i -t 128 -p 256 "docker ps -aq | xargs -r docker rm -f"
+pssh -h big-test-hosts -i -t 0 -p 256 "docker ps -aq | xargs -r docker rm -f"
 
 exit 0
diff --git a/tests/http/big-test-ready.sh b/tests/http/big-test-ready.sh
index 391ca2a1..aa540bba 100755
--- a/tests/http/big-test-ready.sh
+++ b/tests/http/big-test-ready.sh
@@ -25,6 +25,6 @@ export PATH=/bin:/usr/bin:/usr/local/bin:/usr/sbin:/sbin
 #	docker run --device=/dev/net/tun --privileged -d $CONTAINER_IMAGE
 #done
 
-pssh -h big-test-hosts -i -t 128 -p 256 "docker pull $CONTAINER_IMAGE"
+pssh -h big-test-hosts -i -t 0 -p 256 "docker pull $CONTAINER_IMAGE"
 
 exit 0
diff --git a/tests/http/big-test-start.sh b/tests/http/big-test-start.sh
index a5e71ef1..43166c6e 100755
--- a/tests/http/big-test-start.sh
+++ b/tests/http/big-test-start.sh
@@ -1,7 +1,7 @@
 #!/bin/bash
 
 # Edit as needed -- note that >1000 per host is likely problematic due to Linux kernel limits
-NUM_CONTAINERS=100
+NUM_CONTAINERS=25
 CONTAINER_IMAGE=zerotier/http-test
 
 #
@@ -25,6 +25,6 @@ export PATH=/bin:/usr/bin:/usr/local/bin:/usr/sbin:/sbin
 #	docker run --device=/dev/net/tun --privileged -d $CONTAINER_IMAGE
 #done
 
-pssh -h big-test-hosts -i -t 128 -p 256 "for ((n=0;n<$NUM_CONTAINERS;n++)); do docker run --device=/dev/net/tun --privileged -d $CONTAINER_IMAGE; sleep 0.25; done"
+pssh -h big-test-hosts -i -t 0 -p 256 "for ((n=0;n<$NUM_CONTAINERS;n++)); do docker run --device=/dev/net/tun --privileged -d $CONTAINER_IMAGE; sleep 0.25; done"
 
 exit 0
-- 
cgit v1.2.3


From 7fbe2f7adf3575f3a21fc1ab3a5a2a036e18e6e2 Mon Sep 17 00:00:00 2001
From: Adam Ierymenko <adam.ierymenko@gmail.com>
Date: Mon, 2 Nov 2015 15:38:53 -0800
Subject: Tweak some more timings for better reliability.

---
 node/Cluster.hpp             |  2 +-
 node/Constants.hpp           | 10 +++++-----
 node/Node.cpp                |  2 +-
 node/Peer.hpp                |  4 ++--
 node/SelfAwareness.cpp       |  2 +-
 node/Switch.cpp              |  6 +++---
 node/Topology.hpp            |  9 ++++++---
 tests/http/big-test-start.sh |  4 ++--
 8 files changed, 21 insertions(+), 18 deletions(-)

(limited to 'tests')

diff --git a/node/Cluster.hpp b/node/Cluster.hpp
index f1caa436..ee220999 100644
--- a/node/Cluster.hpp
+++ b/node/Cluster.hpp
@@ -55,7 +55,7 @@
 /**
  * How often should we announce that we have a peer?
  */
-#define ZT_CLUSTER_HAVE_PEER_ANNOUNCE_PERIOD (ZT_PEER_DIRECT_PING_DELAY / 2)
+#define ZT_CLUSTER_HAVE_PEER_ANNOUNCE_PERIOD ZT_PEER_DIRECT_PING_DELAY
 
 /**
  * Desired period between doPeriodicTasks() in milliseconds
diff --git a/node/Constants.hpp b/node/Constants.hpp
index bb62484d..552688a6 100644
--- a/node/Constants.hpp
+++ b/node/Constants.hpp
@@ -267,14 +267,14 @@
 #define ZT_PEER_DIRECT_PING_DELAY 60000
 
 /**
- * Delay between requests for updated network autoconf information
+ * Timeout for overall peer activity (measured from last receive)
  */
-#define ZT_NETWORK_AUTOCONF_DELAY 60000
+#define ZT_PEER_ACTIVITY_TIMEOUT ((ZT_PEER_DIRECT_PING_DELAY * 4) + ZT_PING_CHECK_INVERVAL)
 
 /**
- * Timeout for overall peer activity (measured from last receive)
+ * Delay between requests for updated network autoconf information
  */
-#define ZT_PEER_ACTIVITY_TIMEOUT ((ZT_PEER_DIRECT_PING_DELAY * 3) + (ZT_PING_CHECK_INVERVAL * 2))
+#define ZT_NETWORK_AUTOCONF_DELAY 60000
 
 /**
  * Minimum interval between attempts by relays to unite peers
@@ -283,7 +283,7 @@
  * a RENDEZVOUS message no more than this often. This instructs the peers
  * to attempt NAT-t and gives each the other's corresponding IP:port pair.
  */
-#define ZT_MIN_UNITE_INTERVAL 60000
+#define ZT_MIN_UNITE_INTERVAL 30000
 
 /**
  * Delay between initial direct NAT-t packet and more aggressive techniques
diff --git a/node/Node.cpp b/node/Node.cpp
index 74acc869..82cb7ddb 100644
--- a/node/Node.cpp
+++ b/node/Node.cpp
@@ -263,7 +263,7 @@ public:
 			}
 
 			lastReceiveFromUpstream = std::max(p->lastReceive(),lastReceiveFromUpstream);
-		} else if (p->alive(_now)) {
+		} else if (p->activelyTransferringFrames(_now)) {
 			// Normal nodes get their preferred link kept alive if the node has generated frame traffic recently
 			p->doPingAndKeepalive(RR,_now,0);
 		}
diff --git a/node/Peer.hpp b/node/Peer.hpp
index e5db3bde..ad4c6746 100644
--- a/node/Peer.hpp
+++ b/node/Peer.hpp
@@ -231,9 +231,9 @@ public:
 	inline uint64_t lastAnnouncedTo() const throw() { return _lastAnnouncedTo; }
 
 	/**
-	 * @return True if peer has received an actual data frame within ZT_PEER_ACTIVITY_TIMEOUT milliseconds
+	 * @return True if this peer is actively sending real network frames
 	 */
-	inline uint64_t alive(uint64_t now) const throw() { return ((now - lastFrame()) < ZT_PEER_ACTIVITY_TIMEOUT); }
+	inline uint64_t activelyTransferringFrames(uint64_t now) const throw() { return ((now - lastFrame()) < ZT_PEER_ACTIVITY_TIMEOUT); }
 
 	/**
 	 * @return Current latency or 0 if unknown (max: 65535)
diff --git a/node/SelfAwareness.cpp b/node/SelfAwareness.cpp
index d8eca071..ce75eb03 100644
--- a/node/SelfAwareness.cpp
+++ b/node/SelfAwareness.cpp
@@ -128,7 +128,7 @@ void SelfAwareness::iam(const Address &reporter,const InetAddress &reporterPhysi
 		// links to be re-established if possible, possibly using a root server or some
 		// other relay.
 		for(std::vector< SharedPtr<Peer> >::const_iterator p(rset.peersReset.begin());p!=rset.peersReset.end();++p) {
-			if ((*p)->alive(now)) {
+			if ((*p)->activelyTransferringFrames(now)) {
 				Packet outp((*p)->address(),RR->identity.address(),Packet::VERB_NOP);
 				RR->sw->send(outp,true,0);
 			}
diff --git a/node/Switch.cpp b/node/Switch.cpp
index 2f72f57a..120ce7a4 100644
--- a/node/Switch.cpp
+++ b/node/Switch.cpp
@@ -442,8 +442,8 @@ unsigned long Switch::doTimerTasks(uint64_t now)
 		Mutex::Lock _l(_contactQueue_m);
 		for(std::list<ContactQueueEntry>::iterator qi(_contactQueue.begin());qi!=_contactQueue.end();) {
 			if (now >= qi->fireAtTime) {
-				if ((!qi->peer->alive(now))||(qi->peer->hasActiveDirectPath(now))) {
-					// Cancel attempt if we've already connected or peer is no longer "alive"
+				if (qi->peer->hasActiveDirectPath(now)) {
+					// Cancel if connection has succeeded
 					_contactQueue.erase(qi++);
 					continue;
 				} else {
@@ -539,7 +539,7 @@ unsigned long Switch::doTimerTasks(uint64_t now)
 		_LastUniteKey *k = (_LastUniteKey *)0;
 		uint64_t *v = (uint64_t *)0;
 		while (i.next(k,v)) {
-			if ((now - *v) >= (ZT_MIN_UNITE_INTERVAL * 16))
+			if ((now - *v) >= (ZT_MIN_UNITE_INTERVAL * 8))
 				_lastUniteAttempt.erase(*k);
 		}
 	}
diff --git a/node/Topology.hpp b/node/Topology.hpp
index 4c1a2ab3..a0c28b0f 100644
--- a/node/Topology.hpp
+++ b/node/Topology.hpp
@@ -81,6 +81,11 @@ public:
 	/**
 	 * Get a peer only if it is presently in memory (no disk cache)
 	 *
+	 * This also does not update the lastUsed() time for peers, which means
+	 * that it won't prevent them from falling out of RAM. This is currently
+	 * used in the Cluster code to update peer info without forcing all peers
+	 * across the entire cluster to remain in memory cache.
+	 *
 	 * @param zta ZeroTier address
 	 * @param now Current time
 	 */
@@ -88,10 +93,8 @@ public:
 	{
 		Mutex::Lock _l(_lock);
 		const SharedPtr<Peer> *const ap = _peers.get(zta);
-		if (ap) {
-			(*ap)->use(now);
+		if (ap)
 			return *ap;
-		}
 		return SharedPtr<Peer>();
 	}
 
diff --git a/tests/http/big-test-start.sh b/tests/http/big-test-start.sh
index 43166c6e..f300ac61 100755
--- a/tests/http/big-test-start.sh
+++ b/tests/http/big-test-start.sh
@@ -1,7 +1,7 @@
 #!/bin/bash
 
 # Edit as needed -- note that >1000 per host is likely problematic due to Linux kernel limits
-NUM_CONTAINERS=25
+NUM_CONTAINERS=50
 CONTAINER_IMAGE=zerotier/http-test
 
 #
@@ -25,6 +25,6 @@ export PATH=/bin:/usr/bin:/usr/local/bin:/usr/sbin:/sbin
 #	docker run --device=/dev/net/tun --privileged -d $CONTAINER_IMAGE
 #done
 
-pssh -h big-test-hosts -i -t 0 -p 256 "for ((n=0;n<$NUM_CONTAINERS;n++)); do docker run --device=/dev/net/tun --privileged -d $CONTAINER_IMAGE; sleep 0.25; done"
+pssh -h big-test-hosts -o big-test-out -t 0 -p 256 "for ((n=0;n<$NUM_CONTAINERS;n++)); do docker run --device=/dev/net/tun --privileged -d $CONTAINER_IMAGE; sleep 0.25; done"
 
 exit 0
-- 
cgit v1.2.3