9 files changed, 193 insertions, 25 deletions
diff --git a/tests/http/agent.js b/tests/http/agent.js
index d0c33917..e11fed60 100644
--- a/tests/http/agent.js
+++ b/tests/http/agent.js
@@ -214,12 +214,22 @@ app.get('/',function(req,res) {
 });
 
 var expressServer = app.listen(AGENT_PORT,function () {
-	registerAndGetPeers(function(err,peers) {
-		if (err) {
-			console.error('FATAL: unable to contact or query server: '+err.toString());
-			process.exit(1);
-		}
-		doTestsAndReport();
-		setInterval(doTestsAndReport,TEST_INTERVAL);
-	});
+	var serverUp = false;
+	async.whilst(
+		function() { return (!serverUp); },
+		function(nextTry) {
+			registerAndGetPeers(function(err,peers) {
+				if ((err)||(!peers)) {
+					setTimeout(nextTry,1000);
+				} else {
+					serverUp = true;
+					return nextTry(null);
+				}
+			});
+		},
+		function(err) {
+			console.log('Server up, starting!');
+			doTestsAndReport();
+			setInterval(doTestsAndReport,TEST_INTERVAL);
+		});
 });
diff --git a/tests/http/big-test-hosts b/tests/http/big-test-hosts
index 27c0c656..93b6f23f 100644
--- a/tests/http/big-test-hosts
+++ b/tests/http/big-test-hosts
@@ -1,4 +1,2 @@
 root@104.156.246.48
 root@104.156.252.136
-root@46.101.72.130
-root@188.166.240.16
diff --git a/tests/http/big-test-kill.sh b/tests/http/big-test-kill.sh
index fbb34c10..917a7791 100755
--- a/tests/http/big-test-kill.sh
+++ b/tests/http/big-test-kill.sh
@@ -13,18 +13,6 @@ CONTAINER_IMAGE=zerotier/http-test
 
 export PATH=/bin:/usr/bin:/usr/local/bin:/usr/sbin:/sbin
 
-# Kill and clean up old test containers if any -- note that this kills all containers on the system!
-#docker ps -q | xargs -n 1 docker kill
-#docker ps -aq | xargs -n 1 docker rm
-
-# Pull latest if needed -- change this to your image name and/or where to pull it from
-#docker pull $CONTAINER_IMAGE
-
-# Run NUM_CONTAINERS
-#for ((n=0;n<$NUM_CONTAINERS;n++)); do
-#	docker run --device=/dev/net/tun --privileged -d $CONTAINER_IMAGE
-#done
-
-pssh -h big-test-hosts -i -p 256 "docker ps -q | xargs -r -n 128 docker kill && docker ps -aq | xargs -r -P 16 -n 1 docker rm"
+pssh -h big-test-hosts -i -t 128 -p 256 "docker ps -q | xargs -r docker kill && docker ps -aq | xargs -r docker rm"
 
 exit 0
diff --git a/tests/http/big-test-out/root@104.156.246.48 b/tests/http/big-test-out/root@104.156.246.48
new file mode 100644
index 00000000..afcda19f
--- /dev/null
+++ b/tests/http/big-test-out/root@104.156.246.48
@@ -0,0 +1,67 @@
+5fc1676570b10196c5e261dac6ff28998c4f66c706b131d6423532a38b8b7a15
+894bbfa612f772aa0170d5c5ddb9362a9edb0f5fbe22c81591b50dfa0a0eadeb
+bae9bf7c37806adbbdfe9e01b4e62034953d01ecae55e0cd450a21b1415f5c00
+26e83eccbd6083f2c7a8532786fd1ec5c96d6a51bb3508ec4fd1919a1883630a
+90e95ac0cd3b4ba6ffa33fe946a1f12a2cdba61168770af9fca7b42df45c9530
+88a11bbc89c0b3579de7faf4db4dbe3f0a5a073aa49fd7a5482eff35c93fab4e
+50be80765f09000085d6c0895958e6d3794f672f562ff5bc27634b15c77b7583
+99630786ef067f4c053f7bca96e2869f14c9ceb9f1d07b7635945c143700b950
+8422d4cb09ed44e34488163a96910ece8f1dd39c280daf3d369d17ce2ac70766
+e0a0af014418076adc351827fabbd9ef6da7404f2d6184f20d601736c4685154
+387cd6dbbd8e8b17d9ccf943fd13dcaddf27c178834a9630e357407a42733fb8
+b06b75cb0c5748321ab4501e1055dd5ff8457e81502548954702ead843011388
+59d874f5e36f15b62e8540b86d552950c72f086d90667780b6d82b5595326fce
+69dc1347f8671ba4bbe1da12f26e8f67b0980f1e2ad73bc0b77cc06e5cbf1b06
+68ef4b43d3f5559b5b0c82ba2f396a5b6dcb6001a67efd3a3b3b2a415c2b7b61
+3ecc9f45151f95a194d8274a88f433f83540f5397523de7a86db714cd9155bb2
+3b24b66b1dcd5e8ce1fa24d33cb2eb3cc55f3a157602a09bca4942089e25790f
+ba34465f6cedd2f306022cc9259ef4e43819959f51f980a8cb94ea33a29c7e34
+4e9a701f18a0ac42ee24f3bf2dd6dc442706a0bf4b6288336447b03752640852
+5865df2182165576a0825644d9b7537314c9fd4323cb2023260382acc2f9d7fd
+f1498fda2dd6d1f1fdfc95d539d6ee511ca8baad65b1a1b44d76309e84015550
+7d9232ecda50856096523dc2d0c1fae46481053067d8bfe024e1dfccf8f9c0fc
+173b47f327a3c21187e25d2d02dbde49760182a40e00cf4a64746c00537a3ff6
+9221fdb92693a29f720a5c41f09791c35f2b220459a2e4bdcc24fd5e3af4ad85
+ef8def0260ba3ff21aa292bb482bedf94b17fd9306e529f6feae5cab04e1bdc2
+7392179b9d51cad626b6cf78b00c355b280ddccccd7013e11a8cecbdd2db1c15
+fc31766af9265f2c496ee3847971ee0f5249aca8ad0cd214620643c7347436b2
+7f8bd6d1fde6948a09b132b4b2c3919ca11c932bfdf8a279c4cbe140daf24287
+ce9596c1d8dd42929820e4d3a56e8b1f8eb2d8b67474c525bbed1b888143cc14
+67c9de4f3d88daa89796e8a03306d2c5f2bcc4c409eef89e67c6ffc6a6282060
+705c1973e4a9ef77ffe6319375e585aec1d762a031c8b93ba9883af98e377590
+c7fdcc10eed007a30da7c7bbb8767d0b6a7287fadd67d935651f0bb265a71e1b
+5102e854fe72cdcd91b228d520a9380a36a273df7949de054355e8f99bd14e95
+77c37e503ccaa1350bf2fe16daaef486ec223348bc8f678b6f6d6bb477ae10c1
+fd97792d7bd61449163fa4e953212446e11cf02b27a45a2073a1de5148da63e8
+7d5b84b290f727713ad02f0817856c9b891996bdcc6ad4d0994005608cf9bbb7
+104c41c9cce7934f0e205cfc90b65f5ba89ca696cab41771a12b4384af7f6805
+75e1751a7e290a34faddea54c98e870b4a1cd5bb37c810cafabda8d3ba1faa5f
+2293a17e82ff54e04218c5aafd904079d15d71e47496249dc125f90f0960039f
+3f3563782349fddb61cdc638cf5f54030f726d9759cb104253f5b8b04ae8e2b0
+2328da71617e1abc0e2335e5974a70f45f8abaedba641292ad4d87e2f27a6b83
+00e79b478925c7b866f7d669ee73af1a7b377fb8bf22a04f2bd5356f256f59ba
+2960f757cca294c32abd51f072798f5457a1552de52bd42ca9233cd075da086f
+efbc248c9a3ce7b7a52724b67f14e27b02f5f37da295bc905d4f8fabe847cd00
+a068d735478e065236e840e50697a078d77aad9b82f906555279e5dee074db5d
+c4a366870dd1f3d1fb776b25f009d9079a6f7d0d83b03cd178b237b412d8dce6
+20340c7bd3bd9d32d6ecb7a451c377fb239bfe2d2c976886e2bd59746ed180ee
+8c522ecbabb9580528794d94f82bdef2982f0458b677598f236acc14e819e480
+8dfcc3407b050a39d82af263eec6e332bd69cb848ffa660344644ea10e3c2221
+74e06d9deba29982f9dccdb841163715dd419dee6a54eaf3ac987cd3458a1a2a
+725d990ca2ea34dc3e9acc02480cc8009a9c2016414c9c9cca3c7b135bd384f4
+120d7eac6a5bd761ee6acfb751f48bf7075c0316123326ce6bbb6ce3fe05c3a7
+91aa44d2650ff308877d9c81357619c51c0c0d05dc9f4c899df9ce460673b2aa
+3c203cd73c6be2606397357154789a94fbfe8670271825f75bbfc6c97fa0e048
+f9018c8390a472798c7a39bfe834cd01bb62bb4b0882dfe1108bf43334a3bd0e
+cdf8afae641e0423b2a7a1ff92cca80c7db478ff8deb1d81032808ac84415921
+9a5364df1df5460f6a9ed15e020b8bd283c47464556a7c5896707cee00c01a14
+df9365b8816e9d67f484adf94eef53aab236a92a588a1f3f650fd36b8073f7bb
+bd0172f67fa20716502e2bee82387de7f426e3cdb19d5ac6ce9dcc177f919cde
+90dd259c03b11625c09b8db614f45759e67edf07fe350681d273bfd988b45443
+9f69d376248b6851aa7837a7d09a1b9eda917601049e5942796815069d09a80f
+1a065bee20e8f4c6a91bb92ec9cca6ca16e8eb434798ed433a5248c48d91f596
+fb5a6a9397546a97ccdc4252603d5e774d8430195b06ec74926c48cf372b9906
+51cf0867773bb298140eb09c88d69587aae3a6da3e63275d32f3d32d98a737d7
+70574f3e616413ce90b045e0e9fd92353766e216eabf8139556fa61efea9c3b4
+ccd21257994f9eaf309d06b2fc5652b14ea80796a79face304fcc8fd1da53423
+7d8417be656f17fbfb779a9803b2de045e2f496f75ea5f1ea69e223572bde2a5
diff --git a/tests/http/big-test-out/root@104.156.252.136 b/tests/http/big-test-out/root@104.156.252.136
new file mode 100644
index 00000000..c11311b4
--- /dev/null
+++ b/tests/http/big-test-out/root@104.156.252.136
@@ -0,0 +1,75 @@
+5ea6049a0b92070494f40a5ccecccaf788a5aafccee7c2eada9b9eb8731bc002
+798f8beecd2e3fbb50df49b7ef57cdd1e8e00c0680046b3c2d53a3554f956fda
+dd20da6340210e1b7612d8922aaa4b045e84da32f264add073a65a15f676a9e4
+0479f14d0aa68e835c07dc5ca413febd9da19b6554fd8bdba7e319e5f4661f80
+df6747868f90cfa069f5f9f954626b7392cd99026e43e6d6c83ab7c16d5cbdc6
+c24799f74233b1bc7d7d936d57699b955000c640531f3db38be8196a87eb262c
+46b00a65d527738c0bfad924051bd2117563e0c6ad74b803b662e74720d8d085
+dc14b9428032771388d30c6002bb5cba05131972cab53360f088c51769786c47
+e7dc364aeccf60bafa5a42787cc6de231612782252f57b9f03ebae3a309b2352
+4e8e578a8948ab384525646a17c2e0cb9f2b9ef67fd0c489ad6aa2bffbfabddc
+cc626d978e32dfe14782011218ba265ae4e69886f44335a2c402001dc0c4c3c7
+36a148254d34f954906a810ba4a8644a4433e8847d3cc30e091b1f63723f0590
+1bdd06b691fa06b3af77d2868b78f2a01b91026b4f483fb278cb8872a9150987
+d6d5709039708a515b295519a4007c3b49361ef67465ddca2dfba9a473b9c37c
+f8a3a0b3dd5ce42bd87cdba28df7469071f948fb28151955cfa75abe0455a000
+f7eb37fecb17571091f05f1bfc66f6fba731ff934988a529813f4751451401af
+1c2f2c77c429095b1dad53032684df672f351489ed6b7e00e1097f7dc1c0ba97
+361dd9cc6764facd2aca0b462e64a2400f6b41e124c4d9be71466d801763270a
+3f5f49ce854439bc672c4b0cf4c1e1b9a978e8f2db14709977ddcfd39b7d6bb3
+5221bec1aec2c9f08ddec548a24b0700e3d0c0568d10caa753564c914b35c95f
+ebdcc27c264d326619262f82b5d7dfcbf102d720ad9bab4428b9725118bf627a
+bc01c863316a29c8f119da7cce1c891185c43d385521d46b06186a89cb6cff9d
+47fb8b119e3d098f22cc6f4a7867f2016f244cc8b114aa66630dadbd4bfb2a0c
+4ca2402d762adbd7ce860c3f3d072e948bb33afb7c2830ad51ae9d3fb2c714df
+328e4fda6dd1befcfda64d5c89c458fcc9386d88375218cb5197d479d3d292d0
+099636c67de66af06ee8493ccd55b588cf8bbacf67352877f37a077a2166117a
+624e0c032e4b8a78af1387c0199c5b02da68e0795e9b6397ba8bdd5ebcf7daa4
+265e038f17e4bc3c99737bf4ac98364c98a12d9d22a28b6a9302ce5a83c7d4f8
+de198984870126801aa20f25c459ea8f89bb7a4782614e91659b820c42a33c93
+ad48788c3f91a1292fb28dde84750f94e27ef150b2ebb52b7807f3d0c7986c4b
+34ab2f912eae5e75bbde977b8bb2952e552e36c83f29c2cd64d7ac3165aa9726
+951efa9c57da7eb2a1f6578927f809ab0c9feac2aa4326ea42f182e3ad74e600
+8b8193d9ebd89c36a728a3fb89282854bfb89b27172d93952747126c22ea6a97
+79a78a9f96f2c961b2d06dfd484a495ddc3809228243c608ecd51715a228e528
+4301dfc330137391585d36ddc4b54971999a1da96b1c12aa43221dd92cc32e79
+f956660447a893adbc0fc4382fe67ad2a7bb8591a647a130faa9e17aa380328a
+0cee22f18107559eff4f4eba20320c349d70d82b72197ae380fc514e6241730c
+d74601767fec3cff13062ad1393fad9a88277bbc6710c4fce6d78c21d001bfa4
+c1c46fd958b7806bf0c22c77c997317b7e4dffd7243d0a5919d4922fafd841a2
+a07e0f4e1cac4a84e3ca922b8c59a37ed8049096d9154be76e6d2d9094ba3714
+6861c27e7584e83a792ce710d004f7dc22213cc732b23f0a2025606cc5e9e325
+ee334626da5143ddf49561522483eb689663d031ba6cf9891204b709b279b28b
+3be351d25381f07b85c7f5c2bb08b80b5bbed80c348515187d727ccbf293b13b
+268da41127aa3b7768bbf6075baad5397f0af4f0ff16f5b8dff2cad9c3019750
+74cc47af92d6cf6b2315f62e0261d4461c6297c5ffc19b50a97784cd6271acfc
+8e52beef1bd61be4d223c460b589d32d1a64d5525406ab179d1962fccd734309
+9471871b2f1ec2331c1f9855b408d212ba868965f99aeee91ddd0b7dd76b2985
+16eb9e7e24b61ade0939345118d5d14032ae496de3b5fc702c0c1356662b8a80
+f503549eb9c03c8dc7e54559bf076e1f7eaf7c8599c84722062851597a4c91c1
+d9dce304b504003c97ffdf9d076ff348343d7d0ce50070038d049b71239bc70e
+2ed388e1730860b7605527213d7d61bbc8f29703a2f586d127b37e7ed8eca708
+d20988195a901d19597bf4ed2b136c3d2aed2d169093409a5d3ea8daf1f983af
+a9be22365634b68f0ff5ba9228550eba9b3923319eff07f5cf5785eec85fa11e
+99b512fe14b21569c81d358b161976c9dce45c608cd1a03da6f6063dbabb6c41
+88f76143cb289e8daf8edbc183a4b760c0d86efd1cc9da3933e59603bf92c539
+62054c1a23221461f1c06b94148614754f7da0cd7145eb85290bb11bdf7b9af8
+8dffffd5b63672f8749160dbff55c489f1f1f6c41ce01e086f2d9aa8bc6150ad
+f86a0512be3ec599065dec982545ecb96bdb0d82dda7285c58b3c3b7666988c9
+773786ea8094e7f013702ef721a2dde7657dcb6c44927f41f5acb32161957d83
+ef981b569d0c98e79bfc5bb387cf00457c4254e13f1ec625d0149b06ef92d5e1
+77841624d9fbba73cf722327f531ea36cfa42be59b4709c3b4ca4e7e453ac7fa
+f90ada865c766ef7e0e47d0a677b64183aa1e612f14f3f1f996d411b7197ea3e
+11c278763c1f2ff8d9b2ee5e3100a40538adcc1b74f6ca7d9668fcc8c1ab2f9a
+4a72c59eda21b43136cb5f308298aba39235bbd227668c2c9f3830fbab7b4a34
+f12a3d9308eb7800a1436b3258d899aae3b643f2c78648258f126266f9707032
+ce651585240975fc5c954c526203a048c6e4326e2d16feb083ca3a3ff4ad682a
+43aa6c22cb636983ff1b6e169057b0d7a70ad75754ef1b3a3bc7f49461c84cdb
+7e71c166a6f285c4c501b2125713e698575d1987f1819b76d4fabbbf246eae6c
+d38478eec0109a5b76da1a6e3de982382cf01bbc871b651d3e258330642cbe27
+c4c6ce60a0b1c4f2553ecb0f551de2c7df2e2d2cf2101e80af48b29e03cfefd5
+38905fb0f59281f1d4aa80894654e56df76653f3d3545a883e37c80053e72977
+1a6ee044aa753748035975b12885e73504ffde1bf129a7ff992f012d9cff111b
+f0d37e0a5ccb7871a30b143fad68a4e07624aa2a153e295022868e68e34ff770
+ee9d0ef6b557bd5e7c8fba9e087a428a98ee5350ec86785205db6ea10493b21c
+53ca280c12ba4d5be4ea78144fb2d411ecd9910f5105d04537d4bec362865c40
diff --git a/tests/http/big-test-out/root@188.166.240.16 b/tests/http/big-test-out/root@188.166.240.16
new file mode 100644
index 00000000..e69de29b
--- /dev/null
+++ b/tests/http/big-test-out/root@188.166.240.16
diff --git a/tests/http/big-test-out/root@46.101.72.130 b/tests/http/big-test-out/root@46.101.72.130
new file mode 100644
index 00000000..e69de29b
--- /dev/null
+++ b/tests/http/big-test-out/root@46.101.72.130
diff --git a/tests/http/big-test-ready.sh b/tests/http/big-test-ready.sh
new file mode 100755
index 00000000..391ca2a1
--- /dev/null
+++ b/tests/http/big-test-ready.sh
@@ -0,0 +1,30 @@
+#!/bin/bash
+
+# Edit as needed -- note that >1000 per host is likely problematic due to Linux kernel limits
+NUM_CONTAINERS=100
+CONTAINER_IMAGE=zerotier/http-test
+
+#
+# This script is designed to be run on Docker hosts to run NUM_CONTAINERS
+#
+# It can then be run on each Docker host via pssh or similar to run very
+# large scale tests.
+#
+
+export PATH=/bin:/usr/bin:/usr/local/bin:/usr/sbin:/sbin
+
+# Kill and clean up old test containers if any -- note that this kills all containers on the system!
+#docker ps -q | xargs -n 1 docker kill
+#docker ps -aq | xargs -n 1 docker rm
+
+# Pull latest if needed -- change this to your image name and/or where to pull it from
+#docker pull $CONTAINER_IMAGE
+
+# Run NUM_CONTAINERS
+#for ((n=0;n<$NUM_CONTAINERS;n++)); do
+#	docker run --device=/dev/net/tun --privileged -d $CONTAINER_IMAGE
+#done
+
+pssh -h big-test-hosts -i -t 128 -p 256 "docker pull $CONTAINER_IMAGE"
+
+exit 0
diff --git a/tests/http/big-test-start.sh b/tests/http/big-test-start.sh
index 79b6f93a..a4c7e6c1 100755
--- a/tests/http/big-test-start.sh
+++ b/tests/http/big-test-start.sh
@@ -1,7 +1,7 @@
 #!/bin/bash
 
 # Edit as needed -- note that >1000 per host is likely problematic due to Linux kernel limits
-NUM_CONTAINERS=100
+NUM_CONTAINERS=64
 CONTAINER_IMAGE=zerotier/http-test
 
 #
@@ -25,6 +25,6 @@ export PATH=/bin:/usr/bin:/usr/local/bin:/usr/sbin:/sbin
 #	docker run --device=/dev/net/tun --privileged -d $CONTAINER_IMAGE
 #done
 
-pssh -o big-test-out -h big-test-hosts -i -p 256 "docker pull $CONTAINER_IMAGE && for ((n=0;n<$NUM_CONTAINERS;n++)); do docker run --device=/dev/net/tun --privileged -d $CONTAINER_IMAGE; done"
+pssh -o big-test-out -h big-test-hosts -i -t 128 -p 256 "for ((n=0;n<$NUM_CONTAINERS;n++)); do docker run --device=/dev/net/tun --privileged -d $CONTAINER_IMAGE; done"
 
 exit 0