From e9007b1f569ce15780ed34b837db987eec457d2f Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Thu, 19 Jan 2017 10:44:26 -0800 Subject: NodeJS migration script for old Sqlite controller.db to new controller data format. --- controller/EmbeddedNetworkController.hpp | 1 + controller/README.md | 10 +- controller/migrate-sqlite/migrate.js | 320 +++++++++++++++++++++++++++++++ controller/migrate-sqlite/package.json | 15 ++ osdep/OSUtils.cpp | 2 +- service/OneService.cpp | 10 +- 6 files changed, 348 insertions(+), 10 deletions(-) create mode 100644 controller/migrate-sqlite/migrate.js create mode 100644 controller/migrate-sqlite/package.json diff --git a/controller/EmbeddedNetworkController.hpp b/controller/EmbeddedNetworkController.hpp index 37c067ff..bd3a6666 100644 --- a/controller/EmbeddedNetworkController.hpp +++ b/controller/EmbeddedNetworkController.hpp @@ -165,6 +165,7 @@ private: if (!network.count("v6AssignMode")) network["v6AssignMode"] = {{"rfc4193",false},{"zt",false},{"6plane",false}}; if (!network.count("authTokens")) network["authTokens"] = nlohmann::json::array(); if (!network.count("capabilities")) network["capabilities"] = nlohmann::json::array(); + if (!network.count("routes")) network["routes"] = nlohmann::json::array(); if (!network.count("ipAssignmentPools")) network["ipAssignmentPools"] = nlohmann::json::array(); if (!network.count("rules")) { // If unspecified, rules are set to allow anything and behave like a flat L2 segment diff --git a/controller/README.md b/controller/README.md index 805641d9..edbdf804 100644 --- a/controller/README.md +++ b/controller/README.md @@ -15,15 +15,9 @@ The migration tool is written in nodeJS and can be used like this: cd migrate-sqlite npm install - node migrate-sqlite.js + node migrate.js -You may need to `sudo node ...` if the ZeroTier working directory is owned by root. - -This code will dump the contents of any `controller.db` in the ZeroTier working directory and recreate its data in the form of JSON objects under `controller.d`. The old `controller.db` will then be renamed to `controller.db.migrated` and the controller will start normally. - -After migrating make sure that the contents of `controller.d` are owned and writable by the user that will be running the ZeroTier controller process! (Usually this is root but controllers that don't also join networks are sometimes run as unprivileged users.) - -If you don't have nodeJS on the machine running ZeroTier it is perfectly fine to just copy `controller.db` to a directory on another machine and run the migration tool there. After running your migration the contents of `controller.d` can be tar'd up and copied back over to the controller. Just remember to rename or remove `controller.db` on the controller machine afterwords so the controller will start. +Very old versions of nodeJS may have issues. We tested it with version 7. ### Scalability and Reliability diff --git a/controller/migrate-sqlite/migrate.js b/controller/migrate-sqlite/migrate.js new file mode 100644 index 00000000..ac9678a7 --- /dev/null +++ b/controller/migrate-sqlite/migrate.js @@ -0,0 +1,320 @@ +'use strict'; + +var sqlite3 = require('sqlite3').verbose(); +var fs = require('fs'); +var async = require('async'); + +function blobToIPv4(b) +{ + if (!b) + return null; + if (b.length !== 16) + return null; + return b.readUInt8(12).toString()+'.'+b.readUInt8(13).toString()+'.'+b.readUInt8(14).toString()+'.'+b.readUInt8(15).toString(); +} +function blobToIPv6(b) +{ + if (!b) + return null; + if (b.length !== 16) + return null; + var s = ''; + for(var i=0;i<16;++i) { + var x = b.readUInt8(i).toString(16); + if (x.length === 1) + s += '0'; + s += x; + if ((((i+1) & 1) === 0)&&(i !== 15)) + s += ':'; + } + return s; +} + +if (process.argv.length !== 4) { + console.log('ZeroTier Old Sqlite3 Controller DB Migration Utility'); + console.log('(c)2017 ZeroTier, Inc. [GPL3]'); + console.log(''); + console.log('Usage: node migrate.js '); + console.log(''); + console.log('The first argument must be the path to the old Sqlite3 controller.db'); + console.log('file. The second must be the path to the EMPTY controller.d database'); + console.log('directory for a new (1.1.17 or newer) controller. If this path does'); + console.log('not exist it will be created.'); + console.log(''); + console.log('WARNING: this will ONLY work correctly on a 1.1.14 controller database.'); + console.log('If your controller is old you should first upgrade to 1.1.14 and run the'); + console.log('controller so that it will brings its Sqlite3 database up to the latest'); + console.log('version before running this migration.'); + console.log(''); + process.exit(1); +} + +var oldDbPath = process.argv[2]; +var newDbPath = process.argv[3]; + +console.log('Starting migrate of "'+oldDbPath+'" to "'+newDbPath+'"...'); +console.log(''); + +var old = new sqlite3.Database(oldDbPath); + +var networks = {}; + +var nodeIdentities = {}; +var networkCount = 0; +var memberCount = 0; +var routeCount = 0; +var ipAssignmentPoolCount = 0; +var ipAssignmentCount = 0; +var ruleCount = 0; +var oldSchemaVersion = -1; + +async.series([function(nextStep) { + + old.each('SELECT v from Config WHERE k = \'schemaVersion\'',function(err,row) { + oldSchemaVersion = parseInt(row.v)||-1; + },nextStep); + +},function(nextStep) { + + if (oldSchemaVersion !== 4) { + console.log('FATAL: this MUST be run on a 1.1.14 controller.db! Upgrade your old'); + console.log('controller to 1.1.14 first and run it once to bring its DB up to date.'); + return process.exit(1); + } + + console.log('Reading networks...'); + old.each('SELECT * FROM Network',function(err,row) { + if ((typeof row.id === 'string')&&(row.id.length === 16)) { + var flags = parseInt(row.flags)||0; + networks[row.id] = { + id: row.id, + nwid: row.id, + objtype: 'network', + authTokens: [], + capabilities: [], + creationTime: parseInt(row.creationTime)||0, + enableBroadcast: !!row.enableBroadcast, + ipAssignmentPools: [], + lastModified: Date.now(), + multicastLimit: row.multicastLimit||32, + name: row.name||'', + private: !!row.private, + revision: parseInt(row.revision)||1, + rules: [{ 'type': 'ACTION_ACCEPT' }], // populated later if there are defined rules, otherwise default is allow all + routes: [], + v4AssignMode: { + 'zt': ((flags & 1) !== 0) + }, + v6AssignMode: { + '6plane': ((flags & 4) !== 0), + 'rfc4193': ((flags & 2) !== 0), + 'zt': ((flags & 8) !== 0) + }, + _members: {} // temporary + }; + ++networkCount; + //console.log(networks[row.id]); + } + },nextStep); + +},function(nextStep) { + + console.log(' '+networkCount+' networks.'); + console.log('Reading network route definitions...'); + old.each('SELECT * from Route WHERE ipVersion = 4 OR ipVersion = 6',function(err,row) { + var network = networks[row.networkId]; + if (network) { + var rt = { + target: (((row.ipVersion == 4) ? blobToIPv4(row.target) : blobToIPv6(row.target))+'/'+row.targetNetmaskBits), + via: ((row.via) ? ((row.ipVersion == 4) ? blobToIPv4(row.via) : blobToIPv6(row.via)) : null) + }; + network.routes.push(rt); + ++routeCount; + } + },nextStep); + +},function(nextStep) { + + console.log(' '+routeCount+' routes in '+networkCount+' networks.'); + console.log('Reading IP assignment pools...'); + old.each('SELECT * FROM IpAssignmentPool WHERE ipVersion = 4 OR ipVersion = 6',function(err,row) { + var network = networks[row.networkId]; + if (network) { + var p = { + ipRangeStart: ((row.ipVersion == 4) ? blobToIPv4(row.ipRangeStart) : blobToIPv6(row.ipRangeStart)), + ipRangeEnd: ((row.ipVersion == 4) ? blobToIPv4(row.ipRangeEnd) : blobToIPv6(row.ipRangeEnd)) + }; + network.ipAssignmentPools.push(p); + ++ipAssignmentPoolCount; + } + },nextStep); + +},function(nextStep) { + + console.log(' '+ipAssignmentPoolCount+' IP assignment pools in '+networkCount+' networks.'); + console.log('Reading known node identities...'); + old.each('SELECT * FROM Node',function(err,row) { + nodeIdentities[row.id] = row.identity; + },nextStep); + +},function(nextStep) { + + console.log(' '+Object.keys(nodeIdentities).length+' known identities.'); + console.log('Reading network members...'); + old.each('SELECT * FROM Member',function(err,row) { + var network = networks[row.networkId]; + if (network) { + network._members[row.nodeId] = { + id: row.nodeId, + address: row.nodeId, + objtype: 'member', + authorized: !!row.authorized, + activeBridge: !!row.activeBridge, + authHistory: [], + capabilities: [], + creationTime: 0, + identity: nodeIdentities[row.nodeId]||null, + ipAssignments: [], + lastAuthorizedTime: (row.authorized) ? Date.now() : 0, + lastDeauthorizedTime: (row.authorized) ? 0 : Date.now(), + lastModified: Date.now(), + lastRequestMetaData: '', + noAutoAssignIps: false, + nwid: row.networkId, + revision: parseInt(row.memberRevision)||1, + tags: [], + recentLog: [] + }; + ++memberCount; + //console.log(network._members[row.nodeId]); + } + },nextStep); + +},function(nextStep) { + + console.log(' '+memberCount+' members of '+networkCount+' networks.'); + console.log('Reading static IP assignments...'); + old.each('SELECT * FROM IpAssignment WHERE ipVersion = 4 OR ipVersion = 6',function(err,row) { + var network = networks[row.networkId]; + if (network) { + var member = network._members[row.nodeId]; + if ((member)&&((member.authorized)||(!network['private']))) { // don't mirror assignments to unauthorized members to avoid conflicts + if (row.ipVersion == 4) { + member.ipAssignments.push(blobToIPv4(row.ip)); + ++ipAssignmentCount; + } else if (row.ipVersion == 6) { + member.ipAssignments.push(blobToIPv6(row.ip)); + ++ipAssignmentCount; + } + } + } + },nextStep); + +},function(nextStep) { + + // Old versions only supported Ethertype whitelisting, so that's + // all we mirror forward. The other fields were always unused. + + console.log(' '+ipAssignmentCount+' IP assignments for '+memberCount+' authorized members of '+networkCount+' networks.'); + console.log('Reading allowed Ethernet types (old basic rules)...'); + var etherTypesByNetwork = {}; + old.each('SELECT DISTINCT networkId,ruleNo,etherType FROM Rule WHERE "action" = \'accept\'',function(err,row) { + if (row.networkId in networks) { + var et = parseInt(row.etherType)||0; + var ets = etherTypesByNetwork[row.networkId]; + if (!ets) + etherTypesByNetwork[row.networkId] = [ et ]; + else ets.push(et); + } + },function(err) { + if (err) return nextStep(err); + for(var nwid in etherTypesByNetwork) { + var ets = etherTypesByNetwork[nwid].sort(); + var network = networks[nwid]; + if (network) { + var rules = []; + if (ets.indexOf(0) >= 0) { + // If 0 is in the list, all Ethernet types are allowed so we accept all. + rules.push({ 'type': 'ACTION_ACCEPT' }); + } else { + // Otherwise we whitelist. + for(var i=0;i 0) { + try { + fs.mkdirSync(nwBase+network.id); + } catch (e) {} + var mbase = nwBase+network.id+'/member'; + try { + fs.mkdirSync(mbase,0o700); + } catch (e) {} + mbase = mbase + '/'; + + for(var mi=0;mi", + "license": "GPL-3.0", + "dependencies": { + "async": "^2.1.4", + "sqlite3": "^3.1.8" + } +} diff --git a/osdep/OSUtils.cpp b/osdep/OSUtils.cpp index 08d67bbe..170e1376 100644 --- a/osdep/OSUtils.cpp +++ b/osdep/OSUtils.cpp @@ -398,7 +398,7 @@ std::string OSUtils::platformDefaultHomePath() // Inline these massive JSON operations in one place only to reduce binary footprint and compile time nlohmann::json OSUtils::jsonParse(const std::string &buf) { return nlohmann::json::parse(buf); } -std::string OSUtils::jsonDump(const nlohmann::json &j) { return j.dump(2); } +std::string OSUtils::jsonDump(const nlohmann::json &j) { return j.dump(1); } uint64_t OSUtils::jsonInt(const nlohmann::json &jv,const uint64_t dfl) { diff --git a/service/OneService.cpp b/service/OneService.cpp index b273aad4..0fd8496d 100644 --- a/service/OneService.cpp +++ b/service/OneService.cpp @@ -625,7 +625,15 @@ public: for(int i=0;i<3;++i) _portsBE[i] = Utils::hton((uint16_t)_ports[i]); - _controller = new EmbeddedNetworkController(_node,(_homePath + ZT_PATH_SEPARATOR_S + ZT_CONTROLLER_DB_PATH).c_str(),(FILE *)0); + // Check for legacy controller.db and terminate if present to prevent nasty surprises for DIY controller folks + if (OSUtils::fileExists((_homePath + ZT_PATH_SEPARATOR_S "controller.db").c_str())) { + Mutex::Lock _l(_termReason_m); + _termReason = ONE_UNRECOVERABLE_ERROR; + _fatalErrorMessage = "controller.db is present in our home path! run migrate-sqlite to migrate to new controller.d format."; + return _termReason; + } + + _controller = new EmbeddedNetworkController(_node,(_homePath + ZT_PATH_SEPARATOR_S ZT_CONTROLLER_DB_PATH).c_str(),(FILE *)0); _node->setNetconfMaster((void *)_controller); #ifdef ZT_ENABLE_CLUSTER -- cgit v1.2.3