diff --git a/README.md b/README.md index 1e1a245..d96f6bf 100644 --- a/README.md +++ b/README.md @@ -199,12 +199,12 @@ Then select the options to upload/download/audit files while connecting to node1 - `batchain -l` 4. You can always run `batchain -h` to review available command and options -## Note: +#### Note: -For `npm`: -1. Run `npm install -g` before running any `batchain` option or command, make sure to +For `npm`: +1. Run `npm install -g` before running any `batchain` option or command, make sure to 2. Need to run `npm install -g` when making bin changes -3. If "chalk" is not working for you, run `npm insatll chalk --save` to make the command line more colorful +3. If "chalk" is not working for you, run `npm install chalk --save` to make the command line more colorful For `yarn`: 1. Run `yarn link` to create a symbolic link between project directory and executable command @@ -223,3 +223,51 @@ For `yarn`: -h, --help output usage information -l, --list view your list of uploaded files in BatChain network ``` + +#### Local CLI demo 2 - upload and audit a file + +First step is to make some temporary changes to allow the code to run locally + +Uncomment the seed node information and comment out the remote seed node info. The file should end up looking like this: + +``` +// For network testing: +// exports.SEED_NODE = ['a678ed17938527be1383388004dbf84246505dbd', { hostname: '167.99.2.1', port: 80 }]; +// exports.CLI_SERVER = {host: 'localhost', port: 1800}; +// exports.BATNODE_SERVER_PORT = 1900; +// exports.KADNODE_PORT = 80; + +// For local testing +exports.SEED_NODE = ['a678ed17938527be1383388004dbf84246505dbd', { hostname: 'localhost', port: 1338 }] +exports.BASELINE_REDUNDANCY = 3; +``` + +Next, change this line of code in the `while` loop + +``` +getClosestBatNodeToShard(shardId, callback){ + this.kadenceNode.iterativeFindNode(shardId, (err, res) => { + let i = 0 + let targetKadNode = res[0]; // res is an array of these tuples: [id, {hostname, port}] + while (targetKadNode[1].hostname === this.kadenceNode.contact.hostname && + (targetKadNode[1].port === this.kadenceNode.contact.port) { +``` + +to this. + +``` + // while (targetKadNode[1].hostname === this.kadenceNode.contact.hostname && + while (targetKadNode[1].port === this.kadenceNode.contact.port) { +``` + +Now we can proceed with the demo. + +1. `cd` into `/audit` directory +2. If you haven't already, run `yarn link` to create a symbolic link between project directory and executable command. This only needs to be done once. +3. Open 3 additional terminal windows or tabs that are also in the `/audit` directory +4. In the first terminal, `cd` into `server` directory. Run `rm/db` first and then run `node node.js` +5. In the second terminal, `cd` into `server2` directory. Run `rm/db` first and then run `node node.js` +6. In the third terminal, `cd` into `client` directory. Run `rm/db` first and then run `node node.js`. This boots up the CLI server which will listen for CLI commands. Wait for a message to log out saying the CLI is ready before issuing any commands. +7. In the fourth terminal, `cd` into `client` as well. Here we can issue `batchain` CLI commands. +8. There should be a example file in the `personal` directory, so run `batchain -u ./personal/example.txt`. Wait a few seconds for the 24 or so shard files to be written to `server` and `server2` `/host` directories. +9. Kill the process manually (Control-C) and run `batchain -a ./manifest/$MANIFESTNAME.batchain`. Replace `$MANIFESTNAME` with the manifest file name generated on `client/manifest` directory. diff --git a/audit/client/node.js b/audit/client/node.js index 9d8fdf0..3e387b9 100644 --- a/audit/client/node.js +++ b/audit/client/node.js @@ -1,4 +1,3 @@ -const bunyan = require('bunyan'); const levelup = require('levelup'); const leveldown = require('leveldown'); const encoding = require('encoding-down'); @@ -6,9 +5,10 @@ const kad = require('@kadenceproject/kadence'); const BatNode = require('../../batnode').BatNode; const kad_bat = require('../../kadence_plugin').kad_bat; const seed = require('../../constants').SEED_NODE +const backoff = require('backoff'); // Create a third batnode kadnode pair -kadnode3 = new kad.KademliaNode({ +const kadnode3 = new kad.KademliaNode({ transport: new kad.HTTPTransport(), storage: levelup(encoding(leveldown('./dbbb'))), contact: { hostname: 'localhost', port: 1252 } @@ -19,12 +19,67 @@ kadnode3.plugin(kad_bat) kadnode3.listen(1252) const batnode3 = new BatNode(kadnode3) kadnode3.batNode = batnode3 -batnode3.createServer(1985, '127.0.0.1') // Join kadnode3.join(seed, () => { console.log('you have joined the network! Ready to accept commands from the CLI!') - batnode3.uploadFile('./personal/example.txt'); + // batnode3.uploadFile('./personal/example.txt'); // batnode3.retrieveFile('./manifest/85a2ea0f0d11634d334886d9fb073b0d64506199.batchain') // batnode3.auditFile('./manifest/85a2ea0f0d11634d334886d9fb073b0d64506199.batchain') }) + +const nodeCLIConnectionCallback = (serverConnection) => { + + serverConnection.on('data', (data) => { + let receivedData = JSON.parse(data); + + if (receivedData.messageType === "CLI_UPLOAD_FILE") { + let filePath = receivedData.filePath; + + batnode3.uploadFile(filePath); + } else if (receivedData.messageType === "CLI_DOWNLOAD_FILE") { + let filePath = receivedData.filePath; + + batnode3.retrieveFile(filePath); + } else if (receivedData.messageType === "CLI_AUDIT_FILE") { + let filePath = receivedData.filePath; + let fibonacciBackoff = backoff.exponential({ + randomisationFactor: 0, + initialDelay: 20, + maxDelay: 2000 + }); + console.log("received path: ", filePath); + batnode3.auditFile(filePath); + + // post audit cleanup + serverConnection.on('close', () => { + batnode3._audit.ready = false; + batnode3._audit.data = null; + batnode3._audit.passed = false; + }); + + fibonacciBackoff.failAfter(10); + + fibonacciBackoff.on('backoff', function(number, delay) { + console.log(number + ' ' + delay + 'ms'); + }); + + fibonacciBackoff.on('ready', function() { + if (!batnode3._audit.ready) { + fibonacciBackoff.backoff(); + } else { + serverConnection.write(JSON.stringify(batnode3._audit.passed)); + return; + } + }); + + fibonacciBackoff.on('fail', function() { + console.log('Timeout: failed to complete audit'); + }); + + fibonacciBackoff.backoff(); + } + }); +} + +batnode3.createCLIServer(1800, 'localhost', nodeCLIConnectionCallback); diff --git a/audit/server/node.js b/audit/server/node.js index 3372191..9b76657 100644 --- a/audit/server/node.js +++ b/audit/server/node.js @@ -29,25 +29,27 @@ const nodeConnectionCallback = (serverConnection) => { }); serverConnection.on('data', (receivedData, error) => { + if (error) { throw error; } receivedData = JSON.parse(receivedData) if (receivedData.messageType === "RETRIEVE_FILE") { - batnode1.readFile(`./hosted/${receivedData.fileName}`, (error, data) => { + batnode1.readFile(`./hosted/${receivedData.fileName}`, (err, data) => { + if (err) { throw err; } serverConnection.write(data) }); } else if (receivedData.messageType === "STORE_FILE") { let fileName = receivedData.fileName; - batnode1.kadenceNode.iterativeStore(fileName, [batnode1.kadenceNode.identity.toString(), batnode1.kadenceNode.contact], (err, stored) => { + batnode1.kadenceNode.iterativeStore(fileName, [batnode1.kadenceNode.identity.toString(), batnode1.kadenceNode.contact], (innerError, stored) => { + if (innerError) { throw innerError; } console.log('nodes who stored this value: ', stored) let fileContent = new Buffer(receivedData.fileContent) batnode1.writeFile(`./hosted/${fileName}`, fileContent, (err) => { - if (err) { - throw err; - } + if (err) { throw err; } serverConnection.write(JSON.stringify({messageType: "SUCCESS"})) }); }); } else if (receivedData.messageType === "AUDIT_FILE") { - fs.readFile(`./hosted/${receivedData.fileName}`, (error, data) => { + fs.readFile(`./hosted/${receivedData.fileName}`, (innerErr, data) => { + if (innerErr) { throw innerErr; } const shardSha1 = fileUtils.sha1HashData(data); serverConnection.write(shardSha1); }); diff --git a/audit/server2/node.js b/audit/server2/node.js index b35d477..83bcc89 100644 --- a/audit/server2/node.js +++ b/audit/server2/node.js @@ -1,4 +1,3 @@ -const bunyan = require('bunyan'); const levelup = require('levelup'); const leveldown = require('leveldown'); const encoding = require('encoding-down'); @@ -27,11 +26,12 @@ const nodeConnectionCallback = (serverConnection) => { console.log('end') }) serverConnection.on('data', (receivedData, error) => { + if (error) { throw error; } receivedData = JSON.parse(receivedData) if (receivedData.messageType === "RETRIEVE_FILE") { - batnode2.readFile(`./hosted/${receivedData.fileName}`, (error, data) => { - if (error) { throw error; } + batnode2.readFile(`./hosted/${receivedData.fileName}`, (err, data) => { + if (err) { throw err; } serverConnection.write(data) }) } else if (receivedData.messageType === "STORE_FILE") { @@ -39,15 +39,14 @@ const nodeConnectionCallback = (serverConnection) => { batnode2.kadenceNode.iterativeStore(fileName, [batnode2.kadenceNode.identity.toString(), batnode2.kadenceNode.contact], (err, stored) => { console.log('nodes who stored this value: ', stored) let fileContent = new Buffer(receivedData.fileContent) - batnode2.writeFile(`./hosted/${fileName}`, fileContent, (err) => { - if (err) { - throw err; - } + batnode2.writeFile(`./hosted/${fileName}`, fileContent, (innerError) => { + if (innerError) { throw innerError; } serverConnection.write(JSON.stringify({messageType: "SUCCESS"})) }) }); } else if (receivedData.messageType === "AUDIT_FILE") { - fs.readFile(`./hosted/${receivedData.fileName}`, (error, data) => { + fs.readFile(`./hosted/${receivedData.fileName}`, (innerErr, data) => { + if (innerErr) { throw innerErr; } console.log('AUDIT_FILE - data: ', data); const shardSha1 = fileUtils.sha1HashData(data); console.log('shardSha1: ', shardSha1); diff --git a/batnode.js b/batnode.js index 5e733db..fd62972 100644 --- a/batnode.js +++ b/batnode.js @@ -4,10 +4,12 @@ const path = require('path'); const PERSONAL_DIR = require('./utils/file').PERSONAL_DIR; const HOSTED_DIR = require('./utils/file').HOSTED_DIR; const fs = require('fs'); +const constants = require('./constants'); class BatNode { constructor(kadenceNode = {}) { this._kadenceNode = kadenceNode; + this._audit = { ready: false, data: null, passed: false }; fileUtils.generateEnvFile() } @@ -206,10 +208,17 @@ class BatNode { const manifest = fileUtils.loadManifest(manifestFilePath); const shards = manifest.chunks; const shaIds = Object.keys(shards); - const fileName = manifest.fileName; + const shardAuditData = this.prepareAuditData(shards, shaIds); let shaIdx = 0; - const shardAuditData = shaIds.reduce((acc, shaId) => { + while (shaIds.length > shaIdx) { + this.auditShardsGroup(shards, shaIds, shaIdx, shardAuditData); + shaIdx += 1; + } + } + + prepareAuditData(shards, shaIds) { + return shaIds.reduce((acc, shaId) => { acc[shaId] = {}; shards[shaId].forEach((shardId) => { @@ -218,11 +227,6 @@ class BatNode { return acc; }, {}); - - while (shaIds.length > shaIdx) { - this.auditShardsGroup(shards, shaIds, shaIdx, shardAuditData); - shaIdx += 1; - } } /** * Tests the redudant copies of the original shard for data integrity. @@ -232,23 +236,24 @@ class BatNode { * @param {shardAuditData} Object - same as shards param except instead of an * array of shard ids it's an object of shard ids and their audit status */ - auditShardsGroup(shards, shaIds, shaIdx, shardAuditData) { + auditShardsGroup(shards, shaIds, shaIdx, shardAuditData, done) { let shardDupIdx = 0; - let duplicatesAudited = 0; const shaId = shaIds[shaIdx]; while (shards[shaId].length > shardDupIdx) { - this.auditShard(shards, shardDupIdx, shaId, shaIdx, shardAuditData); + this.auditShard(shards, shardDupIdx, shaId, shaIdx, shardAuditData, done); shardDupIdx += 1; } } - auditShard(shards, shardDupIdx, shaId, shaIdx, shardAuditData) { + auditShard(shards, shardDupIdx, shaId, shaIdx, shardAuditData, done) { const shardId = shards[shaId][shardDupIdx]; - this.kadenceNode.iterativeFindValue(shardId, (err, value, responder) => { + this.kadenceNode.iterativeFindValue(shardId, (error, value, responder) => { + if (error) { throw error; } let kadNodeTarget = value.value; - this.kadenceNode.getOtherBatNodeContact(kadNodeTarget, (error, batNode) => { + this.kadenceNode.getOtherBatNodeContact(kadNodeTarget, (err, batNode) => { + if (err) { throw err; } this.auditShardData(batNode, shards, shaIdx, shardDupIdx, shardAuditData) }) }) @@ -281,24 +286,33 @@ class BatNode { } if (finalShaGroup && finalShard) { - this.auditResults(shardAuditData, shaKeys); + const hasBaselineRedundancy = this.auditResults(shardAuditData, shaKeys); + this._audit.ready = true; + this._audit.data = shardAuditData; + this._audit.passed = hasBaselineRedundancy; + + console.log(shardAuditData); + if (hasBaselineRedundancy) { + console.log('Passed audit!'); + } else { + console.log('Failed Audit'); + } } }) } - auditResults(shardAuditData, shaKeys) { - const dataValid = shaKeys.every((shaId) => { - // For each key in the values object for the shaId key - return Object.keys(shardAuditData[shaId]).every((shardId) => { - return shardAuditData[shaId][shardId] === true; - }) - }); - console.log(shardAuditData); - if (dataValid) { - console.log('Passed audit!'); - } else { - console.log('Failed Audit'); + auditResults(auditData, shaKeys) { + const isRedundant = (shaId) => { + let validShards = 0; + // For each key (shardId) under the shard content's shaId key + Object.keys(auditData[shaId]).forEach((shardId) => { + if (auditData[shaId][shardId] === true) { validShards += 1; } + }); + + return validShards >= constants.BASELINE_REDUNDANCY ? true : false; } + + return shaKeys.every(isRedundant); } } diff --git a/bin/index.js b/bin/index.js index 01f4a59..0cf6edb 100755 --- a/bin/index.js +++ b/bin/index.js @@ -48,10 +48,14 @@ function sendAuditMessage() { messageType: "CLI_AUDIT_FILE", filePath: batchain.audit, }; - - console.log("message: ", message); - + client.write(JSON.stringify(message)); + + client.on('data', (data, error) => { + if (error) { throw error; } + const manifest = fileSystem.loadManifest(batchain.audit); + console.log(`File name: ${manifest.fileName} | Manifest: ${batchain.audit} | Data integrity: ${data.toString('utf8')}`); + }) } function displayFileList() { diff --git a/constants.js b/constants.js index 691c938..6667e35 100644 --- a/constants.js +++ b/constants.js @@ -5,4 +5,5 @@ exports.BATNODE_SERVER_PORT = 1900; exports.KADNODE_PORT = 80; // For local testing -//exports.SEED_NODE = ['a678ed17938527be1383388004dbf84246505dbd', { hostname: 'localhost', port: 1338 }] \ No newline at end of file +// exports.SEED_NODE = ['a678ed17938527be1383388004dbf84246505dbd', { hostname: 'localhost', port: 1338 }] +exports.BASELINE_REDUNDANCY = 3; diff --git a/package.json b/package.json index 1243465..b97ece5 100644 --- a/package.json +++ b/package.json @@ -9,6 +9,7 @@ }, "dependencies": { "@kadenceproject/kadence": "^3.1.2", + "backoff": "^2.5.0", "chalk": "^2.3.2", "dotenv": "^5.0.1", "public-ip": "^2.4.0" diff --git a/start.js b/start.js index 223942f..181b0db 100644 --- a/start.js +++ b/start.js @@ -12,6 +12,7 @@ const kadNodePort = require('./constants').KADNODE_PORT const publicIp = require('public-ip'); const fs = require('fs'); const fileUtils = require('./utils/file').fileSystem; +const backoff = require('backoff'); publicIp.v4().then(ip => { const kademliaNode = new kad.KademliaNode({ @@ -30,6 +31,7 @@ publicIp.v4().then(ip => { console.log('end') }) serverConnection.on('data', (receivedData, error) => { + if (error) { throw error; } receivedData = JSON.parse(receivedData) console.log("received data: ", receivedData) @@ -72,9 +74,42 @@ publicIp.v4().then(ip => { batNode.retrieveFile(filePath); } else if (receivedData.messageType === "CLI_AUDIT_FILE") { let filePath = receivedData.filePath; - - console.log("received path: ", filePath); + let fibonacciBackoff = backoff.exponential({ + randomisationFactor: 0, + initialDelay: 20, + maxDelay: 2000 + }); + + console.log("received path: ", filePath); batNode.auditFile(filePath); + + // post audit cleanup + serverConnection.on('close', () => { + batnode._audit.ready = false; + batnode._audit.data = null; + batnode._audit.passed = false; + }); + + fibonacciBackoff.failAfter(10); + + fibonacciBackoff.on('backoff', function(number, delay) { + console.log(number + ' ' + delay + 'ms'); + }); + + fibonacciBackoff.on('ready', function() { + if (!batnode._audit.ready) { + fibonacciBackoff.backoff(); + } else { + serverConnection.write(JSON.stringify(batnode._audit.passed)); + return; + } + }); + + fibonacciBackoff.on('fail', function() { + console.log('Timeout: failed to complete audit'); + }); + + fibonacciBackoff.backoff(); } }); } diff --git a/utils/file.js b/utils/file.js index dbb86a0..243c857 100644 --- a/utils/file.js +++ b/utils/file.js @@ -4,6 +4,7 @@ const zlib = require('zlib'); const algorithm = 'aes-256-cbc'; const path = require('path'); const dotenv = require('dotenv'); +const constants = require('../constants'); exports.PERSONAL_DIR = 'personal' @@ -108,11 +109,11 @@ exports.fileSystem = (function(){ addShardsToManifest(manifest, file, manifestName, dir, callback); } const createRedundantShardIds = (chunk, chunkId, manifest) => { - const copyNum = 3; + const shardsToCreate = constants.BASELINE_REDUNDANCY; let copyShardContent; let appendBytes; - for (let i = 1; i <= copyNum; i++) { + for (let i = 1; i <= shardsToCreate; i++) { appendBytes = crypto.randomBytes(2).toString('hex'); copyShardContent = chunk + appendBytes; @@ -173,4 +174,4 @@ exports.fileSystem = (function(){ sha1HashData, sha1Hash, } -})(); \ No newline at end of file +})(); diff --git a/yarn.lock b/yarn.lock index 651004a..d6cce42 100644 --- a/yarn.lock +++ b/yarn.lock @@ -192,6 +192,12 @@ babel-code-frame@^6.22.0: esutils "^2.0.2" js-tokens "^3.0.2" +backoff@^2.5.0: + version "2.5.0" + resolved "https://registry.yarnpkg.com/backoff/-/backoff-2.5.0.tgz#f616eda9d3e4b66b8ca7fca79f695722c5f8e26f" + dependencies: + precond "0.2" + balanced-match@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/balanced-match/-/balanced-match-1.0.0.tgz#89b4d199ab2bee49de164ea02b89ce462d71b767" @@ -1965,6 +1971,10 @@ prebuild-install@^2.1.0: tunnel-agent "^0.6.0" which-pm-runs "^1.0.0" +precond@0.2: + version "0.2.3" + resolved "https://registry.yarnpkg.com/precond/-/precond-0.2.3.tgz#aa9591bcaa24923f1e0f4849d240f47efc1075ac" + prelude-ls@~1.1.2: version "1.1.2" resolved "https://registry.yarnpkg.com/prelude-ls/-/prelude-ls-1.1.2.tgz#21932a549f5e52ffd9a827f570e04be62a97da54"