RanchiMall/SuperNodeStorage
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Improvement: streamed DB.readAllData

Browse Source 
- DB.readAllData is now a streamed function (readlAllDataStream)
- Large number data from database can be processed without any issue
This commit is contained in:
sairajzero 2022-11-30 03:50:12 +05:30
parent fd2aa002c0
commit ee8089f858
3 changed files with 117 additions and 96 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

75
src/database.js
View File

@ -65,7 +65,7 @@ const F_struct = {
function Database(user, password, dbname, host = 'localhost') { function Database(user, password, dbname, host = 'localhost') {
const db = {}; const db = {};
db.createBase = function() { db.createBase = function () {
return new Promise((resolve, reject) => { return new Promise((resolve, reject) => {
let statements = []; let statements = [];
for (let t in Base_Tables) for (let t in Base_Tables)
@ -77,7 +77,7 @@ function Database(user, password, dbname, host = 'localhost') {
}); });
}; };
db.setLastTx = function(id, n) { db.setLastTx = function (id, n) {
return new Promise((resolve, reject) => { return new Promise((resolve, reject) => {
let statement = "INSERT INTO LastTxs (ID, N) VALUES (?, ?)" + let statement = "INSERT INTO LastTxs (ID, N) VALUES (?, ?)" +
" ON DUPLICATE KEY UPDATE N=?"; " ON DUPLICATE KEY UPDATE N=?";
@ -87,7 +87,7 @@ function Database(user, password, dbname, host = 'localhost') {
}); });
}; };
db.setConfig = function(name, value) { db.setConfig = function (name, value) {
return new Promise((resolve, reject) => { return new Promise((resolve, reject) => {
let statement = "INSERT INTO Configs (NAME, VAL) VALUES (?, ?)" + let statement = "INSERT INTO Configs (NAME, VAL) VALUES (?, ?)" +
" ON DUPLICATE KEY UPDATE VAL=?"; " ON DUPLICATE KEY UPDATE VAL=?";
@ -97,7 +97,7 @@ function Database(user, password, dbname, host = 'localhost') {
}); });
}; };
db.addSuperNode = function(id, pubKey, uri) { db.addSuperNode = function (id, pubKey, uri) {
return new Promise((resolve, reject) => { return new Promise((resolve, reject) => {
let statement = "INSERT INTO SuperNodes (FLO_ID, PUB_KEY, URI) VALUES (?, ?, ?)" + let statement = "INSERT INTO SuperNodes (FLO_ID, PUB_KEY, URI) VALUES (?, ?, ?)" +
" ON DUPLICATE KEY UPDATE URI=?"; " ON DUPLICATE KEY UPDATE URI=?";
@ -107,7 +107,7 @@ function Database(user, password, dbname, host = 'localhost') {
}); });
}; };
db.rmSuperNode = function(id) { db.rmSuperNode = function (id) {
return new Promise((resolve, reject) => { return new Promise((resolve, reject) => {
let statement = "DELETE FROM SuperNodes" + let statement = "DELETE FROM SuperNodes" +
" WHERE FLO_ID=?"; " WHERE FLO_ID=?";
@ -117,7 +117,7 @@ function Database(user, password, dbname, host = 'localhost') {
}); });
}; };
db.setSubAdmin = function(appName, subAdmins) { db.setSubAdmin = function (appName, subAdmins) {
return new Promise((resolve, reject) => { return new Promise((resolve, reject) => {
let statement = "UPDATE Applications" + let statement = "UPDATE Applications" +
" SET SUB_ADMINS=?" + " SET SUB_ADMINS=?" +
@ -128,7 +128,7 @@ function Database(user, password, dbname, host = 'localhost') {
}); });
}; };
db.addApp = function(appName, adminID) { db.addApp = function (appName, adminID) {
return new Promise((resolve, reject) => { return new Promise((resolve, reject) => {
let statement = "INSERT INTO Applications (APP_NAME, ADMIN_ID) VALUES (?, ?)" + let statement = "INSERT INTO Applications (APP_NAME, ADMIN_ID) VALUES (?, ?)" +
" ON DUPLICATE KEY UPDATE ADMIN_ID=?"; " ON DUPLICATE KEY UPDATE ADMIN_ID=?";
@ -138,7 +138,7 @@ function Database(user, password, dbname, host = 'localhost') {
}); });
}; };
db.rmApp = function(appName) { db.rmApp = function (appName) {
return new Promise((resolve, reject) => { return new Promise((resolve, reject) => {
let statement = "DELETE FROM Applications" + let statement = "DELETE FROM Applications" +
" WHERE APP_NAME=" + appName; " WHERE APP_NAME=" + appName;
@ -148,7 +148,7 @@ function Database(user, password, dbname, host = 'localhost') {
}); });
}; };
db.getBase = function() { db.getBase = function () {
return new Promise((resolve, reject) => { return new Promise((resolve, reject) => {
let tables = Object.keys(Base_Tables); let tables = Object.keys(Base_Tables);
Promise.all(tables.map(t => db.query("SELECT * FROM " + t))).then(result => { Promise.all(tables.map(t => db.query("SELECT * FROM " + t))).then(result => {
@ -167,7 +167,7 @@ function Database(user, password, dbname, host = 'localhost') {
}); });
}; };
db.createTable = function(snID) { db.createTable = function (snID) {
return new Promise((resolve, reject) => { return new Promise((resolve, reject) => {
let statement = "CREATE TABLE IF NOT EXISTS _" + snID + " ( " + let statement = "CREATE TABLE IF NOT EXISTS _" + snID + " ( " +
H_struct.VECTOR_CLOCK + " VARCHAR(88) NOT NULL, " + H_struct.VECTOR_CLOCK + " VARCHAR(88) NOT NULL, " +
@ -199,7 +199,7 @@ function Database(user, password, dbname, host = 'localhost') {
}); });
}; };
db.dropTable = function(snID) { db.dropTable = function (snID) {
return new Promise((resolve, reject) => { return new Promise((resolve, reject) => {
let statement = "DROP TABLE _" + snID; let statement = "DROP TABLE _" + snID;
db.query(statement) db.query(statement)
@ -208,7 +208,7 @@ function Database(user, password, dbname, host = 'localhost') {
}); });
}; };
db.addData = function(snID, data) { db.addData = function (snID, data) {
return new Promise((resolve, reject) => { return new Promise((resolve, reject) => {
data[L_struct.STATUS] = 1; data[L_struct.STATUS] = 1;
data[L_struct.LOG_TIME] = Date.now(); data[L_struct.LOG_TIME] = Date.now();
@ -228,7 +228,7 @@ function Database(user, password, dbname, host = 'localhost') {
}); });
}; };
db.getData = function(snID, vectorClock) { db.getData = function (snID, vectorClock) {
return new Promise((resolve, reject) => { return new Promise((resolve, reject) => {
let statement = "SELECT * FROM _" + snID + let statement = "SELECT * FROM _" + snID +
" WHERE " + H_struct.VECTOR_CLOCK + "=?"; " WHERE " + H_struct.VECTOR_CLOCK + "=?";
@ -238,7 +238,7 @@ function Database(user, password, dbname, host = 'localhost') {
}) })
}; };
db.tagData = function(snID, vectorClock, tag, tagTime, tagKey, tagSign) { db.tagData = function (snID, vectorClock, tag, tagTime, tagKey, tagSign) {
return new Promise((resolve, reject) => { return new Promise((resolve, reject) => {
let data = { let data = {
[T_struct.TAG]: tag, [T_struct.TAG]: tag,
@ -259,7 +259,7 @@ function Database(user, password, dbname, host = 'localhost') {
}); });
}; };
db.noteData = function(snID, vectorClock, note, noteTime, noteKey, noteSign) { db.noteData = function (snID, vectorClock, note, noteTime, noteKey, noteSign) {
return new Promise((resolve, reject) => { return new Promise((resolve, reject) => {
let data = { let data = {
[F_struct.NOTE]: note, [F_struct.NOTE]: note,
@ -280,7 +280,7 @@ function Database(user, password, dbname, host = 'localhost') {
}); });
} }
db.searchData = function(snID, request) { db.searchData = function (snID, request) {
return new Promise((resolve, reject) => { return new Promise((resolve, reject) => {
let conditionArr = []; let conditionArr = [];
if (request.lowerVectorClock || request.upperVectorClock || request.atVectorClock) { if (request.lowerVectorClock || request.upperVectorClock || request.atVectorClock) {
@ -322,7 +322,7 @@ function Database(user, password, dbname, host = 'localhost') {
}); });
}; };
db.lastLogTime = function(snID) { db.lastLogTime = function (snID) {
return new Promise((resolve, reject) => { return new Promise((resolve, reject) => {
let attr = "MAX(" + L_struct.LOG_TIME + ")"; let attr = "MAX(" + L_struct.LOG_TIME + ")";
let statement = "SELECT " + attr + " FROM _" + snID; let statement = "SELECT " + attr + " FROM _" + snID;
@ -332,7 +332,7 @@ function Database(user, password, dbname, host = 'localhost') {
}); });
}; };
db.createGetLastLog = function(snID) { db.createGetLastLog = function (snID) {
return new Promise((resolve, reject) => { return new Promise((resolve, reject) => {
db.createTable(snID).then(result => { db.createTable(snID).then(result => {
db.lastLogTime(snID) db.lastLogTime(snID)
@ -342,18 +342,18 @@ function Database(user, password, dbname, host = 'localhost') {
}); });
}; };
db.readAllData = function(snID, logtime) { db.readAllDataStream = function (snID, logtime, callback) {
return new Promise((resolve, reject) => { return new Promise((resolve, reject) => {
let statement = "SELECT * FROM _" + snID + let statement = "SELECT * FROM _" + snID +
" WHERE " + L_struct.LOG_TIME + ">" + logtime + " WHERE " + L_struct.LOG_TIME + ">" + logtime +
" ORDER BY " + L_struct.LOG_TIME; " ORDER BY " + L_struct.LOG_TIME;
db.query(statement) db.query_stream(statement, callback)
.then(result => resolve(result)) .then(result => resolve(result))
.catch(error => reject(error)); .catch(error => reject(error));
}); });
}; };
db.storeData = function(snID, data, updateLogTime = false) { db.storeData = function (snID, data, updateLogTime = false) {
return new Promise((resolve, reject) => { return new Promise((resolve, reject) => {
if (updateLogTime) if (updateLogTime)
data[L_struct.LOG_TIME] = Date.now(); data[L_struct.LOG_TIME] = Date.now();
@ -373,7 +373,7 @@ function Database(user, password, dbname, host = 'localhost') {
}); });
}; };
db.storeTag = function(snID, data) { db.storeTag = function (snID, data) {
return new Promise((resolve, reject) => { return new Promise((resolve, reject) => {
let attr = Object.keys(T_struct).map(a => T_struct[a]).concat(L_struct.LOG_TIME); let attr = Object.keys(T_struct).map(a => T_struct[a]).concat(L_struct.LOG_TIME);
let values = attr.map(a => data[a]).concat(data[H_struct.VECTOR_CLOCK]); let values = attr.map(a => data[a]).concat(data[H_struct.VECTOR_CLOCK]);
@ -386,7 +386,7 @@ function Database(user, password, dbname, host = 'localhost') {
}); });
}; };
db.storeNote = function(snID, data) { db.storeNote = function (snID, data) {
let attr = Object.keys(F_struct).map(a => F_struct[a]).concat(L_struct.LOG_TIME); let attr = Object.keys(F_struct).map(a => F_struct[a]).concat(L_struct.LOG_TIME);
let values = attr.map(a => data[a]).concat(data[H_struct.VECTOR_CLOCK]); let values = attr.map(a => data[a]).concat(data[H_struct.VECTOR_CLOCK]);
let statement = "UPDATE _" + snID + let statement = "UPDATE _" + snID +
@ -397,7 +397,7 @@ function Database(user, password, dbname, host = 'localhost') {
.catch(error => reject(error)); .catch(error => reject(error));
} }
db.deleteData = function(snID, vectorClock) { db.deleteData = function (snID, vectorClock) {
return new Promise((resolve, reject) => { return new Promise((resolve, reject) => {
let statement = "DELETE FROM _" + snID + let statement = "DELETE FROM _" + snID +
" WHERE " + H_struct.VECTOR_CLOCK + "=?"; " WHERE " + H_struct.VECTOR_CLOCK + "=?";
@ -407,7 +407,7 @@ function Database(user, password, dbname, host = 'localhost') {
}); });
}; };
db.clearAuthorisedAppData = function(snID, app, adminID, subAdmins, timestamp) { db.clearAuthorisedAppData = function (snID, app, adminID, subAdmins, timestamp) {
return new Promise((resolve, reject) => { return new Promise((resolve, reject) => {
let statement = "DELETE FROM _" + snID + let statement = "DELETE FROM _" + snID +
" WHERE ( " + H_struct.TIME + "<? AND " + " WHERE ( " + H_struct.TIME + "<? AND " +
@ -423,7 +423,7 @@ function Database(user, password, dbname, host = 'localhost') {
}); });
}; };
db.clearUnauthorisedAppData = function(snID, appList, timestamp) { db.clearUnauthorisedAppData = function (snID, appList, timestamp) {
return new Promise((resolve, reject) => { return new Promise((resolve, reject) => {
let statement = "DELETE FROM _" + snID + let statement = "DELETE FROM _" + snID +
" WHERE " + H_struct.TIME + "<?" + " WHERE " + H_struct.TIME + "<?" +
@ -462,6 +462,29 @@ function Database(user, password, dbname, host = 'localhost') {
}) })
}); });
Object.defineProperty(db, "query_stream", {
value: (sql, value, callback) => new Promise((resolve, reject) => {
db.connect.then(conn => {
if (!callback && value instanceof Function) {
callback = value;
value = undefined;
}
var err_flag, row_count = 0;
(value ? conn.query(sql, values) : conn.query(sql))
.on('error', err => err_flag = err)
.on('result', row => {
row_count++;
conn.pause();
callback(row);
conn.resume();
}).on('end', _ => {
conn.release();
err_flag ? reject(err_flag) : resolve(row_count);
});
})
})
})
return new Promise((resolve, reject) => { return new Promise((resolve, reject) => {
db.pool = mysql.createPool({ db.pool = mysql.createPool({
host: host, host: host,

126
src/intra.js
View File

@ -517,25 +517,24 @@ orderBackup.requestData = function (req_sync, new_order) {
function sendStoredData(lastlogs, node) { function sendStoredData(lastlogs, node) {
for (let n in lastlogs) { for (let n in lastlogs) {
if (_list.stored.includes(n)) { if (_list.stored.includes(n)) {
DB.readAllData(n, lastlogs[n]).then(result => { node.send(packet_.construct({
node.send(packet_.construct({ type: DATA_SYNC,
type: DATA_SYNC, id: n,
id: n, status: true
status: true }));
})); console.info(`START: ${n} data sync(send) to ${node.id}`);
console.info(`START: ${n} data sync(send) to ${node.id}`); DB.readAllDataStream(n, lastlogs[n], d => node.send(packet_.construct({
//TODO: efficiently handle large number of data instead of loading all into memory type: STORE_BACKUP_DATA,
result.forEach(d => node.send(packet_.construct({ data: d
type: STORE_BACKUP_DATA, }))).then(result => console.info(`END: ${n} data sync(send) to ${node.id} (${result} records)`))
data: d .catch(error => {
}))); console.info(`ERROR: ${n} data sync(send) to ${node.id}`)
console.info(`END: ${n} data sync(send) to ${node.id}`); console.error(error);
node.send(packet_.construct({ }).finally(_ => node.send(packet_.construct({
type: DATA_SYNC, type: DATA_SYNC,
id: n, id: n,
status: false status: false
})); })));
}).catch(error => console.error(error));
}; };
}; };
}; };
@ -664,29 +663,29 @@ dataMigration.process_del = async function (del_nodes, old_kb) {
connectToAllActiveNodes().then(ws_connections => { connectToAllActiveNodes().then(ws_connections => {
let remaining = process_nodes.length; let remaining = process_nodes.length;
process_nodes.forEach(n => { process_nodes.forEach(n => {
DB.readAllData(n, 0).then(result => { console.info(`START: Data migration (del) for ${n}`);
console.info(`START: Data migration for ${n}`); DB.readAllDataStream(n, 0, d => {
//TODO: efficiently handle large number of data instead of loading all into memory let closest = cloud.closestNode(d.receiverID);
result.forEach(d => { if (_list.serving.includes(closest)) {
let closest = cloud.closestNode(d.receiverID); DB.storeData(closest, d, true).then(_ => null).catch(e => console.error(e));
if (_list.serving.includes(closest)) { if (_nextNode.id)
DB.storeData(closest, d, true).then(_ => null).catch(e => console.error(e)); _nextNode.send(packet_.construct({
if (_nextNode.id) type: STORE_BACKUP_DATA,
_nextNode.send(packet_.construct({
type: STORE_BACKUP_DATA,
data: d
}));
} else
ws_connections[closest].send(packet_.construct({
type: STORE_MIGRATED_DATA,
data: d data: d
})); }));
}); } else
console.info(`END: Data migration for ${n}`); ws_connections[closest].send(packet_.construct({
type: STORE_MIGRATED_DATA,
data: d
}));
}).then(result => {
console.info(`END: Data migration (del) for ${n}`);
_list.delete(n); _list.delete(n);
DB.dropTable(n).then(_ => null).catch(e => console.error(e)); DB.dropTable(n).then(_ => null).catch(e => console.error(e));
remaining--; }).catch(error => {
}).catch(error => console.error(error)); console.info(`ERROR: Data migration (del) for ${n}`);
console.error(error);
}).finally(_ => remaining--);
}); });
const interval = setInterval(() => { const interval = setInterval(() => {
if (remaining <= 0) { if (remaining <= 0) {
@ -714,37 +713,38 @@ dataMigration.process_new = async function (new_nodes) {
let process_nodes = _list.serving, let process_nodes = _list.serving,
remaining = process_nodes.length; remaining = process_nodes.length;
process_nodes.forEach(n => { process_nodes.forEach(n => {
DB.readAllData(n, 0).then(result => { console.info(`START: Data migration (re-new) for ${n}`);
//TODO: efficiently handle large number of data instead of loading all into memory DB.readAllDataStream(n, 0, d => {
result.forEach(d => { let closest = cloud.closestNode(d.receiverID);
let closest = cloud.closestNode(d.receiverID); if (new_nodes.includes(closest)) {
if (new_nodes.includes(closest)) { if (_list.serving.includes(closest)) {
if (_list.serving.includes(closest)) { DB.storeData(closest, d, true).then(_ => null).catch(e => console.error(e));
DB.storeData(closest, d, true).then(_ => null).catch(e => console.error(e));
if (_nextNode.id)
_nextNode.send(packet_.construct({
type: STORE_BACKUP_DATA,
data: d
}));
} else
ws_connections[closest].send(packet_.construct({
type: STORE_MIGRATED_DATA,
data: d
}));
DB.deleteData(n, d.vectorClock).then(_ => null).catch(e => console.error(e));
if (_nextNode.id) if (_nextNode.id)
_nextNode.send(packet_.construct({ _nextNode.send(packet_.construct({
type: DELETE_MIGRATED_DATA, type: STORE_BACKUP_DATA,
data: { data: d
vectorClock: d.vectorClock,
receiverID: d.receiverID,
snID: n
}
})); }));
}; } else
}); ws_connections[closest].send(packet_.construct({
remaining--; type: STORE_MIGRATED_DATA,
}).catch(error => console.error(error)); data: d
}));
DB.deleteData(n, d.vectorClock).then(_ => null).catch(e => console.error(e));
if (_nextNode.id)
_nextNode.send(packet_.construct({
type: DELETE_MIGRATED_DATA,
data: {
vectorClock: d.vectorClock,
receiverID: d.receiverID,
snID: n
}
}));
};
}).then(result => console.info(`END: Data migration (re-new) for ${n}`))
.catch(error => {
console.info(`ERROR: Data migration (re-new) for ${n}`);
console.error(error);
}).finally(_ => remaining--);
}); });
const interval = setInterval(() => { const interval = setInterval(() => {
if (remaining <= 0) { if (remaining <= 0) {

12
src/main.js
View File

@ -266,13 +266,11 @@ function selfDiskMigration(node_change) {
disks.forEach(n => { disks.forEach(n => {
if (node_change[n] === false) if (node_change[n] === false)
DB.dropTable(n).then(_ => null).catch(e => console.error(e)); DB.dropTable(n).then(_ => null).catch(e => console.error(e));
DB.readAllData(n, 0).then(result => { DB.readAllDataStream(n, 0, d => {
result.forEach(d => { let closest = cloud.closestNode(d.receiverID);
let closest = cloud.closestNode(d.receiverID); if (closest !== n)
if (closest !== n) DB.deleteData(n, d.vectorClock).then(_ => null).catch(e => console.error(e));
DB.deleteData(n, d.vectorClock).then(_ => null).catch(e => console.error(e)); }).then(result => console.debug(`Completed self-disk migration for ${n}`)).catch(error => console.error(error));
});
}).catch(error => console.error(error));
}); });
}).catch(error => console.error(error)); }).catch(error => console.error(error));
}; };