RanchiMall/exchangemarket
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Improve backup hash algo

Browse Source
This commit is contained in:
sairajzero 2023-03-12 00:45:02 +05:30
parent 6538d9e2d6
commit 8e30b1a34d
2 changed files with 26 additions and 18 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
src/backup/slave.js
View File

@ -2,6 +2,7 @@
const keys = require("../keys");
const DB = require("../database");
const { getTableHashes } = require("./sync");
const {
BACKUP_INTERVAL,
@ -375,17 +376,9 @@ function requestHash(tables) {
}
function verifyHash(hashes) {
const getHash = table => new Promise((res, rej) => {
DB.query("SHOW COLUMNS FROM ??", [table]).then(result => {
let columns = result.map(r => r["Field"]).sort();
DB.query(`SELECT CEIL(id/${HASH_N_ROW}) as group_id, MD5(GROUP_CONCAT(${columns.map(c => `IFNULL(${c}, "NULL")`).join()})) as hash FROM ${table} GROUP BY group_id ORDER BY group_id`)
.then(result => res(Object.fromEntries(result.map(r => [r.group_id, r.hash]))))
.catch(error => rej(error))
}).catch(error => rej(error))
});
const convertIntArray = obj => Object.keys(obj).map(i => parseInt(i));
const checkHash = (table, hash_ref) => new Promise((res, rej) => {
getHash(table).then(hash_cur => {
getTableHashes(table).then(hash_cur => {
for (let i in hash_ref)
if (hash_ref[i] === hash_cur[i]) {
delete hash_ref[i];

33
src/backup/sync.js
View File

@ -119,15 +119,7 @@ function backupSync_checksum(ws) {
}
function sendTableHash(tables, ws) {
const getHash = table => new Promise((res, rej) => {
DB.query("SHOW COLUMNS FROM ??", [table]).then(result => {
let columns = result.map(r => r["Field"]).sort();
DB.query(`SELECT CEIL(id/${HASH_N_ROW}) as group_id, MD5(GROUP_CONCAT(${columns.map(c => `IFNULL(${c}, "NULL")`).join()})) as hash FROM ${table} GROUP BY group_id ORDER BY group_id`)
.then(result => res(Object.fromEntries(result.map(r => [r.group_id, r.hash]))))
.catch(error => rej(error))
}).catch(error => rej(error))
});
Promise.allSettled(tables.map(t => getHash(t))).then(result => {
Promise.allSettled(tables.map(t => getTableHashes(t))).then(result => {
let hashes = {};
for (let i in tables)
if (result[i].status === "fulfilled")
@ -141,6 +133,28 @@ function sendTableHash(tables, ws) {
})
}
function getTableHashes(table) {
return new Promise((resolve, reject) => {
DB.query("SHOW COLUMNS FROM ??", [table]).then(result => {
//columns
let columns = result.map(r => r["Field"]).sort();
//select statement
let statement = "SELECT CEIL(id/?) as group_id";
let query_values = [HASH_N_ROW];
//aggregate column values
let col_aggregate = columns.map(c => "IFNULL(CRC32(??), 0)").join('+');
columns.forEach(c => query_values.push(c));
//aggregate rows via group by
statement += " SUM(CRC32(MD5(" + col_aggregate + "))) as hash FROM ?? GROUP BY group_id ORDER BY group_id";
query_values.push(table);
//query
DB.query(statement, query_values)
.then(result => resolve(Object.fromEntries(result.map(r => [r.group_id, r.hash]))))
.catch(error => reject(error))
}).catch(error => reject(error))
})
}
function sendTableData(tables, ws) {
let promises = [
tableSync_data(tables, ws),
@ -261,6 +275,7 @@ function tableSync_checksum(tables, ws) {
}
module.exports = {
getTableHashes,
sendBackupData,
sendTableHash,
sendTableData