From 8e30b1a34d3f172216509c2e6b7f34c1819b78c1 Mon Sep 17 00:00:00 2001
From: sairajzero <sairaj0630@gmail.com>
Date: Sun, 12 Mar 2023 00:45:02 +0530
Subject: [PATCH] Improve backup hash algo

---
 src/backup/slave.js | 11 ++---------
 src/backup/sync.js  | 33 ++++++++++++++++++++++++---------
 2 files changed, 26 insertions(+), 18 deletions(-)

diff --git a/src/backup/slave.js b/src/backup/slave.js
index 1dee013..fd860d2 100644
--- a/src/backup/slave.js
+++ b/src/backup/slave.js
@@ -2,6 +2,7 @@
 
 const keys = require("../keys");
 const DB = require("../database");
+const { getTableHashes } = require("./sync");
 
 const {
     BACKUP_INTERVAL,
@@ -375,17 +376,9 @@ function requestHash(tables) {
 }
 
 function verifyHash(hashes) {
-    const getHash = table => new Promise((res, rej) => {
-        DB.query("SHOW COLUMNS FROM ??", [table]).then(result => {
-            let columns = result.map(r => r["Field"]).sort();
-            DB.query(`SELECT CEIL(id/${HASH_N_ROW}) as group_id, MD5(GROUP_CONCAT(${columns.map(c => `IFNULL(${c}, "NULL")`).join()})) as hash FROM ${table} GROUP BY group_id ORDER BY group_id`)
-                .then(result => res(Object.fromEntries(result.map(r => [r.group_id, r.hash]))))
-                .catch(error => rej(error))
-        }).catch(error => rej(error))
-    });
     const convertIntArray = obj => Object.keys(obj).map(i => parseInt(i));
     const checkHash = (table, hash_ref) => new Promise((res, rej) => {
-        getHash(table).then(hash_cur => {
+        getTableHashes(table).then(hash_cur => {
             for (let i in hash_ref)
                 if (hash_ref[i] === hash_cur[i]) {
                     delete hash_ref[i];
diff --git a/src/backup/sync.js b/src/backup/sync.js
index 86e0eee..087233f 100644
--- a/src/backup/sync.js
+++ b/src/backup/sync.js
@@ -119,15 +119,7 @@ function backupSync_checksum(ws) {
 }
 
 function sendTableHash(tables, ws) {
-    const getHash = table => new Promise((res, rej) => {
-        DB.query("SHOW COLUMNS FROM ??", [table]).then(result => {
-            let columns = result.map(r => r["Field"]).sort();
-            DB.query(`SELECT CEIL(id/${HASH_N_ROW}) as group_id, MD5(GROUP_CONCAT(${columns.map(c => `IFNULL(${c}, "NULL")`).join()})) as hash FROM ${table} GROUP BY group_id ORDER BY group_id`)
-                .then(result => res(Object.fromEntries(result.map(r => [r.group_id, r.hash]))))
-                .catch(error => rej(error))
-        }).catch(error => rej(error))
-    });
-    Promise.allSettled(tables.map(t => getHash(t))).then(result => {
+    Promise.allSettled(tables.map(t => getTableHashes(t))).then(result => {
         let hashes = {};
         for (let i in tables)
             if (result[i].status === "fulfilled")
@@ -141,6 +133,28 @@ function sendTableHash(tables, ws) {
     })
 }
 
+function getTableHashes(table) {
+    return new Promise((resolve, reject) => {
+        DB.query("SHOW COLUMNS FROM ??", [table]).then(result => {
+            //columns
+            let columns = result.map(r => r["Field"]).sort();
+            //select statement
+            let statement = "SELECT CEIL(id/?) as group_id";
+            let query_values = [HASH_N_ROW];
+            //aggregate column values
+            let col_aggregate = columns.map(c => "IFNULL(CRC32(??), 0)").join('+');
+            columns.forEach(c => query_values.push(c));
+            //aggregate rows via group by
+            statement += " SUM(CRC32(MD5(" + col_aggregate + "))) as hash FROM ?? GROUP BY group_id ORDER BY group_id";
+            query_values.push(table);
+            //query
+            DB.query(statement, query_values)
+                .then(result => resolve(Object.fromEntries(result.map(r => [r.group_id, r.hash]))))
+                .catch(error => reject(error))
+        }).catch(error => reject(error))
+    })
+}
+
 function sendTableData(tables, ws) {
     let promises = [
         tableSync_data(tables, ws),
@@ -261,6 +275,7 @@ function tableSync_checksum(tables, ws) {
 }
 
 module.exports = {
+    getTableHashes,
     sendBackupData,
     sendTableHash,
     sendTableData