From e93e0ae10d8d2b868a4beea8694b4b3f50a5719b Mon Sep 17 00:00:00 2001 From: Vivek Teega Date: Wed, 16 Feb 2022 12:25:50 +0000 Subject: [PATCH] Helper scripts : Addition of convert_db for bootstrap and changes in test_rebuild --- convert_db.py | 43 +++++++++++++++++++++++++++++++++++++++++++ test_rebuild.py | 13 +++++++++---- 2 files changed, 52 insertions(+), 4 deletions(-) create mode 100644 convert_db.py diff --git a/convert_db.py b/convert_db.py new file mode 100644 index 0000000..b596e97 --- /dev/null +++ b/convert_db.py @@ -0,0 +1,43 @@ +from models import SystemData, ActiveTable, ConsumedTable, TransferLogs, TransactionHistory, RejectedTransactionHistory, Base, ContractStructure, ContractBase, ContractParticipants, SystemBase, ActiveContracts, ContractAddressMapping, LatestCacheBase, ContractTransactionHistory, RejectedContractTransactionHistory, TokenContractAssociation, ContinuosContractBase, ContractStructure1, ContractParticipants1, ContractDeposits1, ContractTransactionHistory1, LatestTransactions, LatestBlocks, DatabaseTypeMapping, TokenAddressMapping +import pdb +from sqlalchemy import create_engine, func +from sqlalchemy.orm import sessionmaker + + +def create_database_session_orm(type, parameters, base): + if type == 'token': + engine = create_engine(f"sqlite:///tokens/{parameters['token_name']}.db", echo=True) + base.metadata.create_all(bind=engine) + session = sessionmaker(bind=engine)() + + elif type == 'smart_contract': + engine = create_engine(f"sqlite:///smartContracts/{parameters['contract_name']}-{parameters['contract_address']}.db", echo=True) + base.metadata.create_all(bind=engine) + session = sessionmaker(bind=engine)() + + elif type == 'system_dbs': + engine = create_engine(f"sqlite:///{parameters['db_name']}.db", echo=False) + base.metadata.create_all(bind=engine) + session = sessionmaker(bind=engine)() + + return session + + +# connect to the database convert_db +convert_db = create_database_session_orm('system_dbs', {'db_name': 'convertdb'}, LatestCacheBase) +latest_blocks = convert_db.query(LatestBlocks).all() +latest_txs = convert_db.query(LatestTransactions).all() + + +# create a new database convert_db_new +convert_db_1 = create_database_session_orm('system_dbs', {'db_name': 'latestCache'}, LatestCacheBase) + +for block in latest_blocks: + convert_db_1.add(LatestBlocks(blockNumber=block.blockNumber, blockHash=block.blockHash, jsonData=block.jsonData)) + +for tx in latest_txs: + convert_db_1.add(LatestTransactions(transactionHash=tx.transactionHash, blockNumber=tx.blockNumber, jsonData=tx.jsonData, transactionType=tx.transactionType, parsedFloData=tx.parsedFloData)) + +convert_db_1.commit() +convert_db_1.close() +convert_db.close() diff --git a/test_rebuild.py b/test_rebuild.py index a1761a3..081bf92 100644 --- a/test_rebuild.py +++ b/test_rebuild.py @@ -1,6 +1,6 @@ from sqlalchemy import create_engine, desc, func from sqlalchemy.orm import sessionmaker -from models import SystemData, ActiveTable, ConsumedTable, TransferLogs, TransactionHistory, RejectedTransactionHistory, Base, ContractStructure, ContractBase, ContractParticipants, SystemBase, ActiveContracts, ContractAddressMapping, LatestCacheBase, ContractTransactionHistory, RejectedContractTransactionHistory, TokenContractAssociation, ContinuosContractBase, ContractStructure1, ContractParticipants1, ContractDeposits1, ContractTransactionHistory1, LatestTransactions, LatestBlocks, DatabaseTypeMapping +from models import SystemData, ActiveTable, ConsumedTable, TransferLogs, TransactionHistory, RejectedTransactionHistory, Base, ContractStructure, ContractBase, ContractParticipants, SystemBase, ActiveContracts, ContractAddressMapping, LatestCacheBase, ContractTransactionHistory, RejectedContractTransactionHistory, TokenContractAssociation, ContinuosContractBase, ContractStructure1, ContractParticipants1, ContractDeposits, ContractTransactionHistory1, LatestTransactions, LatestBlocks, DatabaseTypeMapping import json from tracktokens_smartcontracts import processTransaction import os @@ -94,7 +94,8 @@ else: latestCache_session = create_database_session_orm('system_dbs', {'db_name':'latestCache'}, LatestCacheBase) forward_block = int(latestCache_session.query(LatestBlocks.blockNumber).order_by(LatestBlocks.blockNumber.desc()).first()[0]) latestCache_session.close() - + + args = parser.parse_args() apppath = os.path.dirname(os.path.realpath(__file__)) @@ -178,6 +179,7 @@ latestCache_session.close() lblocks_dict = {} for block in lblocks: block_dict = block.__dict__ + print(block_dict['blockNumber']) lblocks_dict[block_dict['blockNumber']] = {'blockHash':f"{block_dict['blockHash']}", 'jsonData':f"{block_dict['jsonData']}"} # process and rebuild all transactions @@ -185,8 +187,11 @@ for transaction in ltransactions: transaction_dict = transaction.__dict__ transaction_data = json.loads(transaction_dict['jsonData']) parsed_flodata = json.loads(transaction_dict['parsedFloData']) - block_info = json.loads(lblocks_dict[transaction_dict['blockNumber']]['jsonData']) - processTransaction(transaction_data, parsed_flodata, block_info) + try: + block_info = json.loads(lblocks_dict[transaction_dict['blockNumber']]['jsonData']) + processTransaction(transaction_data, parsed_flodata, block_info) + except: + continue # copy the old block data old_latest_cache = create_database_connection('system_dbs', {'db_name':'latestCache1'})