216 lines
8.7 KiB
Python
216 lines
8.7 KiB
Python
from sqlalchemy import create_engine, desc, func
|
|
from sqlalchemy.orm import sessionmaker
|
|
from models import SystemData, ActiveTable, ConsumedTable, TransferLogs, TransactionHistory, RejectedTransactionHistory, Base, ContractStructure, ContractBase, ContractParticipants, SystemBase, ActiveContracts, ContractAddressMapping, LatestCacheBase, ContractTransactionHistory, RejectedContractTransactionHistory, TokenContractAssociation, ContinuosContractBase, ContractStructure1, ContractParticipants1, ContractDeposits, ContractTransactionHistory1, LatestTransactions, LatestBlocks, DatabaseTypeMapping, ConsumedInfo
|
|
import json
|
|
from tracktokens_smartcontracts import processTransaction
|
|
import os
|
|
import logging
|
|
import argparse
|
|
import configparser
|
|
import pdb
|
|
import shutil
|
|
import sys
|
|
|
|
|
|
# helper functions
|
|
def check_database_existence(type, parameters):
|
|
if type == 'token':
|
|
return os.path.isfile(f"./tokens/{parameters['token_name']}.db")
|
|
|
|
if type == 'smart_contract':
|
|
return os.path.isfile(f"./smartContracts/{parameters['contract_name']}-{parameters['contract_address']}.db")
|
|
|
|
|
|
def create_database_connection(type, parameters):
|
|
if type == 'token':
|
|
engine = create_engine(f"sqlite:///tokens/{parameters['token_name']}.db", echo=True)
|
|
elif type == 'smart_contract':
|
|
engine = create_engine(f"sqlite:///smartContracts/{parameters['contract_name']}-{parameters['contract_address']}.db", echo=True)
|
|
elif type == 'system_dbs':
|
|
engine = create_engine(f"sqlite:///{parameters['db_name']}.db", echo=False)
|
|
|
|
connection = engine.connect()
|
|
return connection
|
|
|
|
|
|
def create_database_session_orm(type, parameters, base):
|
|
if type == 'token':
|
|
engine = create_engine(f"sqlite:///tokens/{parameters['token_name']}.db", echo=True)
|
|
base.metadata.create_all(bind=engine)
|
|
session = sessionmaker(bind=engine)()
|
|
|
|
elif type == 'smart_contract':
|
|
engine = create_engine(f"sqlite:///smartContracts/{parameters['contract_name']}-{parameters['contract_address']}.db", echo=True)
|
|
base.metadata.create_all(bind=engine)
|
|
session = sessionmaker(bind=engine)()
|
|
|
|
elif type == 'system_dbs':
|
|
engine = create_engine(f"sqlite:///{parameters['db_name']}.db", echo=False)
|
|
base.metadata.create_all(bind=engine)
|
|
session = sessionmaker(bind=engine)()
|
|
|
|
return session
|
|
|
|
|
|
# MAIN EXECUTION STARTS
|
|
# Configuration of required variables
|
|
logger = logging.getLogger(__name__)
|
|
logger.setLevel(logging.DEBUG)
|
|
|
|
formatter = logging.Formatter('%(asctime)s:%(name)s:%(message)s')
|
|
|
|
file_handler = logging.FileHandler('tracking.log')
|
|
file_handler.setLevel(logging.INFO)
|
|
file_handler.setFormatter(formatter)
|
|
|
|
stream_handler = logging.StreamHandler()
|
|
stream_handler.setFormatter(formatter)
|
|
|
|
logger.addHandler(file_handler)
|
|
logger.addHandler(stream_handler)
|
|
|
|
|
|
# Rule 1 - Read command line arguments to reset the databases as blank
|
|
# Rule 2 - Read config to set testnet/mainnet
|
|
# Rule 3 - Set flo blockexplorer location depending on testnet or mainnet
|
|
# Rule 4 - Set the local flo-cli path depending on testnet or mainnet ( removed this feature | Flosights are the only source )
|
|
# Rule 5 - Set the block number to scan from
|
|
|
|
|
|
# Read command line arguments
|
|
parser = argparse.ArgumentParser(description='Script tracks RMT using FLO data on the FLO blockchain - https://flo.cash')
|
|
parser.add_argument('-rb', '--toblocknumer', nargs='?', type=int, help='Forward to the specified block number')
|
|
parser.add_argument('-r', '--blockcount', nargs='?', type=int, help='Forward to the specified block count')
|
|
args = parser.parse_args()
|
|
|
|
if (args.blockcount and args.toblocknumber):
|
|
print("You can only specify one of the options -b or -c")
|
|
sys.exit(0)
|
|
elif args.blockcount:
|
|
forward_block = lastscannedblock + args.blockcount
|
|
elif args.toblocknumer:
|
|
forward_block = args.toblocknumer
|
|
else:
|
|
latestCache_session = create_database_session_orm('system_dbs', {'db_name':'latestCache'}, LatestCacheBase)
|
|
forward_block = int(latestCache_session.query(LatestBlocks.blockNumber).order_by(LatestBlocks.blockNumber.desc()).first()[0])
|
|
latestCache_session.close()
|
|
|
|
|
|
args = parser.parse_args()
|
|
|
|
apppath = os.path.dirname(os.path.realpath(__file__))
|
|
dirpath = os.path.join(apppath, 'tokens')
|
|
if not os.path.isdir(dirpath):
|
|
os.mkdir(dirpath)
|
|
dirpath = os.path.join(apppath, 'smartContracts')
|
|
if not os.path.isdir(dirpath):
|
|
os.mkdir(dirpath)
|
|
|
|
# rename all the old databases
|
|
# system.db , latestCache.db, smartContracts, tokens
|
|
if os.path.isfile('./system.db'):
|
|
os.rename('system.db', 'system1.db')
|
|
if os.path.isfile('./latestCache.db'):
|
|
os.rename('latestCache.db', 'latestCache1.db')
|
|
if os.path.isfile('./smartContracts'):
|
|
os.rename('smartContracts', 'smartContracts1')
|
|
if os.path.isfile('./tokens'):
|
|
os.rename('tokens', 'tokens1')
|
|
|
|
# Read configuration
|
|
config = configparser.ConfigParser()
|
|
config.read('config.ini')
|
|
|
|
# todo - write all assertions to make sure default configs are right
|
|
if (config['DEFAULT']['NET'] != 'mainnet') and (config['DEFAULT']['NET'] != 'testnet'):
|
|
logger.error("NET parameter in config.ini invalid. Options are either 'mainnet' or 'testnet'. Script is exiting now")
|
|
sys.exit(0)
|
|
|
|
# Specify mainnet and testnet server list for API calls and websocket calls
|
|
serverlist = None
|
|
if config['DEFAULT']['NET'] == 'mainnet':
|
|
serverlist = config['DEFAULT']['MAINNET_FLOSIGHT_SERVER_LIST']
|
|
elif config['DEFAULT']['NET'] == 'testnet':
|
|
serverlist = config['DEFAULT']['TESTNET_FLOSIGHT_SERVER_LIST']
|
|
serverlist = serverlist.split(',')
|
|
neturl = config['DEFAULT']['FLOSIGHT_NETURL']
|
|
tokenapi_sse_url = config['DEFAULT']['TOKENAPI_SSE_URL']
|
|
|
|
# Delete database and smartcontract directory if reset is set to 1
|
|
#if args.reset == 1:
|
|
logger.info("Resetting the database. ")
|
|
apppath = os.path.dirname(os.path.realpath(__file__))
|
|
dirpath = os.path.join(apppath, 'tokens')
|
|
shutil.rmtree(dirpath)
|
|
os.mkdir(dirpath)
|
|
dirpath = os.path.join(apppath, 'smartContracts')
|
|
shutil.rmtree(dirpath)
|
|
os.mkdir(dirpath)
|
|
dirpath = os.path.join(apppath, 'system.db')
|
|
if os.path.exists(dirpath):
|
|
os.remove(dirpath)
|
|
dirpath = os.path.join(apppath, 'latestCache.db')
|
|
if os.path.exists(dirpath):
|
|
os.remove(dirpath)
|
|
|
|
# Read start block no
|
|
startblock = int(config['DEFAULT']['START_BLOCK'])
|
|
session = create_database_session_orm('system_dbs', {'db_name': "system"}, SystemBase)
|
|
session.add(SystemData(attribute='lastblockscanned', value=startblock - 1))
|
|
session.commit()
|
|
session.close()
|
|
|
|
# Initialize latest cache DB
|
|
session = create_database_session_orm('system_dbs', {'db_name': "latestCache"}, LatestCacheBase)
|
|
session.commit()
|
|
session.close()
|
|
|
|
# get all blocks and transaction data
|
|
latestCache_session = create_database_session_orm('system_dbs', {'db_name':'latestCache1'}, LatestCacheBase)
|
|
if forward_block:
|
|
lblocks = latestCache_session.query(LatestBlocks).filter(LatestBlocks.blockNumber <= forward_block).all()
|
|
ltransactions = latestCache_session.query(LatestTransactions).filter(LatestTransactions.blockNumber <= forward_block).all()
|
|
else:
|
|
lblocks = latestCache_session.query(LatestBlocks).all()
|
|
ltransactions = latestCache_session.query(LatestTransactions).all()
|
|
latestCache_session.close()
|
|
|
|
|
|
lblocks_dict = {}
|
|
for block in lblocks:
|
|
block_dict = block.__dict__
|
|
print(block_dict['blockNumber'])
|
|
lblocks_dict[block_dict['blockNumber']] = {'blockHash':f"{block_dict['blockHash']}", 'jsonData':f"{block_dict['jsonData']}"}
|
|
|
|
# process and rebuild all transactions
|
|
for transaction in ltransactions:
|
|
transaction_dict = transaction.__dict__
|
|
transaction_data = json.loads(transaction_dict['jsonData'])
|
|
parsed_flodata = json.loads(transaction_dict['parsedFloData'])
|
|
try:
|
|
block_info = json.loads(lblocks_dict[transaction_dict['blockNumber']]['jsonData'])
|
|
processTransaction(transaction_data, parsed_flodata, block_info)
|
|
except:
|
|
continue
|
|
|
|
# copy the old block data
|
|
old_latest_cache = create_database_connection('system_dbs', {'db_name':'latestCache1'})
|
|
old_latest_cache.execute("ATTACH DATABASE 'latestCache.db' AS new_db")
|
|
old_latest_cache.execute("INSERT INTO new_db.latestBlocks SELECT * FROM latestBlocks WHERE blockNumber <= ?", (forward_block,))
|
|
old_latest_cache.close()
|
|
|
|
# delete
|
|
# system.db , latestCache.db, smartContracts, tokens
|
|
if os.path.isfile('./system1.db'):
|
|
os.remove('system1.db')
|
|
if os.path.isfile('./latestCache1.db'):
|
|
os.remove('latestCache1.db')
|
|
if os.path.isfile('./smartContracts1'):
|
|
shutil.rmtree('smartContracts1')
|
|
if os.path.isfile('./tokens1'):
|
|
shutil.rmtree('tokens1')
|
|
|
|
# Update system.db's last scanned block
|
|
connection = create_database_connection('system_dbs', {'db_name': "system"})
|
|
connection.execute(f"UPDATE systemData SET value = {int(list(lblocks_dict.keys())[-1])} WHERE attribute = 'lastblockscanned';")
|
|
connection.close() |