247 lines
10 KiB
Python
247 lines
10 KiB
Python
from sqlalchemy import create_engine, desc, func
|
|
from sqlalchemy.orm import sessionmaker
|
|
from models import SystemData, TokenBase, ActiveTable, ConsumedTable, TransferLogs, TransactionHistory, TokenContractAssociation, ContractBase, ContractStructure, ContractParticipants, ContractTransactionHistory, ContractDeposits, ConsumedInfo, ContractWinners, ContinuosContractBase, ContractStructure2, ContractParticipants2, ContractDeposits2, ContractTransactionHistory2, SystemBase, ActiveContracts, SystemData, ContractAddressMapping, TokenAddressMapping, DatabaseTypeMapping, TimeActions, RejectedContractTransactionHistory, RejectedTransactionHistory, LatestCacheBase, LatestTransactions, LatestBlocks
|
|
import json
|
|
from tracktokens_smartcontracts import processTransaction, checkLocal_expiry_trigger_deposit, newMultiRequest
|
|
import os
|
|
import logging
|
|
import argparse
|
|
import configparser
|
|
import shutil
|
|
import sys
|
|
import pdb
|
|
|
|
|
|
# helper functions
|
|
def check_database_existence(type, parameters):
|
|
if type == 'token':
|
|
return os.path.isfile(f"./tokens/{parameters['token_name']}.db")
|
|
|
|
if type == 'smart_contract':
|
|
return os.path.isfile(f"./smartContracts/{parameters['contract_name']}-{parameters['contract_address']}.db")
|
|
|
|
|
|
def create_database_connection(type, parameters):
|
|
if type == 'token':
|
|
engine = create_engine(f"sqlite:///tokens/{parameters['token_name']}.db", echo=True)
|
|
elif type == 'smart_contract':
|
|
engine = create_engine(f"sqlite:///smartContracts/{parameters['contract_name']}-{parameters['contract_address']}.db", echo=True)
|
|
elif type == 'system_dbs':
|
|
engine = create_engine(f"sqlite:///{parameters['db_name']}.db", echo=False)
|
|
|
|
connection = engine.connect()
|
|
return connection
|
|
|
|
|
|
def create_database_session_orm(type, parameters, base):
|
|
if type == 'token':
|
|
engine = create_engine(f"sqlite:///tokens/{parameters['token_name']}.db", echo=True)
|
|
base.metadata.create_all(bind=engine)
|
|
session = sessionmaker(bind=engine)()
|
|
|
|
elif type == 'smart_contract':
|
|
engine = create_engine(f"sqlite:///smartContracts/{parameters['contract_name']}-{parameters['contract_address']}.db", echo=True)
|
|
base.metadata.create_all(bind=engine)
|
|
session = sessionmaker(bind=engine)()
|
|
|
|
elif type == 'system_dbs':
|
|
engine = create_engine(f"sqlite:///{parameters['db_name']}.db", echo=False)
|
|
base.metadata.create_all(bind=engine)
|
|
session = sessionmaker(bind=engine)()
|
|
|
|
return session
|
|
|
|
|
|
# MAIN EXECUTION STARTS
|
|
# Configuration of required variables
|
|
logger = logging.getLogger(__name__)
|
|
logger.setLevel(logging.DEBUG)
|
|
|
|
formatter = logging.Formatter('%(asctime)s:%(name)s:%(message)s')
|
|
|
|
file_handler = logging.FileHandler('tracking.log')
|
|
file_handler.setLevel(logging.INFO)
|
|
file_handler.setFormatter(formatter)
|
|
|
|
stream_handler = logging.StreamHandler()
|
|
stream_handler.setFormatter(formatter)
|
|
|
|
logger.addHandler(file_handler)
|
|
logger.addHandler(stream_handler)
|
|
|
|
|
|
# Rule 1 - Read command line arguments to reset the databases as blank
|
|
# Rule 2 - Read config to set testnet/mainnet
|
|
# Rule 3 - Set flo blockexplorer location depending on testnet or mainnet
|
|
# Rule 4 - Set the local flo-cli path depending on testnet or mainnet ( removed this feature | Flosights are the only source )
|
|
# Rule 5 - Set the block number to scan from
|
|
|
|
|
|
# Read command line arguments
|
|
parser = argparse.ArgumentParser(description='Script tracks RMT using FLO data on the FLO blockchain - https://flo.cash')
|
|
parser.add_argument('-rb', '--toblocknumer', nargs='?', type=int, help='Forward to the specified block number')
|
|
parser.add_argument('-r', '--blockcount', nargs='?', type=int, help='Forward to the specified block count')
|
|
parser.add_argument('-to', '--to_blockNumber', nargs='?', type=int, help='Process until the specified block number') # New argument
|
|
args = parser.parse_args()
|
|
|
|
|
|
if (args.blockcount and args.toblocknumber):
|
|
print("You can only specify one of the options -b or -c")
|
|
sys.exit(0)
|
|
elif args.blockcount:
|
|
forward_block = lastscannedblock + args.blockcount
|
|
elif args.toblocknumer:
|
|
forward_block = args.toblocknumer
|
|
else:
|
|
latestCache_session = create_database_session_orm('system_dbs', {'db_name':'latestCache'}, LatestCacheBase)
|
|
forward_block = int(latestCache_session.query(LatestBlocks.blockNumber).order_by(LatestBlocks.blockNumber.desc()).first()[0])
|
|
latestCache_session.close()
|
|
|
|
|
|
args = parser.parse_args()
|
|
|
|
apppath = os.path.dirname(os.path.realpath(__file__))
|
|
dirpath = os.path.join(apppath, 'tokens')
|
|
if not os.path.isdir(dirpath):
|
|
os.mkdir(dirpath)
|
|
dirpath = os.path.join(apppath, 'smartContracts')
|
|
if not os.path.isdir(dirpath):
|
|
os.mkdir(dirpath)
|
|
|
|
# rename all the old databases
|
|
# system.db , latestCache.db, smartContracts, tokens
|
|
if os.path.isfile('./system.db'):
|
|
os.rename('system.db', 'system1.db')
|
|
if os.path.isfile('./latestCache.db'):
|
|
os.rename('latestCache.db', 'latestCache1.db')
|
|
if os.path.isfile('./smartContracts'):
|
|
os.rename('smartContracts', 'smartContracts1')
|
|
if os.path.isfile('./tokens'):
|
|
os.rename('tokens', 'tokens1')
|
|
|
|
# Read configuration
|
|
config = configparser.ConfigParser()
|
|
config.read('config.ini')
|
|
|
|
# todo - write all assertions to make sure default configs are right
|
|
if (config['DEFAULT']['NET'] != 'mainnet') and (config['DEFAULT']['NET'] != 'testnet'):
|
|
logger.error("NET parameter in config.ini invalid. Options are either 'mainnet' or 'testnet'. Script is exiting now")
|
|
sys.exit(0)
|
|
|
|
# Specify mainnet and testnet server list for API calls and websocket calls
|
|
serverlist = None
|
|
if config['DEFAULT']['NET'] == 'mainnet':
|
|
serverlist = config['DEFAULT']['MAINNET_FLOSIGHT_SERVER_LIST']
|
|
elif config['DEFAULT']['NET'] == 'testnet':
|
|
serverlist = config['DEFAULT']['TESTNET_FLOSIGHT_SERVER_LIST']
|
|
serverlist = serverlist.split(',')
|
|
neturl = config['DEFAULT']['FLOSIGHT_NETURL']
|
|
tokenapi_sse_url = config['DEFAULT']['TOKENAPI_SSE_URL']
|
|
|
|
# Delete database and smartcontract directory if reset is set to 1
|
|
#if args.reset == 1:
|
|
logger.info("Resetting the database. ")
|
|
apppath = os.path.dirname(os.path.realpath(__file__))
|
|
dirpath = os.path.join(apppath, 'tokens')
|
|
shutil.rmtree(dirpath)
|
|
os.mkdir(dirpath)
|
|
dirpath = os.path.join(apppath, 'smartContracts')
|
|
shutil.rmtree(dirpath)
|
|
os.mkdir(dirpath)
|
|
dirpath = os.path.join(apppath, 'system.db')
|
|
if os.path.exists(dirpath):
|
|
os.remove(dirpath)
|
|
dirpath = os.path.join(apppath, 'latestCache.db')
|
|
if os.path.exists(dirpath):
|
|
os.remove(dirpath)
|
|
|
|
# Read start block no
|
|
startblock = int(config['DEFAULT']['START_BLOCK'])
|
|
session = create_database_session_orm('system_dbs', {'db_name': "system"}, SystemBase)
|
|
session.add(SystemData(attribute='lastblockscanned', value=startblock - 1))
|
|
session.commit()
|
|
session.close()
|
|
|
|
# Initialize latest cache DB
|
|
session = create_database_session_orm('system_dbs', {'db_name': "latestCache"}, LatestCacheBase)
|
|
session.commit()
|
|
session.close()
|
|
|
|
# get all blocks and transaction data
|
|
latestCache_session = create_database_session_orm('system_dbs', {'db_name':'latestCache1'}, LatestCacheBase)
|
|
if forward_block:
|
|
lblocks = latestCache_session.query(LatestBlocks).filter(LatestBlocks.blockNumber <= forward_block).all()
|
|
ltransactions = latestCache_session.query(LatestTransactions).filter(LatestTransactions.blockNumber <= forward_block).all()
|
|
else:
|
|
lblocks = latestCache_session.query(LatestBlocks).all()
|
|
ltransactions = latestCache_session.query(LatestTransactions).all()
|
|
latestCache_session.close()
|
|
|
|
# make a list of all internal tx block numbers
|
|
systemDb_session = create_database_session_orm('system_dbs', {'db_name':'system1'}, SystemBase)
|
|
internal_action_blocks = systemDb_session.query(ActiveContracts.blockNumber).all()
|
|
internal_action_blocks = [block[0] for block in internal_action_blocks]
|
|
internal_action_blocks = sorted(internal_action_blocks)
|
|
|
|
lblocks_dict = {}
|
|
for block in lblocks:
|
|
block_dict = block.__dict__
|
|
print(block_dict['blockNumber'])
|
|
lblocks_dict[block_dict['blockNumber']] = {'blockHash':f"{block_dict['blockHash']}", 'jsonData':f"{block_dict['jsonData']}"}
|
|
|
|
# process and rebuild all transactions
|
|
prev_block = 0
|
|
|
|
for transaction in ltransactions:
|
|
transaction_dict = transaction.__dict__
|
|
current_block = transaction_dict['blockNumber']
|
|
|
|
# Check if any internal action block lies between prev_block and current_block
|
|
for internal_block in internal_action_blocks:
|
|
|
|
if prev_block < internal_block <= current_block:
|
|
logger.info(f'Processing block {internal_block}')
|
|
# Get block details
|
|
response = newMultiRequest(f"block-index/{internal_block}")
|
|
blockhash = response['blockHash']
|
|
blockinfo = newMultiRequest(f"block/{blockhash}")
|
|
# Call your function here, passing the internal block to it
|
|
checkLocal_expiry_trigger_deposit(blockinfo)
|
|
|
|
transaction_data = json.loads(transaction_dict['jsonData'])
|
|
transaction_data = newMultiRequest(f"tx/{transaction_dict['transactionHash']}")
|
|
parsed_flodata = json.loads(transaction_dict['parsedFloData'])
|
|
try:
|
|
block_info = json.loads(lblocks_dict[transaction_dict['blockNumber']]['jsonData'])
|
|
processTransaction(transaction_data, parsed_flodata, block_info)
|
|
prev_block = current_block
|
|
except:
|
|
prev_block = current_block
|
|
continue
|
|
|
|
# Check if the current block exceeds the specified "to_blockNumber"
|
|
if current_block >= args.to_blockNumber:
|
|
logger.info(f"Reached the specified block number {args.to_blockNumber}. Stopping processing.")
|
|
break
|
|
|
|
# copy the old block data
|
|
old_latest_cache = create_database_connection('system_dbs', {'db_name':'latestCache1'})
|
|
old_latest_cache.execute("ATTACH DATABASE 'latestCache.db' AS new_db")
|
|
old_latest_cache.execute("INSERT INTO new_db.latestBlocks SELECT * FROM latestBlocks WHERE blockNumber <= ?", (forward_block,))
|
|
old_latest_cache.close()
|
|
|
|
# delete
|
|
# system.db , latestCache.db, smartContracts, tokens
|
|
if os.path.isfile('./system1.db'):
|
|
os.remove('system1.db')
|
|
if os.path.isfile('./latestCache1.db'):
|
|
os.remove('latestCache1.db')
|
|
if os.path.isfile('./smartContracts1'):
|
|
shutil.rmtree('smartContracts1')
|
|
if os.path.isfile('./tokens1'):
|
|
shutil.rmtree('tokens1')
|
|
|
|
# Update system.db's last scanned block
|
|
connection = create_database_connection('system_dbs', {'db_name': "system"})
|
|
connection.execute(f"UPDATE systemData SET value = {int(list(lblocks_dict.keys())[-1])} WHERE attribute = 'lastblockscanned';")
|
|
connection.close() |