RanchiMall/flo-token-tracking
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Update for combining with api

Browse Source
This commit is contained in:
Sai Raj 2024-07-15 02:22:52 -04:00
parent 134553c104
commit 46f2b5696b
1 changed files with 118 additions and 107 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

221
src/backend/tracktokens_smartcontracts.py
View File

@ -208,26 +208,26 @@ def find_sender_receiver(transaction_data):
def check_database_existence(type, parameters): def check_database_existence(type, parameters):
if type == 'token': if type == 'token':
path = os.path.join(config['DEFAULT']['DATA_PATH'], 'tokens', f'{parameters["token_name"]}.db') path = os.path.join(_config['DEFAULT']['DATA_PATH'], 'tokens', f'{parameters["token_name"]}.db')
return os.path.isfile(path) return os.path.isfile(path)
if type == 'smart_contract': if type == 'smart_contract':
path = os.path.join(config['DEFAULT']['DATA_PATH'], 'smartContracts', f"{parameters['contract_name']}-{parameters['contract_address']}.db") path = os.path.join(_config['DEFAULT']['DATA_PATH'], 'smartContracts', f"{parameters['contract_name']}-{parameters['contract_address']}.db")
return os.path.isfile(path) return os.path.isfile(path)
def create_database_connection(type, parameters=None): def create_database_connection(type, parameters=None):
if type == 'token': if type == 'token':
path = os.path.join(config['DEFAULT']['DATA_PATH'], 'tokens', f"{parameters['token_name']}.db") path = os.path.join(_config['DEFAULT']['DATA_PATH'], 'tokens', f"{parameters['token_name']}.db")
engine = create_engine(f"sqlite:///{path}", echo=True) engine = create_engine(f"sqlite:///{path}", echo=True)
elif type == 'smart_contract': elif type == 'smart_contract':
path = os.path.join(config['DEFAULT']['DATA_PATH'], 'smartContracts', f"{parameters['contract_name']}-{parameters['contract_address']}.db") path = os.path.join(_config['DEFAULT']['DATA_PATH'], 'smartContracts', f"{parameters['contract_name']}-{parameters['contract_address']}.db")
engine = create_engine(f"sqlite:///{path}", echo=True) engine = create_engine(f"sqlite:///{path}", echo=True)
elif type == 'system_dbs': elif type == 'system_dbs':
path = os.path.join(config['DEFAULT']['DATA_PATH'], f"system.db") path = os.path.join(_config['DEFAULT']['DATA_PATH'], f"system.db")
engine = create_engine(f"sqlite:///{path}", echo=False) engine = create_engine(f"sqlite:///{path}", echo=False)
elif type == 'latest_cache': elif type == 'latest_cache':
path = os.path.join(config['DEFAULT']['DATA_PATH'], f"latestCache.db") path = os.path.join(_config['DEFAULT']['DATA_PATH'], f"latestCache.db")
engine = create_engine(f"sqlite:///{path}", echo=False) engine = create_engine(f"sqlite:///{path}", echo=False)
connection = engine.connect() connection = engine.connect()
@ -236,19 +236,19 @@ def create_database_connection(type, parameters=None):
def create_database_session_orm(type, parameters, base): def create_database_session_orm(type, parameters, base):
if type == 'token': if type == 'token':
path = os.path.join(config['DEFAULT']['DATA_PATH'], 'tokens', f"{parameters['token_name']}.db") path = os.path.join(_config['DEFAULT']['DATA_PATH'], 'tokens', f"{parameters['token_name']}.db")
engine = create_engine(f"sqlite:///{path}", echo=True) engine = create_engine(f"sqlite:///{path}", echo=True)
base.metadata.create_all(bind=engine) base.metadata.create_all(bind=engine)
session = sessionmaker(bind=engine)() session = sessionmaker(bind=engine)()
elif type == 'smart_contract': elif type == 'smart_contract':
path = os.path.join(config['DEFAULT']['DATA_PATH'], 'smartContracts', f"{parameters['contract_name']}-{parameters['contract_address']}.db") path = os.path.join(_config['DEFAULT']['DATA_PATH'], 'smartContracts', f"{parameters['contract_name']}-{parameters['contract_address']}.db")
engine = create_engine(f"sqlite:///{path}", echo=True) engine = create_engine(f"sqlite:///{path}", echo=True)
base.metadata.create_all(bind=engine) base.metadata.create_all(bind=engine)
session = sessionmaker(bind=engine)() session = sessionmaker(bind=engine)()
elif type == 'system_dbs': elif type == 'system_dbs':
path = os.path.join(config['DEFAULT']['DATA_PATH'], f"{parameters['db_name']}.db") path = os.path.join(_config['DEFAULT']['DATA_PATH'], f"{parameters['db_name']}.db")
engine = create_engine(f"sqlite:///{path}", echo=False) engine = create_engine(f"sqlite:///{path}", echo=False)
base.metadata.create_all(bind=engine) base.metadata.create_all(bind=engine)
session = sessionmaker(bind=engine)() session = sessionmaker(bind=engine)()
@ -258,7 +258,7 @@ def create_database_session_orm(type, parameters, base):
def delete_contract_database(parameters): def delete_contract_database(parameters):
if check_database_existence('smart_contract', {'contract_name':f"{parameters['contract_name']}", 'contract_address':f"{parameters['contract_address']}"}): if check_database_existence('smart_contract', {'contract_name':f"{parameters['contract_name']}", 'contract_address':f"{parameters['contract_address']}"}):
path = os.path.join(config['DEFAULT']['DATA_PATH'], 'smartContracts', f"{parameters['contract_name']}-{parameters['contract_address']}.db") path = os.path.join(_config['DEFAULT']['DATA_PATH'], 'smartContracts', f"{parameters['contract_name']}-{parameters['contract_address']}.db")
os.remove(path) os.remove(path)
@ -532,7 +532,7 @@ def processBlock(blockindex=None, blockhash=None):
text = text.replace("\n", " \n ") text = text.replace("\n", " \n ")
# todo Rule 9 - Reject all noise transactions. Further rules are in parsing.py # todo Rule 9 - Reject all noise transactions. Further rules are in parsing.py
returnval = None returnval = None
parsed_data = parsing.parse_flodata(text, blockinfo, config['DEFAULT']['NET']) parsed_data = parsing.parse_flodata(text, blockinfo, _config['DEFAULT']['NET'])
if parsed_data['type'] not in ['noise', None, '']: if parsed_data['type'] not in ['noise', None, '']:
logger.info(f"Processing transaction {transaction}") logger.info(f"Processing transaction {transaction}")
logger.info(f"flodata {text} is parsed to {parsed_data}") logger.info(f"flodata {text} is parsed to {parsed_data}")
@ -2547,84 +2547,6 @@ async def connect_to_websocket(uri):
await asyncio.sleep(5) # You can adjust the delay as needed await asyncio.sleep(5) # You can adjust the delay as needed
scanBlockchain() scanBlockchain()
# MAIN EXECUTION STARTS
# Configuration of required variables
config = configparser.ConfigParser()
config.read('config.ini')
logger = logging.getLogger(__name__)
logger.setLevel(logging.DEBUG)
formatter = logging.Formatter('%(asctime)s:%(name)s:%(message)s')
file_handler = logging.FileHandler(os.path.join(config['DEFAULT']['DATA_PATH'],'tracking.log'))
file_handler.setLevel(logging.INFO)
file_handler.setFormatter(formatter)
stream_handler = logging.StreamHandler()
stream_handler.setFormatter(formatter)
logger.addHandler(file_handler)
logger.addHandler(stream_handler)
# Rule 1 - Read command line arguments to reset the databases as blank
# Rule 2 - Read config to set testnet/mainnet
# Rule 3 - Set flo blockexplorer location depending on testnet or mainnet
# Rule 4 - Set the local flo-cli path depending on testnet or mainnet ( removed this feature | Flosights are the only source )
# Rule 5 - Set the block number to scan from
# Read command line arguments
parser = argparse.ArgumentParser(description='Script tracks RMT using FLO data on the FLO blockchain - https://flo.cash')
parser.add_argument('-r', '--reset', nargs='?', const=1, type=int, help='Purge existing db and rebuild it from scratch')
parser.add_argument('-rb', '--rebuild', nargs='?', const=1, type=int, help='Rebuild it')
parser.add_argument("--testnet", action="store_true", help="Use the testnet URL")
args = parser.parse_args()
dirpath = os.path.join(config['DEFAULT']['DATA_PATH'], 'tokens')
if not os.path.isdir(dirpath):
os.mkdir(dirpath)
dirpath = os.path.join(config['DEFAULT']['DATA_PATH'], 'smartContracts')
if not os.path.isdir(dirpath):
os.mkdir(dirpath)
# Read configuration
# todo - write all assertions to make sure default configs are right
if (config['DEFAULT']['NET'] != 'mainnet') and (config['DEFAULT']['NET'] != 'testnet'):
logger.error("NET parameter in config.ini invalid. Options are either 'mainnet' or 'testnet'. Script is exiting now")
sys.exit(0)
# Specify mainnet and testnet server list for API calls and websocket calls
# Specify ADMIN ID
serverlist = None
if config['DEFAULT']['NET'] == 'mainnet':
serverlist = config['DEFAULT']['MAINNET_FLOSIGHT_SERVER_LIST']
APP_ADMIN = 'FNcvkz9PZNZM3HcxM1XTrVL4tgivmCkHp9'
websocket_uri = get_websocket_uri(testnet=False)
elif config['DEFAULT']['NET'] == 'testnet':
serverlist = config['DEFAULT']['TESTNET_FLOSIGHT_SERVER_LIST']
APP_ADMIN = 'oWooGLbBELNnwq8Z5YmjoVjw8GhBGH3qSP'
websocket_uri = get_websocket_uri(testnet=True)
serverlist = serverlist.split(',')
neturl = config['DEFAULT']['FLOSIGHT_NETURL']
api_url = neturl
tokenapi_sse_url = config['DEFAULT']['TOKENAPI_SSE_URL']
API_VERIFY = config['DEFAULT']['API_VERIFY']
if API_VERIFY == 'False':
API_VERIFY = False
elif API_VERIFY == 'True':
API_VERIFY = True
else:
API_VERIFY = True
IGNORE_BLOCK_LIST = config['DEFAULT']['IGNORE_BLOCK_LIST'].split(',')
IGNORE_BLOCK_LIST = [int(s) for s in IGNORE_BLOCK_LIST]
IGNORE_TRANSACTION_LIST = config['DEFAULT']['IGNORE_TRANSACTION_LIST'].split(',')
def create_dir_if_not_exist(dir_path, reset = False): def create_dir_if_not_exist(dir_path, reset = False):
if os.path.exists(dir_path): if os.path.exists(dir_path):
if reset: if reset:
@ -2647,23 +2569,24 @@ def init_lastestcache_db():
session.close() session.close()
def init_storage_if_not_exist(reset = False): def init_storage_if_not_exist(reset = False):
# Delete database and smartcontract directory if reset is set to True
token_dir_path = os.path.join(config['DEFAULT']['DATA_PATH'], 'tokens') token_dir_path = os.path.join(_config['DEFAULT']['DATA_PATH'], 'tokens')
create_dir_if_not_exist(token_dir_path, reset) create_dir_if_not_exist(token_dir_path, reset)
smart_contract_dir_path = os.path.join(config['DEFAULT']['DATA_PATH'], 'smartContracts') smart_contract_dir_path = os.path.join(_config['DEFAULT']['DATA_PATH'], 'smartContracts')
create_dir_if_not_exist(smart_contract_dir_path, reset) create_dir_if_not_exist(smart_contract_dir_path, reset)
system_db_path = os.path.join(config['DEFAULT']['DATA_PATH'], 'system.db') system_db_path = os.path.join(_config['DEFAULT']['DATA_PATH'], 'system.db')
if os.path.exists(system_db_path): if os.path.exists(system_db_path):
if reset: if reset:
os.remove(system_db_path) os.remove(system_db_path)
init_system_db(int(config['DEFAULT']['START_BLOCK'])) init_system_db(int(_config['DEFAULT']['START_BLOCK']))
else: else:
init_system_db(int(config['DEFAULT']['START_BLOCK'])) init_system_db(int(_config['DEFAULT']['START_BLOCK']))
latestCache_db_path = os.path.join(config['DEFAULT']['DATA_PATH'], 'latestCache.db') latestCache_db_path = os.path.join(_config['DEFAULT']['DATA_PATH'], 'latestCache.db')
if os.path.exists(latestCache_db_path): if os.path.exists(latestCache_db_path):
if reset: if reset:
os.remove(latestCache_db_path) os.remove(latestCache_db_path)
@ -2671,24 +2594,112 @@ def init_storage_if_not_exist(reset = False):
else: else:
init_lastestcache_db() init_lastestcache_db()
# Delete database and smartcontract directory if reset is set to 1
if args.reset == 1: def initiate_process():
logger.info("Resetting the database. ") # MAIN EXECUTION STARTS
init_storage_if_not_exist(reset=True) global logger
else: logger = logging.getLogger(__name__)
init_storage_if_not_exist() logger.setLevel(logging.DEBUG)
formatter = logging.Formatter('%(asctime)s:%(name)s:%(message)s')
file_handler = logging.FileHandler(os.path.join(_config['DEFAULT']['DATA_PATH'],'tracking.log'))
file_handler.setLevel(logging.INFO)
file_handler.setFormatter(formatter)
stream_handler = logging.StreamHandler()
stream_handler.setFormatter(formatter)
logger.addHandler(file_handler)
logger.addHandler(stream_handler)
# Determine API source for block and transaction information dirpath = os.path.join(_config['DEFAULT']['DATA_PATH'], 'tokens')
if __name__ == "__main__": if not os.path.isdir(dirpath):
os.mkdir(dirpath)
dirpath = os.path.join(_config['DEFAULT']['DATA_PATH'], 'smartContracts')
if not os.path.isdir(dirpath):
os.mkdir(dirpath)
# Read configuration
# todo - write all assertions to make sure default configs are right
if (_config['DEFAULT']['NET'] != 'mainnet') and (_config['DEFAULT']['NET'] != 'testnet'):
logger.error("NET parameter in _config.ini invalid. Options are either 'mainnet' or 'testnet'. Script is exiting now")
sys.exit(0)
# Specify mainnet and testnet server list for API calls and websocket calls
# Specify ADMIN ID
global serverlist, APP_ADMIN, websocket_uri
serverlist = None
if _config['DEFAULT']['NET'] == 'mainnet':
serverlist = _config['DEFAULT']['MAINNET_FLOSIGHT_SERVER_LIST']
APP_ADMIN = 'FNcvkz9PZNZM3HcxM1XTrVL4tgivmCkHp9'
websocket_uri = get_websocket_uri(testnet=False)
elif _config['DEFAULT']['NET'] == 'testnet':
serverlist = _config['DEFAULT']['TESTNET_FLOSIGHT_SERVER_LIST']
APP_ADMIN = 'oWooGLbBELNnwq8Z5YmjoVjw8GhBGH3qSP'
websocket_uri = get_websocket_uri(testnet=True)
serverlist = serverlist.split(',')
global neturl
neturl = _config['DEFAULT']['FLOSIGHT_NETURL']
global api_url
api_url = neturl
global tokenapi_sse_url
tokenapi_sse_url = _config['DEFAULT']['TOKENAPI_SSE_URL']
global API_VERIFY
API_VERIFY = _config['DEFAULT']['API_VERIFY']
if API_VERIFY == 'False':
API_VERIFY = False
elif API_VERIFY == 'True':
API_VERIFY = True
else:
API_VERIFY = True
global IGNORE_BLOCK_LIST, IGNORE_TRANSACTION_LIST
IGNORE_BLOCK_LIST = _config['DEFAULT']['IGNORE_BLOCK_LIST'].split(',')
IGNORE_BLOCK_LIST = [int(s) for s in IGNORE_BLOCK_LIST]
IGNORE_TRANSACTION_LIST = _config['DEFAULT']['IGNORE_TRANSACTION_LIST'].split(',')
def start_backend_process(config, reset = False):
global _config
_config = config
initiate_process()
init_storage_if_not_exist(reset)
# MAIN LOGIC STARTS # MAIN LOGIC STARTS
# scan from the latest block saved locally to latest network block # scan from the latest block saved locally to latest network block
scanBlockchain() scanBlockchain()
logger.debug("Completed first scan") logger.debug("Completed first scan")
# At this point the script has updated to the latest block # At this point the script has updated to the latest block
# Now we connect to flosight's websocket API to get information about the latest blocks # Now we connect to flosight's websocket API to get information about the latest blocks
# Neturl is the URL for Flosight API whose websocket endpoint is being connected to # Neturl is the URL for Flosight API whose websocket endpoint is being connected to
asyncio.get_event_loop().run_until_complete(connect_to_websocket(websocket_uri)) asyncio.get_event_loop().run_until_complete(connect_to_websocket(websocket_uri))
# Determine API source for block and transaction information
if __name__ == "__main__":
# Rule 1 - Read command line arguments to reset the databases as blank
# Rule 2 - Read config to set testnet/mainnet
# Rule 3 - Set flo blockexplorer location depending on testnet or mainnet
# Rule 4 - Set the local flo-cli path depending on testnet or mainnet ( removed this feature | Flosights are the only source )
# Rule 5 - Set the block number to scan from
# Configuration of required variables
config = configparser.ConfigParser()
config.read('config.ini')
# Read command line arguments
parser = argparse.ArgumentParser(description='Script tracks RMT using FLO data on the FLO blockchain - https://flo.cash')
parser.add_argument('-r', '--reset', nargs='?', const=1, type=int, help='Purge existing db and rebuild it from scratch')
parser.add_argument('-rb', '--rebuild', nargs='?', const=1, type=int, help='Rebuild it')
parser.add_argument("--testnet", action="store_true", help="Use the testnet URL")
args = parser.parse_args()
if args.reset == 1:
logger.info("Resetting the database. ")
start_backend_process(config, reset=True)
else:
start_backend_process(config)