From 46f2b5696b43528d7e30b7673ce7612c5b8b5832 Mon Sep 17 00:00:00 2001 From: Sai Raj <39055732+sairajzero@users.noreply.github.com> Date: Mon, 15 Jul 2024 02:22:52 -0400 Subject: [PATCH] Update for combining with api --- src/backend/tracktokens_smartcontracts.py | 225 ++++++++++++---------- 1 file changed, 118 insertions(+), 107 deletions(-) diff --git a/src/backend/tracktokens_smartcontracts.py b/src/backend/tracktokens_smartcontracts.py index 568b601..0c96669 100644 --- a/src/backend/tracktokens_smartcontracts.py +++ b/src/backend/tracktokens_smartcontracts.py @@ -208,26 +208,26 @@ def find_sender_receiver(transaction_data): def check_database_existence(type, parameters): if type == 'token': - path = os.path.join(config['DEFAULT']['DATA_PATH'], 'tokens', f'{parameters["token_name"]}.db') + path = os.path.join(_config['DEFAULT']['DATA_PATH'], 'tokens', f'{parameters["token_name"]}.db') return os.path.isfile(path) if type == 'smart_contract': - path = os.path.join(config['DEFAULT']['DATA_PATH'], 'smartContracts', f"{parameters['contract_name']}-{parameters['contract_address']}.db") + path = os.path.join(_config['DEFAULT']['DATA_PATH'], 'smartContracts', f"{parameters['contract_name']}-{parameters['contract_address']}.db") return os.path.isfile(path) def create_database_connection(type, parameters=None): if type == 'token': - path = os.path.join(config['DEFAULT']['DATA_PATH'], 'tokens', f"{parameters['token_name']}.db") + path = os.path.join(_config['DEFAULT']['DATA_PATH'], 'tokens', f"{parameters['token_name']}.db") engine = create_engine(f"sqlite:///{path}", echo=True) elif type == 'smart_contract': - path = os.path.join(config['DEFAULT']['DATA_PATH'], 'smartContracts', f"{parameters['contract_name']}-{parameters['contract_address']}.db") + path = os.path.join(_config['DEFAULT']['DATA_PATH'], 'smartContracts', f"{parameters['contract_name']}-{parameters['contract_address']}.db") engine = create_engine(f"sqlite:///{path}", echo=True) elif type == 'system_dbs': - path = os.path.join(config['DEFAULT']['DATA_PATH'], f"system.db") + path = os.path.join(_config['DEFAULT']['DATA_PATH'], f"system.db") engine = create_engine(f"sqlite:///{path}", echo=False) elif type == 'latest_cache': - path = os.path.join(config['DEFAULT']['DATA_PATH'], f"latestCache.db") + path = os.path.join(_config['DEFAULT']['DATA_PATH'], f"latestCache.db") engine = create_engine(f"sqlite:///{path}", echo=False) connection = engine.connect() @@ -236,19 +236,19 @@ def create_database_connection(type, parameters=None): def create_database_session_orm(type, parameters, base): if type == 'token': - path = os.path.join(config['DEFAULT']['DATA_PATH'], 'tokens', f"{parameters['token_name']}.db") + path = os.path.join(_config['DEFAULT']['DATA_PATH'], 'tokens', f"{parameters['token_name']}.db") engine = create_engine(f"sqlite:///{path}", echo=True) base.metadata.create_all(bind=engine) session = sessionmaker(bind=engine)() elif type == 'smart_contract': - path = os.path.join(config['DEFAULT']['DATA_PATH'], 'smartContracts', f"{parameters['contract_name']}-{parameters['contract_address']}.db") + path = os.path.join(_config['DEFAULT']['DATA_PATH'], 'smartContracts', f"{parameters['contract_name']}-{parameters['contract_address']}.db") engine = create_engine(f"sqlite:///{path}", echo=True) base.metadata.create_all(bind=engine) session = sessionmaker(bind=engine)() elif type == 'system_dbs': - path = os.path.join(config['DEFAULT']['DATA_PATH'], f"{parameters['db_name']}.db") + path = os.path.join(_config['DEFAULT']['DATA_PATH'], f"{parameters['db_name']}.db") engine = create_engine(f"sqlite:///{path}", echo=False) base.metadata.create_all(bind=engine) session = sessionmaker(bind=engine)() @@ -258,7 +258,7 @@ def create_database_session_orm(type, parameters, base): def delete_contract_database(parameters): if check_database_existence('smart_contract', {'contract_name':f"{parameters['contract_name']}", 'contract_address':f"{parameters['contract_address']}"}): - path = os.path.join(config['DEFAULT']['DATA_PATH'], 'smartContracts', f"{parameters['contract_name']}-{parameters['contract_address']}.db") + path = os.path.join(_config['DEFAULT']['DATA_PATH'], 'smartContracts', f"{parameters['contract_name']}-{parameters['contract_address']}.db") os.remove(path) @@ -532,7 +532,7 @@ def processBlock(blockindex=None, blockhash=None): text = text.replace("\n", " \n ") # todo Rule 9 - Reject all noise transactions. Further rules are in parsing.py returnval = None - parsed_data = parsing.parse_flodata(text, blockinfo, config['DEFAULT']['NET']) + parsed_data = parsing.parse_flodata(text, blockinfo, _config['DEFAULT']['NET']) if parsed_data['type'] not in ['noise', None, '']: logger.info(f"Processing transaction {transaction}") logger.info(f"flodata {text} is parsed to {parsed_data}") @@ -2547,84 +2547,6 @@ async def connect_to_websocket(uri): await asyncio.sleep(5) # You can adjust the delay as needed scanBlockchain() - -# MAIN EXECUTION STARTS -# Configuration of required variables -config = configparser.ConfigParser() -config.read('config.ini') - -logger = logging.getLogger(__name__) -logger.setLevel(logging.DEBUG) - -formatter = logging.Formatter('%(asctime)s:%(name)s:%(message)s') -file_handler = logging.FileHandler(os.path.join(config['DEFAULT']['DATA_PATH'],'tracking.log')) -file_handler.setLevel(logging.INFO) -file_handler.setFormatter(formatter) - -stream_handler = logging.StreamHandler() -stream_handler.setFormatter(formatter) - -logger.addHandler(file_handler) -logger.addHandler(stream_handler) - - -# Rule 1 - Read command line arguments to reset the databases as blank -# Rule 2 - Read config to set testnet/mainnet -# Rule 3 - Set flo blockexplorer location depending on testnet or mainnet -# Rule 4 - Set the local flo-cli path depending on testnet or mainnet ( removed this feature | Flosights are the only source ) -# Rule 5 - Set the block number to scan from - - -# Read command line arguments -parser = argparse.ArgumentParser(description='Script tracks RMT using FLO data on the FLO blockchain - https://flo.cash') -parser.add_argument('-r', '--reset', nargs='?', const=1, type=int, help='Purge existing db and rebuild it from scratch') -parser.add_argument('-rb', '--rebuild', nargs='?', const=1, type=int, help='Rebuild it') -parser.add_argument("--testnet", action="store_true", help="Use the testnet URL") -args = parser.parse_args() - -dirpath = os.path.join(config['DEFAULT']['DATA_PATH'], 'tokens') -if not os.path.isdir(dirpath): - os.mkdir(dirpath) -dirpath = os.path.join(config['DEFAULT']['DATA_PATH'], 'smartContracts') -if not os.path.isdir(dirpath): - os.mkdir(dirpath) - -# Read configuration - -# todo - write all assertions to make sure default configs are right -if (config['DEFAULT']['NET'] != 'mainnet') and (config['DEFAULT']['NET'] != 'testnet'): - logger.error("NET parameter in config.ini invalid. Options are either 'mainnet' or 'testnet'. Script is exiting now") - sys.exit(0) - -# Specify mainnet and testnet server list for API calls and websocket calls -# Specify ADMIN ID -serverlist = None -if config['DEFAULT']['NET'] == 'mainnet': - serverlist = config['DEFAULT']['MAINNET_FLOSIGHT_SERVER_LIST'] - APP_ADMIN = 'FNcvkz9PZNZM3HcxM1XTrVL4tgivmCkHp9' - websocket_uri = get_websocket_uri(testnet=False) -elif config['DEFAULT']['NET'] == 'testnet': - serverlist = config['DEFAULT']['TESTNET_FLOSIGHT_SERVER_LIST'] - APP_ADMIN = 'oWooGLbBELNnwq8Z5YmjoVjw8GhBGH3qSP' - websocket_uri = get_websocket_uri(testnet=True) -serverlist = serverlist.split(',') -neturl = config['DEFAULT']['FLOSIGHT_NETURL'] -api_url = neturl -tokenapi_sse_url = config['DEFAULT']['TOKENAPI_SSE_URL'] -API_VERIFY = config['DEFAULT']['API_VERIFY'] -if API_VERIFY == 'False': - API_VERIFY = False -elif API_VERIFY == 'True': - API_VERIFY = True -else: - API_VERIFY = True - - -IGNORE_BLOCK_LIST = config['DEFAULT']['IGNORE_BLOCK_LIST'].split(',') -IGNORE_BLOCK_LIST = [int(s) for s in IGNORE_BLOCK_LIST] -IGNORE_TRANSACTION_LIST = config['DEFAULT']['IGNORE_TRANSACTION_LIST'].split(',') - - def create_dir_if_not_exist(dir_path, reset = False): if os.path.exists(dir_path): if reset: @@ -2647,23 +2569,24 @@ def init_lastestcache_db(): session.close() def init_storage_if_not_exist(reset = False): - - token_dir_path = os.path.join(config['DEFAULT']['DATA_PATH'], 'tokens') + # Delete database and smartcontract directory if reset is set to True + + token_dir_path = os.path.join(_config['DEFAULT']['DATA_PATH'], 'tokens') create_dir_if_not_exist(token_dir_path, reset) - smart_contract_dir_path = os.path.join(config['DEFAULT']['DATA_PATH'], 'smartContracts') + smart_contract_dir_path = os.path.join(_config['DEFAULT']['DATA_PATH'], 'smartContracts') create_dir_if_not_exist(smart_contract_dir_path, reset) - system_db_path = os.path.join(config['DEFAULT']['DATA_PATH'], 'system.db') + system_db_path = os.path.join(_config['DEFAULT']['DATA_PATH'], 'system.db') if os.path.exists(system_db_path): if reset: os.remove(system_db_path) - init_system_db(int(config['DEFAULT']['START_BLOCK'])) + init_system_db(int(_config['DEFAULT']['START_BLOCK'])) else: - init_system_db(int(config['DEFAULT']['START_BLOCK'])) + init_system_db(int(_config['DEFAULT']['START_BLOCK'])) - latestCache_db_path = os.path.join(config['DEFAULT']['DATA_PATH'], 'latestCache.db') + latestCache_db_path = os.path.join(_config['DEFAULT']['DATA_PATH'], 'latestCache.db') if os.path.exists(latestCache_db_path): if reset: os.remove(latestCache_db_path) @@ -2671,24 +2594,112 @@ def init_storage_if_not_exist(reset = False): else: init_lastestcache_db() -# Delete database and smartcontract directory if reset is set to 1 -if args.reset == 1: - logger.info("Resetting the database. ") - init_storage_if_not_exist(reset=True) -else: - init_storage_if_not_exist() + +def initiate_process(): + # MAIN EXECUTION STARTS + global logger + logger = logging.getLogger(__name__) + logger.setLevel(logging.DEBUG) + + formatter = logging.Formatter('%(asctime)s:%(name)s:%(message)s') + file_handler = logging.FileHandler(os.path.join(_config['DEFAULT']['DATA_PATH'],'tracking.log')) + file_handler.setLevel(logging.INFO) + file_handler.setFormatter(formatter) + + stream_handler = logging.StreamHandler() + stream_handler.setFormatter(formatter) + + logger.addHandler(file_handler) + logger.addHandler(stream_handler) -# Determine API source for block and transaction information -if __name__ == "__main__": + dirpath = os.path.join(_config['DEFAULT']['DATA_PATH'], 'tokens') + if not os.path.isdir(dirpath): + os.mkdir(dirpath) + dirpath = os.path.join(_config['DEFAULT']['DATA_PATH'], 'smartContracts') + if not os.path.isdir(dirpath): + os.mkdir(dirpath) + + # Read configuration + + # todo - write all assertions to make sure default configs are right + if (_config['DEFAULT']['NET'] != 'mainnet') and (_config['DEFAULT']['NET'] != 'testnet'): + logger.error("NET parameter in _config.ini invalid. Options are either 'mainnet' or 'testnet'. Script is exiting now") + sys.exit(0) + + # Specify mainnet and testnet server list for API calls and websocket calls + # Specify ADMIN ID + global serverlist, APP_ADMIN, websocket_uri + serverlist = None + if _config['DEFAULT']['NET'] == 'mainnet': + serverlist = _config['DEFAULT']['MAINNET_FLOSIGHT_SERVER_LIST'] + APP_ADMIN = 'FNcvkz9PZNZM3HcxM1XTrVL4tgivmCkHp9' + websocket_uri = get_websocket_uri(testnet=False) + elif _config['DEFAULT']['NET'] == 'testnet': + serverlist = _config['DEFAULT']['TESTNET_FLOSIGHT_SERVER_LIST'] + APP_ADMIN = 'oWooGLbBELNnwq8Z5YmjoVjw8GhBGH3qSP' + websocket_uri = get_websocket_uri(testnet=True) + serverlist = serverlist.split(',') + + global neturl + neturl = _config['DEFAULT']['FLOSIGHT_NETURL'] + global api_url + api_url = neturl + global tokenapi_sse_url + tokenapi_sse_url = _config['DEFAULT']['TOKENAPI_SSE_URL'] + global API_VERIFY + API_VERIFY = _config['DEFAULT']['API_VERIFY'] + if API_VERIFY == 'False': + API_VERIFY = False + elif API_VERIFY == 'True': + API_VERIFY = True + else: + API_VERIFY = True + + + global IGNORE_BLOCK_LIST, IGNORE_TRANSACTION_LIST + IGNORE_BLOCK_LIST = _config['DEFAULT']['IGNORE_BLOCK_LIST'].split(',') + IGNORE_BLOCK_LIST = [int(s) for s in IGNORE_BLOCK_LIST] + IGNORE_TRANSACTION_LIST = _config['DEFAULT']['IGNORE_TRANSACTION_LIST'].split(',') + +def start_backend_process(config, reset = False): + global _config + _config = config + initiate_process() + init_storage_if_not_exist(reset) # MAIN LOGIC STARTS # scan from the latest block saved locally to latest network block scanBlockchain() - logger.debug("Completed first scan") - # At this point the script has updated to the latest block # Now we connect to flosight's websocket API to get information about the latest blocks # Neturl is the URL for Flosight API whose websocket endpoint is being connected to - asyncio.get_event_loop().run_until_complete(connect_to_websocket(websocket_uri)) \ No newline at end of file + asyncio.get_event_loop().run_until_complete(connect_to_websocket(websocket_uri)) + +# Determine API source for block and transaction information +if __name__ == "__main__": + + # Rule 1 - Read command line arguments to reset the databases as blank + # Rule 2 - Read config to set testnet/mainnet + # Rule 3 - Set flo blockexplorer location depending on testnet or mainnet + # Rule 4 - Set the local flo-cli path depending on testnet or mainnet ( removed this feature | Flosights are the only source ) + # Rule 5 - Set the block number to scan from + + # Configuration of required variables + config = configparser.ConfigParser() + config.read('config.ini') + + # Read command line arguments + parser = argparse.ArgumentParser(description='Script tracks RMT using FLO data on the FLO blockchain - https://flo.cash') + parser.add_argument('-r', '--reset', nargs='?', const=1, type=int, help='Purge existing db and rebuild it from scratch') + parser.add_argument('-rb', '--rebuild', nargs='?', const=1, type=int, help='Rebuild it') + parser.add_argument("--testnet", action="store_true", help="Use the testnet URL") + args = parser.parse_args() + + if args.reset == 1: + logger.info("Resetting the database. ") + start_backend_process(config, reset=True) + else: + start_backend_process(config) +