Update for combining with api
This commit is contained in:
Sai Raj
parent
134553c104
commit
46f2b5696b
@ -208,26 +208,26 @@ def find_sender_receiver(transaction_data):
|
||||
|
||||
def check_database_existence(type, parameters):
|
||||
if type == 'token':
|
||||
path = os.path.join(config['DEFAULT']['DATA_PATH'], 'tokens', f'{parameters["token_name"]}.db')
|
||||
path = os.path.join(_config['DEFAULT']['DATA_PATH'], 'tokens', f'{parameters["token_name"]}.db')
|
||||
return os.path.isfile(path)
|
||||
|
||||
if type == 'smart_contract':
|
||||
path = os.path.join(config['DEFAULT']['DATA_PATH'], 'smartContracts', f"{parameters['contract_name']}-{parameters['contract_address']}.db")
|
||||
path = os.path.join(_config['DEFAULT']['DATA_PATH'], 'smartContracts', f"{parameters['contract_name']}-{parameters['contract_address']}.db")
|
||||
return os.path.isfile(path)
|
||||
|
||||
|
||||
def create_database_connection(type, parameters=None):
|
||||
if type == 'token':
|
||||
path = os.path.join(config['DEFAULT']['DATA_PATH'], 'tokens', f"{parameters['token_name']}.db")
|
||||
path = os.path.join(_config['DEFAULT']['DATA_PATH'], 'tokens', f"{parameters['token_name']}.db")
|
||||
engine = create_engine(f"sqlite:///{path}", echo=True)
|
||||
elif type == 'smart_contract':
|
||||
path = os.path.join(config['DEFAULT']['DATA_PATH'], 'smartContracts', f"{parameters['contract_name']}-{parameters['contract_address']}.db")
|
||||
path = os.path.join(_config['DEFAULT']['DATA_PATH'], 'smartContracts', f"{parameters['contract_name']}-{parameters['contract_address']}.db")
|
||||
engine = create_engine(f"sqlite:///{path}", echo=True)
|
||||
elif type == 'system_dbs':
|
||||
path = os.path.join(config['DEFAULT']['DATA_PATH'], f"system.db")
|
||||
path = os.path.join(_config['DEFAULT']['DATA_PATH'], f"system.db")
|
||||
engine = create_engine(f"sqlite:///{path}", echo=False)
|
||||
elif type == 'latest_cache':
|
||||
path = os.path.join(config['DEFAULT']['DATA_PATH'], f"latestCache.db")
|
||||
path = os.path.join(_config['DEFAULT']['DATA_PATH'], f"latestCache.db")
|
||||
engine = create_engine(f"sqlite:///{path}", echo=False)
|
||||
|
||||
connection = engine.connect()
|
||||
@ -236,19 +236,19 @@ def create_database_connection(type, parameters=None):
|
||||
|
||||
def create_database_session_orm(type, parameters, base):
|
||||
if type == 'token':
|
||||
path = os.path.join(config['DEFAULT']['DATA_PATH'], 'tokens', f"{parameters['token_name']}.db")
|
||||
path = os.path.join(_config['DEFAULT']['DATA_PATH'], 'tokens', f"{parameters['token_name']}.db")
|
||||
engine = create_engine(f"sqlite:///{path}", echo=True)
|
||||
base.metadata.create_all(bind=engine)
|
||||
session = sessionmaker(bind=engine)()
|
||||
|
||||
elif type == 'smart_contract':
|
||||
path = os.path.join(config['DEFAULT']['DATA_PATH'], 'smartContracts', f"{parameters['contract_name']}-{parameters['contract_address']}.db")
|
||||
path = os.path.join(_config['DEFAULT']['DATA_PATH'], 'smartContracts', f"{parameters['contract_name']}-{parameters['contract_address']}.db")
|
||||
engine = create_engine(f"sqlite:///{path}", echo=True)
|
||||
base.metadata.create_all(bind=engine)
|
||||
session = sessionmaker(bind=engine)()
|
||||
|
||||
elif type == 'system_dbs':
|
||||
path = os.path.join(config['DEFAULT']['DATA_PATH'], f"{parameters['db_name']}.db")
|
||||
path = os.path.join(_config['DEFAULT']['DATA_PATH'], f"{parameters['db_name']}.db")
|
||||
engine = create_engine(f"sqlite:///{path}", echo=False)
|
||||
base.metadata.create_all(bind=engine)
|
||||
session = sessionmaker(bind=engine)()
|
||||
@ -258,7 +258,7 @@ def create_database_session_orm(type, parameters, base):
|
||||
|
||||
def delete_contract_database(parameters):
|
||||
if check_database_existence('smart_contract', {'contract_name':f"{parameters['contract_name']}", 'contract_address':f"{parameters['contract_address']}"}):
|
||||
path = os.path.join(config['DEFAULT']['DATA_PATH'], 'smartContracts', f"{parameters['contract_name']}-{parameters['contract_address']}.db")
|
||||
path = os.path.join(_config['DEFAULT']['DATA_PATH'], 'smartContracts', f"{parameters['contract_name']}-{parameters['contract_address']}.db")
|
||||
os.remove(path)
|
||||
|
||||
|
||||
@ -532,7 +532,7 @@ def processBlock(blockindex=None, blockhash=None):
|
||||
text = text.replace("\n", " \n ")
|
||||
# todo Rule 9 - Reject all noise transactions. Further rules are in parsing.py
|
||||
returnval = None
|
||||
parsed_data = parsing.parse_flodata(text, blockinfo, config['DEFAULT']['NET'])
|
||||
parsed_data = parsing.parse_flodata(text, blockinfo, _config['DEFAULT']['NET'])
|
||||
if parsed_data['type'] not in ['noise', None, '']:
|
||||
logger.info(f"Processing transaction {transaction}")
|
||||
logger.info(f"flodata {text} is parsed to {parsed_data}")
|
||||
@ -2547,84 +2547,6 @@ async def connect_to_websocket(uri):
|
||||
await asyncio.sleep(5) # You can adjust the delay as needed
|
||||
scanBlockchain()
|
||||
|
||||
|
||||
# MAIN EXECUTION STARTS
|
||||
# Configuration of required variables
|
||||
config = configparser.ConfigParser()
|
||||
config.read('config.ini')
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
logger.setLevel(logging.DEBUG)
|
||||
|
||||
formatter = logging.Formatter('%(asctime)s:%(name)s:%(message)s')
|
||||
file_handler = logging.FileHandler(os.path.join(config['DEFAULT']['DATA_PATH'],'tracking.log'))
|
||||
file_handler.setLevel(logging.INFO)
|
||||
file_handler.setFormatter(formatter)
|
||||
|
||||
stream_handler = logging.StreamHandler()
|
||||
stream_handler.setFormatter(formatter)
|
||||
|
||||
logger.addHandler(file_handler)
|
||||
logger.addHandler(stream_handler)
|
||||
|
||||
|
||||
# Rule 1 - Read command line arguments to reset the databases as blank
|
||||
# Rule 2 - Read config to set testnet/mainnet
|
||||
# Rule 3 - Set flo blockexplorer location depending on testnet or mainnet
|
||||
# Rule 4 - Set the local flo-cli path depending on testnet or mainnet ( removed this feature | Flosights are the only source )
|
||||
# Rule 5 - Set the block number to scan from
|
||||
|
||||
|
||||
# Read command line arguments
|
||||
parser = argparse.ArgumentParser(description='Script tracks RMT using FLO data on the FLO blockchain - https://flo.cash')
|
||||
parser.add_argument('-r', '--reset', nargs='?', const=1, type=int, help='Purge existing db and rebuild it from scratch')
|
||||
parser.add_argument('-rb', '--rebuild', nargs='?', const=1, type=int, help='Rebuild it')
|
||||
parser.add_argument("--testnet", action="store_true", help="Use the testnet URL")
|
||||
args = parser.parse_args()
|
||||
|
||||
dirpath = os.path.join(config['DEFAULT']['DATA_PATH'], 'tokens')
|
||||
if not os.path.isdir(dirpath):
|
||||
os.mkdir(dirpath)
|
||||
dirpath = os.path.join(config['DEFAULT']['DATA_PATH'], 'smartContracts')
|
||||
if not os.path.isdir(dirpath):
|
||||
os.mkdir(dirpath)
|
||||
|
||||
# Read configuration
|
||||
|
||||
# todo - write all assertions to make sure default configs are right
|
||||
if (config['DEFAULT']['NET'] != 'mainnet') and (config['DEFAULT']['NET'] != 'testnet'):
|
||||
logger.error("NET parameter in config.ini invalid. Options are either 'mainnet' or 'testnet'. Script is exiting now")
|
||||
sys.exit(0)
|
||||
|
||||
# Specify mainnet and testnet server list for API calls and websocket calls
|
||||
# Specify ADMIN ID
|
||||
serverlist = None
|
||||
if config['DEFAULT']['NET'] == 'mainnet':
|
||||
serverlist = config['DEFAULT']['MAINNET_FLOSIGHT_SERVER_LIST']
|
||||
APP_ADMIN = 'FNcvkz9PZNZM3HcxM1XTrVL4tgivmCkHp9'
|
||||
websocket_uri = get_websocket_uri(testnet=False)
|
||||
elif config['DEFAULT']['NET'] == 'testnet':
|
||||
serverlist = config['DEFAULT']['TESTNET_FLOSIGHT_SERVER_LIST']
|
||||
APP_ADMIN = 'oWooGLbBELNnwq8Z5YmjoVjw8GhBGH3qSP'
|
||||
websocket_uri = get_websocket_uri(testnet=True)
|
||||
serverlist = serverlist.split(',')
|
||||
neturl = config['DEFAULT']['FLOSIGHT_NETURL']
|
||||
api_url = neturl
|
||||
tokenapi_sse_url = config['DEFAULT']['TOKENAPI_SSE_URL']
|
||||
API_VERIFY = config['DEFAULT']['API_VERIFY']
|
||||
if API_VERIFY == 'False':
|
||||
API_VERIFY = False
|
||||
elif API_VERIFY == 'True':
|
||||
API_VERIFY = True
|
||||
else:
|
||||
API_VERIFY = True
|
||||
|
||||
|
||||
IGNORE_BLOCK_LIST = config['DEFAULT']['IGNORE_BLOCK_LIST'].split(',')
|
||||
IGNORE_BLOCK_LIST = [int(s) for s in IGNORE_BLOCK_LIST]
|
||||
IGNORE_TRANSACTION_LIST = config['DEFAULT']['IGNORE_TRANSACTION_LIST'].split(',')
|
||||
|
||||
|
||||
def create_dir_if_not_exist(dir_path, reset = False):
|
||||
if os.path.exists(dir_path):
|
||||
if reset:
|
||||
@ -2647,23 +2569,24 @@ def init_lastestcache_db():
|
||||
session.close()
|
||||
|
||||
def init_storage_if_not_exist(reset = False):
|
||||
|
||||
token_dir_path = os.path.join(config['DEFAULT']['DATA_PATH'], 'tokens')
|
||||
# Delete database and smartcontract directory if reset is set to True
|
||||
|
||||
token_dir_path = os.path.join(_config['DEFAULT']['DATA_PATH'], 'tokens')
|
||||
create_dir_if_not_exist(token_dir_path, reset)
|
||||
|
||||
smart_contract_dir_path = os.path.join(config['DEFAULT']['DATA_PATH'], 'smartContracts')
|
||||
smart_contract_dir_path = os.path.join(_config['DEFAULT']['DATA_PATH'], 'smartContracts')
|
||||
create_dir_if_not_exist(smart_contract_dir_path, reset)
|
||||
|
||||
system_db_path = os.path.join(config['DEFAULT']['DATA_PATH'], 'system.db')
|
||||
system_db_path = os.path.join(_config['DEFAULT']['DATA_PATH'], 'system.db')
|
||||
if os.path.exists(system_db_path):
|
||||
if reset:
|
||||
os.remove(system_db_path)
|
||||
init_system_db(int(config['DEFAULT']['START_BLOCK']))
|
||||
init_system_db(int(_config['DEFAULT']['START_BLOCK']))
|
||||
else:
|
||||
init_system_db(int(config['DEFAULT']['START_BLOCK']))
|
||||
init_system_db(int(_config['DEFAULT']['START_BLOCK']))
|
||||
|
||||
|
||||
latestCache_db_path = os.path.join(config['DEFAULT']['DATA_PATH'], 'latestCache.db')
|
||||
latestCache_db_path = os.path.join(_config['DEFAULT']['DATA_PATH'], 'latestCache.db')
|
||||
if os.path.exists(latestCache_db_path):
|
||||
if reset:
|
||||
os.remove(latestCache_db_path)
|
||||
@ -2671,24 +2594,112 @@ def init_storage_if_not_exist(reset = False):
|
||||
else:
|
||||
init_lastestcache_db()
|
||||
|
||||
# Delete database and smartcontract directory if reset is set to 1
|
||||
if args.reset == 1:
|
||||
logger.info("Resetting the database. ")
|
||||
init_storage_if_not_exist(reset=True)
|
||||
else:
|
||||
init_storage_if_not_exist()
|
||||
|
||||
def initiate_process():
|
||||
# MAIN EXECUTION STARTS
|
||||
global logger
|
||||
logger = logging.getLogger(__name__)
|
||||
logger.setLevel(logging.DEBUG)
|
||||
|
||||
formatter = logging.Formatter('%(asctime)s:%(name)s:%(message)s')
|
||||
file_handler = logging.FileHandler(os.path.join(_config['DEFAULT']['DATA_PATH'],'tracking.log'))
|
||||
file_handler.setLevel(logging.INFO)
|
||||
file_handler.setFormatter(formatter)
|
||||
|
||||
stream_handler = logging.StreamHandler()
|
||||
stream_handler.setFormatter(formatter)
|
||||
|
||||
logger.addHandler(file_handler)
|
||||
logger.addHandler(stream_handler)
|
||||
|
||||
|
||||
# Determine API source for block and transaction information
|
||||
if __name__ == "__main__":
|
||||
dirpath = os.path.join(_config['DEFAULT']['DATA_PATH'], 'tokens')
|
||||
if not os.path.isdir(dirpath):
|
||||
os.mkdir(dirpath)
|
||||
dirpath = os.path.join(_config['DEFAULT']['DATA_PATH'], 'smartContracts')
|
||||
if not os.path.isdir(dirpath):
|
||||
os.mkdir(dirpath)
|
||||
|
||||
# Read configuration
|
||||
|
||||
# todo - write all assertions to make sure default configs are right
|
||||
if (_config['DEFAULT']['NET'] != 'mainnet') and (_config['DEFAULT']['NET'] != 'testnet'):
|
||||
logger.error("NET parameter in _config.ini invalid. Options are either 'mainnet' or 'testnet'. Script is exiting now")
|
||||
sys.exit(0)
|
||||
|
||||
# Specify mainnet and testnet server list for API calls and websocket calls
|
||||
# Specify ADMIN ID
|
||||
global serverlist, APP_ADMIN, websocket_uri
|
||||
serverlist = None
|
||||
if _config['DEFAULT']['NET'] == 'mainnet':
|
||||
serverlist = _config['DEFAULT']['MAINNET_FLOSIGHT_SERVER_LIST']
|
||||
APP_ADMIN = 'FNcvkz9PZNZM3HcxM1XTrVL4tgivmCkHp9'
|
||||
websocket_uri = get_websocket_uri(testnet=False)
|
||||
elif _config['DEFAULT']['NET'] == 'testnet':
|
||||
serverlist = _config['DEFAULT']['TESTNET_FLOSIGHT_SERVER_LIST']
|
||||
APP_ADMIN = 'oWooGLbBELNnwq8Z5YmjoVjw8GhBGH3qSP'
|
||||
websocket_uri = get_websocket_uri(testnet=True)
|
||||
serverlist = serverlist.split(',')
|
||||
|
||||
global neturl
|
||||
neturl = _config['DEFAULT']['FLOSIGHT_NETURL']
|
||||
global api_url
|
||||
api_url = neturl
|
||||
global tokenapi_sse_url
|
||||
tokenapi_sse_url = _config['DEFAULT']['TOKENAPI_SSE_URL']
|
||||
global API_VERIFY
|
||||
API_VERIFY = _config['DEFAULT']['API_VERIFY']
|
||||
if API_VERIFY == 'False':
|
||||
API_VERIFY = False
|
||||
elif API_VERIFY == 'True':
|
||||
API_VERIFY = True
|
||||
else:
|
||||
API_VERIFY = True
|
||||
|
||||
|
||||
global IGNORE_BLOCK_LIST, IGNORE_TRANSACTION_LIST
|
||||
IGNORE_BLOCK_LIST = _config['DEFAULT']['IGNORE_BLOCK_LIST'].split(',')
|
||||
IGNORE_BLOCK_LIST = [int(s) for s in IGNORE_BLOCK_LIST]
|
||||
IGNORE_TRANSACTION_LIST = _config['DEFAULT']['IGNORE_TRANSACTION_LIST'].split(',')
|
||||
|
||||
def start_backend_process(config, reset = False):
|
||||
global _config
|
||||
_config = config
|
||||
initiate_process()
|
||||
init_storage_if_not_exist(reset)
|
||||
# MAIN LOGIC STARTS
|
||||
# scan from the latest block saved locally to latest network block
|
||||
scanBlockchain()
|
||||
|
||||
logger.debug("Completed first scan")
|
||||
|
||||
# At this point the script has updated to the latest block
|
||||
# Now we connect to flosight's websocket API to get information about the latest blocks
|
||||
# Neturl is the URL for Flosight API whose websocket endpoint is being connected to
|
||||
|
||||
asyncio.get_event_loop().run_until_complete(connect_to_websocket(websocket_uri))
|
||||
asyncio.get_event_loop().run_until_complete(connect_to_websocket(websocket_uri))
|
||||
|
||||
# Determine API source for block and transaction information
|
||||
if __name__ == "__main__":
|
||||
|
||||
# Rule 1 - Read command line arguments to reset the databases as blank
|
||||
# Rule 2 - Read config to set testnet/mainnet
|
||||
# Rule 3 - Set flo blockexplorer location depending on testnet or mainnet
|
||||
# Rule 4 - Set the local flo-cli path depending on testnet or mainnet ( removed this feature | Flosights are the only source )
|
||||
# Rule 5 - Set the block number to scan from
|
||||
|
||||
# Configuration of required variables
|
||||
config = configparser.ConfigParser()
|
||||
config.read('config.ini')
|
||||
|
||||
# Read command line arguments
|
||||
parser = argparse.ArgumentParser(description='Script tracks RMT using FLO data on the FLO blockchain - https://flo.cash')
|
||||
parser.add_argument('-r', '--reset', nargs='?', const=1, type=int, help='Purge existing db and rebuild it from scratch')
|
||||
parser.add_argument('-rb', '--rebuild', nargs='?', const=1, type=int, help='Rebuild it')
|
||||
parser.add_argument("--testnet", action="store_true", help="Use the testnet URL")
|
||||
args = parser.parse_args()
|
||||
|
||||
if args.reset == 1:
|
||||
logger.info("Resetting the database. ")
|
||||
start_backend_process(config, reset=True)
|
||||
else:
|
||||
start_backend_process(config)
|
||||
|
||||
|
||||
Loading…
Reference in New Issue
Block a user