From a80a412ef5154580ec272217466110cc72f775ca Mon Sep 17 00:00:00 2001 From: Vivek Teega Date: Tue, 15 Sep 2020 09:33:44 +0530 Subject: [PATCH 1/4] Change in reconnection functions for websocket API and flosight API --- tracktokens-smartcontracts.py | 59 +++++++++++++++++++++-------------- 1 file changed, 36 insertions(+), 23 deletions(-) diff --git a/tracktokens-smartcontracts.py b/tracktokens-smartcontracts.py index ee3623a..bbc3186 100755 --- a/tracktokens-smartcontracts.py +++ b/tracktokens-smartcontracts.py @@ -11,6 +11,7 @@ import requests import socketio from sqlalchemy import create_engine, func from sqlalchemy.orm import sessionmaker +import time import parsing from config import * @@ -18,22 +19,22 @@ from models import SystemData, ActiveTable, ConsumedTable, TransferLogs, Transac Base, ContractStructure, ContractBase, ContractParticipants, SystemBase, ActiveContracts, ContractAddressMapping, \ LatestCacheBase, ContractTransactionHistory, RejectedContractTransactionHistory, TokenContractAssociation + def retryRequest(tempserverlist, apicall): - if len(tempserverlist) != 0: + while len(tempserverlist) != 0: try: response = requests.get('{}api/{}'.format(tempserverlist[0], apicall)) except: tempserverlist.pop(0) - return retryRequest(tempserverlist, apicall) else: if response.status_code == 200: return json.loads(response.content) else: tempserverlist.pop(0) - return retryRequest(tempserverlist, apicall) - else: - logger.error("None of the APIs are responding for the call {}".format(apicall)) - sys.exit(0) + + if len(tempserverlist) == 0: + logger.error("None of the APIs are responding for the call {}".format(apicall)) + return 0 def multiRequest(apicall, net): @@ -149,6 +150,7 @@ def processApiBlock(blockhash): # Check smartContracts which will be triggered locally, and not by the contract committee checkLocaltriggerContracts(blockinfo) + def updateLatestTransaction(transactionData, parsed_data): # connect to latest transaction db conn = sqlite3.connect('latestCache.db') @@ -2200,13 +2202,23 @@ def scanBlockchain(): session.close() # todo Rule 6 - Find current block height - # Rule 7 - Start analysing the block contents from starting block to current height + # Rule 7 - Start analysing the block contents from starting block to current height # Find current block height - response = multiRequest('blocks?limit=1', config['DEFAULT']['NET']) - current_index = response['blocks'][0]['height'] - logger.debug("Current block height is %s" % str(current_index)) - + current_index = -1 + while(current_index == -1): + response = multiRequest('blocks?limit=1', config['DEFAULT']['NET']) + try: + current_index = response['blocks'][0]['height'] + except: + logger.debug('Latest block count response from multiRequest() is not in the right format. Displaying the data received in the log below') + logger.debug(response) + logger.debug('Program will wait for 10 seconds and try to reconnect') + time.sleep(10) + else: + logger.debug("Current block height is %s" % str(current_index)) + break + for blockindex in range(startblock, current_index): processBlock(blockindex) @@ -2303,7 +2315,7 @@ if args.reset == 1: # MAIN LOGIC # scan from the latest block saved locally to latest network block scanBlockchain() -scanBlockchain() + def switchNeturl(neturl): testserverlist = ['http://0.0.0.0:9000/', 'https://testnet-flosight.duckdns.org/', 'https://testnet.flocha.in/'] @@ -2321,16 +2333,15 @@ def switchNeturl(neturl): else: return testserverlist[neturlindex+1] -def reconnectSSE(neturl): +def reconnectSSE(neturl): # Connect to Flosight websocket to get data on new incoming blocks sio = socketio.Client(reconnection=False) try: sio.connect( neturl + "socket.io/socket.io.js") except: logger.debug(f"Could not connect to the websocket endpoint {neturl}. Switching & retrying to next") - neturl = switchNeturl(neturl) - reconnectSSE(neturl) + return @sio.on('connect') def on_connect(): @@ -2340,17 +2351,12 @@ def reconnectSSE(neturl): @sio.on('disconnect') def disconnect(): logger.debug(f"Token Tracker disconnected from websocket endpoint {neturl}") - logger.debug('The script will rescan from the latest block in local db to latest server on FLO network') - scanBlockchain() - logger.debug("Rescan completed") - logger.debug('Attempting reconnect to websocket ...') - reconnectSSE(neturl) + return @sio.on('connect_error') def connect_error(): logger.debug(f"CONNECTION_ERROR to the websocket endpoint {neturl}. Switching & retrying to next") - neturl = switchNeturl(neturl) - reconnectSSE(neturl) + return @sio.on('block') def on_block(data): @@ -2358,7 +2364,14 @@ def reconnectSSE(neturl): logger.debug(str(data)) processApiBlock(data) + # At this point the script has updated to the latest block # Now we connect to flosight's websocket API to get information about the latest blocks neturl = 'https://flosight.duckdns.org/' -reconnectSSE(neturl) +while(True): + logger.debug('The script will rescan from the latest block in local db to latest server on FLO network') + scanBlockchain() + logger.debug("Rescan completed") + logger.debug('Attempting reconnect to websocket ...') + reconnectSSE(neturl) + neturl = switchNeturl(neturl) \ No newline at end of file From b1c694d13742ac66b2009339fd818e2346fd7bd6 Mon Sep 17 00:00:00 2001 From: Vivek Teega Date: Tue, 29 Sep 2020 03:39:54 +0530 Subject: [PATCH 2/4] Update Readme --- README.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index f33b765..dd11b25 100644 --- a/README.md +++ b/README.md @@ -1 +1,3 @@ -# flo-token-tracking +# FLO Token & Smart Contract System + +The python script scans the FLO Blockchain for Token and Smart Contract activity and creates/updates local SQLite databases accordingly. From 2ea0996e26819fb66255dbf4a82211816804d036 Mon Sep 17 00:00:00 2001 From: Vivek Teega Date: Fri, 9 Oct 2020 10:00:41 +0530 Subject: [PATCH 3/4] Websocket reconnection - new logic --- tracktokens-smartcontracts.py | 78 ++++++++++++++++++----------------- 1 file changed, 41 insertions(+), 37 deletions(-) diff --git a/tracktokens-smartcontracts.py b/tracktokens-smartcontracts.py index bbc3186..bbdf50e 100755 --- a/tracktokens-smartcontracts.py +++ b/tracktokens-smartcontracts.py @@ -12,7 +12,6 @@ import socketio from sqlalchemy import create_engine, func from sqlalchemy.orm import sessionmaker import time - import parsing from config import * from models import SystemData, ActiveTable, ConsumedTable, TransferLogs, TransactionHistory, RejectedTransactionHistory, \ @@ -2316,7 +2315,6 @@ if args.reset == 1: # scan from the latest block saved locally to latest network block scanBlockchain() - def switchNeturl(neturl): testserverlist = ['http://0.0.0.0:9000/', 'https://testnet-flosight.duckdns.org/', 'https://testnet.flocha.in/'] mainserverlist = ['http://0.0.0.0:9001/', 'https://flosight.duckdns.org/', 'https://explorer.mediciland.com/', 'https://livenet.flocha.in/'] @@ -2335,43 +2333,49 @@ def switchNeturl(neturl): def reconnectSSE(neturl): + # Switch a to different flosight + # neturl = switchNeturl(neturl) # Connect to Flosight websocket to get data on new incoming blocks - sio = socketio.Client(reconnection=False) - try: - sio.connect( neturl + "socket.io/socket.io.js") - except: - logger.debug(f"Could not connect to the websocket endpoint {neturl}. Switching & retrying to next") - return - - @sio.on('connect') - def on_connect(): - logger.debug(f"Token Tracker has connected to websocket endpoint {neturl}") - sio.emit('subscribe', 'inv') - - @sio.on('disconnect') - def disconnect(): - logger.debug(f"Token Tracker disconnected from websocket endpoint {neturl}") - return - - @sio.on('connect_error') - def connect_error(): - logger.debug(f"CONNECTION_ERROR to the websocket endpoint {neturl}. Switching & retrying to next") - return - - @sio.on('block') - def on_block(data): - logger.debug('New block received') - logger.debug(str(data)) - processApiBlock(data) + connection_status = False + while (not connection_status): + logger.debug('The script will rescan from the latest block in local db to latest server on FLO network') + scanBlockchain() + logger.debug("Rescan completed") + try: + logger.debug(f"Attempting websocket connection to {neturl}") + sio.connect(neturl + "socket.io/socket.io.js") + connection_status = True + return neturl + except: + logger.debug(f"Could not connect to the websocket endpoint {neturl}") + neturl = switchNeturl(neturl) # At this point the script has updated to the latest block # Now we connect to flosight's websocket API to get information about the latest blocks -neturl = 'https://flosight.duckdns.org/' -while(True): - logger.debug('The script will rescan from the latest block in local db to latest server on FLO network') - scanBlockchain() - logger.debug("Rescan completed") - logger.debug('Attempting reconnect to websocket ...') - reconnectSSE(neturl) - neturl = switchNeturl(neturl) \ No newline at end of file +neturl = 'https://flosight.duckdns.org/' # Neturl is the URL for Flosight API whose websocket endpoint is being connected to +sio = socketio.Client(reconnection=False) + +# Connect to the websocket endpoint and return the new neturl +neturl = reconnectSSE(neturl) + +@sio.on('connect') +def on_connect(): + logger.debug(f"Token Tracker has connected to websocket endpoint {neturl}") + sio.emit('subscribe', 'inv') + +@sio.on('disconnect') +def disconnect(): + logger.debug(f"Token Tracker disconnected from websocket endpoint {neturl}") + neturl = reconnectSSE(neturl) + +@sio.on('connect_error') +def connect_error(): + logger.debug(f"CONNECTION_ERROR to the websocket endpoint {neturl}. Switching & retrying to next") + neturl = reconnectSSE(neturl) + +@sio.on('block') +def on_block(data): + logger.debug('New block received') + logger.debug(str(data)) + processApiBlock(data) \ No newline at end of file From 94c35cd593d54718ad5290f6a3fef23c7ca4d12c Mon Sep 17 00:00:00 2001 From: Vivek Teega Date: Sat, 10 Oct 2020 04:34:00 +0530 Subject: [PATCH 4/4] Get neturl to global namespace --- tracktokens-smartcontracts.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/tracktokens-smartcontracts.py b/tracktokens-smartcontracts.py index bbdf50e..bb5dcfa 100755 --- a/tracktokens-smartcontracts.py +++ b/tracktokens-smartcontracts.py @@ -2332,7 +2332,7 @@ def switchNeturl(neturl): return testserverlist[neturlindex+1] -def reconnectSSE(neturl): +def reconnectSSE(websocketApi): # Switch a to different flosight # neturl = switchNeturl(neturl) # Connect to Flosight websocket to get data on new incoming blocks @@ -2342,18 +2342,18 @@ def reconnectSSE(neturl): scanBlockchain() logger.debug("Rescan completed") try: - logger.debug(f"Attempting websocket connection to {neturl}") - sio.connect(neturl + "socket.io/socket.io.js") + logger.debug(f"Attempting websocket connection to {websocketApi}") + sio.connect(websocketApi + "socket.io/socket.io.js") connection_status = True - return neturl + return websocketApi except: - logger.debug(f"Could not connect to the websocket endpoint {neturl}") - neturl = switchNeturl(neturl) + logger.debug(f"Could not connect to the websocket endpoint {websocketApi}") + websocketApi = switchNeturl(websocketApi) # At this point the script has updated to the latest block # Now we connect to flosight's websocket API to get information about the latest blocks -neturl = 'https://flosight.duckdns.org/' # Neturl is the URL for Flosight API whose websocket endpoint is being connected to +global neturl = 'https://flosight.duckdns.org/' # Neturl is the URL for Flosight API whose websocket endpoint is being connected to sio = socketio.Client(reconnection=False) # Connect to the websocket endpoint and return the new neturl