RanchiMall/electrumx
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Improve daemon error handling

Browse Source
This commit is contained in:
Neil Booth 2018-08-13 18:33:26 +09:00
parent 3f69595fbd
commit ab2691563f
1 changed files with 32 additions and 27 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

59
electrumx/server/daemon.py
View File

@ -29,6 +29,10 @@ class DaemonError(Exception):
'''Raised when the daemon returns an error in its results.''' '''Raised when the daemon returns an error in its results.'''
class WorkQueueFullError(Exception):
pass
class Daemon(object): class Daemon(object):
'''Handles connections to a daemon at the given URL.''' '''Handles connections to a daemon at the given URL.'''
@ -87,12 +91,16 @@ class Daemon(object):
async with self.workqueue_semaphore: async with self.workqueue_semaphore:
async with self.client_session() as session: async with self.client_session() as session:
async with session.post(self.current_url(), data=data) as resp: async with session.post(self.current_url(), data=data) as resp:
# If bitcoind can't find a tx, for some reason kind = resp.headers.get('Content-Type', None)
# it returns 500 but fills out the JSON. if kind == 'application/json':
# Should still return 200 IMO.
if resp.status in (200, 404, 500):
return await resp.json() return await resp.json()
return (resp.status, resp.reason) # bitcoind's HTTP protocol "handling" is a bad joke
text = await resp.text()
if 'Work queue depth exceeded' in text:
raise WorkQueueFullError
text = text.strip() or resp.reason
self.logger.error(text)
raise DaemonError(text)
async def _send(self, payload, processor): async def _send(self, payload, processor):
'''Send a payload to be converted to JSON. '''Send a payload to be converted to JSON.
@ -101,48 +109,45 @@ class Daemon(object):
are raise through DaemonError. are raise through DaemonError.
''' '''
def log_error(error): def log_error(error):
nonlocal down, last_error_time nonlocal last_error_log, secs
down = True
now = time.time() now = time.time()
prior_time = last_error_time if now - last_error_log > 60:
if now - prior_time > 60:
last_error_time = now last_error_time = now
if prior_time and self.failover(): self.logger.error(f'{error} Retrying occasionally...')
secs = 0 if secs == max_secs and self.failover():
else: secs = 0.25
self.logger.error(f'{error} Retrying occasionally...')
down = False on_good_message = None
last_error_time = 0 last_error_log = 0
data = json.dumps(payload) data = json.dumps(payload)
secs = 1 secs = 0.25
max_secs = 4 max_secs = 4
while True: while True:
try: try:
result = await self._send_data(data) result = await self._send_data(data)
if not isinstance(result, tuple): result = processor(result)
result = processor(result) if on_good_message:
if down: self.logger.info(on_good_message)
self.logger.info('connection restored') return result
return result
log_error(f'HTTP error code {result[0]}: {result[1]}')
except asyncio.TimeoutError: except asyncio.TimeoutError:
log_error('timeout error.') log_error('timeout error.')
except aiohttp.ServerDisconnectedError: except aiohttp.ServerDisconnectedError:
log_error('disconnected.') log_error('disconnected.')
on_good_message = 'connection restored'
except aiohttp.ClientPayloadError: except aiohttp.ClientPayloadError:
log_error('payload encoding error.') log_error('payload encoding error.')
except aiohttp.ClientConnectionError: except aiohttp.ClientConnectionError:
log_error('connection problem - is your daemon running?') log_error('connection problem - is your daemon running?')
on_good_message = 'connection restored'
except self.DaemonWarmingUpError: except self.DaemonWarmingUpError:
log_error('starting up checking blocks.') log_error('starting up checking blocks.')
except (asyncio.CancelledError, DaemonError): on_good_message = 'running normally'
raise except WorkQueueFullError:
except Exception as e: log_error('work queue full.')
self.logger.exception(f'uncaught exception: {e}') on_good_message = 'running normally'
await asyncio.sleep(secs) await asyncio.sleep(secs)
secs = min(max_secs, secs * 2, 1) secs = min(max_secs, secs * 2)
async def _send_single(self, method, params=None): async def _send_single(self, method, params=None):
'''Send a single request to the daemon.''' '''Send a single request to the daemon.'''