From 292073f2c7ab65a9507f25964951cb0018053afe Mon Sep 17 00:00:00 2001 From: Neil Booth Date: Sat, 26 Nov 2016 09:32:29 +0900 Subject: [PATCH] Log large requests and reject them --- docs/ENV-NOTES | 23 ++++++++++++----------- lib/jsonrpc.py | 35 +++++++++++++++++++++++++++++++---- server/env.py | 2 +- server/protocol.py | 19 +++++++++---------- 4 files changed, 53 insertions(+), 26 deletions(-) diff --git a/docs/ENV-NOTES b/docs/ENV-NOTES index 1a30a2c..c93d15e 100644 --- a/docs/ENV-NOTES +++ b/docs/ENV-NOTES @@ -44,17 +44,18 @@ in ElectrumX are very cheap - they consume about 100 bytes of memory each and are processed efficiently. I feel the defaults are low and encourage you to raise them. -MAX_HIST - maximum number of historical transactions to serve for - a single address. The current Electrum protocol requires - address histories be served en-masse or not at all, - an obvious avenue for abuse. This limit is a - stop-gap until the protocol is improved to admit - incremental history requests. The default value is - 2,000 which should be ample for most legitimate - users. Increasing to around 10,000 is likely fine - but bear in mind one client can request multiple - addresses. I welcome your experiences and suggestions - for an appropriate value. +MAX_SEND - maximum size of a response message to send over the wire, + in bytes. Defaults to 250,000. The current Electrum + protocol has a flaw in that address histories must be + served all at once or not at all, an obvious avenue for + abuse. This limit is a stop-gap until the protocol is + improved to admit incremental history requests. + Each history entry is appoximately 100 bytes so the + default is equivalent to a history limit of around 2,500 + entries, which should be ample for most legitimate + users. Increasing by a single-digit factor is likely fine + but bear in mind one client can request history for + multiple addresses. MAX_SUBS - maximum number of address subscriptions across all sessions. Defaults to 250,000. MAX_SESSION_SUBS - maximum number of address subscriptions permitted to a diff --git a/lib/jsonrpc.py b/lib/jsonrpc.py index fda25f4..ee416e0 100644 --- a/lib/jsonrpc.py +++ b/lib/jsonrpc.py @@ -72,6 +72,10 @@ class JSONRPC(asyncio.Protocol, LoggedClass): self.msg = msg self.code = code + class LargeRequestError(Exception): + '''Raised if a large request was prevented from being sent.''' + + def __init__(self): super().__init__() self.start = time.time() @@ -87,6 +91,20 @@ class JSONRPC(asyncio.Protocol, LoggedClass): self.error_count = 0 self.peer_info = None self.messages = asyncio.Queue() + # Sends longer than max_send are prevented, instead returning + # an oversized request error to other end of the network + # connection. The request causing it is logged. Values under + # 1000 are treated as 1000. + self.max_send = 0 + self.anon_logs = False + + def peername(self, *, for_log=True): + '''Return the peer name of this connection.''' + if not self.peer_info: + return 'unknown' + if for_log and self.anon_logs: + return 'xx.xx.xx.xx:xx' + return '{}:{}'.format(self.peer_info[0], self.peer_info[1]) def connection_made(self, transport): '''Handle an incoming client connection.''' @@ -175,9 +193,14 @@ class JSONRPC(asyncio.Protocol, LoggedClass): self.logger.error(msg) self.send_json_error(msg, self.INTERNAL_ERROR, payload.get('id')) else: - self.send_count += 1 - self.send_size += len(data) - self.transport.write(data) + if len(data) > max(1000, self.max_send): + self.send_json_error('request too large', self.INVALID_REQUEST, + payload.get('id')) + raise self.LargeRequestError + else: + self.send_count += 1 + self.send_size += len(data) + self.transport.write(data) async def handle_message(self, message): '''Asynchronously handle a JSON request or response. @@ -190,7 +213,11 @@ class JSONRPC(asyncio.Protocol, LoggedClass): payload = await self.single_payload(message) if payload: - self.send_json(payload) + try: + self.send_json(payload) + except self.LargeRequestError: + self.logger.warning('blocked large request from {}: {}' + .format(self.peername(), message)) async def batch_payload(self, batch): '''Return the JSON payload corresponding to a batch JSON request.''' diff --git a/server/env.py b/server/env.py index b62b8fa..4c6716e 100644 --- a/server/env.py +++ b/server/env.py @@ -45,7 +45,7 @@ class Env(LoggedClass): self.donation_address = self.default('DONATION_ADDRESS', '') self.db_engine = self.default('DB_ENGINE', 'leveldb') # Server limits to help prevent DoS - self.max_hist = self.integer('MAX_HIST', 2000) + self.max_send = self.integer('MAX_SEND', 250000) self.max_subs = self.integer('MAX_SUBS', 250000) self.max_session_subs = self.integer('MAX_SESSION_SUBS', 50000) # IRC diff --git a/server/protocol.py b/server/protocol.py index 9af9e06..0e5bbf8 100644 --- a/server/protocol.py +++ b/server/protocol.py @@ -227,6 +227,8 @@ class ServerManager(util.LoggedClass): self.max_subs = env.max_subs self.subscription_count = 0 self.futures = [] + env.max_send = max(1000, env.max_send) + self.logger.info('max response size {:,d} bytes'.format(env.max_send)) self.logger.info('max subscriptions across all sessions: {:,d}' .format(self.max_subs)) self.logger.info('max subscriptions per session: {:,d}' @@ -421,6 +423,8 @@ class Session(JSONRPC): self.coin = bp.coin self.kind = kind self.client = 'unknown' + self.anon_logs = env.anon_logs + self.max_send = env.max_send def connection_made(self, transport): '''Handle an incoming client connection.''' @@ -463,14 +467,6 @@ class Session(JSONRPC): self.logger.error('error handling request {}'.format(message)) traceback.print_exc() - def peername(self, *, for_log=True): - if not self.peer_info: - return 'unknown' - # Anonymize IP addresses that will be logged - if for_log and self.env.anon_logs: - return 'xx.xx.xx.xx:xx' - return '{}:{}'.format(self.peer_info[0], self.peer_info[1]) - def sub_count(self): return 0 @@ -674,8 +670,11 @@ class ElectrumX(Session): return self.bp.read_headers(start_height, count).hex() async def async_get_history(self, hash168): - # Apply DoS limit - limit = self.env.max_hist + # History DoS limit. Each element of history is about 99 + # bytes when encoded as JSON. This limits resource usage on + # bloated history requests, and uses a smaller divisor so + # large requests are logged before refusing them. + limit = self.max_send // 97 # Python 3.6: use async generators; update callers history = [] for item in self.bp.get_history(hash168, limit=limit):