RanchiMall/electrumx
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Begin work on proper server

Browse Source
This commit is contained in:
Neil Booth 2016-10-16 11:11:18 +09:00
parent edc33febe8
commit 334ffdaa4f
18 changed files with 412 additions and 144 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

35
electrumx_rpc.py
View File

@ -6,16 +6,29 @@
import argparse
import asyncio
import json
from functools import partial
from os import environ
import aiohttp
class RPCClient(asyncio.Protocol):
async def send(url, payload):
data = json.dumps(payload)
def __init__(self, loop):
self.loop = loop
async with aiohttp.post(url, data = data) as resp:
return await resp.json()
def connection_made(self, transport):
self.transport = transport
def connection_lost(self, exc):
self.loop.stop()
def send(self, payload):
data = json.dumps(payload) + '\n'
self.transport.write(data.encode())
def data_received(self, data):
payload = json.loads(data.decode())
self.transport.close()
print(json.dumps(payload, indent=4, sort_keys=True))
def main():
@ -30,18 +43,20 @@ def main():
if args.port is None:
args.port = int(environ.get('ELECTRUMX_RPC_PORT', 8000))
url = 'http://127.0.0.1:{:d}/'.format(args.port)
payload = {'method': args.command[0], 'params': args.command[1:]}
task = send(url, payload)
loop = asyncio.get_event_loop()
proto_factory = partial(RPCClient, loop)
coro = loop.create_connection(proto_factory, 'localhost', args.port)
try:
result = loop.run_until_complete(task)
transport, protocol = loop.run_until_complete(coro)
protocol.send(payload)
loop.run_forever()
except OSError:
print('error connecting - is ElectrumX running?')
finally:
loop.close()
print(result)
if __name__ == '__main__':
main()

7
lib/hash.py
View File

@ -31,6 +31,13 @@ def hash160(x):
return ripemd160(sha256(x))
def hash_to_str(x):
'''Converts a big-endian binary hash to a little-endian hex string, as
shown in block explorers, etc.
'''
return bytes(reversed(x)).hex()
class InvalidBase58String(Exception):
pass

8
lib/util.py
View File

@ -2,10 +2,18 @@
# and warranty status of this software.
import array
import logging
import sys
from collections import Container, Mapping
class LoggedClass(object):
def __init__(self):
self.logger = logging.getLogger(self.__class__.__name__)
self.logger.setLevel(logging.INFO)
# Method decorator. To be used for calculations that will always
# deliver the same result. The method cannot take any arguments
# and should be accessed as an attribute.

5
query.py
View File

@ -8,6 +8,7 @@ import sys
from server.env import Env
from server.db import DB
from lib.hash import hash_to_str
def main():
@ -27,13 +28,13 @@ def main():
n = None
for n, (tx_hash, height) in enumerate(db.get_history(hash168, limit)):
print('History #{:d}: hash: {} height: {:d}'
.format(n + 1, bytes(reversed(tx_hash)).hex(), height))
.format(n + 1, hash_to_str(tx_hash), height))
if n is None:
print('No history')
n = None
for n, utxo in enumerate(db.get_utxos(hash168, limit)):
print('UTXOs #{:d}: hash: {} pos: {:d} height: {:d} value: {:d}'
.format(n + 1, bytes(reversed(utxo.tx_hash)).hex(),
.format(n + 1, hash_to_str(utxo.tx_hash),
utxo.tx_pos, utxo.height, utxo.value))
if n is None:
print('No UTXOs')

0
samples/scripts/env/RPC_HOST → samples/scripts/env/DAEMON_HOST vendored
View File

0
samples/scripts/env/RPC_PASSWORD → samples/scripts/env/DAEMON_PASSWORD vendored
View File

1
samples/scripts/env/DAEMON_PORT vendored Normal file
View File

@ -0,0 +1 @@
8332

0
samples/scripts/env/RPC_USERNAME → samples/scripts/env/DAEMON_USERNAME vendored
View File

2
samples/scripts/env/RPC_PORT vendored
View File

@ -1 +1 @@
8332
8000

1
samples/scripts/env/SSL_PORT vendored Normal file
View File

@ -0,0 +1 @@
50002

0
samples/scripts/env/TCP_PORT vendored Normal file
View File

130
server/server.py → server/controller.py
View File

@ -3,76 +3,134 @@
import asyncio
import json
import logging
import signal
import traceback
from functools import partial
import aiohttp
from server.db import DB
from server.rpc import ElectrumRPCServer
from server.protocol import ElectrumX, LocalRPC
from lib.hash import sha256, hash_to_str, Base58
from lib.util import LoggedClass
class Server(object):
class Controller(LoggedClass):
def __init__(self, env):
super().__init__()
self.env = env
self.db = DB(env)
self.block_cache = BlockCache(env, self.db)
self.rpc_server = ElectrumRPCServer(self)
self.servers = []
self.sessions = set()
self.addresses = {}
self.jobs = set()
self.peers = {}
def start(self, loop):
env = self.env
protocol = partial(LocalRPC, self)
if env.rpc_port is not None:
host = 'localhost'
rpc_server = loop.create_server(protocol, host, env.rpc_port)
self.servers.append(loop.run_until_complete(rpc_server))
self.logger.info('RPC server listening on {}:{:d}'
.format(host, env.rpc_port))
protocol = partial(ElectrumX, self, env)
if env.tcp_port is not None:
tcp_server = loop.create_server(protocol, env.host, env.tcp_port)
self.servers.append(loop.run_until_complete(tcp_server))
self.logger.info('TCP server listening on {}:{:d}'
.format(env.host, env.tcp_port))
if env.ssl_port is not None:
ssl_server = loop.create_server(protocol, env.host, env.ssl_port)
self.servers.append(loop.run_until_complete(ssl_server))
self.logger.info('SSL server listening on {}:{:d}'
.format(env.host, env.ssl_port))
coros = [
self.reap_jobs(),
self.block_cache.catch_up(),
self.block_cache.process_cache()
]
self.tasks = [asyncio.ensure_future(coro) for coro in coros]
# Signal handlers
loop = asyncio.get_event_loop()
for signame in ('SIGINT', 'SIGTERM'):
loop.add_signal_handler(getattr(signal, signame),
partial(self.on_signal, signame))
coros = self.rpc_server.tasks(env.electrumx_rpc_port)
coros += [self.block_cache.catch_up(),
self.block_cache.process_cache()]
self.tasks = [asyncio.ensure_future(coro) for coro in coros]
return self.tasks
async def handle_rpc_getinfo(self, params):
return None, {
'blocks': self.db.height,
'peers': 0,
'sessions': 0,
'watched': 0,
'cached': 0,
}
def stop(self):
for server in self.servers:
server.close()
async def handle_rpc_sessions(self, params):
return None, []
def add_session(self, session):
self.sessions.add(session)
async def handle_rpc_numsessions(self, params):
return None, 0
def remove_session(self, session):
self.sessions.remove(session)
async def handle_rpc_peers(self, params):
return None, []
def add_job(self, coro):
'''Queue a job for asynchronous processing.'''
self.jobs.add(asyncio.ensure_future(coro))
async def handle_rpc_banner_update(self, params):
return None, 'FIXME'
async def reap_jobs(self):
while True:
jobs = set()
for job in self.jobs:
if job.done():
try:
job.result()
except Exception as e:
traceback.print_exc()
else:
jobs.add(job)
self.logger.info('reaped {:d} jobs, {:d} jobs pending'
.format(len(self.jobs) - len(jobs), len(jobs)))
self.jobs = jobs
await asyncio.sleep(5)
def on_signal(self, signame):
logging.warning('received {} signal, preparing to shut down'
.format(signame))
self.logger.warning('received {} signal, preparing to shut down'
.format(signame))
for task in self.tasks:
task.cancel()
def async_tasks(self):
return self.tasks
def address_status(self, hash168):
'''Returns status as 32 bytes.'''
status = self.addresses.get(hash168)
if status is None:
status = ''.join(
'{}:{:d}:'.format(hash_to_str(tx_hash), height)
for tx_hash, height in self.db.get_history(hash168)
)
if status:
status = sha256(status.encode())
self.addresses[hash168] = status
return status
def get_peers(self):
'''Returns a dictionary of IRC nick to (ip, host, ports) tuples, one
per peer.'''
return self.peers
class BlockCache(object):
class BlockCache(LoggedClass):
'''Requests blocks ahead of time from the daemon. Serves them
to the blockchain processor.'''
def __init__(self, env, db):
self.logger = logging.getLogger('BlockCache')
self.logger.setLevel(logging.INFO)
super().__init__()
self.db = db
self.rpc_url = env.rpc_url
self.daemon_url = env.daemon_url
# Cache target size is in MB. Has little effect on sync time.
self.cache_limit = 10
self.daemon_height = 0
@ -82,7 +140,7 @@ class BlockCache(object):
self.recent_sizes = []
self.ave_size = 0
self.logger.info('using RPC URL {}'.format(self.rpc_url))
self.logger.info('using daemon URL {}'.format(self.daemon_url))
async def process_cache(self):
while True:
@ -173,7 +231,7 @@ class BlockCache(object):
data = json.dumps(payload)
while True:
try:
async with aiohttp.post(self.rpc_url, data = data) as resp:
async with aiohttp.post(self.daemon_url, data = data) as resp:
result = await resp.json()
except asyncio.CancelledError:
raise

52
server/db.py
View File

@ -7,16 +7,16 @@ import itertools
import os
import struct
import time
from binascii import hexlify, unhexlify
from bisect import bisect_right
from collections import defaultdict, namedtuple
from functools import partial
import logging
import plyvel
from lib.coins import Bitcoin
from lib.script import ScriptPubKey
from lib.util import LoggedClass
from lib.hash import hash_to_str
# History can hold approx. 65536 * HIST_ENTRIES_PER_KEY entries
HIST_ENTRIES_PER_KEY = 1024
@ -25,14 +25,13 @@ ADDR_TX_HASH_LEN = 4
UTXO_TX_HASH_LEN = 4
UTXO = namedtuple("UTXO", "tx_num tx_pos tx_hash height value")
def formatted_time(t):
t = int(t)
return '{:d}d {:02d}h {:02d}m {:02d}s'.format(
t // 86400, (t % 86400) // 3600, (t % 3600) // 60, t % 60)
class UTXOCache(object):
class UTXOCache(LoggedClass):
'''An in-memory UTXO cache, representing all changes to UTXO state
since the last DB flush.
@ -85,8 +84,7 @@ class UTXOCache(object):
'''
def __init__(self, parent, db, coin):
self.logger = logging.getLogger('UTXO')
self.logger.setLevel(logging.INFO)
super().__init__()
self.parent = parent
self.coin = coin
self.cache = {}
@ -126,7 +124,7 @@ class UTXOCache(object):
# d5d27987d2a3dfc724e359870c6644b40e497bdc0589a033220fe15429d88599
#if key in self.cache:
# self.logger.info('duplicate tx hash {}'
# .format(bytes(reversed(tx_hash)).hex()))
# .format(hash_to_str(tx_hash)))
self.cache[key] = hash168 + tx_numb + pack('<Q', txout.value)
@ -160,7 +158,7 @@ class UTXOCache(object):
if data is None:
# Uh-oh, this should not happen...
self.logger.error('found no UTXO for {} / {:d} key {}'
.format(bytes(reversed(prevout.hash)).hex(),
.format(hash_to_str(prevout.hash),
prevout.n, bytes(key).hex()))
return hash168
@ -194,7 +192,7 @@ class UTXOCache(object):
# Assuming the DB is not corrupt, this indicates a
# successful spend of a non-standard script
# self.logger.info('ignoring spend of non-standard UTXO {} / {:d}'
# .format(bytes(reversed(tx_hash)).hex(), idx)))
# .format(hash_to_str(tx_hash), idx)))
return None
if len(data) == 25:
@ -277,14 +275,13 @@ class UTXOCache(object):
self.adds = self.cache_hits = self.db_deletes = 0
class DB(object):
class DB(LoggedClass):
class Error(Exception):
pass
def __init__(self, env):
self.logger = logging.getLogger('DB')
self.logger.setLevel(logging.INFO)
super().__init__()
# Meta
self.tx_hash_file_size = 16 * 1024 * 1024
@ -293,6 +290,7 @@ class DB(object):
self.next_cache_check = 0
self.last_flush = time.time()
self.coin = env.coin
self.current_header = None
# Chain state (initialize to genesis in case of new DB)
self.db_height = -1
@ -358,7 +356,7 @@ class DB(object):
def read_state(self, db):
state = db.get(b'state')
state = ast.literal_eval(state.decode('ascii'))
state = ast.literal_eval(state.decode())
if state['genesis'] != self.coin.GENESIS_HASH:
raise self.Error('DB genesis hash {} does not match coin {}'
.format(state['genesis_hash'],
@ -448,7 +446,7 @@ class DB(object):
'utxo_flush_count': self.utxo_flush_count,
'wall_time': self.wall_time,
}
batch.put(b'state', repr(state).encode('ascii'))
batch.put(b'state', repr(state).encode())
def flush_utxos(self, batch):
self.logger.info('flushing UTXOs: {:,d} txs and {:,d} blocks'
@ -675,3 +673,27 @@ class DB(object):
'''Returns all the UTXOs for an address sorted by height and
position in the block.'''
return sorted(self.get_utxos(hash168, limit=None))
def encode_header(self):
if self.height == -1:
return None
header = self.read_headers(self.height, 1)
unpack = struct.unpack
version, = unpack('<I', header[:4])
timestamp, bits, nonce = unpack('<III', header[68:80])
return {
'block_height': self.height,
'version': version,
'prev_block_hash': hash_to_str(header[4:36]),
'merkle_root': hash_to_str(header[36:68]),
'timestamp': timestamp,
'bits': bits,
'nonce': nonce,
}
def get_current_header(self):
# FIXME: clear current_header on new block
if self.current_header is None:
self.current_header = self.encode_header()
return self.current_header

38
server/env.py
View File

@ -1,29 +1,35 @@
# See the file "LICENSE" for information about the copyright
# and warranty status of this software.
import logging
from os import environ
from lib.coins import Coin
from lib.util import LoggedClass
class Env(object):
class Env(LoggedClass):
'''Wraps environment configuration.'''
class Error(Exception):
pass
def __init__(self):
self.logger = logging.getLogger('Env')
self.logger.setLevel(logging.INFO)
super().__init__()
coin_name = self.default('COIN', 'Bitcoin')
network = self.default('NETWORK', 'mainnet')
self.coin = Coin.lookup_coin_class(coin_name, network)
self.db_dir = self.required('DB_DIRECTORY')
self.utxo_MB = self.integer('UTXO_MB', 1000)
self.hist_MB = self.integer('HIST_MB', 250)
self.electrumx_rpc_port = self.integer('ELECTRUMX_RPC_PORT', 8000)
self.rpc_url = self.build_rpc_url()
self.host = self.default('HOST', 'localhost')
self.tcp_port = self.integer('TCP_PORT', None)
self.ssl_port = self.integer('SSL_PORT', None)
self.rpc_port = self.integer('RPC_PORT', 8000)
self.daemon_url = self.build_daemon_url()
self.max_subscriptions = self.integer('MAX_SUBSCRIPTIONS', 10000)
self.banner_file = self.default('BANNER_FILE', None)
# The electrum client takes the empty string as unspecified
self.donation_address = self.default('DONATION_ADDRESS', '')
def default(self, envvar, default):
return environ.get(envvar, default)
@ -44,13 +50,13 @@ class Env(object):
raise self.Error('cannot convert envvar {} value {} to an integer'
.format(envvar, value))
def build_rpc_url(self):
rpc_url = environ.get('RPC_URL')
if not rpc_url:
rpc_username = self.required('RPC_USERNAME')
rpc_password = self.required('RPC_PASSWORD')
rpc_host = self.required('RPC_HOST')
rpc_port = self.default('RPC_PORT', self.coin.DEFAULT_RPC_PORT)
rpc_url = ('http://{}:{}@{}:{}/'
.format(rpc_username, rpc_password, rpc_host, rpc_port))
return rpc_url
def build_daemon_url(self):
daemon_url = environ.get('DAEMON_URL')
if not daemon_url:
username = self.required('DAEMON_USERNAME')
password = self.required('DAEMON_PASSWORD')
host = self.required('DAEMON_HOST')
port = self.default('DAEMON_PORT', self.coin.DEFAULT_RPC_PORT)
daemon_url = ('http://{}:{}@{}:{}/'
.format(username, password, host, port))
return daemon_url

206
server/protocol.py Normal file
View File

@ -0,0 +1,206 @@
# See the file "LICENSE" for information about the copyright
# and warranty status of this software.
import asyncio
import codecs
import json
import traceback
from functools import partial
from lib.hash import hash_to_str
from lib.util import LoggedClass
from server.version import VERSION
class Error(Exception):
BAD_REQUEST = 1
INTERNAL_ERROR = 2
class JSONRPC(asyncio.Protocol, LoggedClass):
def __init__(self, controller):
super().__init__()
self.controller = controller
self.parts = []
def connection_made(self, transport):
self.transport = transport
peername = transport.get_extra_info('peername')
self.logger.info('connection from {}'.format(peername))
self.controller.add_session(self)
def connection_lost(self, exc):
self.logger.info('disconnected')
self.controller.remove_session(self)
def data_received(self, data):
while True:
npos = data.find(ord('\n'))
if npos == -1:
break
tail, data = data[:npos], data[npos + 1:]
parts = self.parts
self.parts = []
parts.append(tail)
self.decode_message(b''.join(parts))
if data:
self.parts.append(data)
def decode_message(self, message):
'''Message is a binary message.'''
try:
message = json.loads(message.decode())
except Exception as e:
self.logger.info('caught exception decoding message'.format(e))
return
job = self.request_handler(message)
self.controller.add_job(job)
async def request_handler(self, request):
'''Called asynchronously.'''
error = result = None
try:
result = await self.json_handler(request)
except Error as e:
error = {'code': e.args[0], 'message': e.args[1]}
except asyncio.CancelledError:
raise
except Exception as e:
# This should be considered a bug and fixed
traceback.print_exc()
error = {'code': Error.INTERNAL_ERROR, 'message': str(e)}
payload = {'id': request.get('id'), 'error': error, 'result': result}
try:
data = json.dumps(payload) + '\n'
except TypeError:
msg = 'cannot JSON encode response to request {}'.format(request)
self.logger.error(msg)
error = {'code': Error.INTERNAL_ERROR, 'message': msg}
payload = {'id': request.get('id'), 'error': error, 'result': None}
data = json.dumps(payload) + '\n'
self.transport.write(data.encode())
async def json_handler(self, request):
method = request.get('method')
handler = None
if isinstance(method, str):
handler_name = 'handle_{}'.format(method.replace('.', '_'))
handler = getattr(self, handler_name, None)
if not handler:
self.logger.info('unknown method: {}'.format(method))
raise Error(Error.BAD_REQUEST, 'unknown method: {}'.format(method))
params = request.get('params', [])
if not isinstance(params, list):
raise Error(Error.BAD_REQUEST, 'params should be an array')
return await handler(params)
class ElectrumX(JSONRPC):
def __init__(self, controller, env):
super().__init__(controller)
self.BC = controller.block_cache
self.db = controller.db
self.env = env
self.addresses = set()
self.subscribe_headers = False
def params_to_hash168(self, params):
if len(params) != 1:
raise Error(Error.BAD_REQUEST,
'params should contain a single address')
address = params[0]
try:
return self.env.coin.address_to_hash168(address)
except:
raise Error(Error.BAD_REQUEST,
'invalid address: {}'.format(address))
async def handle_blockchain_address_get_history(self, params):
hash168 = self.params_to_hash168(params)
history = [
{'tx_hash': hash_to_str(tx_hash), 'height': height}
for tx_hash, height in self.db.get_history(hash168, limit=None)
]
return history
async def handle_blockchain_address_subscribe(self, params):
hash168 = self.params_to_hash168(params)
status = self.controller.address_status(hash168)
return status.hex() if status else None
async def handle_blockchain_estimatefee(self, params):
result = await self.BC.send_single('estimatefee', params)
return result
async def handle_blockchain_headers_subscribe(self, params):
self.subscribe_headers = True
return self.db.get_current_header()
async def handle_blockchain_relayfee(self, params):
'''The minimum fee a low-priority tx must pay in order to be accepted
to this daemon's memory pool.
'''
net_info = await self.BC.send_single('getnetworkinfo')
return net_info['relayfee']
async def handle_server_banner(self, params):
'''Return the server banner.'''
banner = 'Welcome to Electrum!'
if self.env.banner_file:
try:
with codecs.open(self.env.banner_file, 'r', 'utf-8') as f:
banner = f.read()
except Exception as e:
self.logger.error('reading banner file {}: {}'
.format(self.env.banner_file, e))
return banner
async def handle_server_donation_address(self, params):
'''Return the donation address as a string.
If none is specified return the empty string.
'''
return self.env.donation_address
async def handle_server_peers_subscribe(self, params):
'''Returns the peer (ip, host, ports) tuples.
Despite the name electrum-server does not treat this as a
subscription.
'''
peers = self.controller.get_peers()
return tuple(peers.values())
async def handle_server_version(self, params):
'''Return the server version as a string.'''
return VERSION
class LocalRPC(JSONRPC):
async def handle_getinfo(self, params):
return {
'blocks': self.controller.db.height,
'peers': len(self.controller.get_peers()),
'sessions': len(self.controller.sessions),
'watched': sum(len(s.addresses) for s in self.controller.sessions
if isinstance(s, ElectrumX)),
'cached': 0,
}
async def handle_sessions(self, params):
return []
async def handle_numsessions(self, params):
return len(self.controller.sessions)
async def handle_peers(self, params):
return tuple(self.controller.get_peers().keys())
async def handle_numpeers(self, params):
return len(self.controller.get_peers())

60
server/rpc.py
View File

@ -1,60 +0,0 @@
# See the file "LICENSE" for information about the copyright
# and warranty status of this software.
import logging
import traceback
from aiohttp import web
class ElectrumRPCServer(object):
'''ElectrumX's RPC server for localhost.'''
def __init__(self, server):
self.logger = logging.getLogger('RPCServer')
self.logger.setLevel(logging.INFO)
self.server = server
async def request_handler(self, request):
json_request = await request.json()
try:
err, result = await self.json_handler(json_request)
except Exception as e:
traceback.print_exc()
err, result = 1, 'caught exception: {}'.format(e)
id_ = request.get('id')
if err is None:
response = {
'id': id_,
'error': None,
'result': result,
}
else:
response = {
'id': id_,
'error': {'code': err, 'message': result},
'result': None,
}
return web.json_response(response)
async def json_handler(self, request):
method = request.get('method')
id_ = request.get('id')
params = request.get('params', [])
handler = getattr(self.server, 'handle_rpc_{}'.format(method), None)
if not handler:
return 1, 'unknown method "{}"'.format(method)
else:
return await handler(params)
def tasks(self, port):
self.logger.info('listening on port {:d}'.format(port))
app = web.Application()
app.router.add_post('/', self.request_handler)
host = '0.0.0.0'
loop = app.loop
handler = app.make_handler()
server = loop.create_server(handler, host, port)
return [server, app.startup()]

1
server/version.py Normal file
View File

@ -0,0 +1 @@
VERSION = "ElectrumX 0.01"

10
server_main.py
View File

@ -9,7 +9,7 @@ import os
import traceback
from server.env import Env
from server.server import Server
from server.controller import Controller
def main_loop():
@ -22,15 +22,17 @@ def main_loop():
logging.info('switching current directory to {}'.format(env.db_dir))
os.chdir(env.db_dir)
server = Server(env)
tasks = server.async_tasks()
loop = asyncio.get_event_loop()
#loop.set_debug(True)
controller = Controller(env)
tasks = controller.start(loop)
try:
loop.run_until_complete(asyncio.gather(*tasks))
except asyncio.CancelledError:
logging.warning('task cancelled; asyncio event loop closing')
finally:
controller.stop()
loop.close()