diff --git a/.gitignore b/.gitignore
index 72519f5..c0894bf 100644
--- a/.gitignore
+++ b/.gitignore
@@ -9,4 +9,6 @@ docs/_build
/build
/dist
/electrumx.egg-info
-.idea/
\ No newline at end of file
+.vscode/
+.mypy_cache/
+.idea/
diff --git a/.travis.yml b/.travis.yml
index 2b22bc7..839d369 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -2,16 +2,11 @@ sudo: required
dist: trusty
language: python
before_install:
- - sudo add-apt-repository -y ppa:giskou/librocksdb
+ - sudo add-apt-repository -y ppa:streetcrypto7/rocksdb
+ - sudo add-apt-repository -y ppa:streetcrypto7/leveldb
- sudo apt-get -qq update
- - sudo apt-get install -yq libsnappy-dev zlib1g-dev libbz2-dev libgflags-dev
- - sudo apt-get install -yq --allow-unauthenticated librocksdb
- - wget https://launchpad.net/ubuntu/+archive/primary/+files/leveldb_1.20.orig.tar.gz
- - tar -xzvf leveldb_1.20.orig.tar.gz
- - pushd leveldb-1.20 && make && sudo mv out-shared/libleveldb.* /usr/local/lib && sudo cp -R include/leveldb /usr/local/include && sudo ldconfig && popd
+ - sudo apt-get install -yq libsnappy-dev zlib1g-dev libbz2-dev libgflags-dev liblz4-dev librocksdb-dev libleveldb-dev
python:
- - "3.6"
- - "3.6-dev"
- "3.7-dev"
- "nightly"
# command to install dependencies
@@ -22,7 +17,7 @@ install:
- pip install plyvel
- pip install pycodestyle
- pip install pylru
- - pip install pyrocksdb
+ - pip install python-rocksdb
- pip install pytest-asyncio
- pip install pytest-cov
- pip install Sphinx
@@ -35,17 +30,22 @@ install:
- pip install xevan_hash
- pip install quark_hash
- pip install groestlcoin_hash
- - pip install git+https://github.com/goacoincore/neoscrypt
- - pip install git+https://github.com/motioncrypto/x16r_hash
+ - pip install neoscrypt
+ - pip install x16r_hash
- pip install pycryptodomex
+ - pip install git+https://github.com/Electra-project/nist5_hash
+ - pip install git+https://github.com/RitoProject/x21s_hash
+ - pip install git+https://github.com/traysi/x16rv2_hash
+ - pip install bell-yespower
+ - pip install cpupower
# command to run tests
script:
- pytest --cov=electrumx
- - pycodestyle electrumx/server/*.py electrumx/lib/*.py
+ - pycodestyle --max-line-length=100 electrumx/server/*.py electrumx/lib/*.py *.py
- sh -c "cd docs && make html"
# Dont report coverage from nightly
after_success:
- - if [[ $(python3 -V 2>&1) == *"Python 3.6"* ]]; then
+ - if [[ $(python3 -V 2>&1) == *"Python 3.7"* ]]; then
pip install python-coveralls;
coveralls;
fi
diff --git a/README.rst b/README.rst
index 12cce47..1d86b50 100644
--- a/README.rst
+++ b/README.rst
@@ -1 +1,22 @@
-Ranchi Mall offers ElectrumX server service for FLO publicly at ranchimall.duckdns.org and ranchimall1.duckdns.org at ports 50001 and 50002
+.. image:: https://travis-ci.org/kyuupichan/electrumx.svg?branch=master
+ :target: https://travis-ci.org/kyuupichan/electrumx
+.. image:: https://coveralls.io/repos/github/kyuupichan/electrumx/badge.svg
+ :target: https://coveralls.io/github/kyuupichan/electrumx
+
+===============================================
+ElectrumX - Reimplementation of electrum-server
+===============================================
+
+For a future network with bigger blocks.
+
+ :Licence: MIT
+ :Language: Python (>= 3.7)
+ :Author: Neil Booth
+
+Documentation
+=============
+
+See `readthedocs `_.
+
+
+**Neil Booth** kyuupichan@gmail.com https://github.com/kyuupichan
diff --git a/contrib/Dockerfile b/contrib/Dockerfile
new file mode 100644
index 0000000..7742dbb
--- /dev/null
+++ b/contrib/Dockerfile
@@ -0,0 +1,35 @@
+# example of Dockerfile that builds release of electrumx-1.13.0
+# ENV variables can be overrided on the `docker run` command
+
+FROM ubuntu:18.04
+
+WORKDIR /
+ADD https://github.com/kyuupichan/electrumx/archive/1.13.0.tar.gz /
+RUN tar zxvf *.tar.gz
+
+RUN apt-get update && \
+ apt-get -y install python3.7 python3-pip librocksdb-dev libsnappy-dev libbz2-dev libz-dev liblz4-dev && \
+ pip3 install aiohttp pylru python-rocksdb
+
+RUN cd /electrumx* && python3 setup.py install
+
+ENV SERVICES="tcp://:50001"
+ENV COIN=BitcoinSV
+ENV DB_DIRECTORY=/db
+ENV DAEMON_URL="http://username:password@hostname:port/"
+ENV ALLOW_ROOT=true
+ENV DB_ENGINE=rocksdb
+ENV MAX_SEND=10000000
+ENV BANDWIDTH_UNIT_COST=50000
+ENV CACHE_MB=2000
+
+VOLUME /db
+
+RUN mkdir -p "$DB_DIRECTORY" && ulimit -n 1048576
+
+CMD ["/usr/bin/python3", "/usr/local/bin/electrumx_server"]
+
+# build it with eg.: `docker build -t electrumx .`
+# run it with eg.:
+# `docker run -d --net=host -v /home/electrumx/db/:/db -e DAEMON_URL="http://youruser:yourpass@localhost:8332" -e REPORT_SERVICES=tcp://example.com:50001 electrumx`
+# for a proper clean shutdown, send TERM signal to the running container eg.: `docker kill --signal="TERM" CONTAINER_ID`
diff --git a/contrib/systemd/electrumx.conf b/contrib/systemd/electrumx.conf
index 51f01c9..9787037 100644
--- a/contrib/systemd/electrumx.conf
+++ b/contrib/systemd/electrumx.conf
@@ -5,5 +5,7 @@ DB_DIRECTORY = /db
# Bitcoin Node RPC Credentials
DAEMON_URL = http://username:password@hostname:port/
+# COIN = BitcoinSegwit
+
# See http://electrumx.readthedocs.io/en/latest/environment.html for
# information about other configuration settings you probably want to consider.
diff --git a/docs/HOWTO.rst b/docs/HOWTO.rst
index 731102e..753c20a 100644
--- a/docs/HOWTO.rst
+++ b/docs/HOWTO.rst
@@ -15,7 +15,7 @@ small - pull requests are welcome.
================ ========================
Package Notes
================ ========================
-Python3 ElectrumX uses asyncio. Python version >= 3.6 is
+Python3 ElectrumX uses asyncio. Python version >= 3.7 is
**required**.
`aiohttp`_ Python library for asynchronous HTTP. Version >=
2.0 required.
@@ -51,11 +51,11 @@ used to either.
When building the database from the genesis block, ElectrumX has to
flush large quantities of data to disk and its DB. You will have a
better experience if the database directory is on an SSD than on an
-HDD. Currently to around height 447,100 of the Bitcoin blockchain the
+HDD. Currently to around height 611,600 of the Bitcoin blockchain the
final size of the leveldb database, and other ElectrumX file metadata
-comes to just over 18.7GB (17.5 GiB). LevelDB needs a bit more for
+comes to just over 46.9GB (43.7 GiB). LevelDB needs a bit more for
brief periods, and the block chain is only getting longer, so I would
-recommend having at least 30-40GB of free space before starting.
+recommend having at least 70-80GB of free space before starting.
Database Engine
===============
@@ -208,14 +208,6 @@ Once configured you may want to start ElectrumX at boot::
:file:`.service` file.
-Installing Python 3.6 under Ubuntu
-----------------------------------
-
-Many Ubuntu distributions have an incompatible Python version baked
-in. Because of this, it is easier to install Python 3.6. See
-`contrib/python3.6/python-3.6.sh`_.
-
-
Installing on Raspberry Pi 3
----------------------------
@@ -371,6 +363,8 @@ The ETA shown is just a rough guide and in the short term can be quite
volatile. It tends to be a little optimistic at first; once you get
to height 280,000 is should be fairly accurate.
+.. _SSL certificates:
+
Creating a self-signed SSL certificate
======================================
@@ -433,6 +427,5 @@ You can then set the port as follows and advertise the service externally on the
.. _`aiohttp`: https://pypi.python.org/pypi/aiohttp
.. _`pylru`: https://pypi.python.org/pypi/pylru
.. _`x11_hash`: https://pypi.python.org/pypi/x11_hash
-.. _`contrib/python3.6/python-3.6.sh`: https://github.com/kyuupichan/electrumx/blob/master/contrib/python3.6/python-3.6.sh
.. _`contrib/raspberrypi3/install_electrumx.sh`: https://github.com/kyuupichan/electrumx/blob/master/contrib/raspberrypi3/install_electrumx.sh
.. _`contrib/raspberrypi3/run_electrumx.sh`: https://github.com/kyuupichan/electrumx/blob/master/contrib/raspberrypi3/run_electrumx.sh
diff --git a/docs/changelog.rst b/docs/changelog.rst
index 23cc13e..47c408a 100644
--- a/docs/changelog.rst
+++ b/docs/changelog.rst
@@ -7,13 +7,122 @@
and memory consumption whilst serving clients. Those problems
should not occur with Python 3.7.
-.. note:: Bitcoin ABC developers have hastily introduced controversial
- changes that break ElectrumX's block processing by requiring it to
- be non-sequential. Unlike others with unique requirements they
- refused to make their code coin-specific. ElectrumX continues to
- require blocks be naturally ordered, and is compatible with any
- non-CToR daemon, such as Bitcoin SV, and Bitcoin Unlimited /
- Bitcoin XT with CToR disabled.
+Version 1.13.0 (26 Sep 2019)
+============================
+
+* daemon: use a single connection for all requests rather than a connection per request.
+ Distinguish handling of JSON and HTTP errors
+* recognise OP_FALSE OP_RETURN scripts as unspendable
+* peers - attempt to bind to correct local IP address
+* improve name support (domob1812)
+* coin additions / updates: BitZeny (y-chan), ZCoin (a-bezrukov), Emercoin (yakimka),
+ BSV (Roger Taylor), Bellcoin (streetcrypto7), Ritocoin (traysi), BTC (Sombernight),
+ PIVX (mrcarlanthony), Monacoin (wakiyamap)), NamecoinRegtest (JeremyRand), Axe (ddude1),
+ Xaya (domob1812), GZRO (MrNaif2018), Ravencoin (standard-error)
+* other: gits7r
+
+Version 1.12.0 (13 May 2019)
+============================
+
+* require aiorpcX 0.18.1. This introduces websocket support. The environment variables
+ changed accordingly; see :envvar:`SERVICES` and :envvar:`REPORT_SERVICES`.
+* work around bug in recent versions of uvloop
+* aiorpcX upgrade fixes from Shane M
+* coin additions / updates: BitcoinSV, Bolivarcoin (Jose Luis Estevez), BTC Testnet (ghost43),
+ Odin (Pixxl)
+
+Version 1.11.0 (18 Apr 2019)
+============================
+
+* require aiorpcX 0.15.x
+* require aiohttp 3.3 or higher; earlier versions had a problematic bug
+* add :envvar:`REQUEST_TIMEOUT` and :envvar:`LOG_LEVEL` environment variables
+* mark 4 old environment variables obsolete. ElectrumX won't start until they are removed
+* getinfo local RPC cleaned up and shows more stats
+* miscellaneous fixes and improvements
+* more efficient handling of some RPC methods, particularly
+ :func:`blockchain.transaction.get_merkle`
+* coin additions / updates: BitcoinSV scaling testnet (Roger Taylor), Dash (zebra lucky),
+* issues resolved: `#566`_, `#731`_, `#795`_
+
+Version 1.10.1 (13 Apr 2019)
+============================
+
+* introduce per-request costing. See environment variables documentation for new
+ variables :envvar:`COST_SOFT_LIMIT`, :envvar:`COST_HARD_LIMIT`, :envvar:`REQUEST_SLEEP`,
+ :envvar:`INITIAL_CONCURRENT`, :envvar:`BANDWIDTH_UNIT_COST`. Sessions are placed in groups
+ with which they share some of their costs. Prior cost is remembered across reconnects.
+* require aiorpcX 0.13.5 for better concurrency handling
+* require clients use protocol 1.4 or higher
+* handle transaction.get_merkle requests more efficiently (ghost43)
+* Windows support (sancoder)
+* peers improvements (ghost43)
+* report mempool and block sizes in logs
+* electrumx_rpc: timeout raised to 30s, fix session request counts
+* other tweaks and improvements by Bjorge Dijkstra, ghost43, peleion,
+* coin additions / updates: ECA (Jenova7), ECCoin (smogm), GXX (DEVCØN), BZX (2INFINITY),
+ DeepOnion (Liam Alford), CivX / EXOS (turcol)
+
+Version 1.10.0 (15 Mar 2019)
+============================
+
+* extra countermeasures to limit BTC phishing effectiveness (ghost43)
+* peers: mark blacklisted peers bad; force retry blacklisted peers (ghost43)
+* coin additions / updates: Monacoin (wakiyamap), Sparks (Mircea Rila), ColossusXT,
+ Polis, MNPCoin, Zcoin, GINCoin (cronos), Grosetlcoin (gruve-p), Dash (konez2k),
+ Bitsend (David), Ravencoin (standard-error), Onixcoin (Jose Estevez), SnowGem
+* coin removals: Gobyte, Moneci (cronos)
+* minor tweaks by d42
+* issues fixed `#660`_ - unclean shutdowns during initial sync
+
+Version 1.9.5 (08 Feb 2019)
+===========================
+
+* server blacklist logic (ecdsa)
+* require aiorpcX 0.10.4
+* remove dead wallet code
+* fix `#727`_ - not listing same peer twice
+
+Version 1.9.4 (07 Feb 2019)
+===========================
+
+* require aiorpcX 0.10.3
+* fix `#713`_
+
+Version 1.9.3 (05 Feb 2019)
+===========================
+
+* ignore potential sybil peers
+* coin additions / updates: BitcoinCashABC (cculianu), Monacoin (wakiyamap)
+
+Version 1.9.2 (03 Feb 2019)
+===========================
+
+* restore protocol version 1.2 and send a warning for old BTC Electrum clients that they
+ need to upgrade. This is an attempt to protect users of old versions of Electrum from
+ the ongoing phishing attacks
+* increase default MAX_SEND for AuxPow Chains. Truncate AuxPow for block heights covered
+ by a checkpoint. (jeremyrand)
+* coin additions / updates: NMC (jeremyrand), Dash (zebra-lucky), PeerCoin (peerchemist),
+ BCH testnet (Mark Lundeberg), Unitus (ChekaZ)
+* tighter RPC param checking (ghost43)
+
+Version 1.9.1 (11 Jan 2019)
+===========================
+
+* fix `#684`_
+
+Version 1.9.0 (10 Jan 2019)
+===========================
+
+* minimum protocol version is now 1.4
+* coin additions / updates: BitcoinSV, SmartCash (rc125), NIX (phamels), Minexcoin (joesixpack),
+ BitcoinABC (mblunderburg), Dash (zebra-lucky), BitcoinABCRegtest (ezegom), AXE (slowdive),
+ NOR (flo071), BitcoinPlus (bushsolo), Myriadcoin (cryptapus), Trezarcoin (ChekaZ),
+ Bitcoin Diamond (John Shine),
+* close `#554`_, `#653`_, `#655`_
+* other minor tweaks (Michael Schmoock, Michael Taborsky)
+
Version 1.8.12 (10 Nov 2018)
============================
@@ -60,20 +169,6 @@ Version 1.8.6 (12 Sep 2018)
* new coin TokenPay (samfiragabriel)
* minor fix: wakiyamap
-Version 1.8.7 (13 Sep 2018)
-===========================
-
-* require aiorpcX 0.8.1
-* fix reorg bug loading blocks from disk (erasmospunk)
-
-Version 1.8.6 (12 Sep 2018)
-===========================
-
-* require aiorpcX 0.8.0
-* suppress socket.send() errors
-* new coin TokenPay (samfiragabriel)
-* minor fix: wakiyamap
-
Version 1.8.5 (18 Aug 2018)
===========================
@@ -128,125 +223,17 @@ Version 1.8 (06 Aug 2018)
Decred (erasmonpsunk)
* other minor (smmalis37)
-Version 1.7.3 (01 Aug 2018)
-============================
-
-* fix `#538`_
-
-Version 1.7.2 (29 Jul 2018)
-============================
-
-* require aiorpcX 0.5.9; 0.5.8 didn't work on Python 3.7
-
-Version 1.7.1 (28 Jul 2018)
-============================
-
-* switch to aiorpcX 0.5.8 which implements some curio task management
- primitives on top of asyncio that make writing correct async code
- much easier, as well as making it simpler to reason about
-* use those primitives to restructure the peer manager, which is now
- fully concurrent again, as well as the block processor and
- controller
-* fix `#534`_ introduced in 1.7
-* minor coin tweaks (ghost43, cipig)
-
-Version 1.7 (25 Jul 2018)
-==========================
-
-* completely overhauled mempool and address notifications
- implementation. Cleaner and a lot more efficient, especially for
- initial synchronization of the mempool. Mempool handling is fully
- asynchronous and doesn't hinder client responses or block
- processing.
-* peer discovery cleaned up, more work remains
-* cleaner shutdown process with clear guarantees
-* aiohttp min version requirement raised to 2.0
-* onion peers are ignored if no tor proxy is available
-* add Motion coin (ocruzv), MinexCoin (joesixpack)
-
-Version 1.6 (19 July 2018)
-===========================
-
-* implement :ref:`version 1.4` of the protocol, with benefit for light
- clients, particularly mobile
-* implement header proofs and merkle caches
-* implement :func:`blockchain.transaction.id_from_pos` (ghost43)
-* large refactoring of session and controller classes
-* recent blocks are now stored on disk. When backing up in a reorg
- ElectrumX uses these rather than asking the daemon for the blocks --
- some daemons cannot correctly handle orphaned block requests after
- a reorg. Fixes `#258`_, `#315`_, `#479`_
-* minor fixes: nijel
-
-Version 1.5.2
-=============
-
-* package renamed from elctrumX-kyuupichan to electrumX
-* split merkle logic out into lib/merkle.py
-* fix `#523`_ for daemons based on older releases of core
-
-Version 1.5.1
-=============
-
-Fixes a couple of issues found in 1.5 after release:
-
-* update peer discovery code for :ref:`version 1.3` of the protocol
-* setup.py would not run in a clean environment (e.g. virtualenv)
-* logging via aiorpcX didn't work with the logging hierarchy updates
-* log Python interpreter version on startup
-
-Version 1.5
-===========
-
-.. note:: The two main scripts, :file:`electrumx_server` and
- :file:`electrumx_rpc` were renamed to drop the `.py` suffix. You
- will probably need to update your run script accordingly.
-
-* support :ref:`version 1.3` of the protocol
-* increase minimum supported protocol version to :ref:`version 1.1`
-* split out history handling in preparation for new DB format
-* force close stubborn connections that refuse to close gracefully
-* RPC getinfo returns server version (erasmospunk)
-* add new masternode methods; document them all (elmora-do)
-* make electrumx a Python package (eukreign)
-* hierarchical logging, Env to take a coin class directly,
- server_listening event (eukreign)
-* decred coin removed as mainnet does not sync
-* issues fixed: `#414`_, `#443`_, `#455`_, `#480`_, `#485`_, `#502`_,
- `#506`_, `#519`_ (wakiyamap)
-* new or updated coins: Feathercoin (lclc), NewYorkCoin Testnet(nicovs),
- BitZeny (wakiyamap), UFO (bushstar), GAME (cipig), MAC (nico205),
- Xuez (ddude), ZCash (wo01), PAC (elmora-do), Koto Testnet (wo01),
- Dash Testnet (ser), BTG all nets (wilsonmeier), Polis + ColossusXT +
- GoByte + Monoeci (cronos-polis), BitcoinCash Regtest (eukreign)
-* minor tweaks: romanz, you21979, SuBPaR42, sangaman, wakiyamap, DaShak
-
-
**Neil Booth** kyuupichan@gmail.com https://github.com/kyuupichan
-bitcoincash:qzxpdlt8ehu9ehftw6rqsy2jgfq4nsltxvhrdmdfpn
-
-.. _#258: https://github.com/kyuupichan/electrumx/issues/258
-.. _#315: https://github.com/kyuupichan/electrumx/issues/315
-.. _#414: https://github.com/kyuupichan/electrumx/issues/414
-.. _#443: https://github.com/kyuupichan/electrumx/issues/443
-.. _#455: https://github.com/kyuupichan/electrumx/issues/455
-.. _#479: https://github.com/kyuupichan/electrumx/issues/479
-.. _#480: https://github.com/kyuupichan/electrumx/issues/480
-.. _#485: https://github.com/kyuupichan/electrumx/issues/485
-.. _#502: https://github.com/kyuupichan/electrumx/issues/50
-.. _#506: https://github.com/kyuupichan/electrumx/issues/506
-.. _#519: https://github.com/kyuupichan/electrumx/issues/519
.. _#521: https://github.com/kyuupichan/electrumx/issues/521
-.. _#523: https://github.com/kyuupichan/electrumx/issues/523
-.. _#534: https://github.com/kyuupichan/electrumx/issues/534
-.. _#538: https://github.com/kyuupichan/electrumx/issues/538
.. _#552: https://github.com/kyuupichan/electrumx/issues/552
+.. _#554: https://github.com/kyuupichan/electrumx/issues/554
.. _#557: https://github.com/kyuupichan/electrumx/issues/557
.. _#559: https://github.com/kyuupichan/electrumx/issues/559
.. _#564: https://github.com/kyuupichan/electrumx/issues/564
.. _#565: https://github.com/kyuupichan/electrumx/issues/565
+.. _#566: https://github.com/kyuupichan/electrumx/issues/566
.. _#567: https://github.com/kyuupichan/electrumx/issues/567
.. _#570: https://github.com/kyuupichan/electrumx/issues/570
.. _#577: https://github.com/kyuupichan/electrumx/issues/577
@@ -254,3 +241,11 @@ bitcoincash:qzxpdlt8ehu9ehftw6rqsy2jgfq4nsltxvhrdmdfpn
.. _#608: https://github.com/kyuupichan/electrumx/issues/608
.. _#630: https://github.com/kyuupichan/electrumx/issues/630
.. _#632: https://github.com/kyuupichan/electrumx/issues/630
+.. _#653: https://github.com/kyuupichan/electrumx/issues/653
+.. _#655: https://github.com/kyuupichan/electrumx/issues/655
+.. _#660: https://github.com/kyuupichan/electrumx/issues/660
+.. _#684: https://github.com/kyuupichan/electrumx/issues/684
+.. _#713: https://github.com/kyuupichan/electrumx/issues/713
+.. _#727: https://github.com/kyuupichan/electrumx/issues/727
+.. _#731: https://github.com/kyuupichan/electrumx/issues/731
+.. _#795: https://github.com/kyuupichan/electrumx/issues/795
diff --git a/docs/conf.py b/docs/conf.py
index fc82fd3..eba3f37 100644
--- a/docs/conf.py
+++ b/docs/conf.py
@@ -15,7 +15,7 @@
import os
import sys
sys.path.insert(0, os.path.abspath('..'))
-VERSION="ElectrumX 1.8.12"
+VERSION="ElectrumX 1.13.0"
# -- Project information -----------------------------------------------------
diff --git a/docs/environment.rst b/docs/environment.rst
index d95e578..f512a2c 100644
--- a/docs/environment.rst
+++ b/docs/environment.rst
@@ -4,15 +4,14 @@
Environment Variables
=====================
-ElectrumX takes no command line arguments, instead its behaviour is
-controlled by environment variables. Only a few are required to be
-given, the rest will have sensible defaults if not specified. Many of
-the defaults around resource usage are conservative; I encourage you
-to review them.
+ElectrumX takes no command line arguments, instead its behaviour is controlled by
+environment variables. Only a few are required to be given, the rest will have sensible
+defaults if not specified. Many of the defaults around resource usage are conservative; I
+encourage you to review them.
-Note: by default the server will only serve to connections from the
-same machine. To be accessible to other users across the internet you
-must set **HOST** appropriately; see below.
+.. note:: set :envvar:`SERVICES` appropriately to be able to connect to your server. For
+ clients across the internet to know what services you offer you must advertize your
+ services with :envvar:`REPORT_SERVICES`.
Required
@@ -46,6 +45,11 @@ These environment variables are always required:
port for :envvar:`COIN` and :envvar:`NET` if omitted.
+.. note:: With the above set your server will run and index the chain. To enable incoming
+ connections you must set :envvar:`SERVICES`, and for others to be aware of your server
+ set :envvar:`REPORT_SERVICES`.
+
+
For the ``run`` script
======================
@@ -61,6 +65,119 @@ The following are required if you use the ``run`` script:
The username the server will run as.
+Services
+========
+
+These two environment variables are comma-separated lists of individual *services*.
+
+A **service** has the general form::
+
+ protocol://host:port
+
+*protocol* is case-insensitive. The recognised protocols are::
+
+ tcp Plaintext TCP sockets
+ ssl SSL-encrypted TCP sockets
+ ws Plaintext websockets
+ wss SSL-encrypted websockets
+ rpc Plaintext RPC
+
+In a services list, a protocol can be specified multiple times, with different hosts or
+ports. This might be useful for multi-homed hosts, or if you offer both Tor and clearnet
+services.
+
+*host* can be a hostname, an IPv4 address, or an IPv6 address enclosed in square brackets.
+
+*port* is an integer from :const:`1` to :const:`65535` inclusive.
+
+Where documented, one or more of *protocol*, *host* and *port* can be omitted, in which
+case a default value will be assumed.
+
+Here are some examples of valid services::
+
+ tcp://host.domain.tld:50001 # Hostname, lowercase protocol, port
+ SSL://23.45.67.78:50002 # An IPv4 address, upper-case protocol, port
+ rpC://localhost # Host as a string, mixed-case protocol, default port
+ ws://[1234:5678:abcd::5601]:8000 # Host as an IPv6 address
+ wss://h3ubaasdlkheryasd.onion:50001 # Host as a Tor ".onion" address
+ rpc://:8000 # Default host, port given
+ host.domain.tld:5151 # Default protocol, hostname, port
+ rpc:// # RPC protocol, default host and port
+
+.. note:: ElectrumX will not serve any incoming connections until it has fully caught up
+ with your bitcoin daemon. The only exception is local **RPC** connections,
+ which are served at any time after the server has initialized.
+
+.. envvar:: SERVICES
+
+ A comma-separated list of services ElectrumX will accept incoming connections for.
+
+ This environment variable determines what interfaces and ports the server listens on, so
+ must be set correctly for any connection to the server to succeed. If unset or empty,
+ ElectrumX will not listen for any incoming connections.
+
+ *protocol* can be any recognised protocol.
+
+ *host* defaults to all of the machine's interfaces, except if the protocol is **rpc**,
+ when it defaults to :const:`localhost`.
+
+ *port* can only be defaulted for **rpc** where the default is :const:`8000`.
+
+ On most Unix systems ports below 1024 require elevated priveleges so choosing a higher
+ port is advisable. On Debian for example, this can be achieved by installinng
+ libcap2-bin package::
+
+ sudo apt-get update && sudo apt-get -y install libcap2-bin
+ sudo setcap cap_net_bind_service=+ep /path/to/electrumx_server
+
+ If any listed service has protocol **ssl** or **wss** then :envvar:`SSL_CERTFILE` and
+ :envvar:`SSL_KEYFILE` must be defined.
+
+ Tor **onion** addresses are invalid in :envvar:`SERVICES`.
+
+ Here is an example value of the :envvar:`SERVICES` environment variable::
+
+ tcp://:50001,ssl://:50002,wss://:50004,rpc://
+
+ This serves **tcp**, **ssl**, **wss** on all interfaces on ports 50001, 50002 and 50004
+ respectively. **rpc** is served on its default host :const:`localhost` and default port
+ :const:`8000`.
+
+.. envvar:: REPORT_SERVICES
+
+ A comma-separated list of services ElectrumX will advertize and other servers in the
+ server network (if peer discovery is enabled), and any successful connection.
+
+ This environment variable must be set correctly, taking account of your network,
+ firewall and router setup, for clients and other servers to see how to connect to your
+ server. If not set or empty, no services are advertized.
+
+ The **rpc** protocol, special IP addresses (inlcuding private ones if peer discovery is
+ enabled), and :const:`localhost` are invalid in :envvar:`REPORT_SERVICES`.
+
+ Here is an example value of the :envvar:`REPORT_SERVICES` environment variable::
+
+ tcp://sv.usebsv.com:50001,ssl://sv.usebsv.com:50002,wss://sv.usebsv.com:50004
+
+ This advertizes **tcp**, **ssl**, **wss** services at :const:`sv.usebsv.com` on ports
+ 50001, 50002 and 50004 respectively.
+
+.. note:: Certificate Authority-signed certificates don't work over Tor, so you should
+ only have Tor services` in :envvar:`REPORT_SERVICES` if yours is self-signed.
+
+.. envvar:: SSL_CERTFILE
+
+ The filesystem path to your SSL certificate file.
+
+ :ref:`SSL certificates`
+
+.. envvar:: SSL_KEYFILE
+
+ The filesystem path to your SSL key file.
+
+ :ref:`SSL certificates`
+
+
Miscellaneous
=============
@@ -72,6 +189,11 @@ These environment variables are optional:
`_
to use. Defaults to ``%(levelname)s:%(name)s:%(message)s``.
+.. envvar:: LOG_LEVEL
+
+ The default Python logging level, a case-insensitive string. Useful values
+ are 'debug', 'info', 'warning' and 'error'.
+
.. envvar:: ALLOW_ROOT
Set this environment variable to anything non-empty to allow running
@@ -89,54 +211,6 @@ These environment variables are optional:
to install the appropriate python package for your engine. The
value is not case sensitive.
-.. envvar:: HOST
-
- The host or IP address that the TCP and SSL servers will use when
- binding listening sockets. Defaults to ``localhost``. To listen on
- multiple specific addresses specify a comma-separated list. Set to
- an empty string to listen on all available interfaces (likely both
- IPv4 and IPv6).
-
-.. envvar:: TCP_PORT
-
- If set ElectrumX will serve TCP clients on
- :envvar:`HOST`\::envvar:`TCP_PORT`.
-
- .. note:: ElectrumX will not serve TCP connections until it has
- fully caught up with your daemon.
-
-.. envvar:: SSL_PORT
-
- If set ElectrumX will serve SSL clients on
- :envvar:`HOST`\::envvar:`SSL_PORT`. If set then
- :envvar:`SSL_CERTFILE` and :envvar:`SSL_KEYFILE` must be defined
- environment variables with values the filesystem paths to those SSL
- files.
-
- .. note:: ElectrumX will not serve SSL connections until it has
- fully caught up with your daemon.
-
-.. envvar:: RPC_HOST
-
- The host or IP address that the RPC server will listen on and
- defaults to ``localhost``. To listen on multiple specific addresses
- specify a comma-separated list. Servers with unusual networking
- setups might want to specify e.g. ``::1`` or ``127.0.0.1``
- explicitly rather than defaulting to ``localhost``.
-
- An empty string (normally indicating all interfaces) is interpreted
- as ``localhost``, because allowing access to the server's RPC
- interface to arbitrary connections across the internet is not a good
- idea.
-
-.. envvar:: RPC_PORT
-
- ElectrumX will listen on this port for local RPC connections.
- ElectrumX listens for RPC connections unless this is explicitly set
- to blank. The default depends on :envvar:`COIN` and :envvar:`NET`
- (e.g., 8000 for Bitcoin mainnet) if not set, as indicated in
- `lib/coins.py`_.
-
.. envvar:: DONATION_ADDRESS
The server donation address reported to Electrum clients. Defaults
@@ -228,9 +302,10 @@ raise them.
.. envvar:: MAX_SEND
The maximum size of a response message to send over the wire, in
- bytes. Defaults to 1,000,000. Values smaller than 350,000 are
- taken as 350,000 because standard Electrum protocol header "chunk"
- requests are almost that large.
+ bytes. Defaults to 1,000,000 (except for AuxPoW coins, which default
+ to 10,000,000). Values smaller than 350,000 are taken as 350,000
+ because standard Electrum protocol header "chunk" requests are almost
+ that large.
The Electrum protocol has a flaw in that address histories must be
served all at once or not at all, an obvious avenue for abuse.
@@ -245,40 +320,69 @@ raise them.
hexadecimal ASCII characters on the wire. Very few transactions on
Bitcoin mainnet are over 500KB in size.
-.. envvar:: MAX_SUBS
+.. envvar:: COST_SOFT_LIMIT
+.. envvar:: COST_HARD_LIMIT
+.. envvar:: REQUEST_SLEEP
+.. envvar:: INITIAL_CONCURRENT
- The maximum number of address subscriptions across all sessions.
- Defaults to 250,000.
+ All values are integers. :envvar:`COST_SOFT_LIMIT` defaults to :const:`1,000`,
+ :envvar:`COST_HARD_LIMIT` to :const:`10,000`, :envvar:`REQUEST_SLEEP` to :const:`2,500`
+ milliseconds, and :envvar:`INITIAL_CONCURRENT` to :const:`10` concurrent requests.
-.. envvar:: MAX_SESSION_SUBS
+ The server prices each request made to it based upon an estimate of the resources needed
+ to process it. Factors include whether the request uses bitcoind, how much bandwidth
+ it uses, and how hard it hits the databases.
- The maximum number of address subscriptions permitted to a single
- session. Defaults to 50,000.
+ To set a base for the units, a :func:`blockchain.scripthash.subscribe` subscription to
+ an address with a history of 2 or fewer transactions is costed at :const:`1.0` before
+ considering the bandwidth consumed. :func:`server.ping` is costed at :const:`0.1`.
-.. envvar:: BANDWIDTH_LIMIT
+ As the total cost of a session goes over the soft limit, its requests start to be
+ throttled in two ways. First, the number of requests for that session that the server
+ will process concurrently is reduced. Second, each request starts to sleep a little
+ before being handled.
- Per-session periodic bandwidth usage limit in bytes. This is a soft,
- not hard, limit. Currently the period is hard-coded to be one hour.
- The default limit value is 2 million bytes.
+ Before throttling starts, the server will process up to :envvar:`INITIAL_CONCURRENT`
+ requests concurrently without sleeping. As the session cost ranges from
+ :envvar:`COST_SOFT_LIMIT` to :envvar:`COST_HARD_LIMIT`, concurrency drops linearly to
+ zero and each request's sleep time increases linearly up to :envvar:`REQUEST_SLEEP`
+ milliseconds. Once the hard limit is reached, the session is disconnected.
- Bandwidth usage over each period is totalled, and when this limit is
- exceeded each subsequent request is stalled by sleeping before
- handling it, effectively giving higher processing priority to other
- sessions.
+ In order that non-abusive sessions can continue to be served, a session's cost gradually
+ decays over time. Subscriptions have an ongoing servicing cost, so the decay is slower
+ as the number of subscriptions increases.
- The more bandwidth usage exceeds this soft limit the longer the next
- request will sleep. Sleep times are a round number of seconds with
- a minimum of 1. Each time the delay changes the event is logged.
+ If a session disconnects, ElectrumX continues to associate its cost with its IP address,
+ so if it immediately reconnects it will re-acquire its previous cost allocation.
- Bandwidth usage is gradually reduced over time by "refunding" a
- proportional part of the limit every now and then.
+ A server operator should experiment with different values according to server loads. It
+ is not necessarily true that e.g. having a low soft limit, decreasing concurrency and
+ increasing sleep will help handling heavy loads, as it will also increase the backlog of
+ requests the server has to manage in memory. It will also give a much worse experience
+ for genuine connections.
+
+.. envvar:: BANDWIDTH_UNIT_COST
+
+ The number of bytes, sent and received, by a session that is deemed to cost :const:`1.0`.
+
+ The default value :const:`5,000` bytes, meaning the bandwidth cost assigned to a response
+ of 100KB is 20. If your bandwidth is cheap you should probably raise this.
+
+.. envvar:: REQUEST_TIMEOUT
+
+ An integer number of seconds defaulting to :const:`30`. If a request takes longer than
+ this to respond to, either because of request limiting or because the request is
+ expensive, the server rejects it and returns a timeout error to the client indicating
+ that the server is busy.
+
+ This can help prevent large backlogs of unprocessed requests building up under heavy load.
.. envvar:: SESSION_TIMEOUT
- An integer number of seconds defaulting to 600. Sessions with no
- activity for longer than this are disconnected. Properly
- functioning Electrum clients by default will send pings roughly
- every 60 seconds.
+ An integer number of seconds defaulting to :const:`600`. Sessions that have not sent a
+ request for longer than this are disconnected. Properly functioning clients should send
+ a :func:`server.ping` request once roughly 450 seconds have passed since the previous
+ request, in order to avoid disconnection.
Peer Discovery
@@ -344,50 +448,10 @@ some of this.
will autodetect any proxy running on the usual ports 9050 (Tor),
9150 (Tor browser bundle) and 1080 (socks).
+.. envvar:: BLACKLIST_URL
-Server Advertising
-==================
-
-These environment variables affect how your server is advertised
-by peer discovery (if enabled).
-
-.. envvar:: REPORT_HOST
-
- The clearnet host to advertise. If not set, no clearnet host is
- advertised.
-
-.. envvar:: REPORT_TCP_PORT
-
- The clearnet TCP port to advertise if :envvar:`REPORT_HOST` is set.
- Defaults to :envvar:`TCP_PORT`. ``0`` disables publishing a TCP
- port.
-
-.. envvar:: REPORT_SSL_PORT
-
- The clearnet SSL port to advertise if :envvar:`REPORT_HOST` is set.
- Defaults to :envvar:`SSL_PORT`. ``0`` disables publishing an SSL
- port.
-
-.. envvar:: REPORT_HOST_TOR
-
- If you wish run a Tor service, this is the Tor host name to
- advertise and must end with ``.onion``.
-
-.. envvar:: REPORT_TCP_PORT_TOR
-
- The Tor TCP port to advertise. The default is the clearnet
- :envvar:`REPORT_TCP_PORT`, unless disabled or it is ``0``, otherwise
- :envvar:`TCP_PORT`. ``0`` disables publishing a Tor TCP port.
-
-.. envvar:: REPORT_SSL_PORT_TOR
-
- The Tor SSL port to advertise. The default is the clearnet
- :envvar:`REPORT_SSL_PORT`, unless disabled or it is ``0``, otherwise
- :envvar:`SSL_PORT`. ``0`` disables publishing a Tor SSL port.
-
- .. note:: Certificate-Authority signed certificates don't work over
- Tor, so you should set :envvar:`REPORT_SSL_PORT_TOR` to
- ``0`` if yours is not self-signed.
+ URL to retrieve a list of blacklisted peers. If not set, a coin-
+ specific default is used.
Cache
diff --git a/docs/index.rst b/docs/index.rst
index d3c3de3..b2fdc78 100644
--- a/docs/index.rst
+++ b/docs/index.rst
@@ -27,7 +27,7 @@ Authors and License
===================
Neil Booth wrote the vast majority of the code; see :ref:`Authors`.
-Python version at least 3.6 is required.
+Python version at least 3.7 is required.
The code is released under the `MIT Licence
`_.
diff --git a/docs/protocol-basics.rst b/docs/protocol-basics.rst
index 606cef2..9dbbd54 100644
--- a/docs/protocol-basics.rst
+++ b/docs/protocol-basics.rst
@@ -4,8 +4,8 @@ Protocol Basics
Message Stream
--------------
-Clients and servers communicate using **JSON RPC** over an unspecified
-underlying stream transport protocol, typically TCP or SSL.
+Clients and servers communicate using **JSON RPC** over an unspecified underlying stream
+transport. Examples include TCP, SSL, WS and WSS.
Two standards `JSON RPC 1.0
`_ and `JSON RPC 2.0
@@ -25,11 +25,12 @@ Clients making batch requests should limit their size depending on the
nature of their query, because servers will limit response size as an
anti-DoS mechanism.
-Each RPC call, and each response, is separated by a single newline in
-their respective streams. The JSON specification does not permit
-control characters within strings, so no confusion is possible there.
-However it does permit newlines as extraneous whitespace between
-elements; client and server MUST NOT use newlines in such a way.
+Over TCP and SSL raw sockets each RPC call, and each response, MUST be terminated by a
+single newline to delimit messages. Websocket messages are already framed so they MUST
+NOT be newline terminated. The JSON specification does not permit control characters
+within strings, so no confusion is possible there. However it does permit newlines as
+extraneous whitespace between elements; client and server MUST NOT use newlines in such a
+way.
If using JSON RPC 2.0's feature of parameter passing by name, the
names shown in the description of the method or notification in
@@ -77,38 +78,6 @@ from and including the server's response to this call will use its
negotiated protocol version.
-.. _deserialized header:
-
-Deserialized Headers
---------------------
-
-A :dfn:`deserialized header` is a dictionary describing a block at a
-given height.
-
-A typical example would be similar to this template::
-
- {
- "block_height": ,
- "version": ,
- "prev_block_hash": ,
- "merkle_root": ,
- "timestamp": ,
- "bits": ,
- "nonce":
- }
-
-.. note:: The precise format of a deserialized block header varies by
- coin, and also potentially by height for the same coin. Detailed
- knowledge of the meaning of a block header is neither necessary nor
- appropriate in the server.
-
- Consequently deserialized headers are deprecated and will be removed
- from the protocol in a future version. Instead, raw headers (as
- hexadecimal strings) along with their height will be returned by new
- RPC calls, and it will be up to the client to interpret the meaning
- of the raw header.
-
-
.. _script hashes:
Script Hashes
diff --git a/docs/protocol-changes.rst b/docs/protocol-changes.rst
index 7020d44..d3faa6e 100644
--- a/docs/protocol-changes.rst
+++ b/docs/protocol-changes.rst
@@ -121,14 +121,8 @@ Deprecated methods
Version 1.4
===========
-<<<<<<< HEAD
-This documents the current intent for protocol version 1.4 which is
-not yet implemented. It removes all support for :ref:`deserialized
-headers `.
-=======
This version removes all support for :ref:`deserialized headers
`.
->>>>>>> upstream/master
Changes
-------
@@ -153,65 +147,22 @@ Removed methods
* :func:`blockchain.block.get_header`
* :func:`blockchain.block.get_chunk`
-Version 1.5
-===========
-
-.. note:: This is a draft of ideas for protocol 1.5; they are not
- implemented
-
-This protocol version makes changes intended to allow clients and
-servers to more easily scale to support queries about busy addresses.
-It has changes to reduce the amount of round-trip queries made in
-common usage, and to make results more compact to reduce bandwidth
-consumption.
-
-RPC calls with potentially large responses have pagination support,
-and the return value of :func:`blockchain.scripthash.subscribe`
-changes. Script hash :ref:`status ` had to be recalculated
-with each new transaction and was undefined if it included more than
-one mempool transaction. Its calculation is linear in history length
-resulting in quadratic complexity as history grows. Its calculation
-for large histories was demanding for both the server to compute and
-the client to check.
-
-RPC calls and notifications that combined the effects of the mempool
-and confirmed history are removed.
-
-The changes are beneficial to clients and servers alike, but will
-require changes to both client-side and server-side logic. In
-particular, the client should track what block (by hash and height)
-wallet data is synchronized to, and if that hash is no longer part of
-the main chain, it will need to remove wallet data for blocks that
-were reorganized away and get updated information as of the first
-reorganized block. The effects are limited to script hashes
-potentially affected by the reorg, and for most clients this will be
-the empty set.
-
-New methods
------------
-
- * :func:`blockchain.scripthash.history`
- * :func:`blockchain.scripthash.utxos`
-
-New notifications
------------------
-
- * :func:`mempool.changes`
+Version 1.4.1
+=============
Changes
-------
- * :func:`blockchain.scripthash.subscribe` has changed its return value
- and the notifications it sends
- * :func:`blockchain.transaction.get` takes an additional optional
- argument *merkle*
+ * :func:`blockchain.block.header` and :func:`blockchain.block.headers` now
+ truncate AuxPoW data (if using an AuxPoW chain) when *cp_height* is
+ nonzero. AuxPoW data is still present when *cp_height* is zero.
+ Non-AuxPoW chains are unaffected.
-Removed methods
----------------
- * :func:`blockchain.scripthash.get_history`. Switch to
- :func:`blockchain.scripthash.history`
- * :func:`blockchain.scripthash.get_mempool`. Switch to
- handling :func:`mempool.changes` notifications
- * :func:`blockchain.scripthash.listunspent`. Switch to
- :func:`blockchain.scripthash.utxos`
+Version 1.4.1
+=============
+
+New methods
+-----------
+
+ * :func:`blockchain.scipthash.unsubscribe` to unsubscribe from a script hash.
diff --git a/docs/protocol-ideas.rst b/docs/protocol-ideas.rst
new file mode 100644
index 0000000..4823a9e
--- /dev/null
+++ b/docs/protocol-ideas.rst
@@ -0,0 +1,283 @@
+==============
+Protocol Ideas
+==============
+
+.. note:: This is a draft of ideas for a future protocol tentatively called 2.0; they are
+ not implemented and it is likely they will change and that protocol 2.0 will be
+ quite different.
+
+This protocol version makes changes intended to allow clients and servers to more easily
+scale to support queries about busy addresses. It has changes to reduce the amount of
+round-trip queries made in common usage, and to make results more compact to reduce
+bandwidth consumption.
+
+RPC calls with potentially large responses have pagination support, and the return value
+of :func:`blockchain.scripthash.subscribe` changes. Script hash :ref:`status `
+had to be recalculated with each new transaction and was undefined if it included more
+than one mempool transaction. Its calculation is linear in history length resulting in
+quadratic complexity as history grows. Its calculation for large histories was demanding
+for both the server to compute and the client to check.
+
+RPC calls and notifications that combined the effects of the mempool and confirmed history
+are removed.
+
+The changes are beneficial to clients and servers alike, but will require changes to both
+client-side and server-side logic. In particular, the client should track what block (by
+hash and height) wallet data is synchronized to, and if that hash is no longer part of the
+main chain, it will need to remove wallet data for blocks that were reorganized away and
+get updated information as of the first reorganized block. The effects are limited to
+script hashes potentially affected by the reorg, and for most clients this will be the
+empty set.
+
+
+blockchain.scripthash.subscribe
+===============================
+
+Subscribe to a script hash.
+
+**Signature**
+
+ .. function:: blockchain_.scripthash.subscribe(scripthash)
+
+ *scripthash*
+
+ The script hash as a hexadecimal string.
+
+**Result**
+
+ .. versionchanged:: 2.0
+
+ As of protocol 2.0, the transaction hash of the last confirmed
+ transaction in blockchain order, or :const:`null` if there are none.
+
+ For protocol versions 1.4 and below, the :ref:`status ` of
+ the script hash.
+
+**Notifications**
+
+ .. versionchanged:: 2.0
+
+ As this is a subscription, the client receives notifications when
+ the confirmed transaction history and/or associated mempool
+ transactions change.
+
+ As of protocol 2.0, the initial mempool and subsequent changes to it
+ are sent with :func:`mempool.changes` notifications. When confirmed
+ history changes, a notification with signature
+
+ .. function:: blockchain_.scripthash.subscribe(scripthash, tx_hash)
+
+ is sent, where *tx_hash* is the hash of the last confirmed
+ transaction in blockchain order.
+
+
+blockchain.scripthash.history
+=============================
+
+Return part of the confirmed history of a :ref:`script hash