lightning: rebased on Jan '18 asyncio

gui/qt/__init__.py

@@ -47,7 +47,7 @@ from electrum.util import UserCancelled, print_error
 # from electrum.wallet import Abstract_Wallet
 
 from .installwizard import InstallWizard, GoBack
-
+from electrum.lightning import LightningUI
 
 try:
     from . import icons_rc
@@ -92,6 +92,10 @@ class ElectrumGui:
         #network.add_jobs([DebugMem([Abstract_Wallet, SPV, Synchronizer,
         #                            ElectrumWindow], interval=5)])
         QtCore.QCoreApplication.setAttribute(QtCore.Qt.AA_X11InitThreads)
+        def setConsoleAndReturnLightning():
+            self.windows[0].wallet.lightning.setConsole(self.windows[0].console)
+            return self.windows[0].wallet.lightning
+        self.lightning = LightningUI(setConsoleAndReturnLightning)
         if hasattr(QtCore.Qt, "AA_ShareOpenGLContexts"):
             QtCore.QCoreApplication.setAttribute(QtCore.Qt.AA_ShareOpenGLContexts)
         self.config = config

gui/qt/console.py

@@ -18,6 +18,8 @@ else:
 
 
 class Console(QtWidgets.QPlainTextEdit):
+    newResult = QtCore.pyqtSignal(str)
+
     def __init__(self, prompt='>> ', startup_message='', parent=None):
         QtWidgets.QPlainTextEdit.__init__(self, parent)
 
@@ -34,6 +36,7 @@ class Console(QtWidgets.QPlainTextEdit):
 
         self.updateNamespace({'run':self.run_script})
         self.set_json(False)
+        self.newResult.connect(self.handleNewResult)
 
     def set_json(self, b):
         self.is_json = b
@@ -45,7 +48,8 @@ class Console(QtWidgets.QPlainTextEdit):
         # eval is generally considered bad practice. use it wisely!
         result = eval(script, self.namespace, self.namespace)
 
-
+    def handleNewResult(self, msg):
+        self.showMessage(msg)
 
     def updateNamespace(self, namespace):
         self.namespace.update(namespace)

gui/qt/main_window.py

@@ -1874,6 +1874,7 @@ class ElectrumWindow(QMainWindow, MessageBoxMixin, PrintError):
         console.updateNamespace({'wallet' : self.wallet,
                                  'network' : self.network,
                                  'plugins' : self.gui_object.plugins,
+                                 'lightning' : self.gui_object.lightning,
                                  'window': self})
         console.updateNamespace({'util' : util, 'bitcoin':bitcoin})
 

lib/bitcoin.py

@@ -70,6 +70,16 @@ XPUB_HEADERS = {
 
 
 class NetworkConstants:
+    @classmethod
+    def set_simnet():
+        cls.TESTNET = True
+        cls.ADDRTYPE_P2PKH = 0x3f
+        cls.ADDRTYPE_P2SH = 0x7b
+        cls.SEGWIT_HRP = "sb"
+        cls.HEADERS_URL = None
+        cls.GENESIS = "683e86bd5c6d110d91b94b97137ba6bfe02dbbdb8e3dff722a669b5d69d77af6"
+        cls.DEFAULT_PORTS = {'t':'50001', 's':'50002'}
+        cls.DEFAULT_SERVERS = { '127.0.0.1': DEFAULT_PORTS, }
 
     @classmethod
     def set_mainnet(cls):
@@ -755,11 +765,10 @@ class EC_KEY(object):
     def get_public_key(self, compressed=True):
         return bh2u(point_to_ser(self.pubkey.point, compressed))
 
-    def sign(self, msg_hash):
+    def sign(self, msg_hash, sigencode=ecdsa.util.sigencode_string):
         private_key = MySigningKey.from_secret_exponent(self.secret, curve = SECP256k1)
         public_key = private_key.get_verifying_key()
-        signature = private_key.sign_digest_deterministic(msg_hash, hashfunc=hashlib.sha256, sigencode = ecdsa.util.sigencode_string)
+        signature = private_key.sign_digest_deterministic(msg_hash, hashfunc=hashlib.sha256, sigencode = sigencode)
-        assert public_key.verify_digest(signature, msg_hash, sigdecode = ecdsa.util.sigdecode_string)
         return signature
 
     def sign_message(self, message, is_compressed):

lib/lightning.py

@@ -0,0 +1,799 @@
+import sys
+import struct
+import traceback
+sys.path.insert(0, "lib/ln")
+from .ln import rpc_pb2
+
+from jsonrpclib import Server
+from google.protobuf import json_format
+import binascii
+import ecdsa.util
+import ecdsa.curves
+import hashlib
+from .bitcoin import EC_KEY
+from . import bitcoin
+from . import transaction
+
+import queue
+
+from .util import ForeverCoroutineJob
+
+import threading
+import json
+import base64
+
+import asyncio
+
+from concurrent.futures import TimeoutError
+
+WALLET = None
+NETWORK = None
+CONFIG = None
+locked = set()
+
+machine = "148.251.87.112"
+
+def SetHdSeed(json):
+    req = rpc_pb2.SetHdSeedRequest()
+    json_format.Parse(json, req)
+    print("set hdseed unimplemented", int.from_bytes(req.hdSeed, "big"))
+    m = rpc_pb2.SetHdSeedResponse()
+    msg = json_format.MessageToJson(m)
+    return msg
+
+
+def ConfirmedBalance(json):
+    request = rpc_pb2.ConfirmedBalanceRequest()
+    json_format.Parse(json, request)
+    m = rpc_pb2.ConfirmedBalanceResponse()
+    confs = request.confirmations
+    witness = request.witness  # bool
+
+    m.amount = sum(WALLET.get_balance())
+    msg = json_format.MessageToJson(m)
+    return msg
+
+
+def NewAddress(json):
+    request = rpc_pb2.NewAddressRequest()
+    json_format.Parse(json, request)
+    m = rpc_pb2.NewAddressResponse()
+    if request.type == rpc_pb2.WITNESS_PUBKEY_HASH:
+        m.address = WALLET.get_unused_address()
+    elif request.type == rpc_pb2.NESTED_PUBKEY_HASH:
+        assert False, "cannot handle nested-pubkey-hash address type generation yet"
+    elif request.type == rpc_pb2.PUBKEY_HASH:
+        assert False, "cannot handle pubkey_hash generation yet"
+    else:
+        assert False, "unknown address type"
+    msg = json_format.MessageToJson(m)
+    return msg
+
+
+def FetchRootKey(json):
+    request = rpc_pb2.FetchRootKeyRequest()
+    json_format.Parse(json, request)
+    m = rpc_pb2.FetchRootKeyResponse()
+    m.rootKey = WALLET.keystore.get_private_key([151,151,151,151], None)[0]
+    msg = json_format.MessageToJson(m)
+    return msg
+
+
+cl = rpc_pb2.ListUnspentWitnessRequest
+
+assert rpc_pb2.WITNESS_PUBKEY_HASH is not None
+
+
+def ListUnspentWitness(json):
+    req = cl()
+    json_format.Parse(json, req)
+    confs = req.minConfirmations #TODO regard this
+
+    unspent = WALLET.get_utxos()
+    m = rpc_pb2.ListUnspentWitnessResponse()
+    for utxo in unspent:
+        # print(utxo)
+        # example:
+        # {'prevout_n': 0,
+        #  'address': 'sb1qt52ccplvtpehz7qvvqft2udf2eaqvfsal08xre',
+        #  'prevout_hash': '0d4caccd6e8a906c8ca22badf597c4dedc6dd7839f3cac3137f8f29212099882',
+        #  'coinbase': False,
+        #  'height': 326,
+        #  'value': 400000000}
+
+        global locked
+        if (utxo["prevout_hash"], utxo["prevout_n"]) in locked:
+            print("SKIPPING LOCKED OUTPOINT", utxo["prevout_hash"])
+            continue
+        towire = m.utxos.add()
+        towire.addressType = rpc_pb2.WITNESS_PUBKEY_HASH
+        towire.redeemScript = b""
+        towire.pkScript = b""
+        towire.witnessScript = bytes(bytearray.fromhex(
+            bitcoin.address_to_script(utxo["address"])))
+        towire.value = utxo["value"]
+        towire.outPoint.hash = utxo["prevout_hash"]
+        towire.outPoint.index = utxo["prevout_n"]
+    return json_format.MessageToJson(m)
+
+
+i = 0
+
+usedAddresses = set()
+
+def NewRawKey(json):
+    global i
+    addresses = WALLET.get_unused_addresses()
+    res = rpc_pb2.NewRawKeyResponse()
+    pubk = None
+    assert len(set(addresses) - usedAddresses) > 0, "used all addresses!"
+    while pubk is None:
+      i = i + 1
+      if i > len(addresses) - 1:
+          i = 0
+      # TODO do not reuse keys!!!!!!!!!!!!!!!!
+      # find out when get_unused_addresses marks an address used...
+      if addresses[i] not in usedAddresses:
+        pubk = addresses[i]
+        usedAddresses.add(pubk)
+    res.publicKey = bytes(bytearray.fromhex(WALLET.get_public_keys(pubk)[0]))
+    return json_format.MessageToJson(res)
+
+
+def LockOutpoint(json):
+    req = rpc_pb2.LockOutpointRequest()
+    json_format.Parse(json, req)
+    global locked
+    locked.add((req.outpoint.hash, req.outpoint.index))
+
+
+def UnlockOutpoint(json):
+    req = rpc_pb2.UnlockOutpointRequest()
+    json_format.Parse(json, req)
+    global locked
+    # throws KeyError if not existing. Use .discard() if we do not care
+    locked.remove((req.outpoint.hash, req.outpoint.index))
+
+HEIGHT = None
+
+def ListTransactionDetails(json):
+    global HEIGHT
+    global WALLET
+    global NETWORK
+    m = rpc_pb2.ListTransactionDetailsResponse()
+    for tx_hash, height, conf, timestamp, delta, balance in WALLET.get_history():
+        if height == 0:
+          print("WARNING", tx_hash, "has zero height!")
+        detail = m.details.add()
+        detail.hash = tx_hash
+        detail.value = delta
+        detail.numConfirmations = conf
+        detail.blockHash = NETWORK.blockchain().get_hash(height)
+        detail.blockHeight = height
+        detail.timestamp = timestamp
+        detail.totalFees = 1337 # TODO
+    return json_format.MessageToJson(m)
+
+def FetchInputInfo(json):
+    req = rpc_pb2.FetchInputInfoRequest()
+    json_format.Parse(json, req)
+    has = req.outPoint.hash
+    idx = req.outPoint.index
+    txoinfo = WALLET.txo.get(has, {})
+    m = rpc_pb2.FetchInputInfoResponse()
+    if has in WALLET.transactions:
+        tx = WALLET.transactions[has]
+        m.mine = True
+    else:
+        tx = WALLET.get_input_tx(has)
+        print("did not find tx with hash", has)
+        print("tx", tx)
+
+        m.mine = False
+        return json_format.MessageToJson(m)
+    outputs = tx.outputs()
+    assert {bitcoin.TYPE_SCRIPT: "SCRIPT", bitcoin.TYPE_ADDRESS: "ADDRESS",
+            bitcoin.TYPE_PUBKEY: "PUBKEY"}[outputs[idx][0]] == "ADDRESS"
+    scr = transaction.Transaction.pay_script(outputs[idx][0], outputs[idx][1])
+    m.txOut.value = outputs[idx][2]  # type, addr, val
+    m.txOut.pkScript = bytes(bytearray.fromhex(scr))
+    msg = json_format.MessageToJson(m)
+    return msg
+
+def SendOutputs(json):
+    global NETWORK, WALLET, CONFIG
+
+    req = rpc_pb2.SendOutputsRequest()
+    json_format.Parse(json, req)
+
+    m = rpc_pb2.SendOutputsResponse()
+
+    elecOutputs = [(bitcoin.TYPE_SCRIPT, binascii.hexlify(txout.pkScript).decode("utf-8"), txout.value) for txout in req.outputs]
+
+    print("ignoring feeSatPerByte", req.feeSatPerByte) # TODO
+
+    tx = None
+    try:
+        #                outputs,     password, config, fee
+        tx = WALLET.mktx(elecOutputs, None,     CONFIG, 1000)
+    except Exception as e:
+        m.success = False
+        m.error = str(e)
+        m.resultHash = ""
+        return json_format.MessageToJson(m)
+
+    suc, has = NETWORK.broadcast(tx)
+    if not suc:
+        m.success = False
+        m.error = "electrum/lightning/SendOutputs: Could not broadcast: " + str(has)
+        m.resultHash = ""
+        return json_format.MessageToJson(m)
+    m.success = True
+    m.error = ""
+    m.resultHash = tx.txid()
+    return json_format.MessageToJson(m)
+
+def isSynced():
+    global NETWORK
+    local_height, server_height = NETWORK.get_status_value("updated")
+    synced = server_height != 0 and NETWORK.is_up_to_date() and local_height >= server_height
+    return synced, local_height, server_height
+
+def IsSynced(json):
+    m = rpc_pb2.IsSyncedResponse()
+    m.synced, _, _ = isSynced()
+    return json_format.MessageToJson(m)
+
+def SignMessage(json):
+    req = rpc_pb2.SignMessageRequest()
+    json_format.Parse(json, req)
+    m = rpc_pb2.SignMessageResponse()
+
+    address = None
+    for adr in usedAddresses:
+      if req.pubKey == bytes(bytearray.fromhex(WALLET.get_public_keys(adr)[0])):
+        address = adr
+        break
+
+    pri = None
+    if address is None:
+        priv_keys = WALLET.storage.get("lightning_extra_keys", [])
+        print("searching lightning_extra_keys", req.pubKey)
+        for i in priv_keys:
+            assert type(i) is int
+            signkey = MySigningKey.from_secret_exponent(i, curve=ecdsa.curves.SECP256k1)
+            pubkeystr = signkey.get_verifying_key().to_string()
+            print("pubkeystr", pubkeystr)
+            if pubkeystr == req.pubKey:
+                pri = signkey
+        if pri is None:
+            assert False, "could not find private key corresponding to " + repr(req.pubKey)
+    else:
+        pri, _ = WALLET.export_private_key(address, None)
+        typ, pri, compressed = bitcoin.deserialize_privkey(pri)
+        pri = EC_KEY(pri)
+
+    m.signature = pri.sign(bitcoin.Hash(req.messageToBeSigned), ecdsa.util.sigencode_der)
+    m.error = ""
+    m.success = True
+    return json_format.MessageToJson(m)
+
+def LEtobytes(x, l):
+    if l == 2:
+        fmt = "<H"
+    elif l == 4:
+        fmt = "<I"
+    elif l == 8:
+        fmt = "<Q"
+    else:
+        assert False, "invalid format for LEtobytes"
+    return struct.pack(fmt, x)
+
+
+def toint(x):
+    if len(x) == 1:
+        return ord(x)
+    elif len(x) == 2:
+        fmt = ">H"
+    elif len(x) == 4:
+        fmt = ">I"
+    elif len(x) == 8:
+        fmt = ">Q"
+    else:
+        assert False, "invalid length for toint(): " + str(len(x))
+    return struct.unpack(fmt, x)[0]
+
+
+class SignDescriptor(object):
+    def __init__(self, pubKey=None, sigHashes=None, inputIndex=None, singleTweak=None, hashType=None, doubleTweak=None, witnessScript=None, output=None):
+        self.pubKey = pubKey
+        self.sigHashes = sigHashes
+        self.inputIndex = inputIndex
+        self.singleTweak = singleTweak
+        self.hashType = hashType
+        self.doubleTweak = doubleTweak
+        self.witnessScript = witnessScript
+        self.output = output
+
+    def __str__(self):
+        return '%s(%s)' % (
+            type(self).__name__,
+            ', '.join('%s=%s' % item for item in vars(self).items())
+        )
+
+
+class TxSigHashes(object):
+    def __init__(self, hashOutputs=None, hashSequence=None, hashPrevOuts=None):
+        self.hashOutputs = hashOutputs
+        self.hashSequence = hashSequence
+        self.hashPrevOuts = hashPrevOuts
+
+
+class Output(object):
+    def __init__(self, value=None, pkScript=None):
+        assert value is not None and pkScript is not None
+        self.value = value
+        self.pkScript = pkScript
+
+
+class InputScript(object):
+    def __init__(self, scriptSig, witness):
+        assert witness is None or type(witness[0]) is type(bytes([]))
+        assert type(scriptSig) is type(bytes([]))
+        self.scriptSig = scriptSig
+        self.witness = witness
+
+
+def tweakPrivKey(basePriv, commitTweak):
+    tweakInt = int.from_bytes(commitTweak, byteorder="big")
+    tweakInt += basePriv.secret # D is secret
+    tweakInt %= ecdsa.curves.SECP256k1.generator.order()
+    return EC_KEY(tweakInt.to_bytes(33, 'big')) # TODO find out if 33 bytes are necessary. private keys are usually only 32 bytes
+
+def singleTweakBytes(commitPoint, basePoint):
+    m = hashlib.sha256()
+    m.update(bytearray.fromhex(commitPoint))
+    m.update(bytearray.fromhex(basePoint))
+    return m.digest()
+
+def deriveRevocationPrivKey(revokeBasePriv, commitSecret):
+    revokeTweakBytes = singleTweakBytes(revokeBasePriv.get_public_key(True),
+                                        commitSecret.get_public_key(True))
+    revokeTweakInt = int.from_bytes(revokeTweakBytes, byteorder="big")
+
+    commitTweakBytes = singleTweakBytes(commitSecret.get_public_key(True),
+                                        revokeBasePriv.get_public_key(True))
+    commitTweakInt = int.from_bytes(commitTweakBytes, byteorder="big")
+
+    revokeHalfPriv = revokeTweakInt * revokeBasePriv.secret # D is secret
+    commitHalfPriv = commitTweakInt * commitSecret.secret
+
+    revocationPriv = revokeHalfPriv + commitHalfPriv
+    revocationPriv %= ecdsa.curves.SECP256k1.generator.order()
+
+    return EC_KEY(revocationPriv.to_bytes(33, byteorder="big")) # TODO find out if 33 bytes are necessary. private keys are usually only 32 bytes
+
+
+def maybeTweakPrivKey(signdesc, pri):
+    if len(signdesc.singleTweak) > 0:
+        pri2 = tweakPrivKey(pri, signdesc.singleTweak)
+    elif len(signdesc.doubleTweak) > 0:
+        pri2 = deriveRevocationPrivKey(pri, EC_KEY(signdesc.doubleTweak))
+    else:
+        pri2 = pri
+
+    if pri2 != pri:
+        have_keys = WALLET.storage.get("lightning_extra_keys", [])
+        if pri2.secret not in have_keys:
+            WALLET.storage.put("lightning_extra_keys", have_keys + [pri2.secret])
+            WALLET.storage.write()
+            print("saved new tweaked key", pri2.secret)
+
+    return pri2
+
+
+def isWitnessPubKeyHash(script):
+    if len(script) != 2:
+        return False
+    haveop0 = (transaction.opcodes.OP_0 == script[0][0])
+    haveopdata20 = (20 == script[1][0])
+    return haveop0 and haveopdata20
+
+#// calcWitnessSignatureHash computes the sighash digest of a transaction's
+#// segwit input using the new, optimized digest calculation algorithm defined
+#// in BIP0143: https://github.com/bitcoin/bips/blob/master/bip-0143.mediawiki.
+#// This function makes use of pre-calculated sighash fragments stored within
+#// the passed HashCache to eliminate duplicate hashing computations when
+#// calculating the final digest, reducing the complexity from O(N^2) to O(N).
+#// Additionally, signatures now cover the input value of the referenced unspent
+#// output. This allows offline, or hardware wallets to compute the exact amount
+#// being spent, in addition to the final transaction fee. In the case the
+#// wallet if fed an invalid input amount, the real sighash will differ causing
+#// the produced signature to be invalid.
+
+
+def calcWitnessSignatureHash(original, sigHashes, hashType, tx, idx, amt):
+    assert len(original) != 0
+    decoded = transaction.deserialize(binascii.hexlify(tx).decode("utf-8"))
+    if idx > len(decoded["inputs"]) - 1:
+        raise Exception("invalid inputIndex")
+    txin = decoded["inputs"][idx]
+    #tohash = transaction.Transaction.serialize_witness(txin)
+    sigHash = LEtobytes(decoded["version"], 4)
+    if toint(hashType) & toint(sigHashAnyOneCanPay) == 0:
+        sigHash += bytes(bytearray.fromhex(sigHashes.hashPrevOuts))[::-1]
+    else:
+        sigHash += b"\x00" * 32
+
+    if toint(hashType) & toint(sigHashAnyOneCanPay) == 0 and toint(hashType) & toint(sigHashMask) != toint(sigHashSingle) and toint(hashType) & toint(sigHashMask) != toint(sigHashNone):
+        sigHash += bytes(bytearray.fromhex(sigHashes.hashSequence))[::-1]
+    else:
+        sigHash += b"\x00" * 32
+
+    sigHash += bytes(bytearray.fromhex(txin["prevout_hash"]))[::-1]
+    sigHash += LEtobytes(txin["prevout_n"], 4)
+    # byte 72
+
+    subscript = list(transaction.script_GetOp(original))
+    if isWitnessPubKeyHash(subscript):
+        sigHash += b"\x19"
+        sigHash += bytes([transaction.opcodes.OP_DUP])
+        sigHash += bytes([transaction.opcodes.OP_HASH160])
+        sigHash += b"\x14"  # 20 bytes
+        assert len(subscript) == 2, subscript
+        opcode, data, length = subscript[1]
+        sigHash += data
+        sigHash += bytes([transaction.opcodes.OP_EQUALVERIFY])
+        sigHash += bytes([transaction.opcodes.OP_CHECKSIG])
+    else:
+        # For p2wsh outputs, and future outputs, the script code is
+        # the original script, with all code separators removed,
+        # serialized with a var int length prefix.
+
+        assert len(sigHash) == 104, len(sigHash)
+        sigHash += bytes(bytearray.fromhex(bitcoin.var_int(len(original))))
+        assert len(sigHash) == 105, len(sigHash)
+
+        sigHash += original
+
+    sigHash += LEtobytes(amt, 8)
+    sigHash += LEtobytes(txin["sequence"], 4)
+
+    if toint(hashType) & toint(sigHashSingle) != toint(sigHashSingle) and toint(hashType) & toint(sigHashNone) != toint(sigHashNone):
+        sigHash += bytes(bytearray.fromhex(sigHashes.hashOutputs))[::-1]
+    elif toint(hashtype) & toint(sigHashMask) == toint(sigHashSingle) and idx < len(decoded["outputs"]):
+        raise Exception("TODO 1")
+    else:
+        raise Exception("TODO 2")
+
+    sigHash += LEtobytes(decoded["lockTime"], 4)
+    sigHash += LEtobytes(toint(hashType), 4)
+
+    return transaction.Hash(sigHash)
+
+#// RawTxInWitnessSignature returns the serialized ECDA signature for the input
+#// idx of the given transaction, with the hashType appended to it. This
+#// function is identical to RawTxInSignature, however the signature generated
+#// signs a new sighash digest defined in BIP0143.
+# func RawTxInWitnessSignature(tx *MsgTx, sigHashes *TxSigHashes, idx int,
+#  amt int64, subScript []byte, hashType SigHashType,
+#  key *btcec.PrivateKey) ([]byte, error) {
+
+
+def rawTxInWitnessSignature(tx, sigHashes, idx, amt, subscript, hashType, key):
+    digest = calcWitnessSignatureHash(
+        subscript, sigHashes, hashType, tx, idx, amt)
+    return key.sign(digest, sigencode=ecdsa.util.sigencode_der) + hashType
+
+# WitnessSignature creates an input witness stack for tx to spend BTC sent
+# from a previous output to the owner of privKey using the p2wkh script
+# template. The passed transaction must contain all the inputs and outputs as
+# dictated by the passed hashType. The signature generated observes the new
+# transaction digest algorithm defined within BIP0143.
+def witnessSignature(tx, sigHashes, idx, amt, subscript, hashType, privKey, compress):
+    sig = rawTxInWitnessSignature(
+        tx, sigHashes, idx, amt, subscript, hashType, privKey)
+
+    pkData = bytes(bytearray.fromhex(
+        privKey.get_public_key(compressed=compress)))
+
+    return sig, pkData
+
+
+sigHashMask = b"\x1f"
+
+sigHashAll = b"\x01"
+sigHashNone = b"\x02"
+sigHashSingle = b"\x03"
+sigHashAnyOneCanPay = b"\x80"
+
+test = rpc_pb2.ComputeInputScriptResponse()
+
+test.witnessScript.append(b"\x01")
+test.witnessScript.append(b"\x02")
+
+
+def SignOutputRaw(json):
+    req = rpc_pb2.SignOutputRawRequest()
+    json_format.Parse(json, req)
+
+    assert len(req.signDesc.pubKey) in [33, 0]
+    assert len(req.signDesc.doubleTweak) in [32, 0]
+    assert len(req.signDesc.sigHashes.hashPrevOuts) == 64
+    assert len(req.signDesc.sigHashes.hashSequence) == 64
+    assert len(req.signDesc.sigHashes.hashOutputs) == 64
+
+    m = rpc_pb2.SignOutputRawResponse()
+
+    m.signature = signOutputRaw(req.tx, req.signDesc)
+
+    msg = json_format.MessageToJson(m)
+    return msg
+
+
+def signOutputRaw(tx, signDesc):
+    adr = bitcoin.pubkey_to_address('p2wpkh', binascii.hexlify(
+        signDesc.pubKey).decode("utf-8"))  # Because this is all NewAddress supports
+    pri = fetchPrivKey(adr)
+    pri2 = maybeTweakPrivKey(signDesc, pri)
+    sig = rawTxInWitnessSignature(tx, signDesc.sigHashes, signDesc.inputIndex,
+                                  signDesc.output.value, signDesc.witnessScript, sigHashAll, pri2)
+    return sig[:len(sig) - 1]
+
+async def PublishTransaction(json):
+    req = rpc_pb2.PublishTransactionRequest()
+    json_format.Parse(json, req)
+    global NETWORK
+    tx = transaction.Transaction(binascii.hexlify(req.tx).decode("utf-8"))
+    suc, has = await NETWORK.broadcast_async(tx)
+    m = rpc_pb2.PublishTransactionResponse()
+    m.success = suc
+    m.error = str(has) if not suc else ""
+    if m.error:
+        print("PublishTransaction", m.error)
+        if "Missing inputs" in m.error:
+            print("inputs", tx.inputs())
+    return json_format.MessageToJson(m)
+
+
+def ComputeInputScript(json):
+    req = rpc_pb2.ComputeInputScriptRequest()
+    json_format.Parse(json, req)
+
+    assert len(req.signDesc.pubKey) in [33, 0]
+    assert len(req.signDesc.doubleTweak) in [32, 0]
+    assert len(req.signDesc.sigHashes.hashPrevOuts) == 64
+    assert len(req.signDesc.sigHashes.hashSequence) == 64
+    assert len(req.signDesc.sigHashes.hashOutputs) == 64
+    # singleTweak , witnessScript variable length
+
+    try:
+        inpscr = computeInputScript(req.tx, req.signDesc)
+    except:
+        print("catched!")
+        traceback.print_exc()
+        return None
+
+    m = rpc_pb2.ComputeInputScriptResponse()
+
+    m.witnessScript.append(inpscr.witness[0])
+    m.witnessScript.append(inpscr.witness[1])
+    m.scriptSig = inpscr.scriptSig
+
+    msg = json_format.MessageToJson(m)
+    return msg
+
+
+def fetchPrivKey(str_address):
+    # TODO FIXME privkey should be retrieved from wallet using also signer_key (in signdesc)
+    pri, redeem_script = WALLET.export_private_key(str_address, None)
+
+    if redeem_script:
+        print("ignoring redeem script", redeem_script)
+
+    typ, pri, compressed = bitcoin.deserialize_privkey(pri)
+    pri = EC_KEY(pri)
+    return pri
+
+
+def computeInputScript(tx, signdesc):
+    typ, str_address = transaction.get_address_from_output_script(
+        signdesc.output.pkScript)
+    assert typ != bitcoin.TYPE_SCRIPT
+
+    pri = fetchPrivKey(str_address)
+
+    isNestedWitness = False  # because NewAddress only does native addresses
+
+    witnessProgram = None
+    ourScriptSig = None
+
+    if isNestedWitness:
+        pub = pri.get_public_key()
+
+        scr = bitcoin.hash_160(pub)
+
+        witnessProgram = b"\x00\x14" + scr
+
+        # \x14 is OP_20
+        ourScriptSig = b"\x16\x00\x14" + scr
+    else:
+        # TODO TEST
+        witnessProgram = signdesc.output.pkScript
+        ourScriptSig = b""
+        print("set empty ourScriptSig")
+        print("witnessProgram", witnessProgram)
+
+    # If a tweak (single or double) is specified, then we'll need to use
+    # this tweak to derive the final private key to be used for signing
+    # this output.
+    pri2 = maybeTweakPrivKey(signdesc, pri)
+
+    #
+    # Generate a valid witness stack for the input.
+    # TODO(roasbeef): adhere to passed HashType
+    witnessScript, pkData = witnessSignature(tx, signdesc.sigHashes,
+                                             signdesc.inputIndex, signdesc.output.value, witnessProgram,
+                                             sigHashAll, pri2, True)
+    return InputScript(witness=(witnessScript, pkData), scriptSig=ourScriptSig)
+
+from collections import namedtuple
+QueueItem = namedtuple("QueueItem", ["methodName", "args"])
+
+class LightningRPC(ForeverCoroutineJob):
+    def __init__(self):
+        super(LightningRPC, self).__init__()
+        self.queue = queue.Queue()
+    # overridden
+    async def run(self, is_running):
+      print("RPC STARTED")
+      while is_running():
+        try:
+            qitem = self.queue.get(block=False)
+        except queue.Empty:
+            await asyncio.sleep(1)
+            pass
+        else:
+            def call(qitem):
+                client = Server("http://" + machine + ":8090")
+                result = getattr(client, qitem.methodName)(base64.b64encode(privateKeyHash[:6]).decode("ascii"), *[str(x) for x in qitem.args])
+                toprint = result
+                try:
+                    if result["stderr"] == "" and result["returncode"] == 0:
+                        toprint = json.loads(result["stdout"])
+                except:
+                    pass
+                self.console.newResult.emit(json.dumps(toprint, indent=4))
+            threading.Thread(target=call, args=(qitem, )).start()
+    def setConsole(self, console):
+        self.console = console
+
+def lightningCall(rpc, methodName):
+    def fun(*args):
+        rpc.queue.put(QueueItem(methodName, args))
+    return fun
+
+class LightningUI():
+    def __init__(self, lightningGetter):
+        self.rpc = lightningGetter
+    def __getattr__(self, nam):
+        synced, local, server = isSynced()
+        if not synced:
+            return lambda *args: "Not synced yet: local/server: {}/{}".format(local, server)
+        return lightningCall(self.rpc(), nam)
+
+privateKeyHash = None
+ip = lambda: "{}.{}.{}.{}".format(privateKeyHash[0], privateKeyHash[1], privateKeyHash[2], privateKeyHash[3])
+port = lambda: int.from_bytes(privateKeyHash[4:6], "big")
+
+class LightningWorker(ForeverCoroutineJob):
+    def __init__(self, wallet, network, config):
+        global privateKeyHash
+        super(LightningWorker, self).__init__()
+        self.server = None
+        self.wallet = wallet
+        self.network = network
+        self.config = config
+        privateKeyHash = bitcoin.Hash(self.wallet().keystore.get_private_key([152,152,152,152], None)[0])
+
+        deser = bitcoin.deserialize_xpub(wallet().keystore.xpub)
+        assert deser[0] == "p2wpkh", deser
+
+    async def run(self, is_running):
+        global WALLET, NETWORK
+        global CONFIG
+
+        wasAlreadyUpToDate = False
+
+        while is_running():
+            WALLET = self.wallet()
+            NETWORK = self.network()
+            CONFIG = self.config()
+
+            synced, local, server = isSynced()
+            if not synced:
+                await asyncio.sleep(5)
+                continue
+            else:
+                if not wasAlreadyUpToDate:
+                    print("UP TO DATE FOR THE FIRST TIME")
+                    print(NETWORK.get_status_value("updated"))
+                wasAlreadyUpToDate = True
+
+            writer = None
+            try:
+                reader, writer = await asyncio.wait_for(asyncio.open_connection(machine, 1080), 5)
+                writer.write(b"MAGIC")
+                writer.write(privateKeyHash[:6])
+                await writer.drain()
+                while is_running():
+                    obj = await readJson(reader, is_running)
+                    if not obj: continue
+                    await readReqAndReply(obj, writer)
+            except:
+                traceback.print_exc()
+                continue
+
+async def readJson(reader, is_running):
+    data = b""
+    while is_running():
+      if data != b"": print("parse failed, data has", data)
+      try:
+        return json.loads(data)
+      except ValueError:
+        try:
+            data += await asyncio.wait_for(reader.read(2048), 1)
+        except TimeoutError:
+            continue
+
+async def readReqAndReply(obj, writer):
+    methods = [FetchRootKey
+    ,ConfirmedBalance
+    ,NewAddress
+    ,ListUnspentWitness
+    ,SetHdSeed
+    ,NewRawKey
+    ,FetchInputInfo
+    ,ComputeInputScript
+    ,SignOutputRaw
+    ,PublishTransaction
+    ,LockOutpoint
+    ,UnlockOutpoint
+    ,ListTransactionDetails
+    ,SendOutputs
+    ,IsSynced
+    ,SignMessage]
+    result = None
+    found = False
+    try:
+        for method in methods:
+            if method.__name__ == obj["method"]:
+                params = obj["params"][0]
+                print("calling method", obj["method"], "with", params)
+                if asyncio.iscoroutinefunction(method):
+                    result = await method(params)
+                else:
+                    result = method(params)
+                found = True
+                break
+    except BaseException as e:
+        traceback.print_exc()
+        print("exception while calling method", obj["method"])
+        writer.write(json.dumps({"id":obj["id"],"error": {"code": -32002, "message": str(e)}}).encode("ascii"))
+        await writer.drain()
+    else:
+        if not found:
+            writer.write(json.dumps({"id":obj["id"],"error": {"code": -32601, "message": "invalid method"}}).encode("ascii"))
+        else:
+            print("result was", result)
+            if result is None:
+                result = "{}"
+            try:
+                assert type({}) is type(json.loads(result))
+            except:
+                traceback.print_exc()
+                print("wrong method implementation")
+                writer.write(json.dumps({"id":obj["id"],"error": {"code": -32000, "message": "wrong return type in electrum-lightning-hub"}}).encode("ascii"))
+            else:
+                writer.write(json.dumps({"id":obj["id"],"result": result}).encode("ascii"))
+        await writer.drain()

lib/network.py

@@ -308,6 +308,9 @@ class Network(util.DaemonThread):
         # Resend unanswered requests
         requests = self.unanswered_requests.values()
         self.unanswered_requests = {}
+        if self.interface.ping_required():
+            params = [ELECTRUM_VERSION, PROTOCOL_VERSION]
+            await self.queue_request('server.version', params, self.interface)
         for request in requests:
             message_id = await self.queue_request(request[0], request[1])
             self.unanswered_requests[message_id] = request
@@ -717,6 +720,11 @@ class Network(util.DaemonThread):
         We distinguish by whether it is in self.interfaces.'''
         async with self.all_server_locks("connection down"):
             if server in self.disconnected_servers:
+                try:
+                    raise Exception("already disconnected " + server + " because " + repr(self.disconnected_servers[server]) + ". new reason: " + repr(reason))
+                except:
+                    traceback.print_exc()
+                    sys.exit(1)
                 return
             self.print_error("connection down", server)
             self.disconnected_servers[server] = reason
@@ -762,6 +770,10 @@ class Network(util.DaemonThread):
             interface.print_error(error or 'bad response')
             return
         index = params[0]
+        # Ignore unsolicited chunks
+        if index not in self.requested_chunks:
+            return
+        self.requested_chunks.remove(index)
         connect = interface.blockchain.connect_chunk(index, result)
         if not connect:
             await self.connection_down(interface.server, "could not connect chunk")
@@ -1114,11 +1126,14 @@ class Network(util.DaemonThread):
                 raise
         asyncio.ensure_future(job())
         run_future = asyncio.Future()
+        self.run_forever_coroutines()
         asyncio.ensure_future(self.run_async(run_future))
 
         loop.run_until_complete(run_future)
+        assert self.forever_coroutines_task.done()
         run_future.exception()
         self.print_error("run future result", run_future.result())
+        loop.close()
 
     async def run_async(self, future):
         try:

lib/transaction.py

@@ -526,7 +526,10 @@ def deserialize(raw):
                     txin['address'] = bitcoin.public_key_to_p2wpkh(bfh(txin['pubkeys'][0]))
                 else:
                     txin['type'] = 'p2wsh'
-                    txin['address'] = bitcoin.script_to_p2wsh(txin['witnessScript'])
+                    try:
+                      txin['address'] = bitcoin.script_to_p2wsh(txin['witnessScript'])
+                    except KeyError:
+                      pass
     d['lockTime'] = vds.read_uint32()
     return d
 

lib/util.py

@@ -28,11 +28,12 @@ from decimal import Decimal
 import traceback
 import urllib
 import threading
-import hmac
 import time
 import json
 import urllib.request, urllib.parse, urllib.error
 import queue
+import asyncio
+import hmac
 
 from .i18n import _
 
@@ -87,6 +88,15 @@ class PrintError(object):
     def print_msg(self, *msg):
         print_msg("[%s]" % self.diagnostic_name(), *msg)
 
+class ForeverCoroutineJob(PrintError):
+    """A job that is run from a thread's main loop.  run() is
+    called from that thread's context.
+    """
+
+    async def run(self, is_running):
+        """Called once from the thread"""
+        pass
+
 class CoroutineJob(PrintError):
     """A job that is run periodically from a thread's main loop.  run() is
     called from that thread's context.
@@ -141,11 +151,28 @@ class DaemonThread(threading.Thread, PrintError):
         self.job_lock = threading.Lock()
         self.jobs = []
         self.coroutines = []
+        self.forever_coroutines_task = None
 
     def add_coroutines(self, jobs):
         for i in jobs: assert isinstance(i, CoroutineJob), i.__class__.__name__ + " does not inherit from CoroutineJob"
         self.coroutines.extend(jobs)
 
+    def set_forever_coroutines(self, jobs):
+        for i in jobs: assert isinstance(i, ForeverCoroutineJob), i.__class__.__name__ + " does not inherit from ForeverCoroutineJob"
+        async def put():
+            await self.forever_coroutines_queue.put(jobs)
+        asyncio.run_coroutine_threadsafe(put(), self.loop)
+
+    def run_forever_coroutines(self):
+        self.forever_coroutines_queue = asyncio.Queue() # making queue here because __init__ is called from non-network thread
+        self.loop = asyncio.get_event_loop()
+        async def getFromQueueAndStart():
+            jobs = await self.forever_coroutines_queue.get()
+            await asyncio.gather(*[i.run(self.is_running) for i in jobs])
+            print("FOREVER JOBS DONE")
+        self.forever_coroutines_task = asyncio.ensure_future(getFromQueueAndStart())
+        return self.forever_coroutines_task
+
     async def run_coroutines(self):
         for coroutine in self.coroutines:
             assert isinstance(coroutine, CoroutineJob)

lib/wallet.py

@@ -56,6 +56,7 @@ from .plugins import run_hook
 from . import bitcoin
 from . import coinchooser
 from .synchronizer import Synchronizer
+from .lightning import LightningRPC
 from .verifier import SPV
 
 from . import paymentrequest
@@ -63,6 +64,8 @@ from .paymentrequest import PR_PAID, PR_UNPAID, PR_UNKNOWN, PR_EXPIRED
 from .paymentrequest import InvoiceStore
 from .contacts import Contacts
 
+from .lightning import LightningWorker
+
 TX_STATUS = [
     _('Replaceable'),
     _('Unconfirmed parent'),
@@ -1004,6 +1007,9 @@ class Abstract_Wallet(PrintError):
             self.verifier = SPV(self.network, self)
             self.synchronizer = Synchronizer(self, network)
             network.add_coroutines([self.verifier, self.synchronizer])
+            self.lightning = LightningRPC()
+            self.lightningworker = LightningWorker(lambda: self, lambda: network, lambda: network.config)
+            network.set_forever_coroutines([self.lightning, self.lightningworker])
         else:
             self.verifier = None
             self.synchronizer = None
@@ -1888,7 +1894,6 @@ class Multisig_Wallet(Deterministic_Wallet):
         txin['signatures'] = [None] * self.n
         txin['num_sig'] = self.m
 
-
 wallet_types = ['standard', 'multisig', 'imported']
 
 def register_wallet_type(category):

protoc_lightning.sh

@@ -0,0 +1,13 @@
+#!/bin/sh -ex
+if [ ! -d $HOME/go/src/github.com/grpc-ecosystem ]; then
+  # from readme in https://github.com/grpc-ecosystem/grpc-gateway
+  go get -u github.com/grpc-ecosystem/grpc-gateway/protoc-gen-grpc-gateway
+  go get -u github.com/grpc-ecosystem/grpc-gateway/protoc-gen-swagger
+  go get -u github.com/golang/protobuf/protoc-gen-go
+fi
+if [ ! -d $HOME/go/src/github.com/lightningnetwork/lnd ]; then
+  echo "You need an lnd with electrum-bridge (ysangkok/lnd maybe?) checked out since we implement the interface from there, and need it to generate code"
+  exit 1
+fi
+protoc -I$HOME/go/src/github.com/grpc-ecosystem/grpc-gateway/third_party/googleapis --python_out=lib/ln $HOME/go/src/github.com/grpc-ecosystem/grpc-gateway/third_party/googleapis/google/api/*.proto
+python3 -m grpc_tools.protoc -I $HOME/go/src/github.com/grpc-ecosystem/grpc-gateway/third_party/googleapis --proto_path $HOME/go/src/github.com/lightningnetwork/lnd/electrum-bridge --python_out=lib/ln --grpc_python_out=lib/ln ~/go/src/github.com/lightningnetwork/lnd/electrum-bridge/rpc.proto