RanchiMall/blockbook
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Support indexing of non UTXO chains

Browse Source
This commit is contained in:
Martin Boehm 2018-03-23 13:15:35 +01:00
parent 32a2c1a417
commit 44fdb5528b
3 changed files with 102 additions and 57 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
bchain/mempool.go
View File

@ -42,12 +42,11 @@ func (m *Mempool) GetTransactions(address string) ([]string, error) {
m.mux.Lock() m.mux.Lock()
defer m.mux.Unlock() defer m.mux.Unlock()
parser := m.chain.GetChainParser() parser := m.chain.GetChainParser()
buf, err := parser.GetAddrIDFromAddress(address) addrID, err := parser.GetAddrIDFromAddress(address)
if err != nil { if err != nil {
return nil, err return nil, err
} }
addrID := parser.UnpackAddrID(buf) outpoints := m.addrIDToTx[string(addrID)]
outpoints := m.addrIDToTx[addrID]
txs := make([]string, 0, len(outpoints)+len(outpoints)/2) txs := make([]string, 0, len(outpoints)+len(outpoints)/2)
for _, o := range outpoints { for _, o := range outpoints {
txs = append(txs, o.txid) txs = append(txs, o.txid)
@ -97,9 +96,13 @@ func (m *Mempool) Resync(onNewTxAddr func(txid string, addr string)) error {
} }
io.outputs = make([]scriptIndex, 0, len(tx.Vout)) io.outputs = make([]scriptIndex, 0, len(tx.Vout))
for _, output := range tx.Vout { for _, output := range tx.Vout {
addrID := parser.GetAddrIDFromVout(&output) addrID, err := parser.GetAddrIDFromVout(&output)
if addrID != "" { if err != nil {
io.outputs = append(io.outputs, scriptIndex{addrID, output.N}) glog.Error("error in addrID in ", txid, " ", output.N, ": ", err)
continue
}
if len(addrID) > 0 {
io.outputs = append(io.outputs, scriptIndex{string(addrID), output.N})
} }
if onNewTxAddr != nil && len(output.ScriptPubKey.Addresses) == 1 { if onNewTxAddr != nil && len(output.ScriptPubKey.Addresses) == 1 {
onNewTxAddr(tx.Txid, output.ScriptPubKey.Addresses[0]) onNewTxAddr(tx.Txid, output.ScriptPubKey.Addresses[0])

1
bchain/types.go
View File

@ -13,6 +13,7 @@ type Vin struct {
Vout uint32 `json:"vout"` Vout uint32 `json:"vout"`
ScriptSig ScriptSig `json:"scriptSig"` ScriptSig ScriptSig `json:"scriptSig"`
Sequence uint32 `json:"sequence"` Sequence uint32 `json:"sequence"`
Addresses []string `json:"addresses,omitempty"`
} }
type ScriptPubKey struct { type ScriptPubKey struct {

143
db/rocksdb.go
View File

@ -156,6 +156,8 @@ func (d *RocksDB) GetTransactions(address string, lower uint32, higher uint32, f
it := d.db.NewIteratorCF(d.ro, d.cfh[cfOutputs]) it := d.db.NewIteratorCF(d.ro, d.cfh[cfOutputs])
defer it.Close() defer it.Close()
isUTXO := d.chainParser.IsUTXOChain()
for it.Seek(kstart); it.Valid(); it.Next() { for it.Seek(kstart); it.Valid(); it.Next() {
key := it.Key().Data() key := it.Key().Data()
val := it.Value().Data() val := it.Value().Data()
@ -170,20 +172,31 @@ func (d *RocksDB) GetTransactions(address string, lower uint32, higher uint32, f
glog.Infof("rocksdb: output %s: %s", hex.EncodeToString(key), hex.EncodeToString(val)) glog.Infof("rocksdb: output %s: %s", hex.EncodeToString(key), hex.EncodeToString(val))
} }
for _, o := range outpoints { for _, o := range outpoints {
if err := fn(o.txid, o.vout, true); err != nil { var vout uint32
var isOutput bool
if o.vout < 0 {
vout = uint32(^o.vout)
isOutput = false
} else {
vout = uint32(o.vout)
isOutput = true
}
if err := fn(o.txid, vout, isOutput); err != nil {
return err return err
} }
stxid, so, err := d.GetSpentOutput(o.txid, o.vout) if isUTXO {
if err != nil { stxid, so, err := d.GetSpentOutput(o.txid, o.vout)
return err if err != nil {
}
if stxid != "" {
if glog.V(2) {
glog.Infof("rocksdb: input %s/%d: %s/%d", o.txid, o.vout, stxid, so)
}
if err := fn(stxid, so, false); err != nil {
return err return err
} }
if stxid != "" {
if glog.V(2) {
glog.Infof("rocksdb: input %s/%d: %s/%d", o.txid, o.vout, stxid, so)
}
if err := fn(stxid, uint32(so), false); err != nil {
return err
}
}
} }
} }
} }
@ -216,14 +229,18 @@ func (d *RocksDB) writeBlock(block *bchain.Block, op int) error {
} }
} }
isUTXO := d.chainParser.IsUTXOChain()
if err := d.writeHeight(wb, block, op); err != nil { if err := d.writeHeight(wb, block, op); err != nil {
return err return err
} }
if err := d.writeOutputs(wb, block, op); err != nil { if err := d.writeOutputs(wb, block, op, isUTXO); err != nil {
return err return err
} }
if err := d.writeInputs(wb, block, op); err != nil { if isUTXO {
return err if err := d.writeInputs(wb, block, op); err != nil {
return err
}
} }
return d.db.Write(d.wo, wb) return d.db.Write(d.wo, wb)
@ -233,30 +250,59 @@ func (d *RocksDB) writeBlock(block *bchain.Block, op int) error {
type outpoint struct { type outpoint struct {
txid string txid string
vout uint32 vout int32
} }
func (d *RocksDB) writeOutputs(wb *gorocksdb.WriteBatch, block *bchain.Block, op int) error { func (d *RocksDB) addAddrIDToRecords(op int, wb *gorocksdb.WriteBatch, records map[string][]outpoint, addrID []byte, txid string, vout int32, bh uint32) error {
if len(addrID) > 0 {
if len(addrID) > 1024 {
glog.Infof("block %d, skipping addrID of length %d", bh, len(addrID))
} else {
strAddrID := string(addrID)
records[strAddrID] = append(records[strAddrID], outpoint{
txid: txid,
vout: vout,
})
if op == opDelete {
// remove transactions from cache
b, err := packTxid(txid)
if err != nil {
return err
}
wb.DeleteCF(d.cfh[cfTransactions], b)
}
}
}
return nil
}
func (d *RocksDB) writeOutputs(wb *gorocksdb.WriteBatch, block *bchain.Block, op int, isUTXO bool) error {
records := make(map[string][]outpoint) records := make(map[string][]outpoint)
for _, tx := range block.Txs { for _, tx := range block.Txs {
for _, output := range tx.Vout { for _, output := range tx.Vout {
addrID := d.chainParser.GetAddrIDFromVout(&output) addrID, err := d.chainParser.GetAddrIDFromVout(&output)
if addrID != "" { if err != nil {
if len(addrID) > 1024 { glog.Warningf("rocksdb: addrID: %v - %d %s", err, block.Height, addrID)
glog.Infof("block %d, skipping addrID of length %d", block.Height, len(addrID)/2) continue
} else { }
records[addrID] = append(records[addrID], outpoint{ err = d.addAddrIDToRecords(op, wb, records, addrID, tx.Txid, int32(output.N), block.Height)
txid: tx.Txid, if err != nil {
vout: output.N, return err
}) }
if op == opDelete { }
// remove transactions from cache if !isUTXO {
b, err := packTxid(tx.Txid) // store inputs in output column in format txid ^index
if err != nil { for _, input := range tx.Vin {
return err for i, a := range input.Addresses {
} addrID, err := d.chainParser.GetAddrIDFromAddress(a)
wb.DeleteCF(d.cfh[cfTransactions], b) if err != nil {
glog.Warningf("rocksdb: addrID: %v - %d %s", err, block.Height, addrID)
continue
}
err = d.addAddrIDToRecords(op, wb, records, addrID, tx.Txid, int32(^i), block.Height)
if err != nil {
return err
} }
} }
} }
@ -264,12 +310,7 @@ func (d *RocksDB) writeOutputs(wb *gorocksdb.WriteBatch, block *bchain.Block, op
} }
for addrID, outpoints := range records { for addrID, outpoints := range records {
bOutid, err := d.chainParser.PackAddrID(addrID) key, err := packOutputKey([]byte(addrID), block.Height)
if err != nil {
glog.Warningf("rocksdb: addrID: %v - %d %s", err, block.Height, addrID)
continue
}
key, err := packOutputKey(bOutid, block.Height)
if err != nil { if err != nil {
glog.Warningf("rocksdb: packOutputKey: %v - %d %s", err, block.Height, addrID) glog.Warningf("rocksdb: packOutputKey: %v - %d %s", err, block.Height, addrID)
continue continue
@ -306,7 +347,7 @@ func packOutputValue(outpoints []outpoint) ([]byte, error) {
if err != nil { if err != nil {
return nil, err return nil, err
} }
bvout := packVaruint(o.vout) bvout := packVarint(o.vout)
buf = append(buf, btxid...) buf = append(buf, btxid...)
buf = append(buf, bvout...) buf = append(buf, bvout...)
} }
@ -321,7 +362,7 @@ func unpackOutputValue(buf []byte) ([]outpoint, error) {
return nil, err return nil, err
} }
i += txIdUnpackedLen i += txIdUnpackedLen
vout, voutLen := unpackVaruint(buf[i:]) vout, voutLen := unpackVarint(buf[i:])
i += voutLen i += voutLen
outpoints = append(outpoints, outpoint{ outpoints = append(outpoints, outpoint{
txid: txid, txid: txid,
@ -343,11 +384,11 @@ func (d *RocksDB) writeInputs(
if input.Coinbase != "" { if input.Coinbase != "" {
continue continue
} }
key, err := packOutpoint(input.Txid, input.Vout) key, err := packOutpoint(input.Txid, int32(input.Vout))
if err != nil { if err != nil {
return err return err
} }
val, err := packOutpoint(tx.Txid, uint32(i)) val, err := packOutpoint(tx.Txid, int32(i))
if err != nil { if err != nil {
return err return err
} }
@ -362,21 +403,21 @@ func (d *RocksDB) writeInputs(
return nil return nil
} }
func packOutpoint(txid string, vout uint32) ([]byte, error) { func packOutpoint(txid string, vout int32) ([]byte, error) {
btxid, err := packTxid(txid) btxid, err := packTxid(txid)
if err != nil { if err != nil {
return nil, err return nil, err
} }
bvout := packVaruint(vout) bvout := packVarint(vout)
buf := make([]byte, 0, len(btxid)+len(bvout)) buf := make([]byte, 0, len(btxid)+len(bvout))
buf = append(buf, btxid...) buf = append(buf, btxid...)
buf = append(buf, bvout...) buf = append(buf, bvout...)
return buf, nil return buf, nil
} }
func unpackOutpoint(buf []byte) (string, uint32, int) { func unpackOutpoint(buf []byte) (string, int32, int) {
txid, _ := unpackTxid(buf[:txIdUnpackedLen]) txid, _ := unpackTxid(buf[:txIdUnpackedLen])
vout, o := unpackVaruint(buf[txIdUnpackedLen:]) vout, o := unpackVarint(buf[txIdUnpackedLen:])
return txid, vout, txIdUnpackedLen + o return txid, vout, txIdUnpackedLen + o
} }
@ -409,7 +450,7 @@ func (d *RocksDB) GetBlockHash(height uint32) (string, error) {
} }
// GetSpentOutput returns output which is spent by input tx // GetSpentOutput returns output which is spent by input tx
func (d *RocksDB) GetSpentOutput(txid string, i uint32) (string, uint32, error) { func (d *RocksDB) GetSpentOutput(txid string, i int32) (string, int32, error) {
b, err := packOutpoint(txid, i) b, err := packOutpoint(txid, i)
if err != nil { if err != nil {
return "", 0, err return "", 0, err
@ -424,7 +465,7 @@ func (d *RocksDB) GetSpentOutput(txid string, i uint32) (string, uint32, error)
return "", 0, err return "", 0, err
} }
var otxid string var otxid string
var oi uint32 var oi int32
for _, i := range p { for _, i := range p {
otxid, oi = i.txid, i.vout otxid, oi = i.txid, i.vout
break break
@ -627,15 +668,15 @@ func unpackFloat64(buf []byte) float64 {
return math.Float64frombits(binary.BigEndian.Uint64(buf)) return math.Float64frombits(binary.BigEndian.Uint64(buf))
} }
func packVaruint(i uint32) []byte { func packVarint(i int32) []byte {
buf := make([]byte, vlq.MaxLen32) buf := make([]byte, vlq.MaxLen32)
ofs := vlq.PutUint(buf, uint64(i)) ofs := vlq.PutInt(buf, int64(i))
return buf[:ofs] return buf[:ofs]
} }
func unpackVaruint(buf []byte) (uint32, int) { func unpackVarint(buf []byte) (int32, int) {
i, ofs := vlq.Uint(buf) i, ofs := vlq.Uint(buf)
return uint32(i), ofs return int32(i), ofs
} }
func packVarint64(i int64) []byte { func packVarint64(i int64) []byte {