RanchiMall/blockbook
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Store addresses in reverse order for newest blocks to be searched first

Browse Source
This commit is contained in:
Martin Boehm 2018-12-20 17:33:13 +01:00
parent e24115da83
commit 4e040cb1f0
5 changed files with 58 additions and 53 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
api/worker.go
View File

@ -355,20 +355,20 @@ func (t *Tx) getAddrVinValue(addrDesc bchain.AddressDescriptor) *big.Int {
return &val return &val
} }
// UniqueTxidsInReverse reverts the order of transactions (so that newest are first) and removes duplicate transactions // GetUniqueTxids removes duplicate transactions
func UniqueTxidsInReverse(txids []string) []string { func GetUniqueTxids(txids []string) []string {
i := len(txids) ut := make([]string, len(txids))
ut := make([]string, i)
txidsMap := make(map[string]struct{}) txidsMap := make(map[string]struct{})
i := 0
for _, txid := range txids { for _, txid := range txids {
_, e := txidsMap[txid] _, e := txidsMap[txid]
if !e { if !e {
i--
ut[i] = txid ut[i] = txid
i++
txidsMap[txid] = struct{}{} txidsMap[txid] = struct{}{}
} }
} }
return ut[i:] return ut[0:i]
} }
func (w *Worker) txFromTxAddress(txid string, ta *db.TxAddresses, bi *db.BlockInfo, bestheight uint32) *Tx { func (w *Worker) txFromTxAddress(txid string, ta *db.TxAddresses, bi *db.BlockInfo, bestheight uint32) *Tx {
@ -578,7 +578,7 @@ func (w *Worker) GetAddress(address string, page int, txsOnPage int, option GetA
if err != nil { if err != nil {
return nil, errors.Annotatef(err, "getAddressTxids %v true", addrDesc) return nil, errors.Annotatef(err, "getAddressTxids %v true", addrDesc)
} }
txm = UniqueTxidsInReverse(txm) txm = GetUniqueTxids(txm)
// if there are only unconfirmed transactions, there is no paging // if there are only unconfirmed transactions, there is no paging
if ba == nil { if ba == nil {
ba = &db.AddrBalance{} ba = &db.AddrBalance{}
@ -589,7 +589,7 @@ func (w *Worker) GetAddress(address string, page int, txsOnPage int, option GetA
if err != nil { if err != nil {
return nil, errors.Annotatef(err, "getAddressTxids %v false", addrDesc) return nil, errors.Annotatef(err, "getAddressTxids %v false", addrDesc)
} }
txc = UniqueTxidsInReverse(txc) txc = GetUniqueTxids(txc)
bestheight, _, err := w.db.GetBestBlock() bestheight, _, err := w.db.GetBestBlock()
if err != nil { if err != nil {
return nil, errors.Annotatef(err, "GetBestBlock") return nil, errors.Annotatef(err, "GetBestBlock")
@ -700,7 +700,7 @@ func (w *Worker) GetAddressUtxo(address string, onlyConfirmed bool) ([]AddressUt
if err != nil { if err != nil {
return nil, errors.Annotatef(err, "getAddressTxids %v true", address) return nil, errors.Annotatef(err, "getAddressTxids %v true", address)
} }
txm = UniqueTxidsInReverse(txm) txm = GetUniqueTxids(txm)
mc := make([]*bchain.Tx, len(txm)) mc := make([]*bchain.Tx, len(txm))
for i, txid := range txm { for i, txid := range txm {
// get mempool txs and process their inputs to detect spends between mempool txs // get mempool txs and process their inputs to detect spends between mempool txs

17
db/rocksdb.go
View File

@ -222,10 +222,10 @@ func (d *RocksDB) GetTransactions(address string, lower uint32, higher uint32, f
} }
// GetAddrDescTransactions finds all input/output transactions for address descriptor // GetAddrDescTransactions finds all input/output transactions for address descriptor
// Transaction are passed to callback function. // Transaction are passed to callback function in the order from newest block to the oldest
func (d *RocksDB) GetAddrDescTransactions(addrDesc bchain.AddressDescriptor, lower uint32, higher uint32, fn func(txid string, vout int32, isOutput bool) error) (err error) { func (d *RocksDB) GetAddrDescTransactions(addrDesc bchain.AddressDescriptor, lower uint32, higher uint32, fn func(txid string, vout int32, isOutput bool) error) (err error) {
kstart := packAddressKey(addrDesc, lower) kstart := packAddressKey(addrDesc, higher)
kstop := packAddressKey(addrDesc, higher) kstop := packAddressKey(addrDesc, lower)
it := d.db.NewIteratorCF(d.ro, d.cfh[cfAddresses]) it := d.db.NewIteratorCF(d.ro, d.cfh[cfAddresses])
defer it.Close() defer it.Close()
@ -1370,10 +1370,10 @@ func (d *RocksDB) ComputeInternalStateColumnStats(stopCompute chan os.Signal) er
// Helpers // Helpers
func packAddressKey(addrDesc bchain.AddressDescriptor, height uint32) []byte { func packAddressKey(addrDesc bchain.AddressDescriptor, height uint32) []byte {
bheight := packUint(height) buf := make([]byte, len(addrDesc)+packedHeightBytes)
buf := make([]byte, 0, len(addrDesc)+len(bheight)) copy(buf, addrDesc)
buf = append(buf, addrDesc...) // pack height as binary complement to achieve ordering from newest to oldest block
buf = append(buf, bheight...) binary.BigEndian.PutUint32(buf[len(addrDesc):], ^height)
return buf return buf
} }
@ -1382,7 +1382,8 @@ func unpackAddressKey(key []byte) ([]byte, uint32, error) {
if i <= 0 { if i <= 0 {
return nil, 0, errors.New("Invalid address key") return nil, 0, errors.New("Invalid address key")
} }
return key[:i], unpackUint(key[i : i+packedHeightBytes]), nil // height is packed in binary complement, convert it
return key[:i], ^unpackUint(key[i : i+packedHeightBytes]), nil
} }
func packUint(i uint32) []byte { func packUint(i uint32) []byte {

30
db/rocksdb_ethereumtype_test.go
View File

@ -34,10 +34,10 @@ func verifyAfterEthereumTypeBlock1(t *testing.T, d *RocksDB, afterDisconnect boo
} }
// the vout is encoded as signed varint, i.e. value * 2 for positive values, abs(value)*2 + 1 for negative values // the vout is encoded as signed varint, i.e. value * 2 for positive values, abs(value)*2 + 1 for negative values
if err := checkColumn(d, cfAddresses, []keyPair{ if err := checkColumn(d, cfAddresses, []keyPair{
keyPair{dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddr3e, d.chainParser) + "0041eee8", dbtestdata.EthTxidB1T1 + "01", nil}, keyPair{addressKeyHex(dbtestdata.EthAddr3e, 4321000, d), dbtestdata.EthTxidB1T1 + "01", nil},
keyPair{dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddr55, d.chainParser) + "0041eee8", dbtestdata.EthTxidB1T1 + "00" + dbtestdata.EthTxidB1T2 + "02", nil}, keyPair{addressKeyHex(dbtestdata.EthAddr55, 4321000, d), dbtestdata.EthTxidB1T1 + "00" + dbtestdata.EthTxidB1T2 + "02", nil},
keyPair{dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddr20, d.chainParser) + "0041eee8", dbtestdata.EthTxidB1T2 + "01" + dbtestdata.EthTxidB1T2 + "03", nil}, keyPair{addressKeyHex(dbtestdata.EthAddr20, 4321000, d), dbtestdata.EthTxidB1T2 + "01" + dbtestdata.EthTxidB1T2 + "03", nil},
keyPair{dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddrContract4a, d.chainParser) + "0041eee8", dbtestdata.EthTxidB1T2 + "00", nil}, keyPair{addressKeyHex(dbtestdata.EthAddrContract4a, 4321000, d), dbtestdata.EthTxidB1T2 + "00", nil},
}); err != nil { }); err != nil {
{ {
t.Fatal(err) t.Fatal(err)
@ -99,15 +99,15 @@ func verifyAfterEthereumTypeBlock2(t *testing.T, d *RocksDB) {
} }
// the vout is encoded as signed varint, i.e. value * 2 for positive values, abs(value)*2 + 1 for negative values // the vout is encoded as signed varint, i.e. value * 2 for positive values, abs(value)*2 + 1 for negative values
if err := checkColumn(d, cfAddresses, []keyPair{ if err := checkColumn(d, cfAddresses, []keyPair{
keyPair{dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddr3e, d.chainParser) + "0041eee8", dbtestdata.EthTxidB1T1 + "01", nil}, keyPair{addressKeyHex(dbtestdata.EthAddr3e, 4321000, d), dbtestdata.EthTxidB1T1 + "01", nil},
keyPair{dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddr55, d.chainParser) + "0041eee8", dbtestdata.EthTxidB1T1 + "00" + dbtestdata.EthTxidB1T2 + "02", nil}, keyPair{addressKeyHex(dbtestdata.EthAddr55, 4321000, d), dbtestdata.EthTxidB1T1 + "00" + dbtestdata.EthTxidB1T2 + "02", nil},
keyPair{dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddr20, d.chainParser) + "0041eee8", dbtestdata.EthTxidB1T2 + "01" + dbtestdata.EthTxidB1T2 + "03", nil}, keyPair{addressKeyHex(dbtestdata.EthAddr20, 4321000, d), dbtestdata.EthTxidB1T2 + "01" + dbtestdata.EthTxidB1T2 + "03", nil},
keyPair{dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddrContract4a, d.chainParser) + "0041eee8", dbtestdata.EthTxidB1T2 + "00", nil}, keyPair{addressKeyHex(dbtestdata.EthAddrContract4a, 4321000, d), dbtestdata.EthTxidB1T2 + "00", nil},
keyPair{dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddr55, d.chainParser) + "0041eee9", dbtestdata.EthTxidB2T1 + "01" + dbtestdata.EthTxidB2T2 + "05" + dbtestdata.EthTxidB2T2 + "02", nil}, keyPair{addressKeyHex(dbtestdata.EthAddr55, 4321001, d), dbtestdata.EthTxidB2T1 + "01" + dbtestdata.EthTxidB2T2 + "05" + dbtestdata.EthTxidB2T2 + "02", nil},
keyPair{dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddr9f, d.chainParser) + "0041eee9", dbtestdata.EthTxidB2T1 + "00", nil}, keyPair{addressKeyHex(dbtestdata.EthAddr9f, 4321001, d), dbtestdata.EthTxidB2T1 + "00", nil},
keyPair{dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddr4b, d.chainParser) + "0041eee9", dbtestdata.EthTxidB2T2 + "01" + dbtestdata.EthTxidB2T2 + "02" + dbtestdata.EthTxidB2T2 + "05" + dbtestdata.EthTxidB2T2 + "04" + dbtestdata.EthTxidB2T2 + "03", nil}, keyPair{addressKeyHex(dbtestdata.EthAddr4b, 4321001, d), dbtestdata.EthTxidB2T2 + "01" + dbtestdata.EthTxidB2T2 + "02" + dbtestdata.EthTxidB2T2 + "05" + dbtestdata.EthTxidB2T2 + "04" + dbtestdata.EthTxidB2T2 + "03", nil},
keyPair{dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddr7b, d.chainParser) + "0041eee9", dbtestdata.EthTxidB2T2 + "03" + dbtestdata.EthTxidB2T2 + "04", nil}, keyPair{addressKeyHex(dbtestdata.EthAddr7b, 4321001, d), dbtestdata.EthTxidB2T2 + "03" + dbtestdata.EthTxidB2T2 + "04", nil},
keyPair{dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddrContract47, d.chainParser) + "0041eee9", dbtestdata.EthTxidB2T2 + "00", nil}, keyPair{addressKeyHex(dbtestdata.EthAddrContract47, 4321001, d), dbtestdata.EthTxidB2T2 + "00", nil},
}); err != nil { }); err != nil {
{ {
t.Fatal(err) t.Fatal(err)
@ -185,11 +185,11 @@ func TestRocksDB_Index_EthereumType(t *testing.T) {
// get transactions for various addresses / low-high ranges // get transactions for various addresses / low-high ranges
verifyGetTransactions(t, d, "0x"+dbtestdata.EthAddr55, 0, 10000000, []txidVoutOutput{ verifyGetTransactions(t, d, "0x"+dbtestdata.EthAddr55, 0, 10000000, []txidVoutOutput{
txidVoutOutput{"0x" + dbtestdata.EthTxidB1T1, 0, true},
txidVoutOutput{"0x" + dbtestdata.EthTxidB1T2, 1, true},
txidVoutOutput{"0x" + dbtestdata.EthTxidB2T1, 0, false}, txidVoutOutput{"0x" + dbtestdata.EthTxidB2T1, 0, false},
txidVoutOutput{"0x" + dbtestdata.EthTxidB2T2, 2, false}, txidVoutOutput{"0x" + dbtestdata.EthTxidB2T2, 2, false},
txidVoutOutput{"0x" + dbtestdata.EthTxidB2T2, 1, true}, txidVoutOutput{"0x" + dbtestdata.EthTxidB2T2, 1, true},
txidVoutOutput{"0x" + dbtestdata.EthTxidB1T1, 0, true},
txidVoutOutput{"0x" + dbtestdata.EthTxidB1T2, 1, true},
}, nil) }, nil)
verifyGetTransactions(t, d, "mtGXQvBowMkBpnhLckhxhbwYK44Gs9eBad", 500000, 1000000, []txidVoutOutput{}, errors.New("Address missing")) verifyGetTransactions(t, d, "mtGXQvBowMkBpnhLckhxhbwYK44Gs9eBad", 500000, 1000000, []txidVoutOutput{}, errors.New("Address missing"))

44
db/rocksdb_test.go
View File

@ -99,6 +99,10 @@ func uintToHex(i uint32) string {
return hex.EncodeToString(buf) return hex.EncodeToString(buf)
} }
func addressKeyHex(a string, height uint32, d *RocksDB) string {
return dbtestdata.AddressToPubKeyHex(a, d.chainParser) + uintToHex(^height)
}
// keyPair is used to compare given key value in DB with expected // keyPair is used to compare given key value in DB with expected
// for more complicated compares it is possible to specify CompareFunc // for more complicated compares it is possible to specify CompareFunc
type keyPair struct { type keyPair struct {
@ -168,11 +172,11 @@ func verifyAfterBitcoinTypeBlock1(t *testing.T, d *RocksDB, afterDisconnect bool
} }
// the vout is encoded as signed varint, i.e. value * 2 for non negative values // the vout is encoded as signed varint, i.e. value * 2 for non negative values
if err := checkColumn(d, cfAddresses, []keyPair{ if err := checkColumn(d, cfAddresses, []keyPair{
keyPair{dbtestdata.AddressToPubKeyHex(dbtestdata.Addr1, d.chainParser) + "000370d5", dbtestdata.TxidB1T1 + "00", nil}, keyPair{addressKeyHex(dbtestdata.Addr1, 225493, d), dbtestdata.TxidB1T1 + "00", nil},
keyPair{dbtestdata.AddressToPubKeyHex(dbtestdata.Addr2, d.chainParser) + "000370d5", dbtestdata.TxidB1T1 + "02", nil}, keyPair{addressKeyHex(dbtestdata.Addr2, 225493, d), dbtestdata.TxidB1T1 + "02", nil},
keyPair{dbtestdata.AddressToPubKeyHex(dbtestdata.Addr3, d.chainParser) + "000370d5", dbtestdata.TxidB1T2 + "00", nil}, keyPair{addressKeyHex(dbtestdata.Addr3, 225493, d), dbtestdata.TxidB1T2 + "00", nil},
keyPair{dbtestdata.AddressToPubKeyHex(dbtestdata.Addr4, d.chainParser) + "000370d5", dbtestdata.TxidB1T2 + "02", nil}, keyPair{addressKeyHex(dbtestdata.Addr4, 225493, d), dbtestdata.TxidB1T2 + "02", nil},
keyPair{dbtestdata.AddressToPubKeyHex(dbtestdata.Addr5, d.chainParser) + "000370d5", dbtestdata.TxidB1T2 + "04", nil}, keyPair{addressKeyHex(dbtestdata.Addr5, 225493, d), dbtestdata.TxidB1T2 + "04", nil},
}); err != nil { }); err != nil {
{ {
t.Fatal(err) t.Fatal(err)
@ -253,20 +257,20 @@ func verifyAfterBitcoinTypeBlock2(t *testing.T, d *RocksDB) {
} }
} }
if err := checkColumn(d, cfAddresses, []keyPair{ if err := checkColumn(d, cfAddresses, []keyPair{
keyPair{dbtestdata.AddressToPubKeyHex(dbtestdata.Addr1, d.chainParser) + "000370d5", dbtestdata.TxidB1T1 + "00", nil}, keyPair{addressKeyHex(dbtestdata.Addr1, 225493, d), dbtestdata.TxidB1T1 + "00", nil},
keyPair{dbtestdata.AddressToPubKeyHex(dbtestdata.Addr2, d.chainParser) + "000370d5", dbtestdata.TxidB1T1 + "02", nil}, keyPair{addressKeyHex(dbtestdata.Addr2, 225493, d), dbtestdata.TxidB1T1 + "02", nil},
keyPair{dbtestdata.AddressToPubKeyHex(dbtestdata.Addr3, d.chainParser) + "000370d5", dbtestdata.TxidB1T2 + "00", nil}, keyPair{addressKeyHex(dbtestdata.Addr3, 225493, d), dbtestdata.TxidB1T2 + "00", nil},
keyPair{dbtestdata.AddressToPubKeyHex(dbtestdata.Addr4, d.chainParser) + "000370d5", dbtestdata.TxidB1T2 + "02", nil}, keyPair{addressKeyHex(dbtestdata.Addr4, 225493, d), dbtestdata.TxidB1T2 + "02", nil},
keyPair{dbtestdata.AddressToPubKeyHex(dbtestdata.Addr5, d.chainParser) + "000370d5", dbtestdata.TxidB1T2 + "04", nil}, keyPair{addressKeyHex(dbtestdata.Addr5, 225493, d), dbtestdata.TxidB1T2 + "04", nil},
keyPair{dbtestdata.AddressToPubKeyHex(dbtestdata.Addr6, d.chainParser) + "000370d6", dbtestdata.TxidB2T1 + "00" + dbtestdata.TxidB2T2 + "01", nil}, keyPair{addressKeyHex(dbtestdata.Addr6, 225494, d), dbtestdata.TxidB2T1 + "00" + dbtestdata.TxidB2T2 + "01", nil},
keyPair{dbtestdata.AddressToPubKeyHex(dbtestdata.Addr7, d.chainParser) + "000370d6", dbtestdata.TxidB2T1 + "02", nil}, keyPair{addressKeyHex(dbtestdata.Addr7, 225494, d), dbtestdata.TxidB2T1 + "02", nil},
keyPair{dbtestdata.AddressToPubKeyHex(dbtestdata.Addr8, d.chainParser) + "000370d6", dbtestdata.TxidB2T2 + "00", nil}, keyPair{addressKeyHex(dbtestdata.Addr8, 225494, d), dbtestdata.TxidB2T2 + "00", nil},
keyPair{dbtestdata.AddressToPubKeyHex(dbtestdata.Addr9, d.chainParser) + "000370d6", dbtestdata.TxidB2T2 + "02", nil}, keyPair{addressKeyHex(dbtestdata.Addr9, 225494, d), dbtestdata.TxidB2T2 + "02", nil},
keyPair{dbtestdata.AddressToPubKeyHex(dbtestdata.Addr3, d.chainParser) + "000370d6", dbtestdata.TxidB2T1 + "01", nil}, keyPair{addressKeyHex(dbtestdata.Addr3, 225494, d), dbtestdata.TxidB2T1 + "01", nil},
keyPair{dbtestdata.AddressToPubKeyHex(dbtestdata.Addr2, d.chainParser) + "000370d6", dbtestdata.TxidB2T1 + "03", nil}, keyPair{addressKeyHex(dbtestdata.Addr2, 225494, d), dbtestdata.TxidB2T1 + "03", nil},
keyPair{dbtestdata.AddressToPubKeyHex(dbtestdata.Addr5, d.chainParser) + "000370d6", dbtestdata.TxidB2T3 + "00" + dbtestdata.TxidB2T3 + "01", nil}, keyPair{addressKeyHex(dbtestdata.Addr5, 225494, d), dbtestdata.TxidB2T3 + "00" + dbtestdata.TxidB2T3 + "01", nil},
keyPair{dbtestdata.AddressToPubKeyHex(dbtestdata.AddrA, d.chainParser) + "000370d6", dbtestdata.TxidB2T4 + "00", nil}, keyPair{addressKeyHex(dbtestdata.AddrA, 225494, d), dbtestdata.TxidB2T4 + "00", nil},
keyPair{dbtestdata.AddressToPubKeyHex(dbtestdata.Addr4, d.chainParser) + "000370d6", dbtestdata.TxidB2T2 + "03", nil}, keyPair{addressKeyHex(dbtestdata.Addr4, 225494, d), dbtestdata.TxidB2T2 + "03", nil},
}); err != nil { }); err != nil {
{ {
t.Fatal(err) t.Fatal(err)
@ -453,8 +457,8 @@ func TestRocksDB_Index_BitcoinType(t *testing.T) {
// get transactions for various addresses / low-high ranges // get transactions for various addresses / low-high ranges
verifyGetTransactions(t, d, dbtestdata.Addr2, 0, 1000000, []txidVoutOutput{ verifyGetTransactions(t, d, dbtestdata.Addr2, 0, 1000000, []txidVoutOutput{
txidVoutOutput{dbtestdata.TxidB1T1, 1, true},
txidVoutOutput{dbtestdata.TxidB2T1, 1, false}, txidVoutOutput{dbtestdata.TxidB2T1, 1, false},
txidVoutOutput{dbtestdata.TxidB1T1, 1, true},
}, nil) }, nil)
verifyGetTransactions(t, d, dbtestdata.Addr2, 225493, 225493, []txidVoutOutput{ verifyGetTransactions(t, d, dbtestdata.Addr2, 225493, 225493, []txidVoutOutput{
txidVoutOutput{dbtestdata.TxidB1T1, 1, true}, txidVoutOutput{dbtestdata.TxidB1T1, 1, true},

2
server/socketio.go
View File

@ -233,7 +233,7 @@ func (s *SocketIoServer) getAddressTxids(addr []string, opts *addrOpts) (res res
} }
} }
} }
res.Result = api.UniqueTxidsInReverse(txids) res.Result = api.GetUniqueTxids(txids)
return res, nil return res, nil
} }