RanchiMall/blockbook
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Store addresses in reverse order for newest blocks to be searched first

Browse Source
This commit is contained in:
Martin Boehm 2018-12-20 17:33:13 +01:00
parent e24115da83
commit 4e040cb1f0
5 changed files with 58 additions and 53 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
api/worker.go
View File

@ -355,20 +355,20 @@ func (t *Tx) getAddrVinValue(addrDesc bchain.AddressDescriptor) *big.Int {
return &val
}
// UniqueTxidsInReverse reverts the order of transactions (so that newest are first) and removes duplicate transactions
func UniqueTxidsInReverse(txids []string) []string {
i := len(txids)
ut := make([]string, i)
// GetUniqueTxids removes duplicate transactions
func GetUniqueTxids(txids []string) []string {
ut := make([]string, len(txids))
txidsMap := make(map[string]struct{})
i := 0
for _, txid := range txids {
_, e := txidsMap[txid]
if !e {
i--
ut[i] = txid
i++
txidsMap[txid] = struct{}{}
}
}
return ut[i:]
return ut[0:i]
}
func (w *Worker) txFromTxAddress(txid string, ta *db.TxAddresses, bi *db.BlockInfo, bestheight uint32) *Tx {
@ -578,7 +578,7 @@ func (w *Worker) GetAddress(address string, page int, txsOnPage int, option GetA
if err != nil {
return nil, errors.Annotatef(err, "getAddressTxids %v true", addrDesc)
}
txm = UniqueTxidsInReverse(txm)
txm = GetUniqueTxids(txm)
// if there are only unconfirmed transactions, there is no paging
if ba == nil {
ba = &db.AddrBalance{}
@ -589,7 +589,7 @@ func (w *Worker) GetAddress(address string, page int, txsOnPage int, option GetA
if err != nil {
return nil, errors.Annotatef(err, "getAddressTxids %v false", addrDesc)
}
txc = UniqueTxidsInReverse(txc)
txc = GetUniqueTxids(txc)
bestheight, _, err := w.db.GetBestBlock()
if err != nil {
return nil, errors.Annotatef(err, "GetBestBlock")
@ -700,7 +700,7 @@ func (w *Worker) GetAddressUtxo(address string, onlyConfirmed bool) ([]AddressUt
if err != nil {
return nil, errors.Annotatef(err, "getAddressTxids %v true", address)
}
txm = UniqueTxidsInReverse(txm)
txm = GetUniqueTxids(txm)
mc := make([]*bchain.Tx, len(txm))
for i, txid := range txm {
// get mempool txs and process their inputs to detect spends between mempool txs

17
db/rocksdb.go
View File

@ -222,10 +222,10 @@ func (d *RocksDB) GetTransactions(address string, lower uint32, higher uint32, f
}
// GetAddrDescTransactions finds all input/output transactions for address descriptor
// Transaction are passed to callback function.
// Transaction are passed to callback function in the order from newest block to the oldest
func (d *RocksDB) GetAddrDescTransactions(addrDesc bchain.AddressDescriptor, lower uint32, higher uint32, fn func(txid string, vout int32, isOutput bool) error) (err error) {
kstart := packAddressKey(addrDesc, lower)
kstop := packAddressKey(addrDesc, higher)
kstart := packAddressKey(addrDesc, higher)
kstop := packAddressKey(addrDesc, lower)
it := d.db.NewIteratorCF(d.ro, d.cfh[cfAddresses])
defer it.Close()
@ -1370,10 +1370,10 @@ func (d *RocksDB) ComputeInternalStateColumnStats(stopCompute chan os.Signal) er
// Helpers
func packAddressKey(addrDesc bchain.AddressDescriptor, height uint32) []byte {
bheight := packUint(height)
buf := make([]byte, 0, len(addrDesc)+len(bheight))
buf = append(buf, addrDesc...)
buf = append(buf, bheight...)
buf := make([]byte, len(addrDesc)+packedHeightBytes)
copy(buf, addrDesc)
// pack height as binary complement to achieve ordering from newest to oldest block
binary.BigEndian.PutUint32(buf[len(addrDesc):], ^height)
return buf
}
@ -1382,7 +1382,8 @@ func unpackAddressKey(key []byte) ([]byte, uint32, error) {
if i <= 0 {
return nil, 0, errors.New("Invalid address key")
}
return key[:i], unpackUint(key[i : i+packedHeightBytes]), nil
// height is packed in binary complement, convert it
return key[:i], ^unpackUint(key[i : i+packedHeightBytes]), nil
}
func packUint(i uint32) []byte {

30
db/rocksdb_ethereumtype_test.go
View File

@ -34,10 +34,10 @@ func verifyAfterEthereumTypeBlock1(t *testing.T, d *RocksDB, afterDisconnect boo
}
// the vout is encoded as signed varint, i.e. value * 2 for positive values, abs(value)*2 + 1 for negative values
if err := checkColumn(d, cfAddresses, []keyPair{
keyPair{dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddr3e, d.chainParser) + "0041eee8", dbtestdata.EthTxidB1T1 + "01", nil},
keyPair{dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddr55, d.chainParser) + "0041eee8", dbtestdata.EthTxidB1T1 + "00" + dbtestdata.EthTxidB1T2 + "02", nil},
keyPair{dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddr20, d.chainParser) + "0041eee8", dbtestdata.EthTxidB1T2 + "01" + dbtestdata.EthTxidB1T2 + "03", nil},
keyPair{dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddrContract4a, d.chainParser) + "0041eee8", dbtestdata.EthTxidB1T2 + "00", nil},
keyPair{addressKeyHex(dbtestdata.EthAddr3e, 4321000, d), dbtestdata.EthTxidB1T1 + "01", nil},
keyPair{addressKeyHex(dbtestdata.EthAddr55, 4321000, d), dbtestdata.EthTxidB1T1 + "00" + dbtestdata.EthTxidB1T2 + "02", nil},
keyPair{addressKeyHex(dbtestdata.EthAddr20, 4321000, d), dbtestdata.EthTxidB1T2 + "01" + dbtestdata.EthTxidB1T2 + "03", nil},
keyPair{addressKeyHex(dbtestdata.EthAddrContract4a, 4321000, d), dbtestdata.EthTxidB1T2 + "00", nil},
}); err != nil {
{
t.Fatal(err)
@ -99,15 +99,15 @@ func verifyAfterEthereumTypeBlock2(t *testing.T, d *RocksDB) {
}
// the vout is encoded as signed varint, i.e. value * 2 for positive values, abs(value)*2 + 1 for negative values
if err := checkColumn(d, cfAddresses, []keyPair{
keyPair{dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddr3e, d.chainParser) + "0041eee8", dbtestdata.EthTxidB1T1 + "01", nil},
keyPair{dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddr55, d.chainParser) + "0041eee8", dbtestdata.EthTxidB1T1 + "00" + dbtestdata.EthTxidB1T2 + "02", nil},
keyPair{dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddr20, d.chainParser) + "0041eee8", dbtestdata.EthTxidB1T2 + "01" + dbtestdata.EthTxidB1T2 + "03", nil},
keyPair{dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddrContract4a, d.chainParser) + "0041eee8", dbtestdata.EthTxidB1T2 + "00", nil},
keyPair{dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddr55, d.chainParser) + "0041eee9", dbtestdata.EthTxidB2T1 + "01" + dbtestdata.EthTxidB2T2 + "05" + dbtestdata.EthTxidB2T2 + "02", nil},
keyPair{dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddr9f, d.chainParser) + "0041eee9", dbtestdata.EthTxidB2T1 + "00", nil},
keyPair{dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddr4b, d.chainParser) + "0041eee9", dbtestdata.EthTxidB2T2 + "01" + dbtestdata.EthTxidB2T2 + "02" + dbtestdata.EthTxidB2T2 + "05" + dbtestdata.EthTxidB2T2 + "04" + dbtestdata.EthTxidB2T2 + "03", nil},
keyPair{dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddr7b, d.chainParser) + "0041eee9", dbtestdata.EthTxidB2T2 + "03" + dbtestdata.EthTxidB2T2 + "04", nil},
keyPair{dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddrContract47, d.chainParser) + "0041eee9", dbtestdata.EthTxidB2T2 + "00", nil},
keyPair{addressKeyHex(dbtestdata.EthAddr3e, 4321000, d), dbtestdata.EthTxidB1T1 + "01", nil},
keyPair{addressKeyHex(dbtestdata.EthAddr55, 4321000, d), dbtestdata.EthTxidB1T1 + "00" + dbtestdata.EthTxidB1T2 + "02", nil},
keyPair{addressKeyHex(dbtestdata.EthAddr20, 4321000, d), dbtestdata.EthTxidB1T2 + "01" + dbtestdata.EthTxidB1T2 + "03", nil},
keyPair{addressKeyHex(dbtestdata.EthAddrContract4a, 4321000, d), dbtestdata.EthTxidB1T2 + "00", nil},
keyPair{addressKeyHex(dbtestdata.EthAddr55, 4321001, d), dbtestdata.EthTxidB2T1 + "01" + dbtestdata.EthTxidB2T2 + "05" + dbtestdata.EthTxidB2T2 + "02", nil},
keyPair{addressKeyHex(dbtestdata.EthAddr9f, 4321001, d), dbtestdata.EthTxidB2T1 + "00", nil},
keyPair{addressKeyHex(dbtestdata.EthAddr4b, 4321001, d), dbtestdata.EthTxidB2T2 + "01" + dbtestdata.EthTxidB2T2 + "02" + dbtestdata.EthTxidB2T2 + "05" + dbtestdata.EthTxidB2T2 + "04" + dbtestdata.EthTxidB2T2 + "03", nil},
keyPair{addressKeyHex(dbtestdata.EthAddr7b, 4321001, d), dbtestdata.EthTxidB2T2 + "03" + dbtestdata.EthTxidB2T2 + "04", nil},
keyPair{addressKeyHex(dbtestdata.EthAddrContract47, 4321001, d), dbtestdata.EthTxidB2T2 + "00", nil},
}); err != nil {
{
t.Fatal(err)
@ -185,11 +185,11 @@ func TestRocksDB_Index_EthereumType(t *testing.T) {
// get transactions for various addresses / low-high ranges
verifyGetTransactions(t, d, "0x"+dbtestdata.EthAddr55, 0, 10000000, []txidVoutOutput{
txidVoutOutput{"0x" + dbtestdata.EthTxidB1T1, 0, true},
txidVoutOutput{"0x" + dbtestdata.EthTxidB1T2, 1, true},
txidVoutOutput{"0x" + dbtestdata.EthTxidB2T1, 0, false},
txidVoutOutput{"0x" + dbtestdata.EthTxidB2T2, 2, false},
txidVoutOutput{"0x" + dbtestdata.EthTxidB2T2, 1, true},
txidVoutOutput{"0x" + dbtestdata.EthTxidB1T1, 0, true},
txidVoutOutput{"0x" + dbtestdata.EthTxidB1T2, 1, true},
}, nil)
verifyGetTransactions(t, d, "mtGXQvBowMkBpnhLckhxhbwYK44Gs9eBad", 500000, 1000000, []txidVoutOutput{}, errors.New("Address missing"))

44
db/rocksdb_test.go
View File

@ -99,6 +99,10 @@ func uintToHex(i uint32) string {
return hex.EncodeToString(buf)
}
func addressKeyHex(a string, height uint32, d *RocksDB) string {
return dbtestdata.AddressToPubKeyHex(a, d.chainParser) + uintToHex(^height)
}
// keyPair is used to compare given key value in DB with expected
// for more complicated compares it is possible to specify CompareFunc
type keyPair struct {
@ -168,11 +172,11 @@ func verifyAfterBitcoinTypeBlock1(t *testing.T, d *RocksDB, afterDisconnect bool
}
// the vout is encoded as signed varint, i.e. value * 2 for non negative values
if err := checkColumn(d, cfAddresses, []keyPair{
keyPair{dbtestdata.AddressToPubKeyHex(dbtestdata.Addr1, d.chainParser) + "000370d5", dbtestdata.TxidB1T1 + "00", nil},
keyPair{dbtestdata.AddressToPubKeyHex(dbtestdata.Addr2, d.chainParser) + "000370d5", dbtestdata.TxidB1T1 + "02", nil},
keyPair{dbtestdata.AddressToPubKeyHex(dbtestdata.Addr3, d.chainParser) + "000370d5", dbtestdata.TxidB1T2 + "00", nil},
keyPair{dbtestdata.AddressToPubKeyHex(dbtestdata.Addr4, d.chainParser) + "000370d5", dbtestdata.TxidB1T2 + "02", nil},
keyPair{dbtestdata.AddressToPubKeyHex(dbtestdata.Addr5, d.chainParser) + "000370d5", dbtestdata.TxidB1T2 + "04", nil},
keyPair{addressKeyHex(dbtestdata.Addr1, 225493, d), dbtestdata.TxidB1T1 + "00", nil},
keyPair{addressKeyHex(dbtestdata.Addr2, 225493, d), dbtestdata.TxidB1T1 + "02", nil},
keyPair{addressKeyHex(dbtestdata.Addr3, 225493, d), dbtestdata.TxidB1T2 + "00", nil},
keyPair{addressKeyHex(dbtestdata.Addr4, 225493, d), dbtestdata.TxidB1T2 + "02", nil},
keyPair{addressKeyHex(dbtestdata.Addr5, 225493, d), dbtestdata.TxidB1T2 + "04", nil},
}); err != nil {
{
t.Fatal(err)
@ -253,20 +257,20 @@ func verifyAfterBitcoinTypeBlock2(t *testing.T, d *RocksDB) {
}
}
if err := checkColumn(d, cfAddresses, []keyPair{
keyPair{dbtestdata.AddressToPubKeyHex(dbtestdata.Addr1, d.chainParser) + "000370d5", dbtestdata.TxidB1T1 + "00", nil},
keyPair{dbtestdata.AddressToPubKeyHex(dbtestdata.Addr2, d.chainParser) + "000370d5", dbtestdata.TxidB1T1 + "02", nil},
keyPair{dbtestdata.AddressToPubKeyHex(dbtestdata.Addr3, d.chainParser) + "000370d5", dbtestdata.TxidB1T2 + "00", nil},
keyPair{dbtestdata.AddressToPubKeyHex(dbtestdata.Addr4, d.chainParser) + "000370d5", dbtestdata.TxidB1T2 + "02", nil},
keyPair{dbtestdata.AddressToPubKeyHex(dbtestdata.Addr5, d.chainParser) + "000370d5", dbtestdata.TxidB1T2 + "04", nil},
keyPair{dbtestdata.AddressToPubKeyHex(dbtestdata.Addr6, d.chainParser) + "000370d6", dbtestdata.TxidB2T1 + "00" + dbtestdata.TxidB2T2 + "01", nil},
keyPair{dbtestdata.AddressToPubKeyHex(dbtestdata.Addr7, d.chainParser) + "000370d6", dbtestdata.TxidB2T1 + "02", nil},
keyPair{dbtestdata.AddressToPubKeyHex(dbtestdata.Addr8, d.chainParser) + "000370d6", dbtestdata.TxidB2T2 + "00", nil},
keyPair{dbtestdata.AddressToPubKeyHex(dbtestdata.Addr9, d.chainParser) + "000370d6", dbtestdata.TxidB2T2 + "02", nil},
keyPair{dbtestdata.AddressToPubKeyHex(dbtestdata.Addr3, d.chainParser) + "000370d6", dbtestdata.TxidB2T1 + "01", nil},
keyPair{dbtestdata.AddressToPubKeyHex(dbtestdata.Addr2, d.chainParser) + "000370d6", dbtestdata.TxidB2T1 + "03", nil},
keyPair{dbtestdata.AddressToPubKeyHex(dbtestdata.Addr5, d.chainParser) + "000370d6", dbtestdata.TxidB2T3 + "00" + dbtestdata.TxidB2T3 + "01", nil},
keyPair{dbtestdata.AddressToPubKeyHex(dbtestdata.AddrA, d.chainParser) + "000370d6", dbtestdata.TxidB2T4 + "00", nil},
keyPair{dbtestdata.AddressToPubKeyHex(dbtestdata.Addr4, d.chainParser) + "000370d6", dbtestdata.TxidB2T2 + "03", nil},
keyPair{addressKeyHex(dbtestdata.Addr1, 225493, d), dbtestdata.TxidB1T1 + "00", nil},
keyPair{addressKeyHex(dbtestdata.Addr2, 225493, d), dbtestdata.TxidB1T1 + "02", nil},
keyPair{addressKeyHex(dbtestdata.Addr3, 225493, d), dbtestdata.TxidB1T2 + "00", nil},
keyPair{addressKeyHex(dbtestdata.Addr4, 225493, d), dbtestdata.TxidB1T2 + "02", nil},
keyPair{addressKeyHex(dbtestdata.Addr5, 225493, d), dbtestdata.TxidB1T2 + "04", nil},
keyPair{addressKeyHex(dbtestdata.Addr6, 225494, d), dbtestdata.TxidB2T1 + "00" + dbtestdata.TxidB2T2 + "01", nil},
keyPair{addressKeyHex(dbtestdata.Addr7, 225494, d), dbtestdata.TxidB2T1 + "02", nil},
keyPair{addressKeyHex(dbtestdata.Addr8, 225494, d), dbtestdata.TxidB2T2 + "00", nil},
keyPair{addressKeyHex(dbtestdata.Addr9, 225494, d), dbtestdata.TxidB2T2 + "02", nil},
keyPair{addressKeyHex(dbtestdata.Addr3, 225494, d), dbtestdata.TxidB2T1 + "01", nil},
keyPair{addressKeyHex(dbtestdata.Addr2, 225494, d), dbtestdata.TxidB2T1 + "03", nil},
keyPair{addressKeyHex(dbtestdata.Addr5, 225494, d), dbtestdata.TxidB2T3 + "00" + dbtestdata.TxidB2T3 + "01", nil},
keyPair{addressKeyHex(dbtestdata.AddrA, 225494, d), dbtestdata.TxidB2T4 + "00", nil},
keyPair{addressKeyHex(dbtestdata.Addr4, 225494, d), dbtestdata.TxidB2T2 + "03", nil},
}); err != nil {
{
t.Fatal(err)
@ -453,8 +457,8 @@ func TestRocksDB_Index_BitcoinType(t *testing.T) {
// get transactions for various addresses / low-high ranges
verifyGetTransactions(t, d, dbtestdata.Addr2, 0, 1000000, []txidVoutOutput{
txidVoutOutput{dbtestdata.TxidB1T1, 1, true},
txidVoutOutput{dbtestdata.TxidB2T1, 1, false},
txidVoutOutput{dbtestdata.TxidB1T1, 1, true},
}, nil)
verifyGetTransactions(t, d, dbtestdata.Addr2, 225493, 225493, []txidVoutOutput{
txidVoutOutput{dbtestdata.TxidB1T1, 1, true},

2
server/socketio.go
View File

@ -233,7 +233,7 @@ func (s *SocketIoServer) getAddressTxids(addr []string, opts *addrOpts) (res res
}
}
}
res.Result = api.UniqueTxidsInReverse(txids)
res.Result = api.GetUniqueTxids(txids)
return res, nil
}