diff --git a/api/worker.go b/api/worker.go index cb91a49b..788a5759 100644 --- a/api/worker.go +++ b/api/worker.go @@ -355,20 +355,20 @@ func (t *Tx) getAddrVinValue(addrDesc bchain.AddressDescriptor) *big.Int { return &val } -// UniqueTxidsInReverse reverts the order of transactions (so that newest are first) and removes duplicate transactions -func UniqueTxidsInReverse(txids []string) []string { - i := len(txids) - ut := make([]string, i) +// GetUniqueTxids removes duplicate transactions +func GetUniqueTxids(txids []string) []string { + ut := make([]string, len(txids)) txidsMap := make(map[string]struct{}) + i := 0 for _, txid := range txids { _, e := txidsMap[txid] if !e { - i-- ut[i] = txid + i++ txidsMap[txid] = struct{}{} } } - return ut[i:] + return ut[0:i] } func (w *Worker) txFromTxAddress(txid string, ta *db.TxAddresses, bi *db.BlockInfo, bestheight uint32) *Tx { @@ -578,7 +578,7 @@ func (w *Worker) GetAddress(address string, page int, txsOnPage int, option GetA if err != nil { return nil, errors.Annotatef(err, "getAddressTxids %v true", addrDesc) } - txm = UniqueTxidsInReverse(txm) + txm = GetUniqueTxids(txm) // if there are only unconfirmed transactions, there is no paging if ba == nil { ba = &db.AddrBalance{} @@ -589,7 +589,7 @@ func (w *Worker) GetAddress(address string, page int, txsOnPage int, option GetA if err != nil { return nil, errors.Annotatef(err, "getAddressTxids %v false", addrDesc) } - txc = UniqueTxidsInReverse(txc) + txc = GetUniqueTxids(txc) bestheight, _, err := w.db.GetBestBlock() if err != nil { return nil, errors.Annotatef(err, "GetBestBlock") @@ -700,7 +700,7 @@ func (w *Worker) GetAddressUtxo(address string, onlyConfirmed bool) ([]AddressUt if err != nil { return nil, errors.Annotatef(err, "getAddressTxids %v true", address) } - txm = UniqueTxidsInReverse(txm) + txm = GetUniqueTxids(txm) mc := make([]*bchain.Tx, len(txm)) for i, txid := range txm { // get mempool txs and process their inputs to detect spends between mempool txs diff --git a/db/rocksdb.go b/db/rocksdb.go index 5fbe956e..f05d2c44 100644 --- a/db/rocksdb.go +++ b/db/rocksdb.go @@ -222,10 +222,10 @@ func (d *RocksDB) GetTransactions(address string, lower uint32, higher uint32, f } // GetAddrDescTransactions finds all input/output transactions for address descriptor -// Transaction are passed to callback function. +// Transaction are passed to callback function in the order from newest block to the oldest func (d *RocksDB) GetAddrDescTransactions(addrDesc bchain.AddressDescriptor, lower uint32, higher uint32, fn func(txid string, vout int32, isOutput bool) error) (err error) { - kstart := packAddressKey(addrDesc, lower) - kstop := packAddressKey(addrDesc, higher) + kstart := packAddressKey(addrDesc, higher) + kstop := packAddressKey(addrDesc, lower) it := d.db.NewIteratorCF(d.ro, d.cfh[cfAddresses]) defer it.Close() @@ -1370,10 +1370,10 @@ func (d *RocksDB) ComputeInternalStateColumnStats(stopCompute chan os.Signal) er // Helpers func packAddressKey(addrDesc bchain.AddressDescriptor, height uint32) []byte { - bheight := packUint(height) - buf := make([]byte, 0, len(addrDesc)+len(bheight)) - buf = append(buf, addrDesc...) - buf = append(buf, bheight...) + buf := make([]byte, len(addrDesc)+packedHeightBytes) + copy(buf, addrDesc) + // pack height as binary complement to achieve ordering from newest to oldest block + binary.BigEndian.PutUint32(buf[len(addrDesc):], ^height) return buf } @@ -1382,7 +1382,8 @@ func unpackAddressKey(key []byte) ([]byte, uint32, error) { if i <= 0 { return nil, 0, errors.New("Invalid address key") } - return key[:i], unpackUint(key[i : i+packedHeightBytes]), nil + // height is packed in binary complement, convert it + return key[:i], ^unpackUint(key[i : i+packedHeightBytes]), nil } func packUint(i uint32) []byte { diff --git a/db/rocksdb_ethereumtype_test.go b/db/rocksdb_ethereumtype_test.go index e57b192d..3f4e68dc 100644 --- a/db/rocksdb_ethereumtype_test.go +++ b/db/rocksdb_ethereumtype_test.go @@ -34,10 +34,10 @@ func verifyAfterEthereumTypeBlock1(t *testing.T, d *RocksDB, afterDisconnect boo } // the vout is encoded as signed varint, i.e. value * 2 for positive values, abs(value)*2 + 1 for negative values if err := checkColumn(d, cfAddresses, []keyPair{ - keyPair{dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddr3e, d.chainParser) + "0041eee8", dbtestdata.EthTxidB1T1 + "01", nil}, - keyPair{dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddr55, d.chainParser) + "0041eee8", dbtestdata.EthTxidB1T1 + "00" + dbtestdata.EthTxidB1T2 + "02", nil}, - keyPair{dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddr20, d.chainParser) + "0041eee8", dbtestdata.EthTxidB1T2 + "01" + dbtestdata.EthTxidB1T2 + "03", nil}, - keyPair{dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddrContract4a, d.chainParser) + "0041eee8", dbtestdata.EthTxidB1T2 + "00", nil}, + keyPair{addressKeyHex(dbtestdata.EthAddr3e, 4321000, d), dbtestdata.EthTxidB1T1 + "01", nil}, + keyPair{addressKeyHex(dbtestdata.EthAddr55, 4321000, d), dbtestdata.EthTxidB1T1 + "00" + dbtestdata.EthTxidB1T2 + "02", nil}, + keyPair{addressKeyHex(dbtestdata.EthAddr20, 4321000, d), dbtestdata.EthTxidB1T2 + "01" + dbtestdata.EthTxidB1T2 + "03", nil}, + keyPair{addressKeyHex(dbtestdata.EthAddrContract4a, 4321000, d), dbtestdata.EthTxidB1T2 + "00", nil}, }); err != nil { { t.Fatal(err) @@ -99,15 +99,15 @@ func verifyAfterEthereumTypeBlock2(t *testing.T, d *RocksDB) { } // the vout is encoded as signed varint, i.e. value * 2 for positive values, abs(value)*2 + 1 for negative values if err := checkColumn(d, cfAddresses, []keyPair{ - keyPair{dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddr3e, d.chainParser) + "0041eee8", dbtestdata.EthTxidB1T1 + "01", nil}, - keyPair{dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddr55, d.chainParser) + "0041eee8", dbtestdata.EthTxidB1T1 + "00" + dbtestdata.EthTxidB1T2 + "02", nil}, - keyPair{dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddr20, d.chainParser) + "0041eee8", dbtestdata.EthTxidB1T2 + "01" + dbtestdata.EthTxidB1T2 + "03", nil}, - keyPair{dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddrContract4a, d.chainParser) + "0041eee8", dbtestdata.EthTxidB1T2 + "00", nil}, - keyPair{dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddr55, d.chainParser) + "0041eee9", dbtestdata.EthTxidB2T1 + "01" + dbtestdata.EthTxidB2T2 + "05" + dbtestdata.EthTxidB2T2 + "02", nil}, - keyPair{dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddr9f, d.chainParser) + "0041eee9", dbtestdata.EthTxidB2T1 + "00", nil}, - keyPair{dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddr4b, d.chainParser) + "0041eee9", dbtestdata.EthTxidB2T2 + "01" + dbtestdata.EthTxidB2T2 + "02" + dbtestdata.EthTxidB2T2 + "05" + dbtestdata.EthTxidB2T2 + "04" + dbtestdata.EthTxidB2T2 + "03", nil}, - keyPair{dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddr7b, d.chainParser) + "0041eee9", dbtestdata.EthTxidB2T2 + "03" + dbtestdata.EthTxidB2T2 + "04", nil}, - keyPair{dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddrContract47, d.chainParser) + "0041eee9", dbtestdata.EthTxidB2T2 + "00", nil}, + keyPair{addressKeyHex(dbtestdata.EthAddr3e, 4321000, d), dbtestdata.EthTxidB1T1 + "01", nil}, + keyPair{addressKeyHex(dbtestdata.EthAddr55, 4321000, d), dbtestdata.EthTxidB1T1 + "00" + dbtestdata.EthTxidB1T2 + "02", nil}, + keyPair{addressKeyHex(dbtestdata.EthAddr20, 4321000, d), dbtestdata.EthTxidB1T2 + "01" + dbtestdata.EthTxidB1T2 + "03", nil}, + keyPair{addressKeyHex(dbtestdata.EthAddrContract4a, 4321000, d), dbtestdata.EthTxidB1T2 + "00", nil}, + keyPair{addressKeyHex(dbtestdata.EthAddr55, 4321001, d), dbtestdata.EthTxidB2T1 + "01" + dbtestdata.EthTxidB2T2 + "05" + dbtestdata.EthTxidB2T2 + "02", nil}, + keyPair{addressKeyHex(dbtestdata.EthAddr9f, 4321001, d), dbtestdata.EthTxidB2T1 + "00", nil}, + keyPair{addressKeyHex(dbtestdata.EthAddr4b, 4321001, d), dbtestdata.EthTxidB2T2 + "01" + dbtestdata.EthTxidB2T2 + "02" + dbtestdata.EthTxidB2T2 + "05" + dbtestdata.EthTxidB2T2 + "04" + dbtestdata.EthTxidB2T2 + "03", nil}, + keyPair{addressKeyHex(dbtestdata.EthAddr7b, 4321001, d), dbtestdata.EthTxidB2T2 + "03" + dbtestdata.EthTxidB2T2 + "04", nil}, + keyPair{addressKeyHex(dbtestdata.EthAddrContract47, 4321001, d), dbtestdata.EthTxidB2T2 + "00", nil}, }); err != nil { { t.Fatal(err) @@ -185,11 +185,11 @@ func TestRocksDB_Index_EthereumType(t *testing.T) { // get transactions for various addresses / low-high ranges verifyGetTransactions(t, d, "0x"+dbtestdata.EthAddr55, 0, 10000000, []txidVoutOutput{ - txidVoutOutput{"0x" + dbtestdata.EthTxidB1T1, 0, true}, - txidVoutOutput{"0x" + dbtestdata.EthTxidB1T2, 1, true}, txidVoutOutput{"0x" + dbtestdata.EthTxidB2T1, 0, false}, txidVoutOutput{"0x" + dbtestdata.EthTxidB2T2, 2, false}, txidVoutOutput{"0x" + dbtestdata.EthTxidB2T2, 1, true}, + txidVoutOutput{"0x" + dbtestdata.EthTxidB1T1, 0, true}, + txidVoutOutput{"0x" + dbtestdata.EthTxidB1T2, 1, true}, }, nil) verifyGetTransactions(t, d, "mtGXQvBowMkBpnhLckhxhbwYK44Gs9eBad", 500000, 1000000, []txidVoutOutput{}, errors.New("Address missing")) diff --git a/db/rocksdb_test.go b/db/rocksdb_test.go index 8367c483..ebc7b16a 100644 --- a/db/rocksdb_test.go +++ b/db/rocksdb_test.go @@ -99,6 +99,10 @@ func uintToHex(i uint32) string { return hex.EncodeToString(buf) } +func addressKeyHex(a string, height uint32, d *RocksDB) string { + return dbtestdata.AddressToPubKeyHex(a, d.chainParser) + uintToHex(^height) +} + // keyPair is used to compare given key value in DB with expected // for more complicated compares it is possible to specify CompareFunc type keyPair struct { @@ -168,11 +172,11 @@ func verifyAfterBitcoinTypeBlock1(t *testing.T, d *RocksDB, afterDisconnect bool } // the vout is encoded as signed varint, i.e. value * 2 for non negative values if err := checkColumn(d, cfAddresses, []keyPair{ - keyPair{dbtestdata.AddressToPubKeyHex(dbtestdata.Addr1, d.chainParser) + "000370d5", dbtestdata.TxidB1T1 + "00", nil}, - keyPair{dbtestdata.AddressToPubKeyHex(dbtestdata.Addr2, d.chainParser) + "000370d5", dbtestdata.TxidB1T1 + "02", nil}, - keyPair{dbtestdata.AddressToPubKeyHex(dbtestdata.Addr3, d.chainParser) + "000370d5", dbtestdata.TxidB1T2 + "00", nil}, - keyPair{dbtestdata.AddressToPubKeyHex(dbtestdata.Addr4, d.chainParser) + "000370d5", dbtestdata.TxidB1T2 + "02", nil}, - keyPair{dbtestdata.AddressToPubKeyHex(dbtestdata.Addr5, d.chainParser) + "000370d5", dbtestdata.TxidB1T2 + "04", nil}, + keyPair{addressKeyHex(dbtestdata.Addr1, 225493, d), dbtestdata.TxidB1T1 + "00", nil}, + keyPair{addressKeyHex(dbtestdata.Addr2, 225493, d), dbtestdata.TxidB1T1 + "02", nil}, + keyPair{addressKeyHex(dbtestdata.Addr3, 225493, d), dbtestdata.TxidB1T2 + "00", nil}, + keyPair{addressKeyHex(dbtestdata.Addr4, 225493, d), dbtestdata.TxidB1T2 + "02", nil}, + keyPair{addressKeyHex(dbtestdata.Addr5, 225493, d), dbtestdata.TxidB1T2 + "04", nil}, }); err != nil { { t.Fatal(err) @@ -253,20 +257,20 @@ func verifyAfterBitcoinTypeBlock2(t *testing.T, d *RocksDB) { } } if err := checkColumn(d, cfAddresses, []keyPair{ - keyPair{dbtestdata.AddressToPubKeyHex(dbtestdata.Addr1, d.chainParser) + "000370d5", dbtestdata.TxidB1T1 + "00", nil}, - keyPair{dbtestdata.AddressToPubKeyHex(dbtestdata.Addr2, d.chainParser) + "000370d5", dbtestdata.TxidB1T1 + "02", nil}, - keyPair{dbtestdata.AddressToPubKeyHex(dbtestdata.Addr3, d.chainParser) + "000370d5", dbtestdata.TxidB1T2 + "00", nil}, - keyPair{dbtestdata.AddressToPubKeyHex(dbtestdata.Addr4, d.chainParser) + "000370d5", dbtestdata.TxidB1T2 + "02", nil}, - keyPair{dbtestdata.AddressToPubKeyHex(dbtestdata.Addr5, d.chainParser) + "000370d5", dbtestdata.TxidB1T2 + "04", nil}, - keyPair{dbtestdata.AddressToPubKeyHex(dbtestdata.Addr6, d.chainParser) + "000370d6", dbtestdata.TxidB2T1 + "00" + dbtestdata.TxidB2T2 + "01", nil}, - keyPair{dbtestdata.AddressToPubKeyHex(dbtestdata.Addr7, d.chainParser) + "000370d6", dbtestdata.TxidB2T1 + "02", nil}, - keyPair{dbtestdata.AddressToPubKeyHex(dbtestdata.Addr8, d.chainParser) + "000370d6", dbtestdata.TxidB2T2 + "00", nil}, - keyPair{dbtestdata.AddressToPubKeyHex(dbtestdata.Addr9, d.chainParser) + "000370d6", dbtestdata.TxidB2T2 + "02", nil}, - keyPair{dbtestdata.AddressToPubKeyHex(dbtestdata.Addr3, d.chainParser) + "000370d6", dbtestdata.TxidB2T1 + "01", nil}, - keyPair{dbtestdata.AddressToPubKeyHex(dbtestdata.Addr2, d.chainParser) + "000370d6", dbtestdata.TxidB2T1 + "03", nil}, - keyPair{dbtestdata.AddressToPubKeyHex(dbtestdata.Addr5, d.chainParser) + "000370d6", dbtestdata.TxidB2T3 + "00" + dbtestdata.TxidB2T3 + "01", nil}, - keyPair{dbtestdata.AddressToPubKeyHex(dbtestdata.AddrA, d.chainParser) + "000370d6", dbtestdata.TxidB2T4 + "00", nil}, - keyPair{dbtestdata.AddressToPubKeyHex(dbtestdata.Addr4, d.chainParser) + "000370d6", dbtestdata.TxidB2T2 + "03", nil}, + keyPair{addressKeyHex(dbtestdata.Addr1, 225493, d), dbtestdata.TxidB1T1 + "00", nil}, + keyPair{addressKeyHex(dbtestdata.Addr2, 225493, d), dbtestdata.TxidB1T1 + "02", nil}, + keyPair{addressKeyHex(dbtestdata.Addr3, 225493, d), dbtestdata.TxidB1T2 + "00", nil}, + keyPair{addressKeyHex(dbtestdata.Addr4, 225493, d), dbtestdata.TxidB1T2 + "02", nil}, + keyPair{addressKeyHex(dbtestdata.Addr5, 225493, d), dbtestdata.TxidB1T2 + "04", nil}, + keyPair{addressKeyHex(dbtestdata.Addr6, 225494, d), dbtestdata.TxidB2T1 + "00" + dbtestdata.TxidB2T2 + "01", nil}, + keyPair{addressKeyHex(dbtestdata.Addr7, 225494, d), dbtestdata.TxidB2T1 + "02", nil}, + keyPair{addressKeyHex(dbtestdata.Addr8, 225494, d), dbtestdata.TxidB2T2 + "00", nil}, + keyPair{addressKeyHex(dbtestdata.Addr9, 225494, d), dbtestdata.TxidB2T2 + "02", nil}, + keyPair{addressKeyHex(dbtestdata.Addr3, 225494, d), dbtestdata.TxidB2T1 + "01", nil}, + keyPair{addressKeyHex(dbtestdata.Addr2, 225494, d), dbtestdata.TxidB2T1 + "03", nil}, + keyPair{addressKeyHex(dbtestdata.Addr5, 225494, d), dbtestdata.TxidB2T3 + "00" + dbtestdata.TxidB2T3 + "01", nil}, + keyPair{addressKeyHex(dbtestdata.AddrA, 225494, d), dbtestdata.TxidB2T4 + "00", nil}, + keyPair{addressKeyHex(dbtestdata.Addr4, 225494, d), dbtestdata.TxidB2T2 + "03", nil}, }); err != nil { { t.Fatal(err) @@ -453,8 +457,8 @@ func TestRocksDB_Index_BitcoinType(t *testing.T) { // get transactions for various addresses / low-high ranges verifyGetTransactions(t, d, dbtestdata.Addr2, 0, 1000000, []txidVoutOutput{ - txidVoutOutput{dbtestdata.TxidB1T1, 1, true}, txidVoutOutput{dbtestdata.TxidB2T1, 1, false}, + txidVoutOutput{dbtestdata.TxidB1T1, 1, true}, }, nil) verifyGetTransactions(t, d, dbtestdata.Addr2, 225493, 225493, []txidVoutOutput{ txidVoutOutput{dbtestdata.TxidB1T1, 1, true}, diff --git a/server/socketio.go b/server/socketio.go index 20022027..dd95e6d4 100644 --- a/server/socketio.go +++ b/server/socketio.go @@ -233,7 +233,7 @@ func (s *SocketIoServer) getAddressTxids(addr []string, opts *addrOpts) (res res } } } - res.Result = api.UniqueTxidsInReverse(txids) + res.Result = api.GetUniqueTxids(txids) return res, nil }