diff --git a/db/rocksdb.go b/db/rocksdb.go index fc48e42e..6b38dd5e 100644 --- a/db/rocksdb.go +++ b/db/rocksdb.go @@ -564,6 +564,10 @@ func (ab *AddrBalance) ReceivedSat() *big.Int { // addUtxo func (ab *AddrBalance) addUtxo(u *Utxo) { ab.Utxos = append(ab.Utxos, *u) + ab.manageUtxoMap(u) +} + +func (ab *AddrBalance) manageUtxoMap(u *Utxo) { l := len(ab.Utxos) if l >= 16 { if len(ab.utxosMap) == 0 { @@ -583,9 +587,45 @@ func (ab *AddrBalance) addUtxo(u *Utxo) { } } +// on disconnect, the added utxos must be inserted in the right position so that utxosMap index works +func (ab *AddrBalance) addUtxoInDisconnect(u *Utxo) { + insert := -1 + if len(ab.utxosMap) > 0 { + if i, e := ab.utxosMap[string(u.BtxID)]; e { + insert = i + } + } else { + for i := range ab.Utxos { + utxo := &ab.Utxos[i] + if *(*int)(unsafe.Pointer(&utxo.BtxID[0])) == *(*int)(unsafe.Pointer(&u.BtxID[0])) && bytes.Equal(utxo.BtxID, u.BtxID) { + insert = i + break + } + } + } + if insert > -1 { + // check if it is necessary to insert the utxo into the array + for i := insert; i < len(ab.Utxos); i++ { + utxo := &ab.Utxos[i] + // either the vout is greater than the inserted vout or it is a different tx + if utxo.Vout > u.Vout || *(*int)(unsafe.Pointer(&utxo.BtxID[0])) != *(*int)(unsafe.Pointer(&u.BtxID[0])) || !bytes.Equal(utxo.BtxID, u.BtxID) { + // found the right place, insert the utxo + ab.Utxos = append(ab.Utxos, *u) + copy(ab.Utxos[i+1:], ab.Utxos[i:]) + ab.Utxos[i] = *u + // reset utxosMap after insert, the index will have to be rebuilt if needed + ab.utxosMap = nil + return + } + } + } + ab.Utxos = append(ab.Utxos, *u) + ab.manageUtxoMap(u) +} + // markUtxoAsSpent finds outpoint btxID:vout in utxos and marks it as spent // for small number of utxos the linear search is done, for larger number there is a hashmap index -// it is much faster than removing the utxo from the slice as it would cause in memory copy operations +// it is much faster than removing the utxo from the slice as it would cause in memory reallocations func (ab *AddrBalance) markUtxoAsSpent(btxID []byte, vout int32) { if len(ab.utxosMap) == 0 { for i := range ab.Utxos { @@ -1360,7 +1400,7 @@ func (d *RocksDB) disconnectTxAddressesInputs(wb *gorocksdb.WriteBatch, btxID [] d.resetValueSatToZero(&balance.SentSat, t.AddrDesc, "sent amount") } balance.BalanceSat.Add(&balance.BalanceSat, &t.ValueSat) - balance.Utxos = append(balance.Utxos, Utxo{ + balance.addUtxoInDisconnect(&Utxo{ BtxID: input.btxID, Vout: input.index, Height: inputHeight, @@ -1420,8 +1460,7 @@ func (d *RocksDB) disconnectBlock(height uint32, blockTxs []blockTxs) error { s := string(addrDesc) b, fb := balances[s] if !fb { - // do not use addressBalanceDetailUTXOIndexed as the utxo may be in wrong order for the helper map - b, err = d.GetAddrDescBalance(addrDesc, AddressBalanceDetailUTXO) + b, err = d.GetAddrDescBalance(addrDesc, addressBalanceDetailUTXOIndexed) if err != nil { return nil, err } diff --git a/db/rocksdb_test.go b/db/rocksdb_test.go index 7326ea80..2f932120 100644 --- a/db/rocksdb_test.go +++ b/db/rocksdb_test.go @@ -192,7 +192,7 @@ func verifyAfterBitcoinTypeBlock1(t *testing.T, d *RocksDB, afterDisconnect bool // the vout is encoded as signed varint, i.e. value * 2 for non negative values if err := checkColumn(d, cfAddresses, []keyPair{ {addressKeyHex(dbtestdata.Addr1, 225493, d), txIndexesHex(dbtestdata.TxidB1T1, []int32{0}), nil}, - {addressKeyHex(dbtestdata.Addr2, 225493, d), txIndexesHex(dbtestdata.TxidB1T1, []int32{1}), nil}, + {addressKeyHex(dbtestdata.Addr2, 225493, d), txIndexesHex(dbtestdata.TxidB1T1, []int32{1, 2}), nil}, {addressKeyHex(dbtestdata.Addr3, 225493, d), txIndexesHex(dbtestdata.TxidB1T2, []int32{0}), nil}, {addressKeyHex(dbtestdata.Addr4, 225493, d), txIndexesHex(dbtestdata.TxidB1T2, []int32{1}), nil}, {addressKeyHex(dbtestdata.Addr5, 225493, d), txIndexesHex(dbtestdata.TxidB1T2, []int32{2}), nil}, @@ -206,8 +206,9 @@ func verifyAfterBitcoinTypeBlock1(t *testing.T, d *RocksDB, afterDisconnect bool dbtestdata.TxidB1T1, varuintToHex(225493) + "00" + - "02" + + "03" + addressToPubKeyHexWithLength(dbtestdata.Addr1, t, d) + bigintToHex(dbtestdata.SatB1T1A1) + + addressToPubKeyHexWithLength(dbtestdata.Addr2, t, d) + bigintToHex(dbtestdata.SatB1T1A2) + addressToPubKeyHexWithLength(dbtestdata.Addr2, t, d) + bigintToHex(dbtestdata.SatB1T1A2), nil, }, @@ -235,8 +236,9 @@ func verifyAfterBitcoinTypeBlock1(t *testing.T, d *RocksDB, afterDisconnect bool }, { dbtestdata.AddressToPubKeyHex(dbtestdata.Addr2, d.chainParser), - "01" + bigintToHex(dbtestdata.SatZero) + bigintToHex(dbtestdata.SatB1T1A2) + - dbtestdata.TxidB1T1 + varuintToHex(1) + varuintToHex(225493) + bigintToHex(dbtestdata.SatB1T1A2), + "01" + bigintToHex(dbtestdata.SatZero) + bigintToHex(dbtestdata.SatB1T1A2Double) + + dbtestdata.TxidB1T1 + varuintToHex(1) + varuintToHex(225493) + bigintToHex(dbtestdata.SatB1T1A2) + + dbtestdata.TxidB1T1 + varuintToHex(2) + varuintToHex(225493) + bigintToHex(dbtestdata.SatB1T1A2), nil, }, { @@ -302,7 +304,7 @@ func verifyAfterBitcoinTypeBlock2(t *testing.T, d *RocksDB) { } if err := checkColumn(d, cfAddresses, []keyPair{ {addressKeyHex(dbtestdata.Addr1, 225493, d), txIndexesHex(dbtestdata.TxidB1T1, []int32{0}), nil}, - {addressKeyHex(dbtestdata.Addr2, 225493, d), txIndexesHex(dbtestdata.TxidB1T1, []int32{1}), nil}, + {addressKeyHex(dbtestdata.Addr2, 225493, d), txIndexesHex(dbtestdata.TxidB1T1, []int32{1, 2}), nil}, {addressKeyHex(dbtestdata.Addr3, 225493, d), txIndexesHex(dbtestdata.TxidB1T2, []int32{0}), nil}, {addressKeyHex(dbtestdata.Addr4, 225493, d), txIndexesHex(dbtestdata.TxidB1T2, []int32{1}), nil}, {addressKeyHex(dbtestdata.Addr5, 225493, d), txIndexesHex(dbtestdata.TxidB1T2, []int32{2}), nil}, @@ -325,9 +327,10 @@ func verifyAfterBitcoinTypeBlock2(t *testing.T, d *RocksDB) { dbtestdata.TxidB1T1, varuintToHex(225493) + "00" + - "02" + + "03" + addressToPubKeyHexWithLength(dbtestdata.Addr1, t, d) + bigintToHex(dbtestdata.SatB1T1A1) + - spentAddressToPubKeyHexWithLength(dbtestdata.Addr2, t, d) + bigintToHex(dbtestdata.SatB1T1A2), + spentAddressToPubKeyHexWithLength(dbtestdata.Addr2, t, d) + bigintToHex(dbtestdata.SatB1T1A2) + + addressToPubKeyHexWithLength(dbtestdata.Addr2, t, d) + bigintToHex(dbtestdata.SatB1T1A2), nil, }, { @@ -395,7 +398,8 @@ func verifyAfterBitcoinTypeBlock2(t *testing.T, d *RocksDB) { }, { dbtestdata.AddressToPubKeyHex(dbtestdata.Addr2, d.chainParser), - "02" + bigintToHex(dbtestdata.SatB1T1A2) + bigintToHex(dbtestdata.SatZero), + "02" + bigintToHex(dbtestdata.SatB1T1A2) + bigintToHex(dbtestdata.SatB1T1A2) + + dbtestdata.TxidB1T1 + varuintToHex(2) + varuintToHex(225493) + bigintToHex(dbtestdata.SatB1T1A2), nil, }, { @@ -563,9 +567,11 @@ func TestRocksDB_Index_BitcoinType(t *testing.T) { verifyGetTransactions(t, d, dbtestdata.Addr2, 0, 1000000, []txidIndex{ {dbtestdata.TxidB2T1, ^1}, {dbtestdata.TxidB1T1, 1}, + {dbtestdata.TxidB1T1, 2}, }, nil) verifyGetTransactions(t, d, dbtestdata.Addr2, 225493, 225493, []txidIndex{ {dbtestdata.TxidB1T1, 1}, + {dbtestdata.TxidB1T1, 2}, }, nil) verifyGetTransactions(t, d, dbtestdata.Addr2, 225494, 1000000, []txidIndex{ {dbtestdata.TxidB2T1, ^1}, @@ -1101,6 +1107,213 @@ func Test_packAddrBalance_unpackAddrBalance(t *testing.T) { } } +func createUtxoMap(ab *AddrBalance) { + l := len(ab.Utxos) + ab.utxosMap = make(map[string]int, 32) + for i := 0; i < l; i++ { + s := string(ab.Utxos[i].BtxID) + if _, e := ab.utxosMap[s]; !e { + ab.utxosMap[s] = i + } + } +} +func TestAddrBalance_utxo_methods(t *testing.T) { + ab := &AddrBalance{ + Txs: 10, + SentSat: *big.NewInt(10000), + BalanceSat: *big.NewInt(1000), + } + + // addUtxo + ab.addUtxo(&Utxo{ + BtxID: hexToBytes(dbtestdata.TxidB1T1), + Vout: 1, + Height: 5000, + ValueSat: *big.NewInt(100), + }) + ab.addUtxo(&Utxo{ + BtxID: hexToBytes(dbtestdata.TxidB1T1), + Vout: 4, + Height: 5000, + ValueSat: *big.NewInt(100), + }) + ab.addUtxo(&Utxo{ + BtxID: hexToBytes(dbtestdata.TxidB1T2), + Vout: 0, + Height: 5001, + ValueSat: *big.NewInt(800), + }) + want := &AddrBalance{ + Txs: 10, + SentSat: *big.NewInt(10000), + BalanceSat: *big.NewInt(1000), + Utxos: []Utxo{ + Utxo{ + BtxID: hexToBytes(dbtestdata.TxidB1T1), + Vout: 1, + Height: 5000, + ValueSat: *big.NewInt(100), + }, + Utxo{ + BtxID: hexToBytes(dbtestdata.TxidB1T1), + Vout: 4, + Height: 5000, + ValueSat: *big.NewInt(100), + }, + Utxo{ + BtxID: hexToBytes(dbtestdata.TxidB1T2), + Vout: 0, + Height: 5001, + ValueSat: *big.NewInt(800), + }, + }, + } + if !reflect.DeepEqual(ab, want) { + t.Errorf("addUtxo, got %+v, want %+v", ab, want) + } + + // addUtxoInDisconnect + ab.addUtxoInDisconnect(&Utxo{ + BtxID: hexToBytes(dbtestdata.TxidB2T1), + Vout: 0, + Height: 5003, + ValueSat: *big.NewInt(800), + }) + ab.addUtxoInDisconnect(&Utxo{ + BtxID: hexToBytes(dbtestdata.TxidB2T1), + Vout: 1, + Height: 5003, + ValueSat: *big.NewInt(800), + }) + ab.addUtxoInDisconnect(&Utxo{ + BtxID: hexToBytes(dbtestdata.TxidB1T1), + Vout: 10, + Height: 5000, + ValueSat: *big.NewInt(100), + }) + ab.addUtxoInDisconnect(&Utxo{ + BtxID: hexToBytes(dbtestdata.TxidB1T1), + Vout: 2, + Height: 5000, + ValueSat: *big.NewInt(100), + }) + ab.addUtxoInDisconnect(&Utxo{ + BtxID: hexToBytes(dbtestdata.TxidB1T1), + Vout: 0, + Height: 5000, + ValueSat: *big.NewInt(100), + }) + want = &AddrBalance{ + Txs: 10, + SentSat: *big.NewInt(10000), + BalanceSat: *big.NewInt(1000), + Utxos: []Utxo{ + Utxo{ + BtxID: hexToBytes(dbtestdata.TxidB1T1), + Vout: 0, + Height: 5000, + ValueSat: *big.NewInt(100), + }, + Utxo{ + BtxID: hexToBytes(dbtestdata.TxidB1T1), + Vout: 1, + Height: 5000, + ValueSat: *big.NewInt(100), + }, + Utxo{ + BtxID: hexToBytes(dbtestdata.TxidB1T1), + Vout: 2, + Height: 5000, + ValueSat: *big.NewInt(100), + }, + Utxo{ + BtxID: hexToBytes(dbtestdata.TxidB1T1), + Vout: 4, + Height: 5000, + ValueSat: *big.NewInt(100), + }, + Utxo{ + BtxID: hexToBytes(dbtestdata.TxidB1T1), + Vout: 10, + Height: 5000, + ValueSat: *big.NewInt(100), + }, + Utxo{ + BtxID: hexToBytes(dbtestdata.TxidB1T2), + Vout: 0, + Height: 5001, + ValueSat: *big.NewInt(800), + }, + Utxo{ + BtxID: hexToBytes(dbtestdata.TxidB2T1), + Vout: 0, + Height: 5003, + ValueSat: *big.NewInt(800), + }, + Utxo{ + BtxID: hexToBytes(dbtestdata.TxidB2T1), + Vout: 1, + Height: 5003, + ValueSat: *big.NewInt(800), + }, + }, + } + if !reflect.DeepEqual(ab, want) { + t.Errorf("addUtxoInDisconnect, got %+v, want %+v", ab, want) + } + + // markUtxoAsSpent + ab.markUtxoAsSpent(hexToBytes(dbtestdata.TxidB2T1), 0) + want.Utxos[6].Vout = -1 + if !reflect.DeepEqual(ab, want) { + t.Errorf("markUtxoAsSpent, got %+v, want %+v", ab, want) + } + + // addUtxo with utxosMap + for i := 0; i < 20; i += 2 { + utxo := Utxo{ + BtxID: hexToBytes(dbtestdata.TxidB2T2), + Vout: int32(i), + Height: 5009, + ValueSat: *big.NewInt(800), + } + ab.addUtxo(&utxo) + want.Utxos = append(want.Utxos, utxo) + } + createUtxoMap(want) + if !reflect.DeepEqual(ab, want) { + t.Errorf("addUtxo with utxosMap, got %+v, want %+v", ab, want) + } + + // markUtxoAsSpent with utxosMap + ab.markUtxoAsSpent(hexToBytes(dbtestdata.TxidB2T1), 1) + want.Utxos[7].Vout = -1 + if !reflect.DeepEqual(ab, want) { + t.Errorf("markUtxoAsSpent with utxosMap, got %+v, want %+v", ab, want) + } + + // addUtxoInDisconnect with utxosMap + ab.addUtxoInDisconnect(&Utxo{ + BtxID: hexToBytes(dbtestdata.TxidB1T1), + Vout: 3, + Height: 5000, + ValueSat: *big.NewInt(100), + }) + want.Utxos = append(want.Utxos, Utxo{}) + copy(want.Utxos[3+1:], want.Utxos[3:]) + want.Utxos[3] = Utxo{ + BtxID: hexToBytes(dbtestdata.TxidB1T1), + Vout: 3, + Height: 5000, + ValueSat: *big.NewInt(100), + } + want.utxosMap = nil + if !reflect.DeepEqual(ab, want) { + t.Errorf("addUtxoInDisconnect with utxosMap, got %+v, want %+v", ab, want) + } + +} + func TestRocksTickers(t *testing.T) { d := setupRocksDB(t, &testBitcoinParser{ BitcoinParser: bitcoinTestnetParser(), diff --git a/server/public_test.go b/server/public_test.go index 0730d08f..6a58947d 100644 --- a/server/public_test.go +++ b/server/public_test.go @@ -236,7 +236,7 @@ func httpTestsBitcoinType(t *testing.T, ts *httptest.Server) { body: []string{ `Fake Coin Explorer`, `