RanchiMall/blockbook
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Use new db column blockaddresses to support UTXO chain block disconnect

Browse Source
This commit is contained in:
Martin Boehm 2018-04-20 13:56:55 +02:00
parent 9ad8a4b873
commit febcba5fbe
6 changed files with 185 additions and 50 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
bchain/baseparser.go
View File

@ -35,6 +35,11 @@ func (p *BaseParser) PackedTxidLen() int {
return 32 return 32
} }
// KeepBlockAddresses returns number of blocks which are to be kept in blockaddresses column
func (p *BaseParser) KeepBlockAddresses() int {
return 100
}
// PackTxid packs txid to byte array // PackTxid packs txid to byte array
func (p *BaseParser) PackTxid(txid string) ([]byte, error) { func (p *BaseParser) PackTxid(txid string) ([]byte, error) {
return hex.DecodeString(txid) return hex.DecodeString(txid)

6
bchain/coins/eth/ethparser.go
View File

@ -272,3 +272,9 @@ func (p *EthereumParser) UnpackBlockHash(buf []byte) (string, error) {
func (p *EthereumParser) IsUTXOChain() bool { func (p *EthereumParser) IsUTXOChain() bool {
return false return false
} }
// KeepBlockAddresses returns number of blocks which are to be kept in blockaddresses column
// do not use the blockaddresses for eth
func (p *EthereumParser) KeepBlockAddresses() int {
return 0
}

4
bchain/types.go
View File

@ -134,6 +134,10 @@ type BlockChainParser interface {
// UTXO chains need "inputs" column in db, that map transactions to transactions that spend them // UTXO chains need "inputs" column in db, that map transactions to transactions that spend them
// non UTXO chains have mapping of address to input and output transactions directly in "outputs" column in db // non UTXO chains have mapping of address to input and output transactions directly in "outputs" column in db
IsUTXOChain() bool IsUTXOChain() bool
// KeepBlockAddresses returns number of blocks which are to be kept in blockaddresses column
// and used in case of fork
// if 0 the blockaddresses column is not used at all (usually non UTXO chains)
KeepBlockAddresses() int
// address id conversions // address id conversions
GetAddrIDFromVout(output *Vout) ([]byte, error) GetAddrIDFromVout(output *Vout) ([]byte, error)
GetAddrIDFromAddress(address string) ([]byte, error) GetAddrIDFromAddress(address string) ([]byte, error)

55
db/rocksdb.go
View File

@ -10,6 +10,7 @@ import (
"github.com/bsm/go-vlq" "github.com/bsm/go-vlq"
"github.com/golang/glog" "github.com/golang/glog"
"github.com/juju/errors"
"github.com/tecbot/gorocksdb" "github.com/tecbot/gorocksdb"
) )
@ -40,9 +41,10 @@ const (
cfAddresses cfAddresses
cfUnspentTxs cfUnspentTxs
cfTransactions cfTransactions
cfBlockAddresses
) )
var cfNames = []string{"default", "height", "addresses", "unspenttxs", "transactions"} var cfNames = []string{"default", "height", "addresses", "unspenttxs", "transactions", "blockaddresses"}
func openDB(path string) (*gorocksdb.DB, []*gorocksdb.ColumnFamilyHandle, error) { func openDB(path string) (*gorocksdb.DB, []*gorocksdb.ColumnFamilyHandle, error) {
c := gorocksdb.NewLRUCache(8 << 30) // 8GB c := gorocksdb.NewLRUCache(8 << 30) // 8GB
@ -80,7 +82,7 @@ func openDB(path string) (*gorocksdb.DB, []*gorocksdb.ColumnFamilyHandle, error)
optsOutputs.SetMaxOpenFiles(25000) optsOutputs.SetMaxOpenFiles(25000)
optsOutputs.SetCompression(gorocksdb.NoCompression) optsOutputs.SetCompression(gorocksdb.NoCompression)
fcOptions := []*gorocksdb.Options{opts, opts, optsOutputs, opts, opts} fcOptions := []*gorocksdb.Options{opts, opts, optsOutputs, opts, opts, opts}
db, cfh, err := gorocksdb.OpenDbColumnFamilies(opts, path, cfNames, fcOptions) db, cfh, err := gorocksdb.OpenDbColumnFamilies(opts, path, cfNames, fcOptions)
if err != nil { if err != nil {
@ -239,6 +241,9 @@ type outpoint struct {
} }
func (d *RocksDB) writeAddressRecords(wb *gorocksdb.WriteBatch, block *bchain.Block, op int, records map[string][]outpoint) error { func (d *RocksDB) writeAddressRecords(wb *gorocksdb.WriteBatch, block *bchain.Block, op int, records map[string][]outpoint) error {
keep := d.chainParser.KeepBlockAddresses()
blockAddresses := make([]byte, 0)
vBuf := make([]byte, vlq.MaxLen32)
for addrID, outpoints := range records { for addrID, outpoints := range records {
key, err := packOutputKey([]byte(addrID), block.Height) key, err := packOutputKey([]byte(addrID), block.Height)
if err != nil { if err != nil {
@ -253,10 +258,36 @@ func (d *RocksDB) writeAddressRecords(wb *gorocksdb.WriteBatch, block *bchain.Bl
continue continue
} }
wb.PutCF(d.cfh[cfAddresses], key, val) wb.PutCF(d.cfh[cfAddresses], key, val)
if keep > 0 {
// collect all addresses to be stored in blockaddresses
vl := packVarint(int32(len([]byte(addrID))), vBuf)
blockAddresses = append(blockAddresses, vBuf[0:vl]...)
blockAddresses = append(blockAddresses, []byte(addrID)...)
}
case opDelete: case opDelete:
wb.DeleteCF(d.cfh[cfAddresses], key) wb.DeleteCF(d.cfh[cfAddresses], key)
} }
} }
if keep > 0 && op == opInsert {
// write new block address
key := packUint(block.Height)
wb.PutCF(d.cfh[cfBlockAddresses], key, blockAddresses)
// cleanup old block address
if block.Height > uint32(keep) {
for rh := block.Height - uint32(keep); rh < block.Height; rh-- {
key = packUint(rh)
val, err := d.db.GetCF(d.ro, d.cfh[cfBlockAddresses], key)
if err != nil {
return err
}
if val.Size() == 0 {
break
}
val.Free()
d.db.DeleteCF(d.wo, d.cfh[cfBlockAddresses], key)
}
}
}
return nil return nil
} }
@ -305,7 +336,7 @@ func appendPackedAddrID(txAddrs []byte, addrID []byte, n uint32, remaining int)
} }
func findAndRemoveUnspentAddr(unspentAddrs []byte, vout uint32) ([]byte, uint32, []byte) { func findAndRemoveUnspentAddr(unspentAddrs []byte, vout uint32) ([]byte, uint32, []byte) {
// the addresses are packed as lenaddrID:addrID:vout, where lenaddrID and vout are varints // the addresses are packed as lenaddrID addrID vout, where lenaddrID and vout are varints
for i := 0; i < len(unspentAddrs); { for i := 0; i < len(unspentAddrs); {
l, lv1 := unpackVarint(unspentAddrs[i:]) l, lv1 := unpackVarint(unspentAddrs[i:])
// index of vout of address in unspentAddrs // index of vout of address in unspentAddrs
@ -326,6 +357,11 @@ func findAndRemoveUnspentAddr(unspentAddrs []byte, vout uint32) ([]byte, uint32,
} }
func (d *RocksDB) writeAddressesUTXO(wb *gorocksdb.WriteBatch, block *bchain.Block, op int) error { func (d *RocksDB) writeAddressesUTXO(wb *gorocksdb.WriteBatch, block *bchain.Block, op int) error {
if op == opDelete {
// block does not contain mapping tx-> input address, which is necessary to recreate
// unspentTxs; therefore it is not possible to DisconnectBlocks this way
return errors.New("DisconnectBlock is not supported for UTXO chains")
}
addresses := make(map[string][]outpoint) addresses := make(map[string][]outpoint)
unspentTxs := make(map[string][]byte) unspentTxs := make(map[string][]byte)
btxIDs := make([][]byte, len(block.Txs)) btxIDs := make([][]byte, len(block.Txs))
@ -355,7 +391,7 @@ func (d *RocksDB) writeAddressesUTXO(wb *gorocksdb.WriteBatch, block *bchain.Blo
} }
unspentTxs[string(btxID)] = txAddrs unspentTxs[string(btxID)] = txAddrs
} }
// locate unspent addresses and add them to addresses map them in format txid ^index // locate addresses spent by this tx and add them to addresses map them in format txid ^index
for txi, tx := range block.Txs { for txi, tx := range block.Txs {
spendingTxid := btxIDs[txi] spendingTxid := btxIDs[txi]
for i, input := range tx.Vin { for i, input := range tx.Vin {
@ -393,15 +429,10 @@ func (d *RocksDB) writeAddressesUTXO(wb *gorocksdb.WriteBatch, block *bchain.Blo
} }
// save unspent txs from current block // save unspent txs from current block
for tx, val := range unspentTxs { for tx, val := range unspentTxs {
switch op { if len(val) == 0 {
case opInsert:
if len(val) == 0 {
wb.DeleteCF(d.cfh[cfUnspentTxs], []byte(tx))
} else {
wb.PutCF(d.cfh[cfUnspentTxs], []byte(tx), val)
}
case opDelete:
wb.DeleteCF(d.cfh[cfUnspentTxs], []byte(tx)) wb.DeleteCF(d.cfh[cfUnspentTxs], []byte(tx))
} else {
wb.PutCF(d.cfh[cfUnspentTxs], []byte(tx), val)
} }
} }
return nil return nil

160
db/rocksdb_test.go
View File

@ -9,6 +9,7 @@ import (
"reflect" "reflect"
"sort" "sort"
"strconv" "strconv"
"strings"
"testing" "testing"
"github.com/juju/errors" "github.com/juju/errors"
@ -41,14 +42,17 @@ func addressToPubKeyHex(addr string, t *testing.T, d *RocksDB) string {
return hex.EncodeToString(b) return hex.EncodeToString(b)
} }
func addressToPubKeyHexWithLenght(addr string, t *testing.T, d *RocksDB) string { func addressToPubKeyHexWithLength(addr string, t *testing.T, d *RocksDB) string {
h := addressToPubKeyHex(addr, t, d) h := addressToPubKeyHex(addr, t, d)
// length is signed varint, therefore 2 times big, we can take len(h) as the correct value // length is signed varint, therefore 2 times big, we can take len(h) as the correct value
return strconv.FormatInt(int64(len(h)), 16) + h return strconv.FormatInt(int64(len(h)), 16) + h
} }
// keyPair is used to compare given key value in DB with expected
// for more complicated compares it is possible to specify CompareFunc
type keyPair struct { type keyPair struct {
Key, Value string Key, Value string
CompareFunc func(string) bool
} }
func checkColumn(d *RocksDB, col int, kp []keyPair) error { func checkColumn(d *RocksDB, col int, kp []keyPair) error {
@ -67,7 +71,13 @@ func checkColumn(d *RocksDB, col int, kp []keyPair) error {
return errors.Errorf("Incorrect key %v found in column %v row %v, expecting %v", key, col, i, kp[i].Key) return errors.Errorf("Incorrect key %v found in column %v row %v, expecting %v", key, col, i, kp[i].Key)
} }
val := hex.EncodeToString(it.Value().Data()) val := hex.EncodeToString(it.Value().Data())
if val != kp[i].Value { var valOK bool
if kp[i].CompareFunc == nil {
valOK = val == kp[i].Value
} else {
valOK = kp[i].CompareFunc(val)
}
if !valOK {
return errors.Errorf("Incorrect value %v found in column %v row %v, expecting %v", val, col, i, kp[i].Value) return errors.Errorf("Incorrect value %v found in column %v row %v, expecting %v", val, col, i, kp[i].Value)
} }
i++ i++
@ -78,7 +88,7 @@ func checkColumn(d *RocksDB, col int, kp []keyPair) error {
return nil return nil
} }
func getTestBlock1(t *testing.T, d *RocksDB) *bchain.Block { func getTestUTXOBlock1(t *testing.T, d *RocksDB) *bchain.Block {
return &bchain.Block{ return &bchain.Block{
BlockHeader: bchain.BlockHeader{ BlockHeader: bchain.BlockHeader{
Height: 225493, Height: 225493,
@ -123,7 +133,7 @@ func getTestBlock1(t *testing.T, d *RocksDB) *bchain.Block {
} }
} }
func getTestBlock2(t *testing.T, d *RocksDB) *bchain.Block { func getTestUTXOBlock2(t *testing.T, d *RocksDB) *bchain.Block {
return &bchain.Block{ return &bchain.Block{
BlockHeader: bchain.BlockHeader{ BlockHeader: bchain.BlockHeader{
Height: 225494, Height: 225494,
@ -188,9 +198,9 @@ func getTestBlock2(t *testing.T, d *RocksDB) *bchain.Block {
} }
} }
func verifyAfterBlock1(t *testing.T, d *RocksDB) { func verifyAfterUTXOBlock1(t *testing.T, d *RocksDB) {
if err := checkColumn(d, cfHeight, []keyPair{ if err := checkColumn(d, cfHeight, []keyPair{
keyPair{"000370d5", "0000000076fbbed90fd75b0e18856aa35baa984e9c9d444cf746ad85e94e2997"}, keyPair{"000370d5", "0000000076fbbed90fd75b0e18856aa35baa984e9c9d444cf746ad85e94e2997", nil},
}); err != nil { }); err != nil {
{ {
t.Fatal(err) t.Fatal(err)
@ -198,10 +208,10 @@ func verifyAfterBlock1(t *testing.T, d *RocksDB) {
} }
// the vout is encoded as signed varint, i.e. value * 2 for non negative values // the vout is encoded as signed varint, i.e. value * 2 for non negative values
if err := checkColumn(d, cfAddresses, []keyPair{ if err := checkColumn(d, cfAddresses, []keyPair{
keyPair{addressToPubKeyHex("mfcWp7DB6NuaZsExybTTXpVgWz559Np4Ti", t, d) + "000370d5", "00b2c06055e5e90e9c82bd4181fde310104391a7fa4f289b1704e5d90caa3840" + "00"}, keyPair{addressToPubKeyHex("mfcWp7DB6NuaZsExybTTXpVgWz559Np4Ti", t, d) + "000370d5", "00b2c06055e5e90e9c82bd4181fde310104391a7fa4f289b1704e5d90caa3840" + "00", nil},
keyPair{addressToPubKeyHex("mtGXQvBowMkBpnhLckhxhbwYK44Gs9eEtz", t, d) + "000370d5", "00b2c06055e5e90e9c82bd4181fde310104391a7fa4f289b1704e5d90caa3840" + "02"}, keyPair{addressToPubKeyHex("mtGXQvBowMkBpnhLckhxhbwYK44Gs9eEtz", t, d) + "000370d5", "00b2c06055e5e90e9c82bd4181fde310104391a7fa4f289b1704e5d90caa3840" + "02", nil},
keyPair{addressToPubKeyHex("mv9uLThosiEnGRbVPS7Vhyw6VssbVRsiAw", t, d) + "000370d5", "effd9ef509383d536b1c8af5bf434c8efbf521a4f2befd4022bbd68694b4ac75" + "00"}, keyPair{addressToPubKeyHex("mv9uLThosiEnGRbVPS7Vhyw6VssbVRsiAw", t, d) + "000370d5", "effd9ef509383d536b1c8af5bf434c8efbf521a4f2befd4022bbd68694b4ac75" + "00", nil},
keyPair{addressToPubKeyHex("2Mz1CYoppGGsLNUGF2YDhTif6J661JitALS", t, d) + "000370d5", "effd9ef509383d536b1c8af5bf434c8efbf521a4f2befd4022bbd68694b4ac75" + "02"}, keyPair{addressToPubKeyHex("2Mz1CYoppGGsLNUGF2YDhTif6J661JitALS", t, d) + "000370d5", "effd9ef509383d536b1c8af5bf434c8efbf521a4f2befd4022bbd68694b4ac75" + "02", nil},
}); err != nil { }); err != nil {
{ {
t.Fatal(err) t.Fatal(err)
@ -210,11 +220,38 @@ func verifyAfterBlock1(t *testing.T, d *RocksDB) {
if err := checkColumn(d, cfUnspentTxs, []keyPair{ if err := checkColumn(d, cfUnspentTxs, []keyPair{
keyPair{ keyPair{
"00b2c06055e5e90e9c82bd4181fde310104391a7fa4f289b1704e5d90caa3840", "00b2c06055e5e90e9c82bd4181fde310104391a7fa4f289b1704e5d90caa3840",
addressToPubKeyHexWithLenght("mfcWp7DB6NuaZsExybTTXpVgWz559Np4Ti", t, d) + "00" + addressToPubKeyHexWithLenght("mtGXQvBowMkBpnhLckhxhbwYK44Gs9eEtz", t, d) + "02", addressToPubKeyHexWithLength("mfcWp7DB6NuaZsExybTTXpVgWz559Np4Ti", t, d) + "00" + addressToPubKeyHexWithLength("mtGXQvBowMkBpnhLckhxhbwYK44Gs9eEtz", t, d) + "02",
nil,
}, },
keyPair{ keyPair{
"effd9ef509383d536b1c8af5bf434c8efbf521a4f2befd4022bbd68694b4ac75", "effd9ef509383d536b1c8af5bf434c8efbf521a4f2befd4022bbd68694b4ac75",
addressToPubKeyHexWithLenght("mv9uLThosiEnGRbVPS7Vhyw6VssbVRsiAw", t, d) + "00" + addressToPubKeyHexWithLenght("2Mz1CYoppGGsLNUGF2YDhTif6J661JitALS", t, d) + "02", addressToPubKeyHexWithLength("mv9uLThosiEnGRbVPS7Vhyw6VssbVRsiAw", t, d) + "00" + addressToPubKeyHexWithLength("2Mz1CYoppGGsLNUGF2YDhTif6J661JitALS", t, d) + "02",
nil,
},
}); err != nil {
{
t.Fatal(err)
}
}
// the values in cfBlockAddresses has random order, must use CompareFunc
if err := checkColumn(d, cfBlockAddresses, []keyPair{
keyPair{"000370d5", "",
func(v string) bool {
expected := []string{
addressToPubKeyHexWithLength("mfcWp7DB6NuaZsExybTTXpVgWz559Np4Ti", t, d),
addressToPubKeyHexWithLength("mtGXQvBowMkBpnhLckhxhbwYK44Gs9eEtz", t, d),
addressToPubKeyHexWithLength("mv9uLThosiEnGRbVPS7Vhyw6VssbVRsiAw", t, d),
addressToPubKeyHexWithLength("2Mz1CYoppGGsLNUGF2YDhTif6J661JitALS", t, d),
}
for _, e := range expected {
lb := len(v)
v = strings.Replace(v, e, "", 1)
if lb == len(v) {
return false
}
}
return len(v) == 0
},
}, },
}); err != nil { }); err != nil {
{ {
@ -223,27 +260,27 @@ func verifyAfterBlock1(t *testing.T, d *RocksDB) {
} }
} }
func verifyAfterBlock2(t *testing.T, d *RocksDB) { func verifyAfterUTXOBlock2(t *testing.T, d *RocksDB) {
if err := checkColumn(d, cfHeight, []keyPair{ if err := checkColumn(d, cfHeight, []keyPair{
keyPair{"000370d5", "0000000076fbbed90fd75b0e18856aa35baa984e9c9d444cf746ad85e94e2997"}, keyPair{"000370d5", "0000000076fbbed90fd75b0e18856aa35baa984e9c9d444cf746ad85e94e2997", nil},
keyPair{"000370d6", "00000000eb0443fd7dc4a1ed5c686a8e995057805f9a161d9a5a77a95e72b7b6"}, keyPair{"000370d6", "00000000eb0443fd7dc4a1ed5c686a8e995057805f9a161d9a5a77a95e72b7b6", nil},
}); err != nil { }); err != nil {
{ {
t.Fatal(err) t.Fatal(err)
} }
} }
if err := checkColumn(d, cfAddresses, []keyPair{ if err := checkColumn(d, cfAddresses, []keyPair{
keyPair{addressToPubKeyHex("mfcWp7DB6NuaZsExybTTXpVgWz559Np4Ti", t, d) + "000370d5", "00b2c06055e5e90e9c82bd4181fde310104391a7fa4f289b1704e5d90caa3840" + "00"}, keyPair{addressToPubKeyHex("mfcWp7DB6NuaZsExybTTXpVgWz559Np4Ti", t, d) + "000370d5", "00b2c06055e5e90e9c82bd4181fde310104391a7fa4f289b1704e5d90caa3840" + "00", nil},
keyPair{addressToPubKeyHex("mtGXQvBowMkBpnhLckhxhbwYK44Gs9eEtz", t, d) + "000370d5", "00b2c06055e5e90e9c82bd4181fde310104391a7fa4f289b1704e5d90caa3840" + "02"}, keyPair{addressToPubKeyHex("mtGXQvBowMkBpnhLckhxhbwYK44Gs9eEtz", t, d) + "000370d5", "00b2c06055e5e90e9c82bd4181fde310104391a7fa4f289b1704e5d90caa3840" + "02", nil},
keyPair{addressToPubKeyHex("mv9uLThosiEnGRbVPS7Vhyw6VssbVRsiAw", t, d) + "000370d5", "effd9ef509383d536b1c8af5bf434c8efbf521a4f2befd4022bbd68694b4ac75" + "00"}, keyPair{addressToPubKeyHex("mv9uLThosiEnGRbVPS7Vhyw6VssbVRsiAw", t, d) + "000370d5", "effd9ef509383d536b1c8af5bf434c8efbf521a4f2befd4022bbd68694b4ac75" + "00", nil},
keyPair{addressToPubKeyHex("2Mz1CYoppGGsLNUGF2YDhTif6J661JitALS", t, d) + "000370d5", "effd9ef509383d536b1c8af5bf434c8efbf521a4f2befd4022bbd68694b4ac75" + "02"}, keyPair{addressToPubKeyHex("2Mz1CYoppGGsLNUGF2YDhTif6J661JitALS", t, d) + "000370d5", "effd9ef509383d536b1c8af5bf434c8efbf521a4f2befd4022bbd68694b4ac75" + "02", nil},
keyPair{addressToPubKeyHex("mzB8cYrfRwFRFAGTDzV8LkUQy5BQicxGhX", t, d) + "000370d6", "7c3be24063f268aaa1ed81b64776798f56088757641a34fb156c4f51ed2e9d25" + "00" + "3d90d15ed026dc45e19ffb52875ed18fa9e8012ad123d7f7212176e2b0ebdb71" + "01"}, keyPair{addressToPubKeyHex("mzB8cYrfRwFRFAGTDzV8LkUQy5BQicxGhX", t, d) + "000370d6", "7c3be24063f268aaa1ed81b64776798f56088757641a34fb156c4f51ed2e9d25" + "00" + "3d90d15ed026dc45e19ffb52875ed18fa9e8012ad123d7f7212176e2b0ebdb71" + "01", nil},
keyPair{addressToPubKeyHex("mtR97eM2HPWVM6c8FGLGcukgaHHQv7THoL", t, d) + "000370d6", "7c3be24063f268aaa1ed81b64776798f56088757641a34fb156c4f51ed2e9d25" + "02"}, keyPair{addressToPubKeyHex("mtR97eM2HPWVM6c8FGLGcukgaHHQv7THoL", t, d) + "000370d6", "7c3be24063f268aaa1ed81b64776798f56088757641a34fb156c4f51ed2e9d25" + "02", nil},
keyPair{addressToPubKeyHex("mwwoKQE5Lb1G4picHSHDQKg8jw424PF9SC", t, d) + "000370d6", "3d90d15ed026dc45e19ffb52875ed18fa9e8012ad123d7f7212176e2b0ebdb71" + "00"}, keyPair{addressToPubKeyHex("mwwoKQE5Lb1G4picHSHDQKg8jw424PF9SC", t, d) + "000370d6", "3d90d15ed026dc45e19ffb52875ed18fa9e8012ad123d7f7212176e2b0ebdb71" + "00", nil},
keyPair{addressToPubKeyHex("mmJx9Y8ayz9h14yd9fgCW1bUKoEpkBAquP", t, d) + "000370d6", "3d90d15ed026dc45e19ffb52875ed18fa9e8012ad123d7f7212176e2b0ebdb71" + "02"}, keyPair{addressToPubKeyHex("mmJx9Y8ayz9h14yd9fgCW1bUKoEpkBAquP", t, d) + "000370d6", "3d90d15ed026dc45e19ffb52875ed18fa9e8012ad123d7f7212176e2b0ebdb71" + "02", nil},
keyPair{addressToPubKeyHex("mv9uLThosiEnGRbVPS7Vhyw6VssbVRsiAw", t, d) + "000370d6", "7c3be24063f268aaa1ed81b64776798f56088757641a34fb156c4f51ed2e9d25" + "01"}, keyPair{addressToPubKeyHex("mv9uLThosiEnGRbVPS7Vhyw6VssbVRsiAw", t, d) + "000370d6", "7c3be24063f268aaa1ed81b64776798f56088757641a34fb156c4f51ed2e9d25" + "01", nil},
keyPair{addressToPubKeyHex("mtGXQvBowMkBpnhLckhxhbwYK44Gs9eEtz", t, d) + "000370d6", "7c3be24063f268aaa1ed81b64776798f56088757641a34fb156c4f51ed2e9d25" + "03"}, keyPair{addressToPubKeyHex("mtGXQvBowMkBpnhLckhxhbwYK44Gs9eEtz", t, d) + "000370d6", "7c3be24063f268aaa1ed81b64776798f56088757641a34fb156c4f51ed2e9d25" + "03", nil},
keyPair{addressToPubKeyHex("2Mz1CYoppGGsLNUGF2YDhTif6J661JitALS", t, d) + "000370d6", "3d90d15ed026dc45e19ffb52875ed18fa9e8012ad123d7f7212176e2b0ebdb71" + "03"}, keyPair{addressToPubKeyHex("2Mz1CYoppGGsLNUGF2YDhTif6J661JitALS", t, d) + "000370d6", "3d90d15ed026dc45e19ffb52875ed18fa9e8012ad123d7f7212176e2b0ebdb71" + "03", nil},
}); err != nil { }); err != nil {
{ {
t.Fatal(err) t.Fatal(err)
@ -252,15 +289,45 @@ func verifyAfterBlock2(t *testing.T, d *RocksDB) {
if err := checkColumn(d, cfUnspentTxs, []keyPair{ if err := checkColumn(d, cfUnspentTxs, []keyPair{
keyPair{ keyPair{
"00b2c06055e5e90e9c82bd4181fde310104391a7fa4f289b1704e5d90caa3840", "00b2c06055e5e90e9c82bd4181fde310104391a7fa4f289b1704e5d90caa3840",
addressToPubKeyHexWithLenght("mfcWp7DB6NuaZsExybTTXpVgWz559Np4Ti", t, d) + "00", addressToPubKeyHexWithLength("mfcWp7DB6NuaZsExybTTXpVgWz559Np4Ti", t, d) + "00",
nil,
}, },
keyPair{ keyPair{
"7c3be24063f268aaa1ed81b64776798f56088757641a34fb156c4f51ed2e9d25", "7c3be24063f268aaa1ed81b64776798f56088757641a34fb156c4f51ed2e9d25",
addressToPubKeyHexWithLenght("mtR97eM2HPWVM6c8FGLGcukgaHHQv7THoL", t, d) + "02", addressToPubKeyHexWithLength("mtR97eM2HPWVM6c8FGLGcukgaHHQv7THoL", t, d) + "02",
nil,
}, },
keyPair{ keyPair{
"3d90d15ed026dc45e19ffb52875ed18fa9e8012ad123d7f7212176e2b0ebdb71", "3d90d15ed026dc45e19ffb52875ed18fa9e8012ad123d7f7212176e2b0ebdb71",
addressToPubKeyHexWithLenght("mwwoKQE5Lb1G4picHSHDQKg8jw424PF9SC", t, d) + "00" + addressToPubKeyHexWithLenght("mmJx9Y8ayz9h14yd9fgCW1bUKoEpkBAquP", t, d) + "02", addressToPubKeyHexWithLength("mwwoKQE5Lb1G4picHSHDQKg8jw424PF9SC", t, d) + "00" + addressToPubKeyHexWithLength("mmJx9Y8ayz9h14yd9fgCW1bUKoEpkBAquP", t, d) + "02",
nil,
},
}); err != nil {
{
t.Fatal(err)
}
}
if err := checkColumn(d, cfBlockAddresses, []keyPair{
keyPair{"000370d6", "",
func(v string) bool {
expected := []string{
addressToPubKeyHexWithLength("mzB8cYrfRwFRFAGTDzV8LkUQy5BQicxGhX", t, d),
addressToPubKeyHexWithLength("mtR97eM2HPWVM6c8FGLGcukgaHHQv7THoL", t, d),
addressToPubKeyHexWithLength("mwwoKQE5Lb1G4picHSHDQKg8jw424PF9SC", t, d),
addressToPubKeyHexWithLength("mmJx9Y8ayz9h14yd9fgCW1bUKoEpkBAquP", t, d),
addressToPubKeyHexWithLength("mv9uLThosiEnGRbVPS7Vhyw6VssbVRsiAw", t, d),
addressToPubKeyHexWithLength("mtGXQvBowMkBpnhLckhxhbwYK44Gs9eEtz", t, d),
addressToPubKeyHexWithLength("2Mz1CYoppGGsLNUGF2YDhTif6J661JitALS", t, d),
}
for _, e := range expected {
lb := len(v)
v = strings.Replace(v, e, "", 1)
if lb == len(v) {
return false
}
}
return len(v) == 0
},
}, },
}); err != nil { }); err != nil {
{ {
@ -291,31 +358,39 @@ func verifyGetTransactions(t *testing.T, d *RocksDB, addr string, low, high uint
} }
} }
type testBitcoinParser struct {
*btc.BitcoinParser
}
// override btc.KeepBlockAddresses to keep only one blockaddress
func (p *testBitcoinParser) KeepBlockAddresses() int {
return 1
}
// TestRocksDB_Index_UTXO is a composite test probing the whole indexing functionality for UTXO chains // TestRocksDB_Index_UTXO is a composite test probing the whole indexing functionality for UTXO chains
// It does the following: // It does the following:
// 1) Connect two blocks (inputs from 2nd block are spending some outputs from the 1st block) // 1) Connect two blocks (inputs from 2nd block are spending some outputs from the 1st block)
// 2) GetTransactions for various addresses / low-high ranges // 2) GetTransactions for various addresses / low-high ranges
// 3) Disconnect block 2 // 3) Disconnect block 2 - expect error
// 4) GetTransactions for various addresses // 4) Disconnect the block 2 using full scan
// 5) Connect the block 2 back
// After each step, the whole content of DB is examined and any difference against expected state is regarded as failure // After each step, the whole content of DB is examined and any difference against expected state is regarded as failure
func TestRocksDB_Index_UTXO(t *testing.T) { func TestRocksDB_Index_UTXO(t *testing.T) {
d := setupRocksDB(t, &btc.BitcoinParser{Params: btc.GetChainParams("test")}) d := setupRocksDB(t, &testBitcoinParser{BitcoinParser: &btc.BitcoinParser{Params: btc.GetChainParams("test")}})
defer closeAnddestroyRocksDB(t, d) defer closeAnddestroyRocksDB(t, d)
// connect 1st block - will log warnings about missing UTXO transactions in cfUnspentTxs column // connect 1st block - will log warnings about missing UTXO transactions in cfUnspentTxs column
block1 := getTestBlock1(t, d) block1 := getTestUTXOBlock1(t, d)
if err := d.ConnectBlock(block1); err != nil { if err := d.ConnectBlock(block1); err != nil {
t.Fatal(err) t.Fatal(err)
} }
verifyAfterBlock1(t, d) verifyAfterUTXOBlock1(t, d)
// connect 2nd block - use some outputs from the 1st block as the inputs and 1 input uses tx from the same block // connect 2nd block - use some outputs from the 1st block as the inputs and 1 input uses tx from the same block
block2 := getTestBlock2(t, d) block2 := getTestUTXOBlock2(t, d)
if err := d.ConnectBlock(block2); err != nil { if err := d.ConnectBlock(block2); err != nil {
t.Fatal(err) t.Fatal(err)
} }
verifyAfterBlock2(t, d) verifyAfterUTXOBlock2(t, d)
// get transactions for various addresses / low-high ranges // get transactions for various addresses / low-high ranges
verifyGetTransactions(t, d, "mtGXQvBowMkBpnhLckhxhbwYK44Gs9eEtz", 0, 1000000, []txidVoutOutput{ verifyGetTransactions(t, d, "mtGXQvBowMkBpnhLckhxhbwYK44Gs9eEtz", 0, 1000000, []txidVoutOutput{
@ -334,4 +409,13 @@ func TestRocksDB_Index_UTXO(t *testing.T) {
}, nil) }, nil)
verifyGetTransactions(t, d, "mtGXQvBowMkBpnhLckhxhbwYK44Gs9eBad", 500000, 1000000, []txidVoutOutput{}, errors.New("checksum mismatch")) verifyGetTransactions(t, d, "mtGXQvBowMkBpnhLckhxhbwYK44Gs9eBad", 500000, 1000000, []txidVoutOutput{}, errors.New("checksum mismatch"))
// DisconnectBlock for UTXO chains is not possible
err := d.DisconnectBlock(block2)
if err == nil || err.Error() != "DisconnectBlock is not supported for UTXO chains" {
t.Fatal(err)
}
verifyAfterUTXOBlock2(t, d)
// disconnect the 2nd block, verify that the db contains only the 1st block
} }

5
db/sync.go
View File

@ -382,6 +382,11 @@ func (w *SyncWorker) getBlockChain(out chan blockResult, done chan struct{}) {
// otherwise doing full scan // otherwise doing full scan
func (w *SyncWorker) DisconnectBlocks(lower uint32, higher uint32, hashes []string) error { func (w *SyncWorker) DisconnectBlocks(lower uint32, higher uint32, hashes []string) error {
glog.Infof("sync: disconnecting blocks %d-%d", lower, higher) glog.Infof("sync: disconnecting blocks %d-%d", lower, higher)
// if the chain uses Block to Addresses mapping, always use DisconnectBlocksFullScan
// the full scan will be optimized using the mapping
if w.chain.GetChainParser().KeepBlockAddresses() > 0 {
return w.db.DisconnectBlocksFullScan(lower, higher)
}
blocks := make([]*bchain.Block, len(hashes)) blocks := make([]*bchain.Block, len(hashes))
var err error var err error
// get all blocks first to see if we can avoid full scan // get all blocks first to see if we can avoid full scan