From febcba5fbe6b0cd5ff1358c58ff2e004c64296eb Mon Sep 17 00:00:00 2001 From: Martin Boehm Date: Fri, 20 Apr 2018 13:56:55 +0200 Subject: [PATCH] Use new db column blockaddresses to support UTXO chain block disconnect --- bchain/baseparser.go | 5 ++ bchain/coins/eth/ethparser.go | 6 ++ bchain/types.go | 4 + db/rocksdb.go | 55 +++++++++--- db/rocksdb_test.go | 160 ++++++++++++++++++++++++++-------- db/sync.go | 5 ++ 6 files changed, 185 insertions(+), 50 deletions(-) diff --git a/bchain/baseparser.go b/bchain/baseparser.go index 1b831b44..8b747ed9 100644 --- a/bchain/baseparser.go +++ b/bchain/baseparser.go @@ -35,6 +35,11 @@ func (p *BaseParser) PackedTxidLen() int { return 32 } +// KeepBlockAddresses returns number of blocks which are to be kept in blockaddresses column +func (p *BaseParser) KeepBlockAddresses() int { + return 100 +} + // PackTxid packs txid to byte array func (p *BaseParser) PackTxid(txid string) ([]byte, error) { return hex.DecodeString(txid) diff --git a/bchain/coins/eth/ethparser.go b/bchain/coins/eth/ethparser.go index be642bd5..5e3ed45f 100644 --- a/bchain/coins/eth/ethparser.go +++ b/bchain/coins/eth/ethparser.go @@ -272,3 +272,9 @@ func (p *EthereumParser) UnpackBlockHash(buf []byte) (string, error) { func (p *EthereumParser) IsUTXOChain() bool { return false } + +// KeepBlockAddresses returns number of blocks which are to be kept in blockaddresses column +// do not use the blockaddresses for eth +func (p *EthereumParser) KeepBlockAddresses() int { + return 0 +} diff --git a/bchain/types.go b/bchain/types.go index 8e2a3d1b..46effc48 100644 --- a/bchain/types.go +++ b/bchain/types.go @@ -134,6 +134,10 @@ type BlockChainParser interface { // UTXO chains need "inputs" column in db, that map transactions to transactions that spend them // non UTXO chains have mapping of address to input and output transactions directly in "outputs" column in db IsUTXOChain() bool + // KeepBlockAddresses returns number of blocks which are to be kept in blockaddresses column + // and used in case of fork + // if 0 the blockaddresses column is not used at all (usually non UTXO chains) + KeepBlockAddresses() int // address id conversions GetAddrIDFromVout(output *Vout) ([]byte, error) GetAddrIDFromAddress(address string) ([]byte, error) diff --git a/db/rocksdb.go b/db/rocksdb.go index 5c5c0aa3..b155fe52 100644 --- a/db/rocksdb.go +++ b/db/rocksdb.go @@ -10,6 +10,7 @@ import ( "github.com/bsm/go-vlq" "github.com/golang/glog" + "github.com/juju/errors" "github.com/tecbot/gorocksdb" ) @@ -40,9 +41,10 @@ const ( cfAddresses cfUnspentTxs cfTransactions + cfBlockAddresses ) -var cfNames = []string{"default", "height", "addresses", "unspenttxs", "transactions"} +var cfNames = []string{"default", "height", "addresses", "unspenttxs", "transactions", "blockaddresses"} func openDB(path string) (*gorocksdb.DB, []*gorocksdb.ColumnFamilyHandle, error) { c := gorocksdb.NewLRUCache(8 << 30) // 8GB @@ -80,7 +82,7 @@ func openDB(path string) (*gorocksdb.DB, []*gorocksdb.ColumnFamilyHandle, error) optsOutputs.SetMaxOpenFiles(25000) optsOutputs.SetCompression(gorocksdb.NoCompression) - fcOptions := []*gorocksdb.Options{opts, opts, optsOutputs, opts, opts} + fcOptions := []*gorocksdb.Options{opts, opts, optsOutputs, opts, opts, opts} db, cfh, err := gorocksdb.OpenDbColumnFamilies(opts, path, cfNames, fcOptions) if err != nil { @@ -239,6 +241,9 @@ type outpoint struct { } func (d *RocksDB) writeAddressRecords(wb *gorocksdb.WriteBatch, block *bchain.Block, op int, records map[string][]outpoint) error { + keep := d.chainParser.KeepBlockAddresses() + blockAddresses := make([]byte, 0) + vBuf := make([]byte, vlq.MaxLen32) for addrID, outpoints := range records { key, err := packOutputKey([]byte(addrID), block.Height) if err != nil { @@ -253,10 +258,36 @@ func (d *RocksDB) writeAddressRecords(wb *gorocksdb.WriteBatch, block *bchain.Bl continue } wb.PutCF(d.cfh[cfAddresses], key, val) + if keep > 0 { + // collect all addresses to be stored in blockaddresses + vl := packVarint(int32(len([]byte(addrID))), vBuf) + blockAddresses = append(blockAddresses, vBuf[0:vl]...) + blockAddresses = append(blockAddresses, []byte(addrID)...) + } case opDelete: wb.DeleteCF(d.cfh[cfAddresses], key) } } + if keep > 0 && op == opInsert { + // write new block address + key := packUint(block.Height) + wb.PutCF(d.cfh[cfBlockAddresses], key, blockAddresses) + // cleanup old block address + if block.Height > uint32(keep) { + for rh := block.Height - uint32(keep); rh < block.Height; rh-- { + key = packUint(rh) + val, err := d.db.GetCF(d.ro, d.cfh[cfBlockAddresses], key) + if err != nil { + return err + } + if val.Size() == 0 { + break + } + val.Free() + d.db.DeleteCF(d.wo, d.cfh[cfBlockAddresses], key) + } + } + } return nil } @@ -305,7 +336,7 @@ func appendPackedAddrID(txAddrs []byte, addrID []byte, n uint32, remaining int) } func findAndRemoveUnspentAddr(unspentAddrs []byte, vout uint32) ([]byte, uint32, []byte) { - // the addresses are packed as lenaddrID:addrID:vout, where lenaddrID and vout are varints + // the addresses are packed as lenaddrID addrID vout, where lenaddrID and vout are varints for i := 0; i < len(unspentAddrs); { l, lv1 := unpackVarint(unspentAddrs[i:]) // index of vout of address in unspentAddrs @@ -326,6 +357,11 @@ func findAndRemoveUnspentAddr(unspentAddrs []byte, vout uint32) ([]byte, uint32, } func (d *RocksDB) writeAddressesUTXO(wb *gorocksdb.WriteBatch, block *bchain.Block, op int) error { + if op == opDelete { + // block does not contain mapping tx-> input address, which is necessary to recreate + // unspentTxs; therefore it is not possible to DisconnectBlocks this way + return errors.New("DisconnectBlock is not supported for UTXO chains") + } addresses := make(map[string][]outpoint) unspentTxs := make(map[string][]byte) btxIDs := make([][]byte, len(block.Txs)) @@ -355,7 +391,7 @@ func (d *RocksDB) writeAddressesUTXO(wb *gorocksdb.WriteBatch, block *bchain.Blo } unspentTxs[string(btxID)] = txAddrs } - // locate unspent addresses and add them to addresses map them in format txid ^index + // locate addresses spent by this tx and add them to addresses map them in format txid ^index for txi, tx := range block.Txs { spendingTxid := btxIDs[txi] for i, input := range tx.Vin { @@ -393,15 +429,10 @@ func (d *RocksDB) writeAddressesUTXO(wb *gorocksdb.WriteBatch, block *bchain.Blo } // save unspent txs from current block for tx, val := range unspentTxs { - switch op { - case opInsert: - if len(val) == 0 { - wb.DeleteCF(d.cfh[cfUnspentTxs], []byte(tx)) - } else { - wb.PutCF(d.cfh[cfUnspentTxs], []byte(tx), val) - } - case opDelete: + if len(val) == 0 { wb.DeleteCF(d.cfh[cfUnspentTxs], []byte(tx)) + } else { + wb.PutCF(d.cfh[cfUnspentTxs], []byte(tx), val) } } return nil diff --git a/db/rocksdb_test.go b/db/rocksdb_test.go index 48edf8ae..b12a5c3f 100644 --- a/db/rocksdb_test.go +++ b/db/rocksdb_test.go @@ -9,6 +9,7 @@ import ( "reflect" "sort" "strconv" + "strings" "testing" "github.com/juju/errors" @@ -41,14 +42,17 @@ func addressToPubKeyHex(addr string, t *testing.T, d *RocksDB) string { return hex.EncodeToString(b) } -func addressToPubKeyHexWithLenght(addr string, t *testing.T, d *RocksDB) string { +func addressToPubKeyHexWithLength(addr string, t *testing.T, d *RocksDB) string { h := addressToPubKeyHex(addr, t, d) // length is signed varint, therefore 2 times big, we can take len(h) as the correct value return strconv.FormatInt(int64(len(h)), 16) + h } +// keyPair is used to compare given key value in DB with expected +// for more complicated compares it is possible to specify CompareFunc type keyPair struct { - Key, Value string + Key, Value string + CompareFunc func(string) bool } func checkColumn(d *RocksDB, col int, kp []keyPair) error { @@ -67,7 +71,13 @@ func checkColumn(d *RocksDB, col int, kp []keyPair) error { return errors.Errorf("Incorrect key %v found in column %v row %v, expecting %v", key, col, i, kp[i].Key) } val := hex.EncodeToString(it.Value().Data()) - if val != kp[i].Value { + var valOK bool + if kp[i].CompareFunc == nil { + valOK = val == kp[i].Value + } else { + valOK = kp[i].CompareFunc(val) + } + if !valOK { return errors.Errorf("Incorrect value %v found in column %v row %v, expecting %v", val, col, i, kp[i].Value) } i++ @@ -78,7 +88,7 @@ func checkColumn(d *RocksDB, col int, kp []keyPair) error { return nil } -func getTestBlock1(t *testing.T, d *RocksDB) *bchain.Block { +func getTestUTXOBlock1(t *testing.T, d *RocksDB) *bchain.Block { return &bchain.Block{ BlockHeader: bchain.BlockHeader{ Height: 225493, @@ -123,7 +133,7 @@ func getTestBlock1(t *testing.T, d *RocksDB) *bchain.Block { } } -func getTestBlock2(t *testing.T, d *RocksDB) *bchain.Block { +func getTestUTXOBlock2(t *testing.T, d *RocksDB) *bchain.Block { return &bchain.Block{ BlockHeader: bchain.BlockHeader{ Height: 225494, @@ -188,9 +198,9 @@ func getTestBlock2(t *testing.T, d *RocksDB) *bchain.Block { } } -func verifyAfterBlock1(t *testing.T, d *RocksDB) { +func verifyAfterUTXOBlock1(t *testing.T, d *RocksDB) { if err := checkColumn(d, cfHeight, []keyPair{ - keyPair{"000370d5", "0000000076fbbed90fd75b0e18856aa35baa984e9c9d444cf746ad85e94e2997"}, + keyPair{"000370d5", "0000000076fbbed90fd75b0e18856aa35baa984e9c9d444cf746ad85e94e2997", nil}, }); err != nil { { t.Fatal(err) @@ -198,10 +208,10 @@ func verifyAfterBlock1(t *testing.T, d *RocksDB) { } // the vout is encoded as signed varint, i.e. value * 2 for non negative values if err := checkColumn(d, cfAddresses, []keyPair{ - keyPair{addressToPubKeyHex("mfcWp7DB6NuaZsExybTTXpVgWz559Np4Ti", t, d) + "000370d5", "00b2c06055e5e90e9c82bd4181fde310104391a7fa4f289b1704e5d90caa3840" + "00"}, - keyPair{addressToPubKeyHex("mtGXQvBowMkBpnhLckhxhbwYK44Gs9eEtz", t, d) + "000370d5", "00b2c06055e5e90e9c82bd4181fde310104391a7fa4f289b1704e5d90caa3840" + "02"}, - keyPair{addressToPubKeyHex("mv9uLThosiEnGRbVPS7Vhyw6VssbVRsiAw", t, d) + "000370d5", "effd9ef509383d536b1c8af5bf434c8efbf521a4f2befd4022bbd68694b4ac75" + "00"}, - keyPair{addressToPubKeyHex("2Mz1CYoppGGsLNUGF2YDhTif6J661JitALS", t, d) + "000370d5", "effd9ef509383d536b1c8af5bf434c8efbf521a4f2befd4022bbd68694b4ac75" + "02"}, + keyPair{addressToPubKeyHex("mfcWp7DB6NuaZsExybTTXpVgWz559Np4Ti", t, d) + "000370d5", "00b2c06055e5e90e9c82bd4181fde310104391a7fa4f289b1704e5d90caa3840" + "00", nil}, + keyPair{addressToPubKeyHex("mtGXQvBowMkBpnhLckhxhbwYK44Gs9eEtz", t, d) + "000370d5", "00b2c06055e5e90e9c82bd4181fde310104391a7fa4f289b1704e5d90caa3840" + "02", nil}, + keyPair{addressToPubKeyHex("mv9uLThosiEnGRbVPS7Vhyw6VssbVRsiAw", t, d) + "000370d5", "effd9ef509383d536b1c8af5bf434c8efbf521a4f2befd4022bbd68694b4ac75" + "00", nil}, + keyPair{addressToPubKeyHex("2Mz1CYoppGGsLNUGF2YDhTif6J661JitALS", t, d) + "000370d5", "effd9ef509383d536b1c8af5bf434c8efbf521a4f2befd4022bbd68694b4ac75" + "02", nil}, }); err != nil { { t.Fatal(err) @@ -210,11 +220,38 @@ func verifyAfterBlock1(t *testing.T, d *RocksDB) { if err := checkColumn(d, cfUnspentTxs, []keyPair{ keyPair{ "00b2c06055e5e90e9c82bd4181fde310104391a7fa4f289b1704e5d90caa3840", - addressToPubKeyHexWithLenght("mfcWp7DB6NuaZsExybTTXpVgWz559Np4Ti", t, d) + "00" + addressToPubKeyHexWithLenght("mtGXQvBowMkBpnhLckhxhbwYK44Gs9eEtz", t, d) + "02", + addressToPubKeyHexWithLength("mfcWp7DB6NuaZsExybTTXpVgWz559Np4Ti", t, d) + "00" + addressToPubKeyHexWithLength("mtGXQvBowMkBpnhLckhxhbwYK44Gs9eEtz", t, d) + "02", + nil, }, keyPair{ "effd9ef509383d536b1c8af5bf434c8efbf521a4f2befd4022bbd68694b4ac75", - addressToPubKeyHexWithLenght("mv9uLThosiEnGRbVPS7Vhyw6VssbVRsiAw", t, d) + "00" + addressToPubKeyHexWithLenght("2Mz1CYoppGGsLNUGF2YDhTif6J661JitALS", t, d) + "02", + addressToPubKeyHexWithLength("mv9uLThosiEnGRbVPS7Vhyw6VssbVRsiAw", t, d) + "00" + addressToPubKeyHexWithLength("2Mz1CYoppGGsLNUGF2YDhTif6J661JitALS", t, d) + "02", + nil, + }, + }); err != nil { + { + t.Fatal(err) + } + } + // the values in cfBlockAddresses has random order, must use CompareFunc + if err := checkColumn(d, cfBlockAddresses, []keyPair{ + keyPair{"000370d5", "", + func(v string) bool { + expected := []string{ + addressToPubKeyHexWithLength("mfcWp7DB6NuaZsExybTTXpVgWz559Np4Ti", t, d), + addressToPubKeyHexWithLength("mtGXQvBowMkBpnhLckhxhbwYK44Gs9eEtz", t, d), + addressToPubKeyHexWithLength("mv9uLThosiEnGRbVPS7Vhyw6VssbVRsiAw", t, d), + addressToPubKeyHexWithLength("2Mz1CYoppGGsLNUGF2YDhTif6J661JitALS", t, d), + } + for _, e := range expected { + lb := len(v) + v = strings.Replace(v, e, "", 1) + if lb == len(v) { + return false + } + } + return len(v) == 0 + }, }, }); err != nil { { @@ -223,27 +260,27 @@ func verifyAfterBlock1(t *testing.T, d *RocksDB) { } } -func verifyAfterBlock2(t *testing.T, d *RocksDB) { +func verifyAfterUTXOBlock2(t *testing.T, d *RocksDB) { if err := checkColumn(d, cfHeight, []keyPair{ - keyPair{"000370d5", "0000000076fbbed90fd75b0e18856aa35baa984e9c9d444cf746ad85e94e2997"}, - keyPair{"000370d6", "00000000eb0443fd7dc4a1ed5c686a8e995057805f9a161d9a5a77a95e72b7b6"}, + keyPair{"000370d5", "0000000076fbbed90fd75b0e18856aa35baa984e9c9d444cf746ad85e94e2997", nil}, + keyPair{"000370d6", "00000000eb0443fd7dc4a1ed5c686a8e995057805f9a161d9a5a77a95e72b7b6", nil}, }); err != nil { { t.Fatal(err) } } if err := checkColumn(d, cfAddresses, []keyPair{ - keyPair{addressToPubKeyHex("mfcWp7DB6NuaZsExybTTXpVgWz559Np4Ti", t, d) + "000370d5", "00b2c06055e5e90e9c82bd4181fde310104391a7fa4f289b1704e5d90caa3840" + "00"}, - keyPair{addressToPubKeyHex("mtGXQvBowMkBpnhLckhxhbwYK44Gs9eEtz", t, d) + "000370d5", "00b2c06055e5e90e9c82bd4181fde310104391a7fa4f289b1704e5d90caa3840" + "02"}, - keyPair{addressToPubKeyHex("mv9uLThosiEnGRbVPS7Vhyw6VssbVRsiAw", t, d) + "000370d5", "effd9ef509383d536b1c8af5bf434c8efbf521a4f2befd4022bbd68694b4ac75" + "00"}, - keyPair{addressToPubKeyHex("2Mz1CYoppGGsLNUGF2YDhTif6J661JitALS", t, d) + "000370d5", "effd9ef509383d536b1c8af5bf434c8efbf521a4f2befd4022bbd68694b4ac75" + "02"}, - keyPair{addressToPubKeyHex("mzB8cYrfRwFRFAGTDzV8LkUQy5BQicxGhX", t, d) + "000370d6", "7c3be24063f268aaa1ed81b64776798f56088757641a34fb156c4f51ed2e9d25" + "00" + "3d90d15ed026dc45e19ffb52875ed18fa9e8012ad123d7f7212176e2b0ebdb71" + "01"}, - keyPair{addressToPubKeyHex("mtR97eM2HPWVM6c8FGLGcukgaHHQv7THoL", t, d) + "000370d6", "7c3be24063f268aaa1ed81b64776798f56088757641a34fb156c4f51ed2e9d25" + "02"}, - keyPair{addressToPubKeyHex("mwwoKQE5Lb1G4picHSHDQKg8jw424PF9SC", t, d) + "000370d6", "3d90d15ed026dc45e19ffb52875ed18fa9e8012ad123d7f7212176e2b0ebdb71" + "00"}, - keyPair{addressToPubKeyHex("mmJx9Y8ayz9h14yd9fgCW1bUKoEpkBAquP", t, d) + "000370d6", "3d90d15ed026dc45e19ffb52875ed18fa9e8012ad123d7f7212176e2b0ebdb71" + "02"}, - keyPair{addressToPubKeyHex("mv9uLThosiEnGRbVPS7Vhyw6VssbVRsiAw", t, d) + "000370d6", "7c3be24063f268aaa1ed81b64776798f56088757641a34fb156c4f51ed2e9d25" + "01"}, - keyPair{addressToPubKeyHex("mtGXQvBowMkBpnhLckhxhbwYK44Gs9eEtz", t, d) + "000370d6", "7c3be24063f268aaa1ed81b64776798f56088757641a34fb156c4f51ed2e9d25" + "03"}, - keyPair{addressToPubKeyHex("2Mz1CYoppGGsLNUGF2YDhTif6J661JitALS", t, d) + "000370d6", "3d90d15ed026dc45e19ffb52875ed18fa9e8012ad123d7f7212176e2b0ebdb71" + "03"}, + keyPair{addressToPubKeyHex("mfcWp7DB6NuaZsExybTTXpVgWz559Np4Ti", t, d) + "000370d5", "00b2c06055e5e90e9c82bd4181fde310104391a7fa4f289b1704e5d90caa3840" + "00", nil}, + keyPair{addressToPubKeyHex("mtGXQvBowMkBpnhLckhxhbwYK44Gs9eEtz", t, d) + "000370d5", "00b2c06055e5e90e9c82bd4181fde310104391a7fa4f289b1704e5d90caa3840" + "02", nil}, + keyPair{addressToPubKeyHex("mv9uLThosiEnGRbVPS7Vhyw6VssbVRsiAw", t, d) + "000370d5", "effd9ef509383d536b1c8af5bf434c8efbf521a4f2befd4022bbd68694b4ac75" + "00", nil}, + keyPair{addressToPubKeyHex("2Mz1CYoppGGsLNUGF2YDhTif6J661JitALS", t, d) + "000370d5", "effd9ef509383d536b1c8af5bf434c8efbf521a4f2befd4022bbd68694b4ac75" + "02", nil}, + keyPair{addressToPubKeyHex("mzB8cYrfRwFRFAGTDzV8LkUQy5BQicxGhX", t, d) + "000370d6", "7c3be24063f268aaa1ed81b64776798f56088757641a34fb156c4f51ed2e9d25" + "00" + "3d90d15ed026dc45e19ffb52875ed18fa9e8012ad123d7f7212176e2b0ebdb71" + "01", nil}, + keyPair{addressToPubKeyHex("mtR97eM2HPWVM6c8FGLGcukgaHHQv7THoL", t, d) + "000370d6", "7c3be24063f268aaa1ed81b64776798f56088757641a34fb156c4f51ed2e9d25" + "02", nil}, + keyPair{addressToPubKeyHex("mwwoKQE5Lb1G4picHSHDQKg8jw424PF9SC", t, d) + "000370d6", "3d90d15ed026dc45e19ffb52875ed18fa9e8012ad123d7f7212176e2b0ebdb71" + "00", nil}, + keyPair{addressToPubKeyHex("mmJx9Y8ayz9h14yd9fgCW1bUKoEpkBAquP", t, d) + "000370d6", "3d90d15ed026dc45e19ffb52875ed18fa9e8012ad123d7f7212176e2b0ebdb71" + "02", nil}, + keyPair{addressToPubKeyHex("mv9uLThosiEnGRbVPS7Vhyw6VssbVRsiAw", t, d) + "000370d6", "7c3be24063f268aaa1ed81b64776798f56088757641a34fb156c4f51ed2e9d25" + "01", nil}, + keyPair{addressToPubKeyHex("mtGXQvBowMkBpnhLckhxhbwYK44Gs9eEtz", t, d) + "000370d6", "7c3be24063f268aaa1ed81b64776798f56088757641a34fb156c4f51ed2e9d25" + "03", nil}, + keyPair{addressToPubKeyHex("2Mz1CYoppGGsLNUGF2YDhTif6J661JitALS", t, d) + "000370d6", "3d90d15ed026dc45e19ffb52875ed18fa9e8012ad123d7f7212176e2b0ebdb71" + "03", nil}, }); err != nil { { t.Fatal(err) @@ -252,15 +289,45 @@ func verifyAfterBlock2(t *testing.T, d *RocksDB) { if err := checkColumn(d, cfUnspentTxs, []keyPair{ keyPair{ "00b2c06055e5e90e9c82bd4181fde310104391a7fa4f289b1704e5d90caa3840", - addressToPubKeyHexWithLenght("mfcWp7DB6NuaZsExybTTXpVgWz559Np4Ti", t, d) + "00", + addressToPubKeyHexWithLength("mfcWp7DB6NuaZsExybTTXpVgWz559Np4Ti", t, d) + "00", + nil, }, keyPair{ "7c3be24063f268aaa1ed81b64776798f56088757641a34fb156c4f51ed2e9d25", - addressToPubKeyHexWithLenght("mtR97eM2HPWVM6c8FGLGcukgaHHQv7THoL", t, d) + "02", + addressToPubKeyHexWithLength("mtR97eM2HPWVM6c8FGLGcukgaHHQv7THoL", t, d) + "02", + nil, }, keyPair{ "3d90d15ed026dc45e19ffb52875ed18fa9e8012ad123d7f7212176e2b0ebdb71", - addressToPubKeyHexWithLenght("mwwoKQE5Lb1G4picHSHDQKg8jw424PF9SC", t, d) + "00" + addressToPubKeyHexWithLenght("mmJx9Y8ayz9h14yd9fgCW1bUKoEpkBAquP", t, d) + "02", + addressToPubKeyHexWithLength("mwwoKQE5Lb1G4picHSHDQKg8jw424PF9SC", t, d) + "00" + addressToPubKeyHexWithLength("mmJx9Y8ayz9h14yd9fgCW1bUKoEpkBAquP", t, d) + "02", + nil, + }, + }); err != nil { + { + t.Fatal(err) + } + } + if err := checkColumn(d, cfBlockAddresses, []keyPair{ + keyPair{"000370d6", "", + func(v string) bool { + expected := []string{ + addressToPubKeyHexWithLength("mzB8cYrfRwFRFAGTDzV8LkUQy5BQicxGhX", t, d), + addressToPubKeyHexWithLength("mtR97eM2HPWVM6c8FGLGcukgaHHQv7THoL", t, d), + addressToPubKeyHexWithLength("mwwoKQE5Lb1G4picHSHDQKg8jw424PF9SC", t, d), + addressToPubKeyHexWithLength("mmJx9Y8ayz9h14yd9fgCW1bUKoEpkBAquP", t, d), + addressToPubKeyHexWithLength("mv9uLThosiEnGRbVPS7Vhyw6VssbVRsiAw", t, d), + addressToPubKeyHexWithLength("mtGXQvBowMkBpnhLckhxhbwYK44Gs9eEtz", t, d), + addressToPubKeyHexWithLength("2Mz1CYoppGGsLNUGF2YDhTif6J661JitALS", t, d), + } + for _, e := range expected { + lb := len(v) + v = strings.Replace(v, e, "", 1) + if lb == len(v) { + return false + } + } + return len(v) == 0 + }, }, }); err != nil { { @@ -291,31 +358,39 @@ func verifyGetTransactions(t *testing.T, d *RocksDB, addr string, low, high uint } } +type testBitcoinParser struct { + *btc.BitcoinParser +} + +// override btc.KeepBlockAddresses to keep only one blockaddress +func (p *testBitcoinParser) KeepBlockAddresses() int { + return 1 +} + // TestRocksDB_Index_UTXO is a composite test probing the whole indexing functionality for UTXO chains // It does the following: // 1) Connect two blocks (inputs from 2nd block are spending some outputs from the 1st block) // 2) GetTransactions for various addresses / low-high ranges -// 3) Disconnect block 2 -// 4) GetTransactions for various addresses -// 5) Connect the block 2 back +// 3) Disconnect block 2 - expect error +// 4) Disconnect the block 2 using full scan // After each step, the whole content of DB is examined and any difference against expected state is regarded as failure func TestRocksDB_Index_UTXO(t *testing.T) { - d := setupRocksDB(t, &btc.BitcoinParser{Params: btc.GetChainParams("test")}) + d := setupRocksDB(t, &testBitcoinParser{BitcoinParser: &btc.BitcoinParser{Params: btc.GetChainParams("test")}}) defer closeAnddestroyRocksDB(t, d) // connect 1st block - will log warnings about missing UTXO transactions in cfUnspentTxs column - block1 := getTestBlock1(t, d) + block1 := getTestUTXOBlock1(t, d) if err := d.ConnectBlock(block1); err != nil { t.Fatal(err) } - verifyAfterBlock1(t, d) + verifyAfterUTXOBlock1(t, d) // connect 2nd block - use some outputs from the 1st block as the inputs and 1 input uses tx from the same block - block2 := getTestBlock2(t, d) + block2 := getTestUTXOBlock2(t, d) if err := d.ConnectBlock(block2); err != nil { t.Fatal(err) } - verifyAfterBlock2(t, d) + verifyAfterUTXOBlock2(t, d) // get transactions for various addresses / low-high ranges verifyGetTransactions(t, d, "mtGXQvBowMkBpnhLckhxhbwYK44Gs9eEtz", 0, 1000000, []txidVoutOutput{ @@ -334,4 +409,13 @@ func TestRocksDB_Index_UTXO(t *testing.T) { }, nil) verifyGetTransactions(t, d, "mtGXQvBowMkBpnhLckhxhbwYK44Gs9eBad", 500000, 1000000, []txidVoutOutput{}, errors.New("checksum mismatch")) + // DisconnectBlock for UTXO chains is not possible + err := d.DisconnectBlock(block2) + if err == nil || err.Error() != "DisconnectBlock is not supported for UTXO chains" { + t.Fatal(err) + } + verifyAfterUTXOBlock2(t, d) + + // disconnect the 2nd block, verify that the db contains only the 1st block + } diff --git a/db/sync.go b/db/sync.go index 339daedb..845a517d 100644 --- a/db/sync.go +++ b/db/sync.go @@ -382,6 +382,11 @@ func (w *SyncWorker) getBlockChain(out chan blockResult, done chan struct{}) { // otherwise doing full scan func (w *SyncWorker) DisconnectBlocks(lower uint32, higher uint32, hashes []string) error { glog.Infof("sync: disconnecting blocks %d-%d", lower, higher) + // if the chain uses Block to Addresses mapping, always use DisconnectBlocksFullScan + // the full scan will be optimized using the mapping + if w.chain.GetChainParser().KeepBlockAddresses() > 0 { + return w.db.DisconnectBlocksFullScan(lower, higher) + } blocks := make([]*bchain.Block, len(hashes)) var err error // get all blocks first to see if we can avoid full scan