diff --git a/bchain/coins/eth/ethparser.go b/bchain/coins/eth/ethparser.go index 7b93fa7d..6232a2a1 100644 --- a/bchain/coins/eth/ethparser.go +++ b/bchain/coins/eth/ethparser.go @@ -235,6 +235,7 @@ func (p *EthereumParser) PackTx(tx *bchain.Tx, height uint32, blockTime int64) ( if pt.Tx.AccountNonce, err = hexutil.DecodeUint64(r.Tx.AccountNonce); err != nil { return nil, errors.Annotatef(err, "AccountNonce %v", r.Tx.AccountNonce) } + // pt.BlockNumber = height if n, err = hexutil.DecodeUint64(r.Tx.BlockNumber); err != nil { return nil, errors.Annotatef(err, "BlockNumber %v", r.Tx.BlockNumber) } diff --git a/common/internalstate.go b/common/internalstate.go index f19fd1f8..9419beaa 100644 --- a/common/internalstate.go +++ b/common/internalstate.go @@ -150,9 +150,7 @@ func (is *InternalState) GetDBColumnStatValues(c int) (int64, int64, int64) { func (is *InternalState) GetAllDBColumnStats() []InternalStateColumn { is.mux.Lock() defer is.mux.Unlock() - rv := make([]InternalStateColumn, len(is.DbColumns)) - copy(rv, is.DbColumns) - return rv + return append(is.DbColumns[:0:0], is.DbColumns...) } // DBSizeTotal sums the computed sizes of all columns diff --git a/db/rocksdb.go b/db/rocksdb.go index e8169e31..7d957fd2 100644 --- a/db/rocksdb.go +++ b/db/rocksdb.go @@ -273,39 +273,20 @@ const ( // ConnectBlock indexes addresses in the block and stores them in db func (d *RocksDB) ConnectBlock(block *bchain.Block) error { - return d.writeBlock(block, opInsert) -} - -// DisconnectBlock removes addresses in the block from the db -func (d *RocksDB) DisconnectBlock(block *bchain.Block) error { - return d.writeBlock(block, opDelete) -} - -func (d *RocksDB) writeBlock(block *bchain.Block, op int) error { wb := gorocksdb.NewWriteBatch() defer wb.Destroy() if glog.V(2) { - switch op { - case opInsert: - glog.Infof("rocksdb: insert %d %s", block.Height, block.Hash) - case opDelete: - glog.Infof("rocksdb: delete %d %s", block.Height, block.Hash) - } + glog.Infof("rocksdb: insert %d %s", block.Height, block.Hash) } chainType := d.chainParser.GetChainType() - if err := d.writeHeightFromBlock(wb, block, op); err != nil { + if err := d.writeHeightFromBlock(wb, block, opInsert); err != nil { return err } addresses := make(map[string][]outpoint) if chainType == bchain.ChainBitcoinType { - if op == opDelete { - // block does not contain mapping tx-> input address, which is necessary to recreate - // unspentTxs; therefore it is not possible to DisconnectBlocks this way - return errors.New("DisconnectBlock is not supported for BitcoinType chains") - } txAddressesMap := make(map[string]*TxAddresses) balances := make(map[string]*AddrBalance) if err := d.processAddressesBitcoinType(block, addresses, txAddressesMap, balances); err != nil { @@ -680,8 +661,7 @@ func (d *RocksDB) getBlockTxs(height uint32) ([]blockTxs, error) { glog.Error("rocksdb: Inconsistent data in blockTxs ", hex.EncodeToString(buf)) return nil, errors.New("Inconsistent data in blockTxs") } - txid := make([]byte, pl) - copy(txid, buf[i:]) + txid := append([]byte(nil), buf[i:i+pl]...) i += pl o, ol, err := d.unpackNOutpoints(buf[i:]) if err != nil { @@ -812,8 +792,7 @@ func unpackTxAddresses(buf []byte) (*TxAddresses, error) { func unpackTxInput(ti *TxInput, buf []byte) int { al, l := unpackVaruint(buf) - ti.AddrDesc = make([]byte, al) - copy(ti.AddrDesc, buf[l:l+int(al)]) + ti.AddrDesc = append([]byte(nil), buf[l:l+int(al)]...) al += uint(l) ti.ValueSat, l = unpackBigint(buf[al:]) return l + int(al) @@ -825,8 +804,7 @@ func unpackTxOutput(to *TxOutput, buf []byte) int { to.Spent = true al = ^al } - to.AddrDesc = make([]byte, al) - copy(to.AddrDesc, buf[l:l+al]) + to.AddrDesc = append([]byte(nil), buf[l:l+al]...) al += l to.ValueSat, l = unpackBigint(buf[al:]) return l + al @@ -1004,51 +982,6 @@ func (d *RocksDB) writeHeight(wb *gorocksdb.WriteBatch, height uint32, bi *Block // Disconnect blocks -func (d *RocksDB) allAddressesScan(lower uint32, higher uint32) ([][]byte, [][]byte, error) { - glog.Infof("db: doing full scan of addresses column") - addrKeys := [][]byte{} - addrValues := [][]byte{} - var totalOutputs, count uint64 - var seekKey []byte - for { - var key []byte - it := d.db.NewIteratorCF(d.ro, d.cfh[cfAddresses]) - if totalOutputs == 0 { - it.SeekToFirst() - } else { - it.Seek(seekKey) - it.Next() - } - for count = 0; it.Valid() && count < refreshIterator; it.Next() { - totalOutputs++ - count++ - key = it.Key().Data() - l := len(key) - if l > packedHeightBytes { - height := unpackUint(key[l-packedHeightBytes : l]) - if height >= lower && height <= higher { - addrKey := make([]byte, len(key)) - copy(addrKey, key) - addrKeys = append(addrKeys, addrKey) - value := it.Value().Data() - addrValue := make([]byte, len(value)) - copy(addrValue, value) - addrValues = append(addrValues, addrValue) - } - } - } - seekKey = make([]byte, len(key)) - copy(seekKey, key) - valid := it.Valid() - it.Close() - if !valid { - break - } - } - glog.Infof("rocksdb: scanned %d addresses, found %d to disconnect", totalOutputs, len(addrKeys)) - return addrKeys, addrValues, nil -} - func (d *RocksDB) disconnectTxAddresses(wb *gorocksdb.WriteBatch, height uint32, txid string, inputs []outpoint, txa *TxAddresses, txAddressesToUpdate map[string]*TxAddresses, balances map[string]*AddrBalance) error { addresses := make(map[string]struct{}) @@ -1136,9 +1069,8 @@ func (d *RocksDB) disconnectTxAddresses(wb *gorocksdb.WriteBatch, height uint32, } // DisconnectBlockRangeBitcoinType removes all data belonging to blocks in range lower-higher -// if they are in the range kept in the cfBlockTxids column +// it is able to disconnect only blocks for which there are data in the blockTxs column func (d *RocksDB) DisconnectBlockRangeBitcoinType(lower uint32, higher uint32) error { - glog.Infof("db: disconnecting blocks %d-%d", lower, higher) blocks := make([][]blockTxs, higher-lower+1) for height := lower; height <= higher; height++ { blockTxs, err := d.getBlockTxs(height) diff --git a/db/rocksdb_ethereumtype.go b/db/rocksdb_ethereumtype.go index 96033c83..b0614e78 100644 --- a/db/rocksdb_ethereumtype.go +++ b/db/rocksdb_ethereumtype.go @@ -65,8 +65,7 @@ func (d *RocksDB) GetAddrDescContracts(addrDesc bchain.AddressDescriptor) (*Addr return nil, errors.New("Invalid data stored in cfAddressContracts for AddrDesc " + addrDesc.String()) } txs, l := unpackVaruint(buf[eth.EthereumTypeAddressDescriptorLen:]) - contract := make(bchain.AddressDescriptor, eth.EthereumTypeAddressDescriptorLen) - copy(contract, buf[:eth.EthereumTypeAddressDescriptorLen]) + contract := append(bchain.AddressDescriptor(nil), buf[:eth.EthereumTypeAddressDescriptorLen]...) c = append(c, AddrContract{ Contract: contract, Txs: txs, @@ -78,6 +77,15 @@ func (d *RocksDB) GetAddrDescContracts(addrDesc bchain.AddressDescriptor) (*Addr Contracts: c}, nil } +func findContractInAddressContracts(contract bchain.AddressDescriptor, contracts []AddrContract) (int, bool) { + for i := range contracts { + if bytes.Equal(contract, contracts[i].Contract) { + return i, true + } + } + return 0, false +} + func (d *RocksDB) addToAddressesAndContractsEthereumType(addrDesc bchain.AddressDescriptor, btxID []byte, index int32, contract bchain.AddressDescriptor, addresses map[string][]outpoint, addressContracts map[string]*AddrContracts) error { var err error strAddrDesc := string(addrDesc) @@ -99,14 +107,7 @@ func (d *RocksDB) addToAddressesAndContractsEthereumType(addrDesc bchain.Address ac.EthTxs++ } else { // locate the contract and set i to the index in the array of contracts - var i int - var found bool - for i = range ac.Contracts { - if bytes.Equal(contract, ac.Contracts[i].Contract) { - found = true - break - } - } + i, found := findContractInAddressContracts(contract, ac.Contracts) if !found { i = len(ac.Contracts) ac.Contracts = append(ac.Contracts, AddrContract{Contract: contract}) @@ -240,31 +241,159 @@ func (d *RocksDB) storeAndCleanupBlockTxsEthereumType(wb *gorocksdb.WriteBatch, return d.cleanupBlockTxs(wb, block) } -// DisconnectBlockRangeNonUTXO performs full range scan to remove a range of blocks -// it is very slow operation -func (d *RocksDB) DisconnectBlockRangeNonUTXO(lower uint32, higher uint32) error { - glog.Infof("db: disconnecting blocks %d-%d", lower, higher) - addrKeys, _, err := d.allAddressesScan(lower, higher) +func (d *RocksDB) getBlockTxsEthereumType(height uint32) ([]ethBlockTx, error) { + pl := d.chainParser.PackedTxidLen() + val, err := d.db.GetCF(d.ro, d.cfh[cfBlockTxs], packUint(height)) if err != nil { - return err + return nil, err + } + defer val.Free() + buf := val.Data() + bt := make([]ethBlockTx, 0, 8) + getAddress := func(i int) (bchain.AddressDescriptor, int, error) { + if len(buf)-i < eth.EthereumTypeAddressDescriptorLen { + glog.Error("rocksdb: Inconsistent data in blockTxs ", hex.EncodeToString(buf)) + return nil, 0, errors.New("Inconsistent data in blockTxs") + } + a := append(bchain.AddressDescriptor(nil), buf[i:i+eth.EthereumTypeAddressDescriptorLen]...) + // return null addresses as nil + for _, b := range a { + if b != 0 { + return a, i + eth.EthereumTypeAddressDescriptorLen, nil + } + } + return nil, i + eth.EthereumTypeAddressDescriptorLen, nil + } + var from, to bchain.AddressDescriptor + for i := 0; i < len(buf); { + if len(buf)-i < pl { + glog.Error("rocksdb: Inconsistent data in blockTxs ", hex.EncodeToString(buf)) + return nil, errors.New("Inconsistent data in blockTxs") + } + txid := append([]byte(nil), buf[i:i+pl]...) + i += pl + from, i, err = getAddress(i) + if err != nil { + return nil, err + } + to, i, err = getAddress(i) + if err != nil { + return nil, err + } + cc, l := unpackVaruint(buf[i:]) + i += l + contracts := make([]ethBlockTxContract, cc) + for j := range contracts { + contracts[j].addr, i, err = getAddress(i) + if err != nil { + return nil, err + } + contracts[j].contract, i, err = getAddress(i) + if err != nil { + return nil, err + } + } + bt = append(bt, ethBlockTx{ + btxID: txid, + from: from, + to: to, + contracts: contracts, + }) + } + return bt, nil +} + +func (d *RocksDB) disconnectBlockTxsEthereumType(wb *gorocksdb.WriteBatch, height uint32, blockTxs []ethBlockTx, contracts map[string]*AddrContracts) error { + glog.Info("Disconnecting block ", height, " containing ", len(blockTxs), " transactions") + addresses := make(map[string]struct{}) + disconnectAddress := func(addrDesc, contract bchain.AddressDescriptor) error { + var err error + s := string(addrDesc) + addresses[s] = struct{}{} + c, fc := contracts[s] + if !fc { + c, err = d.GetAddrDescContracts(addrDesc) + if err != nil { + return err + } + contracts[s] = c + } + if c != nil { + if contract == nil { + if c.EthTxs > 0 { + c.EthTxs-- + } else { + glog.Warning("AddressContracts ", addrDesc, ", EthTxs would be negative") + } + } else { + i, found := findContractInAddressContracts(contract, c.Contracts) + if found { + if c.Contracts[i].Txs > 0 { + c.Contracts[i].Txs-- + if c.Contracts[i].Txs == 0 { + c.Contracts = append(c.Contracts[:i], c.Contracts[i+1:]...) + } + } else { + glog.Warning("AddressContracts ", addrDesc, ", contract ", i, " Txs would be negative") + } + } else { + glog.Warning("AddressContracts ", addrDesc, ", contract ", contract, " not found") + } + } + } else { + glog.Warning("AddressContracts ", addrDesc, " not found") + } + return nil + } + for i := range blockTxs { + blockTx := &blockTxs[i] + if err := disconnectAddress(blockTx.from, nil); err != nil { + return err + } + if err := disconnectAddress(blockTx.to, nil); err != nil { + return err + } + for _, c := range blockTx.contracts { + if err := disconnectAddress(c.addr, c.contract); err != nil { + return err + } + } + wb.DeleteCF(d.cfh[cfTransactions], blockTx.btxID) + } + for a := range addresses { + key := packAddressKey([]byte(a), height) + wb.DeleteCF(d.cfh[cfAddresses], key) + } + return nil +} + +// DisconnectBlockRangeEthereumType removes all data belonging to blocks in range lower-higher +// it is able to disconnect only blocks for which there are data in the blockTxs column +func (d *RocksDB) DisconnectBlockRangeEthereumType(lower uint32, higher uint32) error { + blocks := make([][]ethBlockTx, higher-lower+1) + for height := lower; height <= higher; height++ { + blockTxs, err := d.getBlockTxsEthereumType(height) + if err != nil { + return err + } + if len(blockTxs) == 0 { + return errors.Errorf("Cannot disconnect blocks with height %v and lower. It is necessary to rebuild index.", height) + } + blocks[height-lower] = blockTxs } - glog.Infof("rocksdb: about to disconnect %d addresses ", len(addrKeys)) wb := gorocksdb.NewWriteBatch() defer wb.Destroy() - for _, addrKey := range addrKeys { - if glog.V(2) { - glog.Info("address ", hex.EncodeToString(addrKey)) + contracts := make(map[string]*AddrContracts) + for height := higher; height >= lower; height-- { + if err := d.disconnectBlockTxsEthereumType(wb, height, blocks[height-lower], contracts); err != nil { + return err } - // delete address:height from the index - wb.DeleteCF(d.cfh[cfAddresses], addrKey) + key := packUint(height) + wb.DeleteCF(d.cfh[cfBlockTxs], key) + wb.DeleteCF(d.cfh[cfHeight], key) } - for height := lower; height <= higher; height++ { - if glog.V(2) { - glog.Info("height ", height) - } - wb.DeleteCF(d.cfh[cfHeight], packUint(height)) - } - err = d.db.Write(d.wo, wb) + d.storeAddressContracts(wb, contracts) + err := d.db.Write(d.wo, wb) if err == nil { glog.Infof("rocksdb: blocks %d-%d disconnected", lower, higher) } diff --git a/db/rocksdb_ethereumtype_test.go b/db/rocksdb_ethereumtype_test.go index 2b05c46d..e57b192d 100644 --- a/db/rocksdb_ethereumtype_test.go +++ b/db/rocksdb_ethereumtype_test.go @@ -5,7 +5,11 @@ package db import ( "blockbook/bchain/coins/eth" "blockbook/tests/dbtestdata" + "encoding/hex" + "reflect" "testing" + + "github.com/juju/errors" ) type testEthereumParser struct { @@ -85,7 +89,7 @@ func verifyAfterEthereumTypeBlock2(t *testing.T, d *RocksDB) { }, keyPair{ "0041eee9", - "00000000eb0443fd7dc4a1ed5c686a8e995057805f9a161d9a5a77a95e72b7b6" + uintToHex(1534859988) + varuintToHex(2) + varuintToHex(2345678), + "2b57e15e93a0ed197417a34c2498b7187df79099572c04a6b6e6ff418f74e6ee" + uintToHex(1534859988) + varuintToHex(2) + varuintToHex(2345678), nil, }, }); err != nil { @@ -156,9 +160,8 @@ func verifyAfterEthereumTypeBlock2(t *testing.T, d *RocksDB) { // 2) GetTransactions for various addresses / low-high ranges // 3) GetBestBlock, GetBlockHash // 4) Test tx caching functionality -// 5) Disconnect block 2 - expect error -// 6) Disconnect the block 2 using BlockTxs column -// 7) Reconnect block 2 and check +// 5) Disconnect the block 2 using BlockTxs column +// 6) Reconnect block 2 and check // After each step, the content of DB is examined and any difference against expected state is regarded as failure func TestRocksDB_Index_EthereumType(t *testing.T) { d := setupRocksDB(t, &testEthereumParser{ @@ -166,18 +169,118 @@ func TestRocksDB_Index_EthereumType(t *testing.T) { }) defer closeAndDestroyRocksDB(t, d) - // connect 1st block - will log warnings about missing UTXO transactions in txAddresses column + // connect 1st block block1 := dbtestdata.GetTestEthereumTypeBlock1(d.chainParser) if err := d.ConnectBlock(block1); err != nil { t.Fatal(err) } verifyAfterEthereumTypeBlock1(t, d, false) - // connect 2nd block - use some outputs from the 1st block as the inputs and 1 input uses tx from the same block + // connect 2nd block block2 := dbtestdata.GetTestEthereumTypeBlock2(d.chainParser) if err := d.ConnectBlock(block2); err != nil { t.Fatal(err) } verifyAfterEthereumTypeBlock2(t, d) + // get transactions for various addresses / low-high ranges + verifyGetTransactions(t, d, "0x"+dbtestdata.EthAddr55, 0, 10000000, []txidVoutOutput{ + txidVoutOutput{"0x" + dbtestdata.EthTxidB1T1, 0, true}, + txidVoutOutput{"0x" + dbtestdata.EthTxidB1T2, 1, true}, + txidVoutOutput{"0x" + dbtestdata.EthTxidB2T1, 0, false}, + txidVoutOutput{"0x" + dbtestdata.EthTxidB2T2, 2, false}, + txidVoutOutput{"0x" + dbtestdata.EthTxidB2T2, 1, true}, + }, nil) + verifyGetTransactions(t, d, "mtGXQvBowMkBpnhLckhxhbwYK44Gs9eBad", 500000, 1000000, []txidVoutOutput{}, errors.New("Address missing")) + + // GetBestBlock + height, hash, err := d.GetBestBlock() + if err != nil { + t.Fatal(err) + } + if height != 4321001 { + t.Fatalf("GetBestBlock: got height %v, expected %v", height, 4321001) + } + if hash != "0x2b57e15e93a0ed197417a34c2498b7187df79099572c04a6b6e6ff418f74e6ee" { + t.Fatalf("GetBestBlock: got hash %v, expected %v", hash, "0x2b57e15e93a0ed197417a34c2498b7187df79099572c04a6b6e6ff418f74e6ee") + } + + // GetBlockHash + hash, err = d.GetBlockHash(4321000) + if err != nil { + t.Fatal(err) + } + if hash != "0xc7b98df95acfd11c51ba25611a39e004fe56c8fdfc1582af99354fcd09c17b11" { + t.Fatalf("GetBlockHash: got hash %v, expected %v", hash, "0xc7b98df95acfd11c51ba25611a39e004fe56c8fdfc1582af99354fcd09c17b11") + } + + // Not connected block + hash, err = d.GetBlockHash(4321002) + if err != nil { + t.Fatal(err) + } + if hash != "" { + t.Fatalf("GetBlockHash: got hash '%v', expected ''", hash) + } + + // GetBlockHash + info, err := d.GetBlockInfo(4321001) + if err != nil { + t.Fatal(err) + } + iw := &BlockInfo{ + Hash: "0x2b57e15e93a0ed197417a34c2498b7187df79099572c04a6b6e6ff418f74e6ee", + Txs: 2, + Size: 2345678, + Time: 1534859988, + Height: 4321001, + } + if !reflect.DeepEqual(info, iw) { + t.Errorf("GetBlockInfo() = %+v, want %+v", info, iw) + } + + // Test tx caching functionality, leave one tx in db to test cleanup in DisconnectBlock + testTxCache(t, d, block1, &block1.Txs[0]) + testTxCache(t, d, block2, &block2.Txs[0]) + if err = d.PutTx(&block2.Txs[1], block2.Height, block2.Txs[1].Blocktime); err != nil { + t.Fatal(err) + } + if err = d.PutTx(&block2.Txs[1], block2.Height, block2.Txs[1].Blocktime); err != nil { + t.Fatal(err) + } + // check that there is only the last tx in the cache + packedTx, err := d.chainParser.PackTx(&block2.Txs[1], block2.Height, block2.Txs[1].Blocktime) + if err := checkColumn(d, cfTransactions, []keyPair{ + keyPair{dbtestdata.EthTxidB2T2, hex.EncodeToString(packedTx), nil}, + }); err != nil { + { + t.Fatal(err) + } + } + // try to disconnect both blocks, however only the last one is kept, it is not possible + err = d.DisconnectBlockRangeEthereumType(4321000, 4321001) + if err == nil || err.Error() != "Cannot disconnect blocks with height 4321000 and lower. It is necessary to rebuild index." { + t.Fatal(err) + } + verifyAfterEthereumTypeBlock2(t, d) + + // disconnect the 2nd block, verify that the db contains only data from the 1st block with restored unspentTxs + // and that the cached tx is removed + err = d.DisconnectBlockRangeEthereumType(4321001, 4321001) + if err != nil { + t.Fatal(err) + } + verifyAfterEthereumTypeBlock1(t, d, true) + if err := checkColumn(d, cfTransactions, []keyPair{}); err != nil { + { + t.Fatal(err) + } + } + + // connect block again and verify the state of db + if err := d.ConnectBlock(block2); err != nil { + t.Fatal(err) + } + verifyAfterEthereumTypeBlock2(t, d) + } diff --git a/db/rocksdb_test.go b/db/rocksdb_test.go index fe8d50af..64eff3b7 100644 --- a/db/rocksdb_test.go +++ b/db/rocksdb_test.go @@ -9,7 +9,6 @@ import ( "blockbook/tests/dbtestdata" "encoding/binary" "encoding/hex" - "fmt" "io/ioutil" "math/big" "os" @@ -415,7 +414,7 @@ func testTxCache(t *testing.T, d *RocksDB, b *bchain.Block, tx *bchain.Tx) { } // Confirmations are not stored in the DB, set them from input tx gtx.Confirmations = tx.Confirmations - if fmt.Sprint(gtx) != fmt.Sprint(tx) { + if !reflect.DeepEqual(gtx, tx) { t.Errorf("GetTx: %v, want %v", gtx, tx) } if err := d.DeleteTx(tx.Txid); err != nil { @@ -429,9 +428,8 @@ func testTxCache(t *testing.T, d *RocksDB, b *bchain.Block, tx *bchain.Tx) { // 2) GetTransactions for various addresses / low-high ranges // 3) GetBestBlock, GetBlockHash // 4) Test tx caching functionality -// 5) Disconnect block 2 - expect error -// 6) Disconnect the block 2 using BlockTxs column -// 7) Reconnect block 2 and check +// 5) Disconnect the block 2 using BlockTxs column +// 6) Reconnect block 2 and check // After each step, the content of DB is examined and any difference against expected state is regarded as failure func TestRocksDB_Index_BitcoinType(t *testing.T) { d := setupRocksDB(t, &testBitcoinParser{ @@ -513,7 +511,7 @@ func TestRocksDB_Index_BitcoinType(t *testing.T) { Height: 225494, } if !reflect.DeepEqual(info, iw) { - t.Errorf("GetAddressBalance() = %+v, want %+v", info, iw) + t.Errorf("GetBlockInfo() = %+v, want %+v", info, iw) } // Test tx caching functionality, leave one tx in db to test cleanup in DisconnectBlock @@ -532,13 +530,6 @@ func TestRocksDB_Index_BitcoinType(t *testing.T) { } } - // DisconnectBlock for BitcoinType chains is not possible - err = d.DisconnectBlock(block2) - if err == nil || err.Error() != "DisconnectBlock is not supported for BitcoinType chains" { - t.Fatal(err) - } - verifyAfterBitcoinTypeBlock2(t, d) - // try to disconnect both blocks, however only the last one is kept, it is not possible err = d.DisconnectBlockRangeBitcoinType(225493, 225494) if err == nil || err.Error() != "Cannot disconnect blocks with height 225493 and lower. It is necessary to rebuild index." { @@ -566,7 +557,6 @@ func TestRocksDB_Index_BitcoinType(t *testing.T) { verifyAfterBitcoinTypeBlock2(t, d) // test public methods for address balance and tx addresses - ab, err := d.GetAddressBalance(dbtestdata.Addr5) if err != nil { t.Fatal(err) @@ -848,17 +838,17 @@ func Test_packTxAddresses_unpackTxAddresses(t *testing.T) { Height: 123456789, Inputs: []TxInput{ { - AddrDesc: []byte{}, + AddrDesc: []byte(nil), ValueSat: *big.NewInt(1234), }, }, Outputs: []TxOutput{ { - AddrDesc: []byte{}, + AddrDesc: []byte(nil), ValueSat: *big.NewInt(5678), }, { - AddrDesc: []byte{}, + AddrDesc: []byte(nil), ValueSat: *big.NewInt(98), Spent: true, }, diff --git a/db/sync.go b/db/sync.go index c408bb72..e755c027 100644 --- a/db/sync.go +++ b/db/sync.go @@ -393,26 +393,11 @@ func (w *SyncWorker) getBlockChain(out chan blockResult, done chan struct{}) { // DisconnectBlocks removes all data belonging to blocks in range lower-higher, func (w *SyncWorker) DisconnectBlocks(lower uint32, higher uint32, hashes []string) error { glog.Infof("sync: disconnecting blocks %d-%d", lower, higher) - // if the chain is ChainBitcoinType, always use DisconnectBlockRange - if w.chain.GetChainParser().GetChainType() == bchain.ChainBitcoinType { + ct := w.chain.GetChainParser().GetChainType() + if ct == bchain.ChainBitcoinType { return w.db.DisconnectBlockRangeBitcoinType(lower, higher) + } else if ct == bchain.ChainEthereumType { + return w.db.DisconnectBlockRangeEthereumType(lower, higher) } - blocks := make([]*bchain.Block, len(hashes)) - var err error - // try to get all blocks first to see if we can avoid full scan - for i, hash := range hashes { - blocks[i], err = w.chain.GetBlock(hash, 0) - if err != nil { - // cannot get a block, we must do full range scan - return w.db.DisconnectBlockRangeNonUTXO(lower, higher) - } - } - // got all blocks to be disconnected, disconnect them one after another - for i, block := range blocks { - glog.Info("Disconnecting block ", (int(higher) - i), " ", block.Hash) - if err = w.db.DisconnectBlock(block); err != nil { - return err - } - } - return nil + return errors.New("Unknown chain type") } diff --git a/server/public_test.go b/server/public_test.go index 6b72853b..41750d6c 100644 --- a/server/public_test.go +++ b/server/public_test.go @@ -396,7 +396,7 @@ func httpTests(t *testing.T, ts *httptest.Server) { status: http.StatusOK, contentType: "application/json; charset=utf-8", body: []string{ - `{"hex":"","txid":"00b2c06055e5e90e9c82bd4181fde310104391a7fa4f289b1704e5d90caa3840","version":0,"locktime":0,"vin":null,"vout":[{"ValueSat":100000000,"value":0,"n":0,"scriptPubKey":{"hex":"76a914010d39800f86122416e28f485029acf77507169288ac","addresses":null}},{"ValueSat":12345,"value":0,"n":1,"scriptPubKey":{"hex":"76a9148bdf0aa3c567aa5975c2e61321b8bebbe7293df688ac","addresses":null}}],"confirmations":2,"time":22549300000,"blocktime":22549300000}`, + `{"hex":"","txid":"00b2c06055e5e90e9c82bd4181fde310104391a7fa4f289b1704e5d90caa3840","version":0,"locktime":0,"vin":[],"vout":[{"ValueSat":100000000,"value":0,"n":0,"scriptPubKey":{"hex":"76a914010d39800f86122416e28f485029acf77507169288ac","addresses":null}},{"ValueSat":12345,"value":0,"n":1,"scriptPubKey":{"hex":"76a9148bdf0aa3c567aa5975c2e61321b8bebbe7293df688ac","addresses":null}}],"confirmations":2,"time":22549300000,"blocktime":22549300000}`, }, }, { diff --git a/tests/dbtestdata/dbtestdata.go b/tests/dbtestdata/dbtestdata.go index 3f0a92d2..c485129f 100644 --- a/tests/dbtestdata/dbtestdata.go +++ b/tests/dbtestdata/dbtestdata.go @@ -66,6 +66,7 @@ func GetTestBitcoinTypeBlock1(parser bchain.BlockChainParser) *bchain.Block { Txs: []bchain.Tx{ bchain.Tx{ Txid: TxidB1T1, + Vin: []bchain.Vin{}, Vout: []bchain.Vout{ bchain.Vout{ N: 0, diff --git a/tests/dbtestdata/dbtestdata_ethereumtype.go b/tests/dbtestdata/dbtestdata_ethereumtype.go index 4f228182..04ca9956 100644 --- a/tests/dbtestdata/dbtestdata_ethereumtype.go +++ b/tests/dbtestdata/dbtestdata_ethereumtype.go @@ -17,13 +17,13 @@ const ( EthAddrContract47 = "479cc461fecd078f766ecc58533d6f69580cf3ac" // non ERC20 EthTxidB1T1 = "cd647151552b5132b2aef7c9be00dc6f73afc5901dde157aab131335baaa853b" - EthTx1Packed = "08889eaf0110fa83c3d5051a6908ece40212050430e234001888a40122081bc0159d530e60003220cd647151552b5132b2aef7c9be00dc6f73afc5901dde157aab131335baaa853b3a14555ee11fbddc0e49a9bab358a8941ad95ffdb48f42143e3a3d69dc66ba10737f531ed088954a9ec89d97480a22070a025208120101" + EthTx1Packed = "08e8dd870210a6a6f0db051a6908ece40212050430e234001888a40122081bc0159d530e60003220cd647151552b5132b2aef7c9be00dc6f73afc5901dde157aab131335baaa853b3a14555ee11fbddc0e49a9bab358a8941ad95ffdb48f42143e3a3d69dc66ba10737f531ed088954a9ec89d97480a22070a025208120101" EthTxidB1T2 = "a9cd088aba2131000da6f38a33c20169baee476218deea6b78720700b895b101" - EthTx2Packed = "08d38388021092f4c1d5051aa20108d001120509502f900018d5e1042a44a9059cbb000000000000000000000000555ee11fbddc0e49a9bab358a8941ad95ffdb48f00000000000000000000000000000000000000000000021e19e0c9bab24000003220a9cd088aba2131000da6f38a33c20169baee476218deea6b78720700b895b1013a144af4114f73d1c1c903ac9e0361b379d1291808a2421420cd153de35d469ba46127a0c8f18626b59a256a22a8010a02cb391201011a9e010a144af4114f73d1c1c903ac9e0361b379d1291808a2122000000000000000000000000000000000000000000000021e19e0c9bab24000001a20ddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef1a2000000000000000000000000020cd153de35d469ba46127a0c8f18626b59a256a1a20000000000000000000000000555ee11fbddc0e49a9bab358a8941ad95ffdb48f" + EthTx2Packed = "08e8dd870210a6a6f0db051aa20108d001120509502f900018d5e1042a44a9059cbb000000000000000000000000555ee11fbddc0e49a9bab358a8941ad95ffdb48f00000000000000000000000000000000000000000000021e19e0c9bab24000003220a9cd088aba2131000da6f38a33c20169baee476218deea6b78720700b895b1013a144af4114f73d1c1c903ac9e0361b379d1291808a2421420cd153de35d469ba46127a0c8f18626b59a256a22a8010a02cb391201011a9e010a144af4114f73d1c1c903ac9e0361b379d1291808a2122000000000000000000000000000000000000000000000021e19e0c9bab24000001a20ddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef1a2000000000000000000000000020cd153de35d469ba46127a0c8f18626b59a256a1a20000000000000000000000000555ee11fbddc0e49a9bab358a8941ad95ffdb48f" EthTxidB2T1 = "c2c3dd1ecb00e8a6d81f793d24387cf2947a313e94ab03b1fb22cd63320f6c91" - EthTx3Packed = "08fda28c0210d8eb95df051a6708c20112050218711a001888a401220710bc3578bd37d83220c2c3dd1ecb00e8a6d81f793d24387cf2947a313e94ab03b1fb22cd63320f6c913a149f4981531fda132e83c44680787dfa7ee31e4f8d4214555ee11fbddc0e49a9bab358a8941ad95ffdb48f480722070a025208120101" + EthTx3Packed = "08e9dd870210d4b5f0db051a6708c20112050218711a001888a401220710bc3578bd37d83220c2c3dd1ecb00e8a6d81f793d24387cf2947a313e94ab03b1fb22cd63320f6c913a149f4981531fda132e83c44680787dfa7ee31e4f8d4214555ee11fbddc0e49a9bab358a8941ad95ffdb48f480722070a025208120101" EthTxidB2T2 = "c92919ad24ffd58f760b18df7949f06e1190cf54a50a0e3745a385608ed3cbf2" - EthTx4Packed = "088795890210b5e1ebde051aa50b08f6be0712043b9aca001890a10f2ac40a4f15078700000000000000000000000000000000000000000000000000000000000000c0000000000000000000000000000000000000000000000000000000000000022000000000000000000000000000000000000000000000000000000000000003c00000000000000000000000000000000000000000000000000000000000000420000000000000000000000000000000000000000000000000000000000000048000000000000000000000000000000000000000000000000000000000000004e00000000000000000000000000000000000000000000000000000000000000002000000000000000000000000555ee11fbddc0e49a9bab358a8941ad95ffdb48f0000000000000000000000004bda106325c335df99eab7fe363cac8a0ba2a24d0000000000000000000000000d0f936ee4c93e25944694d6c121de94d9760f110000000000000000000000004af4114f73d1c1c903ac9e0361b379d1291808a200000000000000000000000000000000000000000000000000000000000000000000000000000000000000007b62eb7fe80350dc7ec945c0b73242cb9877fb1b0000000000000000000000004bda106325c335df99eab7fe363cac8a0ba2a24d0000000000000000000000004af4114f73d1c1c903ac9e0361b379d1291808a20000000000000000000000000d0f936ee4c93e25944694d6c121de94d9760f110000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000200000000000000000000000000000000000000000000000a5ef5a7656bfb0000000000000000000000000000000000000000000000000000004ba78398d5c5000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000166cfe0b9579b4ecf7a2801880f644009a324671a79754ea57c3a103c6e70d3dbef6ba69a08000000000000000000000000000000000000000000000000004f937d86afb90000000000000000000000000000000000000000000000000ab280fd8037d500000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000166cfb784b7c1f3fbe8b75484603ab8adc58aaee3a46245a6579fac7077b5570018b4e0d4eb0000000000000000000000000000000000000000000000000000000000000002000000000000000000000000000000000000000000000000000308fd0e798ac00000000000000000000000000000000000000000000000006a8313d60b1f80000000000000000000000000000000000000000000000000000000000000000002000000000000000000000000000000000000000000000000000000000000001b000000000000000000000000000000000000000000000000000000000000001b00000000000000000000000000000000000000000000000000000000000000029de0ccec59e8948e3d905b40e5542335ebc1eb4674db517d2f6392ec7fdeb3d45f3449d313ee2589819c6c79eb1c1b047adae68565c1608e3a1d1d70823febb0000000000000000000000000000000000000000000000000000000000000000234d06fe17f1202e8b07177a30eb64d14adc08cdb3fa1b3e3e0bea0f9672c02175b77c01c51d3c7e460723b27ecbc7801fd6482559a8c9999593f9a4d149c73843220c92919ad24ffd58f760b18df7949f06e1190cf54a50a0e3745a385608ed3cbf23a14479cc461fecd078f766ecc58533d6f69580cf3ac42144bda106325c335df99eab7fe363cac8a0ba2a24d482422d40b0a03034d301201011a9e010a140d0f936ee4c93e25944694d6c121de94d9760f1112200000000000000000000000000000000000000000000000006a8313d60b1f606b1a20ddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef1a20000000000000000000000000555ee11fbddc0e49a9bab358a8941ad95ffdb48f1a200000000000000000000000004bda106325c335df99eab7fe363cac8a0ba2a24d1a9e010a144af4114f73d1c1c903ac9e0361b379d1291808a21220000000000000000000000000000000000000000000000000000308fd0e798ac01a20ddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef1a200000000000000000000000004bda106325c335df99eab7fe363cac8a0ba2a24d1a20000000000000000000000000555ee11fbddc0e49a9bab358a8941ad95ffdb48f1aa1030a14479cc461fecd078f766ecc58533d6f69580cf3ac1280020000000000000000000000004bda106325c335df99eab7fe363cac8a0ba2a24d0000000000000000000000000d0f936ee4c93e25944694d6c121de94d9760f110000000000000000000000004af4114f73d1c1c903ac9e0361b379d1291808a20000000000000000000000000000000000000000000000006a8313d60b1f606b000000000000000000000000000000000000000000000000000308fd0e798ac0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000005e083a16f4b092c5729a49f9c3ed3cc171bb3d3d0c22e20b1de6063c32f399ac1a200d0b9391970d9a25552f37d436d2aae2925e2bfe1b2a923754bada030c498cb31a20000000000000000000000000555ee11fbddc0e49a9bab358a8941ad95ffdb48f1a2000000000000000000000000000000000000000000000000000000000000000001a205af266c0a89a07c1917deaa024414577e6c3c31c8907d079e13eb448c082594f1a9e010a144af4114f73d1c1c903ac9e0361b379d1291808a2122000000000000000000000000000000000000000000000000000031855667df7a81a20ddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef1a200000000000000000000000007b62eb7fe80350dc7ec945c0b73242cb9877fb1b1a200000000000000000000000004bda106325c335df99eab7fe363cac8a0ba2a24d1a9e010a140d0f936ee4c93e25944694d6c121de94d9760f1112200000000000000000000000000000000000000000000000006a8313d60b1f80001a20ddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef1a200000000000000000000000004bda106325c335df99eab7fe363cac8a0ba2a24d1a200000000000000000000000007b62eb7fe80350dc7ec945c0b73242cb9877fb1b1aa1030a14479cc461fecd078f766ecc58533d6f69580cf3ac1280020000000000000000000000004bda106325c335df99eab7fe363cac8a0ba2a24d0000000000000000000000004af4114f73d1c1c903ac9e0361b379d1291808a20000000000000000000000000d0f936ee4c93e25944694d6c121de94d9760f1100000000000000000000000000000000000000000000000000031855667df7a80000000000000000000000000000000000000000000000006a8313d60b1f800000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000f2b0d62c44ed08f2a5adef40c875d20310a42a9d4f488bd26323256fe01c7f481a200d0b9391970d9a25552f37d436d2aae2925e2bfe1b2a923754bada030c498cb31a200000000000000000000000007b62eb7fe80350dc7ec945c0b73242cb9877fb1b1a2000000000000000000000000000000000000000000000000000000000000000001a20b0b69dad58df6032c3b266e19b1045b19c87acd2c06fb0c598090f44b8e263aa" + EthTx4Packed = "08e9dd870210d4b5f0db051aa50b08f6be0712043b9aca001890a10f2ac40a4f15078700000000000000000000000000000000000000000000000000000000000000c0000000000000000000000000000000000000000000000000000000000000022000000000000000000000000000000000000000000000000000000000000003c00000000000000000000000000000000000000000000000000000000000000420000000000000000000000000000000000000000000000000000000000000048000000000000000000000000000000000000000000000000000000000000004e00000000000000000000000000000000000000000000000000000000000000002000000000000000000000000555ee11fbddc0e49a9bab358a8941ad95ffdb48f0000000000000000000000004bda106325c335df99eab7fe363cac8a0ba2a24d0000000000000000000000000d0f936ee4c93e25944694d6c121de94d9760f110000000000000000000000004af4114f73d1c1c903ac9e0361b379d1291808a200000000000000000000000000000000000000000000000000000000000000000000000000000000000000007b62eb7fe80350dc7ec945c0b73242cb9877fb1b0000000000000000000000004bda106325c335df99eab7fe363cac8a0ba2a24d0000000000000000000000004af4114f73d1c1c903ac9e0361b379d1291808a20000000000000000000000000d0f936ee4c93e25944694d6c121de94d9760f110000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000200000000000000000000000000000000000000000000000a5ef5a7656bfb0000000000000000000000000000000000000000000000000000004ba78398d5c5000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000166cfe0b9579b4ecf7a2801880f644009a324671a79754ea57c3a103c6e70d3dbef6ba69a08000000000000000000000000000000000000000000000000004f937d86afb90000000000000000000000000000000000000000000000000ab280fd8037d500000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000166cfb784b7c1f3fbe8b75484603ab8adc58aaee3a46245a6579fac7077b5570018b4e0d4eb0000000000000000000000000000000000000000000000000000000000000002000000000000000000000000000000000000000000000000000308fd0e798ac00000000000000000000000000000000000000000000000006a8313d60b1f80000000000000000000000000000000000000000000000000000000000000000002000000000000000000000000000000000000000000000000000000000000001b000000000000000000000000000000000000000000000000000000000000001b00000000000000000000000000000000000000000000000000000000000000029de0ccec59e8948e3d905b40e5542335ebc1eb4674db517d2f6392ec7fdeb3d45f3449d313ee2589819c6c79eb1c1b047adae68565c1608e3a1d1d70823febb0000000000000000000000000000000000000000000000000000000000000000234d06fe17f1202e8b07177a30eb64d14adc08cdb3fa1b3e3e0bea0f9672c02175b77c01c51d3c7e460723b27ecbc7801fd6482559a8c9999593f9a4d149c73843220c92919ad24ffd58f760b18df7949f06e1190cf54a50a0e3745a385608ed3cbf23a14479cc461fecd078f766ecc58533d6f69580cf3ac42144bda106325c335df99eab7fe363cac8a0ba2a24d482422d40b0a03034d301201011a9e010a140d0f936ee4c93e25944694d6c121de94d9760f1112200000000000000000000000000000000000000000000000006a8313d60b1f606b1a20ddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef1a20000000000000000000000000555ee11fbddc0e49a9bab358a8941ad95ffdb48f1a200000000000000000000000004bda106325c335df99eab7fe363cac8a0ba2a24d1a9e010a144af4114f73d1c1c903ac9e0361b379d1291808a21220000000000000000000000000000000000000000000000000000308fd0e798ac01a20ddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef1a200000000000000000000000004bda106325c335df99eab7fe363cac8a0ba2a24d1a20000000000000000000000000555ee11fbddc0e49a9bab358a8941ad95ffdb48f1aa1030a14479cc461fecd078f766ecc58533d6f69580cf3ac1280020000000000000000000000004bda106325c335df99eab7fe363cac8a0ba2a24d0000000000000000000000000d0f936ee4c93e25944694d6c121de94d9760f110000000000000000000000004af4114f73d1c1c903ac9e0361b379d1291808a20000000000000000000000000000000000000000000000006a8313d60b1f606b000000000000000000000000000000000000000000000000000308fd0e798ac0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000005e083a16f4b092c5729a49f9c3ed3cc171bb3d3d0c22e20b1de6063c32f399ac1a200d0b9391970d9a25552f37d436d2aae2925e2bfe1b2a923754bada030c498cb31a20000000000000000000000000555ee11fbddc0e49a9bab358a8941ad95ffdb48f1a2000000000000000000000000000000000000000000000000000000000000000001a205af266c0a89a07c1917deaa024414577e6c3c31c8907d079e13eb448c082594f1a9e010a144af4114f73d1c1c903ac9e0361b379d1291808a2122000000000000000000000000000000000000000000000000000031855667df7a81a20ddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef1a200000000000000000000000007b62eb7fe80350dc7ec945c0b73242cb9877fb1b1a200000000000000000000000004bda106325c335df99eab7fe363cac8a0ba2a24d1a9e010a140d0f936ee4c93e25944694d6c121de94d9760f1112200000000000000000000000000000000000000000000000006a8313d60b1f80001a20ddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef1a200000000000000000000000004bda106325c335df99eab7fe363cac8a0ba2a24d1a200000000000000000000000007b62eb7fe80350dc7ec945c0b73242cb9877fb1b1aa1030a14479cc461fecd078f766ecc58533d6f69580cf3ac1280020000000000000000000000004bda106325c335df99eab7fe363cac8a0ba2a24d0000000000000000000000004af4114f73d1c1c903ac9e0361b379d1291808a20000000000000000000000000d0f936ee4c93e25944694d6c121de94d9760f1100000000000000000000000000000000000000000000000000031855667df7a80000000000000000000000000000000000000000000000006a8313d60b1f800000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000f2b0d62c44ed08f2a5adef40c875d20310a42a9d4f488bd26323256fe01c7f481a200d0b9391970d9a25552f37d436d2aae2925e2bfe1b2a923754bada030c498cb31a200000000000000000000000007b62eb7fe80350dc7ec945c0b73242cb9877fb1b1a2000000000000000000000000000000000000000000000000000000000000000001a20b0b69dad58df6032c3b266e19b1045b19c87acd2c06fb0c598090f44b8e263aa" ) func unpackTxs(packed []string, parser bchain.BlockChainParser) []bchain.Tx { @@ -59,7 +59,7 @@ func GetTestEthereumTypeBlock2(parser bchain.BlockChainParser) *bchain.Block { return &bchain.Block{ BlockHeader: bchain.BlockHeader{ Height: 4321001, - Hash: "00000000eb0443fd7dc4a1ed5c686a8e995057805f9a161d9a5a77a95e72b7b6", + Hash: "0x2b57e15e93a0ed197417a34c2498b7187df79099572c04a6b6e6ff418f74e6ee", Size: 2345678, Time: 1534859988, Confirmations: 1,