From c9471bf867fcd59f1b11348dbf161c5ddd1356b5 Mon Sep 17 00:00:00 2001 From: Martin Boehm Date: Tue, 21 Aug 2018 16:36:14 +0200 Subject: [PATCH] Store extended info about block in heigth column --- bchain/coins/btc/bitcoinparser.go | 8 +++- bchain/coins/eth/ethrpc.go | 4 +- bchain/types.go | 2 + db/rocksdb.go | 73 ++++++++++++++++++++++++++++--- db/rocksdb_test.go | 54 +++++++++++++++++++++-- 5 files changed, 130 insertions(+), 11 deletions(-) diff --git a/bchain/coins/btc/bitcoinparser.go b/bchain/coins/btc/bitcoinparser.go index 767e8d08..6e2f4c69 100644 --- a/bchain/coins/btc/bitcoinparser.go +++ b/bchain/coins/btc/bitcoinparser.go @@ -184,7 +184,13 @@ func (p *BitcoinParser) ParseBlock(b []byte) (*bchain.Block, error) { txs[ti] = p.TxFromMsgTx(t, false) } - return &bchain.Block{Txs: txs}, nil + return &bchain.Block{ + BlockHeader: bchain.BlockHeader{ + Size: len(b), + Time: w.Header.Timestamp.Unix(), + }, + Txs: txs, + }, nil } // PackTx packs transaction to byte array diff --git a/bchain/coins/eth/ethrpc.go b/bchain/coins/eth/ethrpc.go index 0a19e10b..2ef90b5d 100644 --- a/bchain/coins/eth/ethrpc.go +++ b/bchain/coins/eth/ethrpc.go @@ -321,9 +321,9 @@ func (b *EthereumRPC) ethHeaderToBlockHeader(h *ethtypes.Header) (*bchain.BlockH Hash: ethHashToHash(h.Hash()), Height: uint32(hn), Confirmations: int(c), + Time: int64(h.Time.Uint64()), // Next // Prev - }, nil } @@ -393,6 +393,8 @@ func (b *EthereumRPC) GetBlock(hash string, height uint32) (*bchain.Block, error return nil, errors.Annotatef(fmt.Errorf("server returned empty transaction list but block header indicates transactions"), "hash %v, height %v", hash, height) } bbh, err := b.ethHeaderToBlockHeader(head) + // TODO - this is probably not the correct size + bbh.Size = len(raw) btxs := make([]bchain.Tx, len(body.Transactions)) for i, tx := range body.Transactions { btx, err := b.Parser.ethTxToTx(&tx, int64(head.Time.Uint64()), uint32(bbh.Confirmations)) diff --git a/bchain/types.go b/bchain/types.go index cfe90771..ba631310 100644 --- a/bchain/types.go +++ b/bchain/types.go @@ -88,6 +88,8 @@ type BlockHeader struct { Next string `json:"nextblockhash"` Height uint32 `json:"height"` Confirmations int `json:"confirmations"` + Size int `json:"size"` + Time int64 `json:"time,omitempty"` } type MempoolEntry struct { diff --git a/db/rocksdb.go b/db/rocksdb.go index 7a34e47a..1ecc4f61 100644 --- a/db/rocksdb.go +++ b/db/rocksdb.go @@ -1091,17 +1091,63 @@ func (d *RocksDB) writeAddressesNonUTXO(wb *gorocksdb.WriteBatch, block *bchain. // Block index +type BlockInfo struct { + Txid string + Time time.Time + Txs uint32 + Size uint32 +} + +func (d *RocksDB) packBlockInfo(block *bchain.Block) ([]byte, error) { + packed := make([]byte, 0, 64) + varBuf := make([]byte, vlq.MaxLen64) + b, err := d.chainParser.PackBlockHash(block.Hash) + if err != nil { + return nil, err + } + packed = append(packed, b...) + packed = append(packed, packUint(uint32(block.Time))...) + l := packVaruint(uint(len(block.Txs)), varBuf) + packed = append(packed, varBuf[:l]...) + l = packVaruint(uint(block.Size), varBuf) + packed = append(packed, varBuf[:l]...) + return packed, nil +} + +func (d *RocksDB) unpackBlockInfo(buf []byte) (*BlockInfo, error) { + pl := d.chainParser.PackedTxidLen() + // minimum length is PackedTxidLen+4 bytes time + 1 byte txs + 1 byte size + if len(buf) < pl+4+2 { + return nil, nil + } + txid, err := d.chainParser.UnpackBlockHash(buf[:pl]) + if err != nil { + return nil, err + } + t := unpackUint(buf[pl:]) + txs, l := unpackVaruint(buf[pl+4:]) + size, _ := unpackVaruint(buf[pl+4+l:]) + return &BlockInfo{ + Txid: txid, + Time: time.Unix(int64(t), 0), + Txs: uint32(txs), + Size: uint32(size), + }, nil +} + // GetBestBlock returns the block hash of the block with highest height in the db func (d *RocksDB) GetBestBlock() (uint32, string, error) { it := d.db.NewIteratorCF(d.ro, d.cfh[cfHeight]) defer it.Close() if it.SeekToLast(); it.Valid() { bestHeight := unpackUint(it.Key().Data()) - val, err := d.chainParser.UnpackBlockHash(it.Value().Data()) - if glog.V(1) { - glog.Infof("rocksdb: bestblock %d %s", bestHeight, val) + info, err := d.unpackBlockInfo(it.Value().Data()) + if info != nil { + if glog.V(1) { + glog.Infof("rocksdb: bestblock %d %+v", bestHeight, info) + } + return bestHeight, info.Txid, err } - return bestHeight, val, err } return 0, "", nil } @@ -1114,7 +1160,22 @@ func (d *RocksDB) GetBlockHash(height uint32) (string, error) { return "", err } defer val.Free() - return d.chainParser.UnpackBlockHash(val.Data()) + info, err := d.unpackBlockInfo(val.Data()) + if info == nil { + return "", err + } + return info.Txid, nil +} + +// GetBlockInfo returns block info stored in db +func (d *RocksDB) GetBlockInfo(height uint32) (*BlockInfo, error) { + key := packUint(height) + val, err := d.db.GetCF(d.ro, d.cfh[cfHeight], key) + if err != nil { + return nil, err + } + defer val.Free() + return d.unpackBlockInfo(val.Data()) } func (d *RocksDB) writeHeight(wb *gorocksdb.WriteBatch, block *bchain.Block, op int) error { @@ -1122,7 +1183,7 @@ func (d *RocksDB) writeHeight(wb *gorocksdb.WriteBatch, block *bchain.Block, op switch op { case opInsert: - val, err := d.chainParser.PackBlockHash(block.Hash) + val, err := d.packBlockInfo(block) if err != nil { return err } diff --git a/db/rocksdb_test.go b/db/rocksdb_test.go index 7c61a5a1..732ae410 100644 --- a/db/rocksdb_test.go +++ b/db/rocksdb_test.go @@ -6,6 +6,7 @@ import ( "blockbook/bchain" "blockbook/bchain/coins/btc" "blockbook/common" + "encoding/binary" "encoding/hex" "fmt" "io/ioutil" @@ -15,6 +16,7 @@ import ( "sort" "strings" "testing" + "time" vlq "github.com/bsm/go-vlq" "github.com/juju/errors" @@ -92,6 +94,12 @@ func varuintToHex(i uint) string { return hex.EncodeToString(b[:l]) } +func uintToHex(i uint32) string { + buf := make([]byte, 4) + binary.BigEndian.PutUint32(buf, i) + return hex.EncodeToString(buf) +} + // keyPair is used to compare given key value in DB with expected // for more complicated compares it is possible to specify CompareFunc type keyPair struct { @@ -187,6 +195,8 @@ func getTestUTXOBlock1(t *testing.T, d *RocksDB) *bchain.Block { BlockHeader: bchain.BlockHeader{ Height: 225493, Hash: "0000000076fbbed90fd75b0e18856aa35baa984e9c9d444cf746ad85e94e2997", + Size: 1234567, + Time: 1534858021, }, Txs: []bchain.Tx{ bchain.Tx{ @@ -247,6 +257,8 @@ func getTestUTXOBlock2(t *testing.T, d *RocksDB) *bchain.Block { BlockHeader: bchain.BlockHeader{ Height: 225494, Hash: "00000000eb0443fd7dc4a1ed5c686a8e995057805f9a161d9a5a77a95e72b7b6", + Size: 2345678, + Time: 1534859123, }, Txs: []bchain.Tx{ bchain.Tx{ @@ -368,7 +380,11 @@ func getTestUTXOBlock2(t *testing.T, d *RocksDB) *bchain.Block { func verifyAfterUTXOBlock1(t *testing.T, d *RocksDB, afterDisconnect bool) { if err := checkColumn(d, cfHeight, []keyPair{ - keyPair{"000370d5", "0000000076fbbed90fd75b0e18856aa35baa984e9c9d444cf746ad85e94e2997", nil}, + keyPair{ + "000370d5", + "0000000076fbbed90fd75b0e18856aa35baa984e9c9d444cf746ad85e94e2997" + uintToHex(1534858021) + varuintToHex(2) + varuintToHex(1234567), + nil, + }, }); err != nil { { t.Fatal(err) @@ -445,8 +461,16 @@ func verifyAfterUTXOBlock1(t *testing.T, d *RocksDB, afterDisconnect bool) { func verifyAfterUTXOBlock2(t *testing.T, d *RocksDB) { if err := checkColumn(d, cfHeight, []keyPair{ - keyPair{"000370d5", "0000000076fbbed90fd75b0e18856aa35baa984e9c9d444cf746ad85e94e2997", nil}, - keyPair{"000370d6", "00000000eb0443fd7dc4a1ed5c686a8e995057805f9a161d9a5a77a95e72b7b6", nil}, + keyPair{ + "000370d5", + "0000000076fbbed90fd75b0e18856aa35baa984e9c9d444cf746ad85e94e2997" + uintToHex(1534858021) + varuintToHex(2) + varuintToHex(1234567), + nil, + }, + keyPair{ + "000370d6", + "00000000eb0443fd7dc4a1ed5c686a8e995057805f9a161d9a5a77a95e72b7b6" + uintToHex(1534859123) + varuintToHex(4) + varuintToHex(2345678), + nil, + }, }); err != nil { { t.Fatal(err) @@ -692,6 +716,30 @@ func TestRocksDB_Index_UTXO(t *testing.T) { t.Fatalf("GetBlockHash: got hash %v, expected %v", hash, "0000000076fbbed90fd75b0e18856aa35baa984e9c9d444cf746ad85e94e2997") } + // Not connected block + hash, err = d.GetBlockHash(225495) + if err != nil { + t.Fatal(err) + } + if hash != "" { + t.Fatalf("GetBlockHash: got hash '%v', expected ''", hash) + } + + // GetBlockHash + info, err := d.GetBlockInfo(225494) + if err != nil { + t.Fatal(err) + } + iw := &BlockInfo{ + Txid: "00000000eb0443fd7dc4a1ed5c686a8e995057805f9a161d9a5a77a95e72b7b6", + Txs: 4, + Size: 2345678, + Time: time.Unix(1534859123, 0), + } + if !reflect.DeepEqual(info, iw) { + t.Errorf("GetAddressBalance() = %+v, want %+v", info, iw) + } + // Test tx caching functionality, leave one tx in db to test cleanup in DisconnectBlock testTxCache(t, d, block1, &block1.Txs[0]) testTxCache(t, d, block2, &block2.Txs[0])