From 328312e48adcc7fda0fbda1cd86e47d1a194bf60 Mon Sep 17 00:00:00 2001 From: Martin Boehm Date: Thu, 23 Aug 2018 21:14:16 +0200 Subject: [PATCH] Add dbcache configuration flag, tune db options --- blockbook.go | 5 +-- configs/coins/bitcoin.json | 4 +-- configs/coins/bitcoin_testnet.json | 2 +- db/bulkconnect.go | 6 ++-- db/dboptions.go | 53 ++++++++++++++---------------- db/rocksdb.go | 8 ++--- db/rocksdb_test.go | 2 +- 7 files changed, 38 insertions(+), 42 deletions(-) diff --git a/blockbook.go b/blockbook.go index 7e3da52b..07e507d5 100644 --- a/blockbook.go +++ b/blockbook.go @@ -38,7 +38,8 @@ const storeInternalStatePeriodMs = 59699 var ( blockchain = flag.String("blockchaincfg", "", "path to blockchain RPC service configuration json file") - dbPath = flag.String("datadir", "./data", "path to database directory") + dbPath = flag.String("datadir", "./data", "path to database directory") + dbCache = flag.Int("dbcache", 1<<30, "size of the rocksdb cache") blockFrom = flag.Int("blockheight", -1, "height of the starting block") blockUntil = flag.Int("blockuntil", -1, "height of the final block") @@ -162,7 +163,7 @@ func main() { glog.Fatal("rpc: ", err) } - index, err = db.NewRocksDB(*dbPath, chain.GetChainParser(), metrics) + index, err = db.NewRocksDB(*dbPath, *dbCache, chain.GetChainParser(), metrics) if err != nil { glog.Fatal("rocksDB: ", err) } diff --git a/configs/coins/bitcoin.json b/configs/coins/bitcoin.json index d5435bba..a957102a 100644 --- a/configs/coins/bitcoin.json +++ b/configs/coins/bitcoin.json @@ -47,8 +47,8 @@ "system_user": "blockbook-bitcoin", "internal_binding_template": ":{{.Ports.BlockbookInternal}}", "public_binding_template": ":{{.Ports.BlockbookPublic}}", - "explorer_url": "https://btc-explorer.trezor.io/", - "additional_params": "", + "explorer_url": "/explorer", + "additional_params": "-dbcache=4294967296", "block_chain": { "parse": true, "mempool_workers": 8, diff --git a/configs/coins/bitcoin_testnet.json b/configs/coins/bitcoin_testnet.json index 3397cb95..760163be 100644 --- a/configs/coins/bitcoin_testnet.json +++ b/configs/coins/bitcoin_testnet.json @@ -47,7 +47,7 @@ "system_user": "blockbook-bitcoin", "internal_binding_template": ":{{.Ports.BlockbookInternal}}", "public_binding_template": ":{{.Ports.BlockbookPublic}}", - "explorer_url": "https://btc-testnet-explorer.trezor.io/", + "explorer_url": "/explorer", "additional_params": "", "block_chain": { "parse": true, diff --git a/db/bulkconnect.go b/db/bulkconnect.go index ce1af0aa..bece9c6b 100644 --- a/db/bulkconnect.go +++ b/db/bulkconnect.go @@ -31,10 +31,10 @@ type BulkConnect struct { } const ( - maxBulkAddresses = 400000 - maxBulkTxAddresses = 1500000 + maxBulkAddresses = 200000 + maxBulkTxAddresses = 500000 partialStoreAddresses = maxBulkTxAddresses / 10 - maxBulkBalances = 2500000 + maxBulkBalances = 800000 partialStoreBalances = maxBulkBalances / 10 ) diff --git a/db/dboptions.go b/db/dboptions.go index 81ff82d3..2e0da942 100644 --- a/db/dboptions.go +++ b/db/dboptions.go @@ -1,47 +1,51 @@ package db -// #include "rocksdb/c.h" -import "C" import ( - "reflect" - "unsafe" - "github.com/tecbot/gorocksdb" ) -func createAndSetDBOptions(bloomBits int, twoLevelIndex bool, c *gorocksdb.Cache) *gorocksdb.Options { - // get access to all db options, gorocksdb does not expose all +/* + possible additional tuning, using options not accessible by gorocksdb + +// #include "rocksdb/c.h" +import "C" - // opts := gorocksdb.NewDefaultOptions() cNativeOpts := C.rocksdb_options_create() opts := &gorocksdb.Options{} cField := reflect.Indirect(reflect.ValueOf(opts)).FieldByName("c") cPtr := (**C.rocksdb_options_t)(unsafe.Pointer(cField.UnsafeAddr())) *cPtr = cNativeOpts - // blockOpts := gorocksdb.NewDefaultBlockBasedTableOptions() cNativeBlockOpts := C.rocksdb_block_based_options_create() blockOpts := &gorocksdb.BlockBasedTableOptions{} cBlockField := reflect.Indirect(reflect.ValueOf(blockOpts)).FieldByName("c") cBlockPtr := (**C.rocksdb_block_based_table_options_t)(unsafe.Pointer(cBlockField.UnsafeAddr())) *cBlockPtr = cNativeBlockOpts + // https://github.com/facebook/rocksdb/wiki/Partitioned-Index-Filters + blockOpts.SetIndexType(gorocksdb.KTwoLevelIndexSearchIndexType) + C.rocksdb_block_based_options_set_partition_filters(cNativeBlockOpts, boolToChar(true)) + C.rocksdb_block_based_options_set_metadata_block_size(cNativeBlockOpts, C.uint64_t(4096)) + C.rocksdb_block_based_options_set_cache_index_and_filter_blocks_with_high_priority(cNativeBlockOpts, boolToChar(true)) + blockOpts.SetPinL0FilterAndIndexBlocksInCache(true) + +// boolToChar converts a bool value to C.uchar. +func boolToChar(b bool) C.uchar { + if b { + return 1 + } + return 0 +} +*/ + +func createAndSetDBOptions(bloomBits int, c *gorocksdb.Cache) *gorocksdb.Options { + blockOpts := gorocksdb.NewDefaultBlockBasedTableOptions() blockOpts.SetBlockSize(32 << 10) // 32kB blockOpts.SetBlockCache(c) if bloomBits > 0 { blockOpts.SetFilterPolicy(gorocksdb.NewBloomFilter(bloomBits)) } - - // https://github.com/facebook/rocksdb/wiki/Partitioned-Index-Filters - if twoLevelIndex { - // blockOpts.SetCacheIndexAndFilterBlocks(true) - blockOpts.SetIndexType(gorocksdb.KTwoLevelIndexSearchIndexType) - C.rocksdb_block_based_options_set_partition_filters(cNativeBlockOpts, boolToChar(true)) - C.rocksdb_block_based_options_set_metadata_block_size(cNativeBlockOpts, C.uint64_t(4096)) - C.rocksdb_block_based_options_set_cache_index_and_filter_blocks_with_high_priority(cNativeBlockOpts, boolToChar(true)) - blockOpts.SetPinL0FilterAndIndexBlocksInCache(true) - } - + opts := gorocksdb.NewDefaultOptions() opts.SetBlockBasedTableFactory(blockOpts) opts.SetCreateIfMissing(true) opts.SetCreateIfMissingColumnFamilies(true) @@ -52,14 +56,5 @@ func createAndSetDBOptions(bloomBits int, twoLevelIndex bool, c *gorocksdb.Cache opts.SetMaxBytesForLevelBase(1 << 27) // 128MB opts.SetMaxOpenFiles(25000) opts.SetCompression(gorocksdb.LZ4HCCompression) - return opts } - -// boolToChar converts a bool value to C.uchar. -func boolToChar(b bool) C.uchar { - if b { - return 1 - } - return 0 -} diff --git a/db/rocksdb.go b/db/rocksdb.go index 2a151ba4..bceb1b4d 100644 --- a/db/rocksdb.go +++ b/db/rocksdb.go @@ -60,10 +60,10 @@ var cfNames = []string{"default", "height", "addresses", "txAddresses", "address func openDB(path string, c *gorocksdb.Cache) (*gorocksdb.DB, []*gorocksdb.ColumnFamilyHandle, error) { // opts with bloom filter - opts := createAndSetDBOptions(10, true, c) + opts := createAndSetDBOptions(10, c) // opts for addresses without bloom filter // from documentation: if most of your queries are executed using iterators, you shouldn't set bloom filter - optsAddresses := createAndSetDBOptions(0, true, c) + optsAddresses := createAndSetDBOptions(0, c) // default, height, addresses, txAddresses, addressBalance, blockTxids, transactions fcOptions := []*gorocksdb.Options{opts, opts, optsAddresses, opts, opts, opts, opts} db, cfh, err := gorocksdb.OpenDbColumnFamilies(opts, path, cfNames, fcOptions) @@ -75,9 +75,9 @@ func openDB(path string, c *gorocksdb.Cache) (*gorocksdb.DB, []*gorocksdb.Column // NewRocksDB opens an internal handle to RocksDB environment. Close // needs to be called to release it. -func NewRocksDB(path string, parser bchain.BlockChainParser, metrics *common.Metrics) (d *RocksDB, err error) { +func NewRocksDB(path string, cacheSize int, parser bchain.BlockChainParser, metrics *common.Metrics) (d *RocksDB, err error) { glog.Infof("rocksdb: open %s, version %v", path, dbVersion) - c := gorocksdb.NewLRUCache(1 << 30) // 1GB + c := gorocksdb.NewLRUCache(cacheSize) db, cfh, err := openDB(path, c) wo := gorocksdb.NewDefaultWriteOptions() ro := gorocksdb.NewDefaultReadOptions() diff --git a/db/rocksdb_test.go b/db/rocksdb_test.go index e74d0664..e36a3ab0 100644 --- a/db/rocksdb_test.go +++ b/db/rocksdb_test.go @@ -37,7 +37,7 @@ func setupRocksDB(t *testing.T, p bchain.BlockChainParser) *RocksDB { if err != nil { t.Fatal(err) } - d, err := NewRocksDB(tmp, p, nil) + d, err := NewRocksDB(tmp, 100000, p, nil) if err != nil { t.Fatal(err) }