RanchiMall/blockbook
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add maxOpenFiles rocksdb flag

Browse Source
This commit is contained in:
Martin Boehm 2018-09-18 11:49:39 +02:00
parent df67d7dcaf
commit 184f2e9ca6
4 changed files with 25 additions and 23 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
blockbook.go
View File

@ -36,8 +36,9 @@ const storeInternalStatePeriodMs = 59699
var ( var (
blockchain = flag.String("blockchaincfg", "", "path to blockchain RPC service configuration json file") blockchain = flag.String("blockchaincfg", "", "path to blockchain RPC service configuration json file")
dbPath = flag.String("datadir", "./data", "path to database directory") dbPath = flag.String("datadir", "./data", "path to database directory")
dbCache = flag.Int("dbcache", 1<<29, "size of the rocksdb cache") dbCache = flag.Int("dbcache", 1<<29, "size of the rocksdb cache")
dbMaxOpenFiles = flag.Int("dbmaxopenfiles", 1<<14, "max open files by rocksdb")
blockFrom = flag.Int("blockheight", -1, "height of the starting block") blockFrom = flag.Int("blockheight", -1, "height of the starting block")
blockUntil = flag.Int("blockuntil", -1, "height of the final block") blockUntil = flag.Int("blockuntil", -1, "height of the final block")
@ -165,7 +166,7 @@ func main() {
glog.Fatal("rpc: ", err) glog.Fatal("rpc: ", err)
} }
index, err = db.NewRocksDB(*dbPath, *dbCache, chain.GetChainParser(), metrics) index, err = db.NewRocksDB(*dbPath, *dbCache, *dbMaxOpenFiles, chain.GetChainParser(), metrics)
if err != nil { if err != nil {
glog.Fatal("rocksDB: ", err) glog.Fatal("rocksDB: ", err)
} }

4
db/dboptions.go
View File

@ -38,7 +38,7 @@ func boolToChar(b bool) C.uchar {
} }
*/ */
func createAndSetDBOptions(bloomBits int, c *gorocksdb.Cache) *gorocksdb.Options { func createAndSetDBOptions(bloomBits int, c *gorocksdb.Cache, maxOpenFiles int) *gorocksdb.Options {
blockOpts := gorocksdb.NewDefaultBlockBasedTableOptions() blockOpts := gorocksdb.NewDefaultBlockBasedTableOptions()
blockOpts.SetBlockSize(32 << 10) // 32kB blockOpts.SetBlockSize(32 << 10) // 32kB
blockOpts.SetBlockCache(c) blockOpts.SetBlockCache(c)
@ -54,7 +54,7 @@ func createAndSetDBOptions(bloomBits int, c *gorocksdb.Cache) *gorocksdb.Options
opts.SetBytesPerSync(8 << 20) // 8MB opts.SetBytesPerSync(8 << 20) // 8MB
opts.SetWriteBufferSize(1 << 27) // 128MB opts.SetWriteBufferSize(1 << 27) // 128MB
opts.SetMaxBytesForLevelBase(1 << 27) // 128MB opts.SetMaxBytesForLevelBase(1 << 27) // 128MB
opts.SetMaxOpenFiles(25000) opts.SetMaxOpenFiles(maxOpenFiles)
opts.SetCompression(gorocksdb.LZ4HCCompression) opts.SetCompression(gorocksdb.LZ4HCCompression)
return opts return opts
} }

35
db/rocksdb.go
View File

@ -35,15 +35,16 @@ func RepairRocksDB(name string) error {
// RocksDB handle // RocksDB handle
type RocksDB struct { type RocksDB struct {
path string path string
db *gorocksdb.DB db *gorocksdb.DB
wo *gorocksdb.WriteOptions wo *gorocksdb.WriteOptions
ro *gorocksdb.ReadOptions ro *gorocksdb.ReadOptions
cfh []*gorocksdb.ColumnFamilyHandle cfh []*gorocksdb.ColumnFamilyHandle
chainParser bchain.BlockChainParser chainParser bchain.BlockChainParser
is *common.InternalState is *common.InternalState
metrics *common.Metrics metrics *common.Metrics
cache *gorocksdb.Cache cache *gorocksdb.Cache
maxOpenFiles int
} }
const ( const (
@ -58,12 +59,12 @@ const (
var cfNames = []string{"default", "height", "addresses", "txAddresses", "addressBalance", "blockTxs", "transactions"} var cfNames = []string{"default", "height", "addresses", "txAddresses", "addressBalance", "blockTxs", "transactions"}
func openDB(path string, c *gorocksdb.Cache) (*gorocksdb.DB, []*gorocksdb.ColumnFamilyHandle, error) { func openDB(path string, c *gorocksdb.Cache, openFiles int) (*gorocksdb.DB, []*gorocksdb.ColumnFamilyHandle, error) {
// opts with bloom filter // opts with bloom filter
opts := createAndSetDBOptions(10, c) opts := createAndSetDBOptions(10, c, openFiles)
// opts for addresses without bloom filter // opts for addresses without bloom filter
// from documentation: if most of your queries are executed using iterators, you shouldn't set bloom filter // from documentation: if most of your queries are executed using iterators, you shouldn't set bloom filter
optsAddresses := createAndSetDBOptions(0, c) optsAddresses := createAndSetDBOptions(0, c, openFiles)
// default, height, addresses, txAddresses, addressBalance, blockTxids, transactions // default, height, addresses, txAddresses, addressBalance, blockTxids, transactions
fcOptions := []*gorocksdb.Options{opts, opts, optsAddresses, opts, opts, opts, opts} fcOptions := []*gorocksdb.Options{opts, opts, optsAddresses, opts, opts, opts, opts}
db, cfh, err := gorocksdb.OpenDbColumnFamilies(opts, path, cfNames, fcOptions) db, cfh, err := gorocksdb.OpenDbColumnFamilies(opts, path, cfNames, fcOptions)
@ -75,13 +76,13 @@ func openDB(path string, c *gorocksdb.Cache) (*gorocksdb.DB, []*gorocksdb.Column
// NewRocksDB opens an internal handle to RocksDB environment. Close // NewRocksDB opens an internal handle to RocksDB environment. Close
// needs to be called to release it. // needs to be called to release it.
func NewRocksDB(path string, cacheSize int, parser bchain.BlockChainParser, metrics *common.Metrics) (d *RocksDB, err error) { func NewRocksDB(path string, cacheSize, maxOpenFiles int, parser bchain.BlockChainParser, metrics *common.Metrics) (d *RocksDB, err error) {
glog.Infof("rocksdb: open %s, version %v", path, dbVersion) glog.Infof("rocksdb: opening %s, required data version %v, cache size %v, max open files %v", path, dbVersion, cacheSize, maxOpenFiles)
c := gorocksdb.NewLRUCache(cacheSize) c := gorocksdb.NewLRUCache(cacheSize)
db, cfh, err := openDB(path, c) db, cfh, err := openDB(path, c, maxOpenFiles)
wo := gorocksdb.NewDefaultWriteOptions() wo := gorocksdb.NewDefaultWriteOptions()
ro := gorocksdb.NewDefaultReadOptions() ro := gorocksdb.NewDefaultReadOptions()
return &RocksDB{path, db, wo, ro, cfh, parser, nil, metrics, c}, nil return &RocksDB{path, db, wo, ro, cfh, parser, nil, metrics, c, maxOpenFiles}, nil
} }
func (d *RocksDB) closeDB() error { func (d *RocksDB) closeDB() error {
@ -119,7 +120,7 @@ func (d *RocksDB) Reopen() error {
return err return err
} }
d.db = nil d.db = nil
db, cfh, err := openDB(d.path, d.cache) db, cfh, err := openDB(d.path, d.cache, d.maxOpenFiles)
if err != nil { if err != nil {
return err return err
} }

2
db/rocksdb_test.go
View File

@ -36,7 +36,7 @@ func setupRocksDB(t *testing.T, p bchain.BlockChainParser) *RocksDB {
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
d, err := NewRocksDB(tmp, 100000, p, nil) d, err := NewRocksDB(tmp, 100000, -1, p, nil)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }