RanchiMall/blockbook
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Change db columns, enable compression on some columns

Browse Source
This commit is contained in:
Martin Boehm 2018-07-27 14:08:20 +02:00
parent 6ce5d04ef0
commit 75d48376e1
1 changed files with 45 additions and 28 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

73
db/rocksdb.go
View File

@ -46,12 +46,17 @@ const (
cfDefault = iota cfDefault = iota
cfHeight cfHeight
cfAddresses cfAddresses
cfUnspentTxs cfTxAddresses
cfAddressBalance
cfBlockTxids
cfTransactions cfTransactions
// to be removed, kept temporarily so that the code is compilable
cfUnspentTxs
cfBlockAddresses cfBlockAddresses
) )
var cfNames = []string{"default", "height", "addresses", "unspenttxs", "transactions", "blockaddresses"} var cfNames = []string{"default", "height", "addresses", "txAddresses", "addressBalance", "blockTxids", "transactions"}
func openDB(path string) (*gorocksdb.DB, []*gorocksdb.ColumnFamilyHandle, error) { func openDB(path string) (*gorocksdb.DB, []*gorocksdb.ColumnFamilyHandle, error) {
c := gorocksdb.NewLRUCache(8 << 30) // 8GB c := gorocksdb.NewLRUCache(8 << 30) // 8GB
@ -61,37 +66,49 @@ func openDB(path string) (*gorocksdb.DB, []*gorocksdb.ColumnFamilyHandle, error)
bbto.SetBlockCache(c) bbto.SetBlockCache(c)
bbto.SetFilterPolicy(fp) bbto.SetFilterPolicy(fp)
opts := gorocksdb.NewDefaultOptions() optsNoCompression := gorocksdb.NewDefaultOptions()
opts.SetBlockBasedTableFactory(bbto) optsNoCompression.SetBlockBasedTableFactory(bbto)
opts.SetCreateIfMissing(true) optsNoCompression.SetCreateIfMissing(true)
opts.SetCreateIfMissingColumnFamilies(true) optsNoCompression.SetCreateIfMissingColumnFamilies(true)
opts.SetMaxBackgroundCompactions(4) optsNoCompression.SetMaxBackgroundCompactions(4)
opts.SetMaxBackgroundFlushes(2) optsNoCompression.SetMaxBackgroundFlushes(2)
opts.SetBytesPerSync(1 << 20) // 1MB optsNoCompression.SetBytesPerSync(1 << 20) // 1MB
opts.SetWriteBufferSize(1 << 27) // 128MB optsNoCompression.SetWriteBufferSize(1 << 27) // 128MB
opts.SetMaxOpenFiles(25000) optsNoCompression.SetMaxOpenFiles(25000)
opts.SetCompression(gorocksdb.NoCompression) optsNoCompression.SetCompression(gorocksdb.NoCompression)
// opts for outputs are different: optsLZ4 := gorocksdb.NewDefaultOptions()
optsLZ4.SetBlockBasedTableFactory(bbto)
optsLZ4.SetCreateIfMissing(true)
optsLZ4.SetCreateIfMissingColumnFamilies(true)
optsLZ4.SetMaxBackgroundCompactions(4)
optsLZ4.SetMaxBackgroundFlushes(2)
optsLZ4.SetBytesPerSync(1 << 20) // 1MB
optsLZ4.SetWriteBufferSize(1 << 27) // 128MB
optsLZ4.SetMaxOpenFiles(25000)
optsLZ4.SetCompression(gorocksdb.LZ4Compression)
// opts for addresses are different:
// no bloom filter - from documentation: If most of your queries are executed using iterators, you shouldn't set bloom filter // no bloom filter - from documentation: If most of your queries are executed using iterators, you shouldn't set bloom filter
bbtoOutputs := gorocksdb.NewDefaultBlockBasedTableOptions() bbtoAddresses := gorocksdb.NewDefaultBlockBasedTableOptions()
bbtoOutputs.SetBlockSize(16 << 10) // 16kB bbtoAddresses.SetBlockSize(16 << 10) // 16kB
bbtoOutputs.SetBlockCache(c) // 8GB bbtoAddresses.SetBlockCache(c) // 8GB
optsOutputs := gorocksdb.NewDefaultOptions() optsAddresses := gorocksdb.NewDefaultOptions()
optsOutputs.SetBlockBasedTableFactory(bbtoOutputs) optsAddresses.SetBlockBasedTableFactory(bbtoAddresses)
optsOutputs.SetCreateIfMissing(true) optsAddresses.SetCreateIfMissing(true)
optsOutputs.SetCreateIfMissingColumnFamilies(true) optsAddresses.SetCreateIfMissingColumnFamilies(true)
optsOutputs.SetMaxBackgroundCompactions(4) optsAddresses.SetMaxBackgroundCompactions(4)
optsOutputs.SetMaxBackgroundFlushes(2) optsAddresses.SetMaxBackgroundFlushes(2)
optsOutputs.SetBytesPerSync(1 << 20) // 1MB optsAddresses.SetBytesPerSync(1 << 20) // 1MB
optsOutputs.SetWriteBufferSize(1 << 27) // 128MB optsAddresses.SetWriteBufferSize(1 << 27) // 128MB
optsOutputs.SetMaxOpenFiles(25000) optsAddresses.SetMaxOpenFiles(25000)
optsOutputs.SetCompression(gorocksdb.NoCompression) optsAddresses.SetCompression(gorocksdb.NoCompression)
fcOptions := []*gorocksdb.Options{opts, opts, optsOutputs, opts, opts, opts} // default, height, addresses, txAddresses, addressBalance, blockTxids, transactions
fcOptions := []*gorocksdb.Options{optsNoCompression, optsNoCompression, optsAddresses, optsLZ4, optsLZ4, optsLZ4, optsLZ4}
db, cfh, err := gorocksdb.OpenDbColumnFamilies(opts, path, cfNames, fcOptions) db, cfh, err := gorocksdb.OpenDbColumnFamilies(optsNoCompression, path, cfNames, fcOptions)
if err != nil { if err != nil {
return nil, nil, err return nil, nil, err
} }