RanchiMall/blockbook
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Use maximum size on disk trigger to invoke DB compaction

Browse Source
This commit is contained in:
Martin Boehm 2018-02-06 09:43:54 +01:00
parent 359d7ee1c4
commit ec87f4ed8d
2 changed files with 16 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
blockbook.go
View File

@ -18,9 +18,6 @@ import (
"github.com/pkg/profile" "github.com/pkg/profile"
) )
// how many blocks are connected before database is compacted in connectBlocksParallel
const compactAfterBlocks = 40000
// resync index at least each resyncIndexPeriodMs (could be more often if invoked by message from ZeroMQ) // resync index at least each resyncIndexPeriodMs (could be more often if invoked by message from ZeroMQ)
const resyncIndexPeriodMs = 935093 const resyncIndexPeriodMs = 935093
@ -51,10 +48,11 @@ var (
repair = flag.Bool("repair", false, "repair the database") repair = flag.Bool("repair", false, "repair the database")
prof = flag.Bool("prof", false, "profile program execution") prof = flag.Bool("prof", false, "profile program execution")
syncChunk = flag.Int("chunk", 100, "block chunk size for processing") syncChunk = flag.Int("chunk", 100, "block chunk size for processing")
syncWorkers = flag.Int("workers", 8, "number of workers to process blocks") syncWorkers = flag.Int("workers", 8, "number of workers to process blocks")
dryRun = flag.Bool("dryrun", false, "do not index blocks, only download") dryRun = flag.Bool("dryrun", false, "do not index blocks, only download")
parse = flag.Bool("parse", false, "use in-process block parsing") parse = flag.Bool("parse", false, "use in-process block parsing")
compactDBTriggerMB = flag.Int64("compact", -1, "invoke compaction when db size exceeds value in MB, default no compaction")
httpServerBinding = flag.String("httpserver", "", "http server binding [address]:port, if missing no http server") httpServerBinding = flag.String("httpserver", "", "http server binding [address]:port, if missing no http server")
@ -481,8 +479,12 @@ func connectBlocksParallel(
hch <- hash hch <- hash
if h > 0 && h%1000 == 0 { if h > 0 && h%1000 == 0 {
glog.Info("connecting block ", h, " ", hash) glog.Info("connecting block ", h, " ", hash)
if bulk { if bulk && *compactDBTriggerMB > 0 {
if h%compactAfterBlocks == 0 { size, err := index.DatabaseSizeOnDisk()
if err != nil {
break
}
if size > *compactDBTriggerMB*1048576 {
// wait for the workers to finish block // wait for the workers to finish block
WaitAgain: WaitAgain:
for { for {

5
db/rocksdb.go
View File

@ -506,6 +506,11 @@ func dirSize(path string) (int64, error) {
return size, err return size, err
} }
// DatabaseSizeOnDisk returns size of the database in bytes
func (d *RocksDB) DatabaseSizeOnDisk() (int64, error) {
return dirSize(d.path)
}
// CompactDatabase compacts the database // CompactDatabase compacts the database
// After unsuccessful experiment with CompactRange method (slow and actually fragmenting the db without compacting) // After unsuccessful experiment with CompactRange method (slow and actually fragmenting the db without compacting)
// the method now closes the db instance and opens it again. // the method now closes the db instance and opens it again.