RanchiMall/blockbook
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Use bulk option only for initial load of DB

Browse Source
This commit is contained in:
Martin Boehm 2018-02-05 17:47:20 +01:00
parent d6e9b5caa4
commit fdd9c9b99f
1 changed files with 27 additions and 18 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

45
blockbook.go
View File

@ -344,6 +344,8 @@ func resyncIndex(bulk bool) error {
glog.Info("resync: local is behind") glog.Info("resync: local is behind")
hash = header.Next hash = header.Next
startHeight = localBestHeight startHeight = localBestHeight
// bulk load is allowed only for empty db, otherwise we could get rocksdb "error db has more levels than options.num_levels"
bulk = false
} else { } else {
// If the local block is missing, we're indexing from the genesis block // If the local block is missing, we're indexing from the genesis block
// or from the start block specified by flags // or from the start block specified by flags
@ -359,7 +361,7 @@ func resyncIndex(bulk bool) error {
// if parallel operation is enabled and the number of blocks to be connected is large, // if parallel operation is enabled and the number of blocks to be connected is large,
// use parallel routine to load majority of blocks // use parallel routine to load majority of blocks
if bulk && *syncWorkers > 1 { if *syncWorkers > 1 {
chainBestHeight, err := chain.GetBestBlockHeight() chainBestHeight, err := chain.GetBestBlockHeight()
if err != nil { if err != nil {
return err return err
@ -370,6 +372,7 @@ func resyncIndex(bulk bool) error {
startHeight, startHeight,
chainBestHeight, chainBestHeight,
*syncWorkers, *syncWorkers,
bulk,
) )
if err != nil { if err != nil {
return err return err
@ -415,10 +418,14 @@ func connectBlocksParallel(
lower uint32, lower uint32,
higher uint32, higher uint32,
numWorkers int, numWorkers int,
bulk bool,
) error { ) error {
err := index.ReopenWithBulk(true) var err error
if err != nil { if bulk {
return err err = index.ReopenWithBulk(true)
if err != nil {
return err
}
} }
var wg sync.WaitGroup var wg sync.WaitGroup
@ -459,21 +466,23 @@ func connectBlocksParallel(
hch <- hash hch <- hash
if h > 0 && h%1000 == 0 { if h > 0 && h%1000 == 0 {
glog.Info("connecting block ", h, " ", hash) glog.Info("connecting block ", h, " ", hash)
if h%50000 == 0 { if bulk {
// wait for the workers to finish block if h%50000 == 0 {
WaitAgain: // wait for the workers to finish block
for { WaitAgain:
for _, r := range running { for {
if r { for _, r := range running {
glog.Info("Waiting ", running) if r {
time.Sleep(time.Millisecond * 500) glog.Info("Waiting ", running)
continue WaitAgain time.Sleep(time.Millisecond * 500)
continue WaitAgain
}
} }
break
}
if err = index.CompactDatabase(bulk); err != nil {
break
} }
break
}
if err = index.CompactDatabase(true); err != nil {
break
} }
} }
} }
@ -481,7 +490,7 @@ func connectBlocksParallel(
close(hch) close(hch)
wg.Wait() wg.Wait()
if err == nil { if err == nil && bulk {
err = index.ReopenWithBulk(false) err = index.ReopenWithBulk(false)
} }