diff --git a/blockbook.go b/blockbook.go index 2e71f3f0..77df065b 100644 --- a/blockbook.go +++ b/blockbook.go @@ -70,6 +70,8 @@ var ( coin = flag.String("coin", "btc", "coin name") noTxCache = flag.Bool("notxcache", false, "disable tx cache") + + computeColumnStats = flag.Bool("computedbstats", false, "compute column stats and exit") ) var ( @@ -162,13 +164,24 @@ func main() { internalState, err = newInternalState(*coin, index) if err != nil { - glog.Fatal("internalState: ", err) + glog.Error("internalState: ", err) + return } index.SetInternalState(internalState) if internalState.DbState != common.DbStateClosed { glog.Warning("internalState: database in not closed state ", internalState.DbState, ", possibly previous ungraceful shutdown") } + if *computeColumnStats { + internalState.DbState = common.DbStateOpen + err = index.ComputeInternalStateColumnStats() + if err != nil { + glog.Error("internalState: ", err) + } + glog.Info("DB size on disk: ", index.DatabaseSizeOnDisk()) + return + } + syncWorker, err = db.NewSyncWorker(index, chain, *syncWorkers, *syncChunk, *blockFrom, *dryRun, chanOsSignal, metrics, internalState) if err != nil { glog.Fatalf("NewSyncWorker %v", err) diff --git a/common/internalstate.go b/common/internalstate.go index d5ca398e..4f77470e 100644 --- a/common/internalstate.go +++ b/common/internalstate.go @@ -105,6 +105,14 @@ func (is *InternalState) AddDBColumnStats(c int, rowsDiff int64, keysSumDiff int is.DbColumns[c].ValuesSum += valuesSumDiff } +func (is *InternalState) SetDBColumnStats(c int, rowsDiff int64, keysSumDiff int64, valuesSumDiff int64) { + is.mux.Lock() + defer is.mux.Unlock() + is.DbColumns[c].Rows = rowsDiff + is.DbColumns[c].KeysSum = keysSumDiff + is.DbColumns[c].ValuesSum = valuesSumDiff +} + func (is *InternalState) Pack() ([]byte, error) { is.mux.Lock() defer is.mux.Unlock() diff --git a/db/rocksdb.go b/db/rocksdb.go index 4ae710ed..f08cf543 100644 --- a/db/rocksdb.go +++ b/db/rocksdb.go @@ -18,7 +18,7 @@ import ( // iterator creates snapshot, which takes lots of resources // when doing huge scan, it is better to close it and reopen from time to time to free the resources -const disconnectBlocksRefreshIterator = uint64(1000000) +const refreshIterator = 5000000 const packedHeightBytes = 4 // RepairRocksDB calls RocksDb db repair function @@ -686,7 +686,7 @@ func (d *RocksDB) allAddressesScan(lower uint32, higher uint32) ([][]byte, [][]b it.Seek(seekKey) it.Next() } - for count = 0; it.Valid() && count < disconnectBlocksRefreshIterator; it.Next() { + for count = 0; it.Valid() && count < refreshIterator; it.Next() { totalOutputs++ count++ key = it.Key().Data() @@ -932,6 +932,50 @@ func (d *RocksDB) StoreInternalState(is *common.InternalState) error { return d.db.PutCF(d.wo, d.cfh[cfDefault], []byte(internalStateKey), buf) } +func (d *RocksDB) computeColumnSize(col int) (int64, int64, int64, error) { + var rows, keysSum, valuesSum int64 + var seekKey []byte + for { + var key []byte + it := d.db.NewIteratorCF(d.ro, d.cfh[col]) + if rows == 0 { + it.SeekToFirst() + } else { + glog.Info("Column ", cfNames[col], ": rows ", rows, ", key bytes ", keysSum, ", value bytes ", valuesSum, ", in progress...") + it.Seek(seekKey) + it.Next() + } + for count := 0; it.Valid() && count < refreshIterator; it.Next() { + key = it.Key().Data() + count++ + rows++ + keysSum += int64(len(key)) + valuesSum += int64(len(it.Value().Data())) + } + seekKey = append([]byte{}, key...) + valid := it.Valid() + it.Close() + if !valid { + break + } + } + return rows, keysSum, valuesSum, nil +} + +// ComputeInternalStateColumnStats computes stats of all db columns and sets them to internal state +// can be very slow operation +func (d *RocksDB) ComputeInternalStateColumnStats() error { + for c := 0; c < len(cfNames); c++ { + rows, keysSum, valuesSum, err := d.computeColumnSize(c) + if err != nil { + return err + } + d.is.SetDBColumnStats(c, rows, keysSum, valuesSum) + glog.Info("Column ", cfNames[c], ": rows ", rows, ", key bytes ", keysSum, ", value bytes ", valuesSum) + } + return nil +} + // Helpers func packAddressKey(addrID []byte, height uint32) []byte {