RanchiMall/blockbook
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Allow multiple block ranges for sync test

Browse Source
This commit is contained in:
Jakub Matys 2018-09-24 15:30:30 +02:00
parent d67eae31bb
commit 2531c789dd
2 changed files with 178 additions and 91 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

156
tests/sync/sync.go
View File

@ -11,7 +11,6 @@ import (
"os" "os"
"path/filepath" "path/filepath"
"reflect" "reflect"
"sort"
"strings" "strings"
"testing" "testing"
) )
@ -28,12 +27,14 @@ type TestHandler struct {
TestData *TestData TestData *TestData
} }
type Range struct {
Lower uint32 `json:"lower"`
Upper uint32 `json:"upper"`
}
type TestData struct { type TestData struct {
ConnectBlocksParallel struct { SyncRanges []Range `json:"syncRanges"`
SyncWorkers int `json:"syncWorkers"` Blocks map[uint32]BlockInfo `json:"blocks"`
SyncChunk int `json:"syncChunk"`
} `json:"connectBlocksParallel"`
Blocks []BlockInfo `json:"blocks"`
} }
type BlockInfo struct { type BlockInfo struct {
@ -111,10 +112,6 @@ func loadTestData(coin string, parser bchain.BlockChainParser) (*TestData, error
} }
} }
sort.Slice(v.Blocks, func(i, j int) bool {
return v.Blocks[i].Height < v.Blocks[j].Height
})
return &v, nil return &v, nil
} }
@ -154,8 +151,23 @@ func makeRocksDB(parser bchain.BlockChainParser, m *common.Metrics, is *common.I
return d, closer, nil return d, closer, nil
} }
func withRocksDBAndSyncWorker(t *testing.T, h *TestHandler, fn func(*db.RocksDB, *db.SyncWorker, chan os.Signal)) { var metricsRegistry = map[string]*common.Metrics{}
m, err := common.GetMetrics(h.Coin)
func getMetrics(name string) (*common.Metrics, error) {
if m, found := metricsRegistry[name]; found {
return m, nil
} else {
m, err := common.GetMetrics(name)
if err != nil {
return nil, err
}
metricsRegistry[name] = m
return m, nil
}
}
func withRocksDBAndSyncWorker(t *testing.T, h *TestHandler, startHeight uint32, fn func(*db.RocksDB, *db.SyncWorker, chan os.Signal)) {
m, err := getMetrics(h.Coin)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
@ -167,13 +179,9 @@ func withRocksDBAndSyncWorker(t *testing.T, h *TestHandler, fn func(*db.RocksDB,
} }
defer closer() defer closer()
if len(h.TestData.Blocks) == 0 {
t.Fatal("No test data")
}
ch := make(chan os.Signal) ch := make(chan os.Signal)
sw, err := db.NewSyncWorker(d, h.Chain, 3, 0, int(h.TestData.Blocks[0].Height), false, ch, m, is) sw, err := db.NewSyncWorker(d, h.Chain, 8, 0, int(startHeight), false, ch, m, is)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
@ -182,42 +190,50 @@ func withRocksDBAndSyncWorker(t *testing.T, h *TestHandler, fn func(*db.RocksDB,
} }
func testConnectBlocksParallel(t *testing.T, h *TestHandler) { func testConnectBlocksParallel(t *testing.T, h *TestHandler) {
withRocksDBAndSyncWorker(t, h, func(d *db.RocksDB, sw *db.SyncWorker, ch chan os.Signal) { for _, rng := range h.TestData.SyncRanges {
lowerHeight := h.TestData.Blocks[0].Height withRocksDBAndSyncWorker(t, h, rng.Lower, func(d *db.RocksDB, sw *db.SyncWorker, ch chan os.Signal) {
upperHeight := h.TestData.Blocks[len(h.TestData.Blocks)-1].Height upperHash, err := h.Chain.GetBlockHash(rng.Upper)
upperHash := h.TestData.Blocks[len(h.TestData.Blocks)-1].Hash if err != nil {
t.Fatal(err)
}
err := sw.ConnectBlocksParallel(lowerHeight, upperHeight) err = sw.ConnectBlocksParallel(rng.Lower, rng.Upper)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
height, hash, err := d.GetBestBlock() height, hash, err := d.GetBestBlock()
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
if height != upperHeight { if height != rng.Upper {
t.Fatalf("Upper block height mismatch: %d != %d", height, upperHeight) t.Fatalf("Upper block height mismatch: %d != %d", height, rng.Upper)
} }
if hash != upperHash { if hash != upperHash {
t.Fatalf("Upper block hash mismatch: %s != %s", hash, upperHash) t.Fatalf("Upper block hash mismatch: %s != %s", hash, upperHash)
} }
t.Run("verifyBlockInfo", func(t *testing.T) { verifyBlockInfo(t, d, h) }) t.Run("verifyBlockInfo", func(t *testing.T) { verifyBlockInfo(t, d, h, rng) })
t.Run("verifyTransactions", func(t *testing.T) { verifyTransactions(t, d, h) }) t.Run("verifyTransactions", func(t *testing.T) { verifyTransactions(t, d, h, rng) })
t.Run("verifyAddresses", func(t *testing.T) { verifyAddresses(t, d, h) }) t.Run("verifyAddresses", func(t *testing.T) { verifyAddresses(t, d, h, rng) })
}) })
}
} }
func verifyBlockInfo(t *testing.T, d *db.RocksDB, h *TestHandler) { func verifyBlockInfo(t *testing.T, d *db.RocksDB, h *TestHandler, rng Range) {
for _, block := range h.TestData.Blocks { for height := rng.Lower; height <= rng.Upper; height++ {
bi, err := d.GetBlockInfo(block.Height) block, found := h.TestData.Blocks[height]
if !found {
continue
}
bi, err := d.GetBlockInfo(height)
if err != nil { if err != nil {
t.Errorf("GetBlockInfo(%d) error: %s", block.Height, err) t.Errorf("GetBlockInfo(%d) error: %s", height, err)
continue continue
} }
if bi == nil { if bi == nil {
t.Errorf("GetBlockInfo(%d) returned nil", block.Height) t.Errorf("GetBlockInfo(%d) returned nil", height)
continue continue
} }
@ -231,7 +247,7 @@ func verifyBlockInfo(t *testing.T, d *db.RocksDB, h *TestHandler) {
} }
} }
func verifyTransactions(t *testing.T, d *db.RocksDB, h *TestHandler) { func verifyTransactions(t *testing.T, d *db.RocksDB, h *TestHandler, rng Range) {
type txInfo struct { type txInfo struct {
txid string txid string
vout uint32 vout uint32
@ -240,26 +256,13 @@ func verifyTransactions(t *testing.T, d *db.RocksDB, h *TestHandler) {
addr2txs := make(map[string][]txInfo) addr2txs := make(map[string][]txInfo)
checkMap := make(map[string][]bool) checkMap := make(map[string][]bool)
for _, block := range h.TestData.Blocks { for height := rng.Lower; height <= rng.Upper; height++ {
block, found := h.TestData.Blocks[height]
if !found {
continue
}
for _, tx := range block.TxDetails { for _, tx := range block.TxDetails {
// for _, vin := range tx.Vin {
// if vin.Txid != "" {
// ta, err := d.GetTxAddresses(vin.Txid)
// if err != nil {
// t.Fatal(err)
// }
// if ta != nil {
// if len(ta.Outputs) > int(vin.Vout) {
// output := &ta.Outputs[vin.Vout]
// voutAddr, _, err := output.Addresses(h.Chain.GetChainParser())
// if err != nil {
// t.Fatal(err)
// }
// t.Logf("XXX: %q", voutAddr)
// }
// }
// }
// }
for _, vin := range tx.Vin { for _, vin := range tx.Vin {
for _, a := range vin.Addresses { for _, a := range vin.Addresses {
addr2txs[a] = append(addr2txs[a], txInfo{tx.Txid, vin.Vout, false}) addr2txs[a] = append(addr2txs[a], txInfo{tx.Txid, vin.Vout, false})
@ -275,10 +278,8 @@ func verifyTransactions(t *testing.T, d *db.RocksDB, h *TestHandler) {
} }
} }
lowerHeight := h.TestData.Blocks[0].Height
upperHeight := h.TestData.Blocks[len(h.TestData.Blocks)-1].Height
for addr, txs := range addr2txs { for addr, txs := range addr2txs {
err := d.GetTransactions(addr, lowerHeight, upperHeight, func(txid string, vout uint32, isOutput bool) error { err := d.GetTransactions(addr, rng.Lower, rng.Upper, func(txid string, vout uint32, isOutput bool) error {
for i, tx := range txs { for i, tx := range txs {
if txid == tx.txid && vout == tx.vout && isOutput == tx.isOutput { if txid == tx.txid && vout == tx.vout && isOutput == tx.isOutput {
checkMap[addr][i] = true checkMap[addr][i] = true
@ -300,10 +301,15 @@ func verifyTransactions(t *testing.T, d *db.RocksDB, h *TestHandler) {
} }
} }
func verifyAddresses(t *testing.T, d *db.RocksDB, h *TestHandler) { func verifyAddresses(t *testing.T, d *db.RocksDB, h *TestHandler, rng Range) {
parser := h.Chain.GetChainParser() parser := h.Chain.GetChainParser()
for _, block := range h.TestData.Blocks { for height := rng.Lower; height <= rng.Upper; height++ {
block, found := h.TestData.Blocks[height]
if !found {
continue
}
for _, tx := range block.TxDetails { for _, tx := range block.TxDetails {
ta, err := d.GetTxAddresses(tx.Txid) ta, err := d.GetTxAddresses(tx.Txid)
if err != nil { if err != nil {
@ -316,8 +322,8 @@ func verifyAddresses(t *testing.T, d *db.RocksDB, h *TestHandler) {
t.Fatal(err) t.Fatal(err)
} }
if ta.Height != block.Height { if ta.Height != height {
t.Errorf("Tx %s: block height mismatch: %d != %d", tx.Txid, ta.Height, block.Height) t.Errorf("Tx %s: block height mismatch: %d != %d", tx.Txid, ta.Height, height)
continue continue
} }
@ -394,11 +400,3 @@ func getTaInfo(parser bchain.BlockChainParser, ta *db.TxAddresses) (*txInfo, err
return info, nil return info, nil
} }
func satToFloat(parser bchain.BlockChainParser, sat *big.Int) *big.Float {
f, ok := new(big.Float).SetString(parser.AmountToDecimalString(sat))
if !ok {
return big.NewFloat(-1)
}
return f
}

113
tests/sync/testdata/bitcoin.json vendored
View File

@ -1,10 +1,10 @@
{ {
"connectBlocksParallel": { "syncRanges": [
"syncWorkers": 3, {"lower": 541224, "upper": 541255},
"syncChunk": 0 {"lower": 542835, "upper": 542838}
}, ],
"blocks": [ "blocks": {
{ "541224": {
"height": 541224, "height": 541224,
"hash": "00000000000000000027fe3aca0241ccee4f3b103b2108457a2dfd7490aa41a8", "hash": "00000000000000000027fe3aca0241ccee4f3b103b2108457a2dfd7490aa41a8",
"noTxs": 1880, "noTxs": 1880,
@ -77,7 +77,7 @@
} }
] ]
}, },
{ "541225": {
"height": 541225, "height": 541225,
"hash": "00000000000000000023e1bf6bc0a71fb1340c00f256de589108a4768ea82c76", "hash": "00000000000000000023e1bf6bc0a71fb1340c00f256de589108a4768ea82c76",
"noTxs": 1971, "noTxs": 1971,
@ -133,7 +133,7 @@
} }
] ]
}, },
{ "541226": {
"height": 541226, "height": 541226,
"hash": "00000000000000000023fc3b42ff5da9f74c6a672a7f793860f7c03b217628ae", "hash": "00000000000000000023fc3b42ff5da9f74c6a672a7f793860f7c03b217628ae",
"noTxs": 2479, "noTxs": 2479,
@ -174,7 +174,7 @@
} }
] ]
}, },
{ "541227": {
"height": 541227, "height": 541227,
"hash": "00000000000000000007d2b42e5ea24bed4aa02d18192c33b9ec7f6685acb16a", "hash": "00000000000000000007d2b42e5ea24bed4aa02d18192c33b9ec7f6685acb16a",
"noTxs": 1815, "noTxs": 1815,
@ -216,7 +216,7 @@
} }
] ]
}, },
{ "541228": {
"height": 541228, "height": 541228,
"hash": "0000000000000000001c2444541142d58b126ef608b5f52081aced02612d078f", "hash": "0000000000000000001c2444541142d58b126ef608b5f52081aced02612d078f",
"noTxs": 2549, "noTxs": 2549,
@ -257,7 +257,7 @@
} }
] ]
}, },
{ "541255": {
"height": 541255, "height": 541255,
"hash": "0000000000000000001375c3e88b4e53b687a70aba4b645ba81c93a4539d724d", "hash": "0000000000000000001375c3e88b4e53b687a70aba4b645ba81c93a4539d724d",
"noTxs": 2633, "noTxs": 2633,
@ -305,6 +305,95 @@
] ]
} }
] ]
},
"542838": {
"height": 542838,
"hash": "00000000000000000001e973be0770d428d3494d4e1b661aafcd8924c892648d",
"noTxs": 31,
"txDetails": [
{
"hex": "0100000007763056a30225a7f9f1af67e74a7ec67b514bcf094f8790da301b931ad2c8110e000000006b48304502210095e355273d421b93123786510757e10301508de7c4d9fb0c10900c43b6d6488a02206ddcd52d5e3e09083b50b8e020e2ce3f520e45b7be748e2cf79f1dafc44e54380121037d662ed0eab74137653a05001ada5b16c9b13ee4c542666d762564ebf68e3c8bffffffffbd4d49ec91d404facffca906661a1f437ba2e01f7004a769f01e3841d753e238000000006b4830450221008c92a4c432f4b822d234ee817f544673a86263a2eb106d1fd4189f0e8aeb4f64022031ecf397a0a15eeb41ed636d2729e422f57cb8269a74ddf7696df85055d35c5b012102940f4b3a9737f208c8cb1fe5ae8b317d06d0ca6e63dca5a04f735eb7d42377e4ffffffff3e851c0f9f70cb40a2492d811d87a899f6e29a1299f93dc7985e39e2db782559000000006a47304402202f8942571366051dc870be96e602e9415996e7583ecb270d7102b51453c50bb9022021c529a26ea348377863f59b03321279d4775cce1877fb37e3d09eb24b39e45b012102940f4b3a9737f208c8cb1fe5ae8b317d06d0ca6e63dca5a04f735eb7d42377e4ffffffffc9f4fc77c61640d945f14cbf9c7ecc2a89421ed03c3c593858915c995a009273000000006b483045022100e83039eb8d55b21f90a0fbae004bb3b0f0681b7e7a4f1bfceadbe5b470cb483702203be7ce3567967a2a1cdaeb3bd47d57cffd645557ab504d1056c3294c08ec08a6012103f3422741129eb0bfb82ca77f3818371749de4c38eb5f8def51a0abfb81cc1957ffffffff395717f7dfdec0df3e2117966af1e43eca5515f7469618d403a1383b5e4bca8a020000006a4730440220071c6128864f4ee9437b27a9ccdec52961f6a481c5ba01ae993a20a1d9afdfa20220248a5209af572f0af3c922e96def6c8b1967e355a8dfeaa891e69f59c9a37e96012102940f4b3a9737f208c8cb1fe5ae8b317d06d0ca6e63dca5a04f735eb7d42377e4ffffffff2e9ea71c6811838ec46444f08fad8298f8fa83cb1190ccaf98950963f0e68199000000006b483045022100d84f351d070103e614487ef7e8a7f5a6d60b6c51566aa1262111f8be1cad47d702203e0f36fec153718d3ba3a1c7dd8dd7f28691f4673498680d7f094b83db357aa4012103c71e9c61b2092b415b8ae4d5107786770cff927196efd12e39ed55477baefcdeffffffff39e161ac068370063eddd3ede85c9301f23e088f1ee72ac8e12603d7e0e8d6d6000000006a473044022053eb77ad34f47bcf00fdc845119d673948e2c6c0d21e6ec82d903598b32194df0220379127f024b85b964b52d3a7f501d7dad0b77321675d5d1ba6f0dc8f09305837012102940f4b3a9737f208c8cb1fe5ae8b317d06d0ca6e63dca5a04f735eb7d42377e4ffffffff02bb430000000000001976a9147270edc07db25cd4a7b99e54fd3c0c76076ba02688ac00e1f5050000000017a9141458b33294f0791f682c0d181ea757730faeeca88700000000",
"txid": "17da91b3b0590ee84b51174a76c0a61126d2ce82830230371cae549e0c070cd0",
"blocktime": 1537791290,
"time": 1537791290,
"locktime": 0,
"version": 1,
"vin": [
{
"txid": "48304502210095e355273d421b93123786510757e10301508de7c4d9fb0c10900c43b6d6488a02206ddcd52d5e3e09083b50b8e020e2ce3f520e45b7be748e2cf79f1dafc44e54380121037d662ed0eab74137653a05001ada5b16c9b13ee4c542666d762564ebf68e3c8b",
"vout": 0,
"sequence": 4294967295,
"scriptSig": {
"hex": "48304502210095e355273d421b93123786510757e10301508de7c4d9fb0c10900c43b6d6488a02206ddcd52d5e3e09083b50b8e020e2ce3f520e45b7be748e2cf79f1dafc44e54380121037d662ed0eab74137653a05001ada5b16c9b13ee4c542666d762564ebf68e3c8b"
}
},
{
"txid": "38e253d741381ef069a704701fe0a27b431f1a6606a9fccffa04d491ec494dbd",
"vout": 0,
"sequence": 4294967295,
"scriptSig": {
"hex": "4830450221008c92a4c432f4b822d234ee817f544673a86263a2eb106d1fd4189f0e8aeb4f64022031ecf397a0a15eeb41ed636d2729e422f57cb8269a74ddf7696df85055d35c5b012102940f4b3a9737f208c8cb1fe5ae8b317d06d0ca6e63dca5a04f735eb7d42377e4"
}
},
{
"txid": "592578dbe2395e98c73df999129ae2f699a8871d812d49a240cb709f0f1c853e",
"vout": 0,
"sequence": 4294967295,
"scriptSig": {
"hex": "47304402202f8942571366051dc870be96e602e9415996e7583ecb270d7102b51453c50bb9022021c529a26ea348377863f59b03321279d4775cce1877fb37e3d09eb24b39e45b012102940f4b3a9737f208c8cb1fe5ae8b317d06d0ca6e63dca5a04f735eb7d42377e4"
}
},
{
"txid": "7392005a995c915838593c3cd01e42892acc7e9cbf4cf145d94016c677fcf4c9",
"vout": 0,
"sequence": 4294967295,
"scriptSig": {
"hex": "483045022100e83039eb8d55b21f90a0fbae004bb3b0f0681b7e7a4f1bfceadbe5b470cb483702203be7ce3567967a2a1cdaeb3bd47d57cffd645557ab504d1056c3294c08ec08a6012103f3422741129eb0bfb82ca77f3818371749de4c38eb5f8def51a0abfb81cc1957"
}
},
{
"txid": "8aca4b5e3b38a103d4189646f71555ca3ee4f16a9617213edfc0dedff7175739",
"vout": 2,
"sequence": 4294967295,
"scriptSig": {
"hex": "4730440220071c6128864f4ee9437b27a9ccdec52961f6a481c5ba01ae993a20a1d9afdfa20220248a5209af572f0af3c922e96def6c8b1967e355a8dfeaa891e69f59c9a37e96012102940f4b3a9737f208c8cb1fe5ae8b317d06d0ca6e63dca5a04f735eb7d42377e4"
}
},
{
"txid": "9981e6f063099598afcc9011cb83faf89882ad8ff04464c48e8311681ca79e2e",
"vout": 0,
"sequence": 4294967295,
"scriptSig": {
"hex": "483045022100d84f351d070103e614487ef7e8a7f5a6d60b6c51566aa1262111f8be1cad47d702203e0f36fec153718d3ba3a1c7dd8dd7f28691f4673498680d7f094b83db357aa4012103c71e9c61b2092b415b8ae4d5107786770cff927196efd12e39ed55477baefcde"
}
},
{
"txid": "d6d6e8e0d70326e1c82ae71e8f083ef201935ce8edd3dd3e06708306ac61e139",
"vout": 0,
"sequence": 4294967295,
"scriptSig": {
"hex": "473044022053eb77ad34f47bcf00fdc845119d673948e2c6c0d21e6ec82d903598b32194df0220379127f024b85b964b52d3a7f501d7dad0b77321675d5d1ba6f0dc8f09305837012102940f4b3a9737f208c8cb1fe5ae8b317d06d0ca6e63dca5a04f735eb7d42377e4"
}
}
],
"vout": [
{
"value": 0.00017339,
"n": 0,
"scriptPubKey": {
"hex": "76a9147270edc07db25cd4a7b99e54fd3c0c76076ba02688ac"
}
},
{
"value": 1.00000000,
"n": 1,
"scriptPubKey": {
"hex": "a9141458b33294f0791f682c0d181ea757730faeeca887"
}
}
]
}
]
} }
] }
} }