From d52832f6f7e8cfce0812bfe6c64673d017d7f5a1 Mon Sep 17 00:00:00 2001 From: Martin Boehm Date: Mon, 6 Feb 2023 00:04:35 +0100 Subject: [PATCH] Add extended index option - spendingTxid --- api/worker.go | 32 +++++++-- blockbook.go | 4 +- common/internalstate.go | 3 +- db/rocksdb.go | 97 +++++++++++++++++++-------- db/rocksdb_test.go | 101 +++++++++++++++++++++++++++-- fiat/fiat_rates_test.go | 2 +- server/public_ethereumtype_test.go | 2 +- server/public_test.go | 96 +++++++++++++++++++++++++-- tests/sync/sync.go | 2 +- 9 files changed, 291 insertions(+), 48 deletions(-) diff --git a/api/worker.go b/api/worker.go index 5f0873a3..005481a1 100644 --- a/api/worker.go +++ b/api/worker.go @@ -101,6 +101,19 @@ func (w *Worker) setSpendingTxToVout(vout *Vout, txid string, height uint32) err // GetSpendingTxid returns transaction id of transaction that spent given output func (w *Worker) GetSpendingTxid(txid string, n int) (string, error) { + if w.db.HasExtendedIndex() { + tsp, err := w.db.GetTxAddresses(txid) + if err != nil { + return "", err + } else if tsp == nil { + glog.Warning("DB inconsistency: tx ", txid, ": not found in txAddresses") + return "", NewAPIError(fmt.Sprintf("Txid %v not found", txid), false) + } + if n >= len(tsp.Outputs) || n < 0 { + return "", NewAPIError(fmt.Sprintf("Passed incorrect vout index %v for tx %v, len vout %v", n, txid, len(tsp.Outputs)), false) + } + return tsp.Outputs[n].SpentTxid, nil + } start := time.Now() tx, err := w.getTransaction(txid, false, false, nil) if err != nil { @@ -368,10 +381,16 @@ func (w *Worker) getTransactionFromBchainTx(bchainTx *bchain.Tx, height int, spe aggregateAddresses(addresses, vout.Addresses, vout.IsAddress) if ta != nil { vout.Spent = ta.Outputs[i].Spent - if spendingTxs && vout.Spent { - err = w.setSpendingTxToVout(vout, bchainTx.Txid, uint32(height)) - if err != nil { - glog.Errorf("setSpendingTxToVout error %v, %v, output %v", err, vout.AddrDesc, vout.N) + if vout.Spent { + if w.db.HasExtendedIndex() { + vout.SpentTxID = ta.Outputs[i].SpentTxid + vout.SpentIndex = int(ta.Outputs[i].SpentIndex) + vout.SpentHeight = int(ta.Outputs[i].SpentHeight) + } else if spendingTxs { + err = w.setSpendingTxToVout(vout, bchainTx.Txid, uint32(height)) + if err != nil { + glog.Errorf("setSpendingTxToVout error %v, %v, output %v", err, vout.AddrDesc, vout.N) + } } } } @@ -839,6 +858,11 @@ func (w *Worker) txFromTxAddress(txid string, ta *db.TxAddresses, bi *db.BlockIn glog.Errorf("tai.Addresses error %v, tx %v, output %v, tao %+v", err, txid, i, tao) } vout.Spent = tao.Spent + if vout.Spent && w.db.HasExtendedIndex() { + vout.SpentTxID = tao.SpentTxid + vout.SpentIndex = int(tao.SpentIndex) + vout.SpentHeight = int(tao.SpentHeight) + } aggregateAddresses(addresses, vout.Addresses, vout.IsAddress) } // for coinbase transactions valIn is 0 diff --git a/blockbook.go b/blockbook.go index aac96964..e9442f3b 100644 --- a/blockbook.go +++ b/blockbook.go @@ -84,6 +84,8 @@ var ( // resync mempool at least each resyncMempoolPeriodMs (could be more often if invoked by message from ZeroMQ) resyncMempoolPeriodMs = flag.Int("resyncmempoolperiod", 60017, "resync mempool period in milliseconds") + + extendedIndex = flag.Bool("extendedindex", false, "if true, create index of input txids and spending transactions") ) var ( @@ -172,7 +174,7 @@ func mainWithExitCode() int { return exitCodeFatal } - index, err = db.NewRocksDB(*dbPath, *dbCache, *dbMaxOpenFiles, chain.GetChainParser(), metrics) + index, err = db.NewRocksDB(*dbPath, *dbCache, *dbMaxOpenFiles, chain.GetChainParser(), metrics, *extendedIndex) if err != nil { glog.Error("rocksDB: ", err) return exitCodeFatal diff --git a/common/internalstate.go b/common/internalstate.go index 3e65d9a3..f12f4729 100644 --- a/common/internalstate.go +++ b/common/internalstate.go @@ -58,7 +58,8 @@ type InternalState struct { CoinLabel string `json:"coinLabel"` Host string `json:"host"` - DbState uint32 `json:"dbState"` + DbState uint32 `json:"dbState"` + ExtendedIndex bool `json:"extendedIndex"` LastStore time.Time `json:"lastStore"` diff --git a/db/rocksdb.go b/db/rocksdb.go index 783cdd12..72aeebf9 100644 --- a/db/rocksdb.go +++ b/db/rocksdb.go @@ -59,17 +59,18 @@ const ( // RocksDB handle type RocksDB struct { - path string - db *grocksdb.DB - wo *grocksdb.WriteOptions - ro *grocksdb.ReadOptions - cfh []*grocksdb.ColumnFamilyHandle - chainParser bchain.BlockChainParser - is *common.InternalState - metrics *common.Metrics - cache *grocksdb.Cache - maxOpenFiles int - cbs connectBlockStats + path string + db *grocksdb.DB + wo *grocksdb.WriteOptions + ro *grocksdb.ReadOptions + cfh []*grocksdb.ColumnFamilyHandle + chainParser bchain.BlockChainParser + is *common.InternalState + metrics *common.Metrics + cache *grocksdb.Cache + maxOpenFiles int + cbs connectBlockStats + extendedIndex bool } const ( @@ -126,7 +127,7 @@ func openDB(path string, c *grocksdb.Cache, openFiles int) (*grocksdb.DB, []*gro // NewRocksDB opens an internal handle to RocksDB environment. Close // needs to be called to release it. -func NewRocksDB(path string, cacheSize, maxOpenFiles int, parser bchain.BlockChainParser, metrics *common.Metrics) (d *RocksDB, err error) { +func NewRocksDB(path string, cacheSize, maxOpenFiles int, parser bchain.BlockChainParser, metrics *common.Metrics, extendedIndex bool) (d *RocksDB, err error) { glog.Infof("rocksdb: opening %s, required data version %v, cache size %v, max open files %v", path, dbVersion, cacheSize, maxOpenFiles) cfNames = append([]string{}, cfBaseNames...) @@ -135,6 +136,7 @@ func NewRocksDB(path string, cacheSize, maxOpenFiles int, parser bchain.BlockCha cfNames = append(cfNames, cfNamesBitcoinType...) } else if chainType == bchain.ChainEthereumType { cfNames = append(cfNames, cfNamesEthereumType...) + extendedIndex = false } else { return nil, errors.New("Unknown chain type") } @@ -146,7 +148,7 @@ func NewRocksDB(path string, cacheSize, maxOpenFiles int, parser bchain.BlockCha } wo := grocksdb.NewDefaultWriteOptions() ro := grocksdb.NewDefaultReadOptions() - return &RocksDB{path, db, wo, ro, cfh, parser, nil, metrics, c, maxOpenFiles, connectBlockStats{}}, nil + return &RocksDB{path, db, wo, ro, cfh, parser, nil, metrics, c, maxOpenFiles, connectBlockStats{}, extendedIndex}, nil } func (d *RocksDB) closeDB() error { @@ -204,6 +206,11 @@ func (d *RocksDB) WriteBatch(wb *grocksdb.WriteBatch) error { return d.db.Write(d.wo, wb) } +// HasExtendedIndex returns true if the DB indexes input txids and spending data +func (d *RocksDB) HasExtendedIndex() bool { + return d.extendedIndex +} + // GetMemoryStats returns memory usage statistics as reported by RocksDB func (d *RocksDB) GetMemoryStats() string { var total, indexAndFilter, memtable uint64 @@ -417,9 +424,12 @@ func (ti *TxInput) Addresses(p bchain.BlockChainParser) ([]string, bool, error) // TxOutput holds output data of the transaction in TxAddresses type TxOutput struct { - AddrDesc bchain.AddressDescriptor - Spent bool - ValueSat big.Int + AddrDesc bchain.AddressDescriptor + Spent bool + ValueSat big.Int + SpentTxid string + SpentIndex uint32 + SpentHeight uint32 } // Addresses converts AddressDescriptor of the output to array of strings @@ -681,6 +691,11 @@ func (d *RocksDB) processAddressesBitcoinType(block *bchain.Block, addresses add tai.ValueSat = spentOutput.ValueSat // mark the output as spent in tx spentOutput.Spent = true + if d.extendedIndex { + spentOutput.SpentTxid = tx.Txid + spentOutput.SpentIndex = uint32(i) + spentOutput.SpentHeight = block.Height + } if len(spentOutput.AddrDesc) == 0 { if !logged { glog.V(1).Infof("rocksdb: height %d, tx %v, input tx %v vout %v skipping empty address", block.Height, tx.Txid, input.Txid, input.Vout) @@ -757,7 +772,7 @@ func (d *RocksDB) storeTxAddresses(wb *grocksdb.WriteBatch, am map[string]*TxAdd varBuf := make([]byte, maxPackedBigintBytes) buf := make([]byte, 1024) for txID, ta := range am { - buf = packTxAddresses(ta, buf, varBuf) + buf = d.packTxAddresses(ta, buf, varBuf) wb.PutCF(d.cfh[cfTxAddresses], []byte(txID), buf) } return nil @@ -901,7 +916,7 @@ func (d *RocksDB) getTxAddresses(btxID []byte) (*TxAddresses, error) { if len(buf) < 3 { return nil, nil } - return unpackTxAddresses(buf) + return d.unpackTxAddresses(buf) } // GetTxAddresses returns TxAddresses for given txid or nil if not found @@ -932,7 +947,7 @@ func (d *RocksDB) AddrDescForOutpoint(outpoint bchain.Outpoint) (bchain.AddressD return ta.Outputs[outpoint.Vout].AddrDesc, &ta.Outputs[outpoint.Vout].ValueSat } -func packTxAddresses(ta *TxAddresses, buf []byte, varBuf []byte) []byte { +func (d *RocksDB) packTxAddresses(ta *TxAddresses, buf []byte, varBuf []byte) []byte { buf = buf[:0] l := packVaruint(uint(ta.Height), varBuf) buf = append(buf, varBuf[:l]...) @@ -944,7 +959,7 @@ func packTxAddresses(ta *TxAddresses, buf []byte, varBuf []byte) []byte { l = packVaruint(uint(len(ta.Outputs)), varBuf) buf = append(buf, varBuf[:l]...) for i := range ta.Outputs { - buf = appendTxOutput(&ta.Outputs[i], buf, varBuf) + buf = d.appendTxOutput(&ta.Outputs[i], buf, varBuf) } return buf } @@ -959,7 +974,7 @@ func appendTxInput(txi *TxInput, buf []byte, varBuf []byte) []byte { return buf } -func appendTxOutput(txo *TxOutput, buf []byte, varBuf []byte) []byte { +func (d *RocksDB) appendTxOutput(txo *TxOutput, buf []byte, varBuf []byte) []byte { la := len(txo.AddrDesc) if txo.Spent { la = ^la @@ -969,6 +984,20 @@ func appendTxOutput(txo *TxOutput, buf []byte, varBuf []byte) []byte { buf = append(buf, txo.AddrDesc...) l = packBigint(&txo.ValueSat, varBuf) buf = append(buf, varBuf[:l]...) + if d.extendedIndex && txo.Spent { + btxID, err := d.chainParser.PackTxid(txo.SpentTxid) + if err != nil { + if err != bchain.ErrTxidMissing { + glog.Error("Cannot pack txid ", txo.SpentTxid) + } + btxID = make([]byte, d.chainParser.PackedTxidLen()) + } + buf = append(buf, btxID...) + l = packVaruint(uint(txo.SpentIndex), varBuf) + buf = append(buf, varBuf[:l]...) + l = packVaruint(uint(txo.SpentHeight), varBuf) + buf = append(buf, varBuf[:l]...) + } return buf } @@ -1034,7 +1063,7 @@ func packAddrBalance(ab *AddrBalance, buf, varBuf []byte) []byte { return buf } -func unpackTxAddresses(buf []byte) (*TxAddresses, error) { +func (d *RocksDB) unpackTxAddresses(buf []byte) (*TxAddresses, error) { ta := TxAddresses{} height, l := unpackVaruint(buf) ta.Height = uint32(height) @@ -1048,7 +1077,7 @@ func unpackTxAddresses(buf []byte) (*TxAddresses, error) { l += ll ta.Outputs = make([]TxOutput, outputs) for i := uint(0); i < outputs; i++ { - l += unpackTxOutput(&ta.Outputs[i], buf[l:]) + l += d.unpackTxOutput(&ta.Outputs[i], buf[l:]) } return &ta, nil } @@ -1061,7 +1090,7 @@ func unpackTxInput(ti *TxInput, buf []byte) int { return l + int(al) } -func unpackTxOutput(to *TxOutput, buf []byte) int { +func (d *RocksDB) unpackTxOutput(to *TxOutput, buf []byte) int { al, l := unpackVarint(buf) if al < 0 { to.Spent = true @@ -1070,7 +1099,20 @@ func unpackTxOutput(to *TxOutput, buf []byte) int { to.AddrDesc = append([]byte(nil), buf[l:l+al]...) al += l to.ValueSat, l = unpackBigint(buf[al:]) - return l + al + al += l + if d.extendedIndex && to.Spent { + l = d.chainParser.PackedTxidLen() + to.SpentTxid, _ = d.chainParser.UnpackTxid(buf[al : al+l]) + al += l + var i uint + i, l = unpackVaruint(buf[al:]) + al += l + to.SpentIndex = uint32(i) + i, l = unpackVaruint(buf[al:]) + to.SpentHeight = uint32(i) + al += l + } + return al } func (d *RocksDB) packTxIndexes(txi []txIndexes) []byte { @@ -1682,7 +1724,7 @@ func (d *RocksDB) LoadInternalState(rpcCoin string) (*common.InternalState, erro data := val.Data() var is *common.InternalState if len(data) == 0 { - is = &common.InternalState{Coin: rpcCoin, UtxoChecked: true} + is = &common.InternalState{Coin: rpcCoin, UtxoChecked: true, ExtendedIndex: d.extendedIndex} } else { is, err = common.UnpackInternalState(data) if err != nil { @@ -1695,6 +1737,9 @@ func (d *RocksDB) LoadInternalState(rpcCoin string) (*common.InternalState, erro } else if is.Coin != rpcCoin { return nil, errors.Errorf("Coins do not match. DB coin %v, RPC coin %v", is.Coin, rpcCoin) } + if is.ExtendedIndex != d.extendedIndex { + return nil, errors.Errorf("ExtendedIndex setting does not match. DB extendedIndex %v, extendedIndex in options %v", is.ExtendedIndex, d.extendedIndex) + } } nc, err := d.checkColumns(is) if err != nil { diff --git a/db/rocksdb_test.go b/db/rocksdb_test.go index 8a287403..4138976d 100644 --- a/db/rocksdb_test.go +++ b/db/rocksdb_test.go @@ -47,7 +47,7 @@ func setupRocksDB(t *testing.T, p bchain.BlockChainParser) *RocksDB { if err != nil { t.Fatal(err) } - d, err := NewRocksDB(tmp, 100000, -1, p, nil) + d, err := NewRocksDB(tmp, 100000, -1, p, nil, false) if err != nil { t.Fatal(err) } @@ -903,9 +903,10 @@ func addressToAddrDesc(addr string, parser bchain.BlockChainParser) []byte { func Test_packTxAddresses_unpackTxAddresses(t *testing.T) { parser := bitcoinTestnetParser() tests := []struct { - name string - hex string - data *TxAddresses + name string + hex string + data *TxAddresses + rocksDB *RocksDB }{ { name: "1", @@ -930,6 +931,7 @@ func Test_packTxAddresses_unpackTxAddresses(t *testing.T) { }, }, }, + rocksDB: &RocksDB{chainParser: parser, extendedIndex: false}, }, { name: "2", @@ -976,6 +978,7 @@ func Test_packTxAddresses_unpackTxAddresses(t *testing.T) { }, }, }, + rocksDB: &RocksDB{chainParser: parser, extendedIndex: false}, }, { name: "empty address", @@ -1000,6 +1003,7 @@ func Test_packTxAddresses_unpackTxAddresses(t *testing.T) { }, }, }, + rocksDB: &RocksDB{chainParser: parser, extendedIndex: false}, }, { name: "empty", @@ -1008,18 +1012,103 @@ func Test_packTxAddresses_unpackTxAddresses(t *testing.T) { Inputs: []TxInput{}, Outputs: []TxOutput{}, }, + rocksDB: &RocksDB{chainParser: parser, extendedIndex: false}, + }, + { + name: "extendedIndex 1", + hex: "e0390317a9149eb21980dc9d413d8eac27314938b9da920ee53e8705021918f2c017a91409f70b896169c37981d2b54b371df0d81a136a2c870501dd7e28c017a914e371782582a4addb541362c55565d2cdf56f6498870501a1e35ec0052fa9141d9ca71efa36d814424ea6ca1437e67287aebe348705012aadcac000b2c06055e5e90e9c82bd4181fde310104391a7fa4f289b1704e5d90caa38400081ce8685592ea91424fbc77cdc62702ade74dcf989c15e5d3f9240bc870501664894c02fa914afbfb74ee994c7d45f6698738bc4226d065266f7870501a1e35ec0effd9ef509383d536b1c8af5bf434c8efbf521a4f2befd4022bbd68694b4ac75ef17a1f4233276a914d2a37ce20ac9ec4f15dd05a7c6e8e9fbdb99850e88ac043b9943603376a9146b2044146a4438e6e5bfbc65f147afeb64d14fbb88ac05012a05f2007c3be24063f268aaa1ed81b64776798f56088757641a34fb156c4f51ed2e9d25a9956d8396f32a", + data: &TxAddresses{ + Height: 12345, + Inputs: []TxInput{ + { + AddrDesc: addressToAddrDesc("2N7iL7AvS4LViugwsdjTB13uN4T7XhV1bCP", parser), + ValueSat: *big.NewInt(9011000000), + }, + { + AddrDesc: addressToAddrDesc("2Mt9v216YiNBAzobeNEzd4FQweHrGyuRHze", parser), + ValueSat: *big.NewInt(8011000000), + }, + { + AddrDesc: addressToAddrDesc("2NDyqJpHvHnqNtL1F9xAeCWMAW8WLJmEMyD", parser), + ValueSat: *big.NewInt(7011000000), + }, + }, + Outputs: []TxOutput{ + { + AddrDesc: addressToAddrDesc("2MuwoFGwABMakU7DCpdGDAKzyj2nTyRagDP", parser), + ValueSat: *big.NewInt(5011000000), + Spent: true, + SpentTxid: dbtestdata.TxidB1T1, + SpentIndex: 0, + SpentHeight: 432112345, + }, + { + AddrDesc: addressToAddrDesc("2Mvcmw7qkGXNWzkfH1EjvxDcNRGL1Kf2tEM", parser), + ValueSat: *big.NewInt(6011000000), + }, + { + AddrDesc: addressToAddrDesc("2N9GVuX3XJGHS5MCdgn97gVezc6EgvzikTB", parser), + ValueSat: *big.NewInt(7011000000), + Spent: true, + SpentTxid: dbtestdata.TxidB1T2, + SpentIndex: 14231, + SpentHeight: 555555, + }, + { + AddrDesc: addressToAddrDesc("mzii3fuRSpExMLJEHdHveW8NmiX8MPgavk", parser), + ValueSat: *big.NewInt(999900000), + }, + { + AddrDesc: addressToAddrDesc("mqHPFTRk23JZm9W1ANuEFtwTYwxjESSgKs", parser), + ValueSat: *big.NewInt(5000000000), + Spent: true, + SpentTxid: dbtestdata.TxidB2T1, + SpentIndex: 674541, + SpentHeight: 6666666, + }, + }, + }, + rocksDB: &RocksDB{chainParser: parser, extendedIndex: true}, + }, + { + name: "extendedIndex empty address", + hex: "baef9a1501000204d2020002162e010162fdd824a780cbb718eeb766eb05d83fdefc793a27082cd5e67f856d69798cf7db03e039", + data: &TxAddresses{ + Height: 123456789, + Inputs: []TxInput{ + { + AddrDesc: []byte(nil), + ValueSat: *big.NewInt(1234), + }, + }, + Outputs: []TxOutput{ + { + AddrDesc: []byte(nil), + ValueSat: *big.NewInt(5678), + }, + { + AddrDesc: []byte(nil), + ValueSat: *big.NewInt(98), + Spent: true, + SpentTxid: dbtestdata.TxidB2T4, + SpentIndex: 3, + SpentHeight: 12345, + }, + }, + }, + rocksDB: &RocksDB{chainParser: parser, extendedIndex: true}, }, } varBuf := make([]byte, maxPackedBigintBytes) buf := make([]byte, 1024) for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - b := packTxAddresses(tt.data, buf, varBuf) + b := tt.rocksDB.packTxAddresses(tt.data, buf, varBuf) hex := hex.EncodeToString(b) if !reflect.DeepEqual(hex, tt.hex) { t.Errorf("packTxAddresses() = %v, want %v", hex, tt.hex) } - got1, err := unpackTxAddresses(b) + got1, err := tt.rocksDB.unpackTxAddresses(b) if err != nil { t.Errorf("unpackTxAddresses() error = %v", err) return diff --git a/fiat/fiat_rates_test.go b/fiat/fiat_rates_test.go index 417c960b..f80d9efa 100644 --- a/fiat/fiat_rates_test.go +++ b/fiat/fiat_rates_test.go @@ -36,7 +36,7 @@ func setupRocksDB(t *testing.T, parser bchain.BlockChainParser) (*db.RocksDB, *c if err != nil { t.Fatal(err) } - d, err := db.NewRocksDB(tmp, 100000, -1, parser, nil) + d, err := db.NewRocksDB(tmp, 100000, -1, parser, nil, false) if err != nil { t.Fatal(err) } diff --git a/server/public_ethereumtype_test.go b/server/public_ethereumtype_test.go index 9094ca17..26b6b49c 100644 --- a/server/public_ethereumtype_test.go +++ b/server/public_ethereumtype_test.go @@ -211,7 +211,7 @@ func Test_PublicServer_EthereumType(t *testing.T) { glog.Fatal("fakechain: ", err) } - s, dbpath := setupPublicHTTPServer(parser, chain, t) + s, dbpath := setupPublicHTTPServer(parser, chain, t, false) defer closeAndDestroyPublicServer(t, s, dbpath) s.ConnectFullPublicInterface() // take the handler of the public server and pass it to the test server diff --git a/server/public_test.go b/server/public_test.go index ad4ec906..f8e76d7b 100644 --- a/server/public_test.go +++ b/server/public_test.go @@ -39,12 +39,12 @@ func TestMain(m *testing.M) { os.Exit(c) } -func setupRocksDB(parser bchain.BlockChainParser, chain bchain.BlockChain, t *testing.T) (*db.RocksDB, *common.InternalState, string) { +func setupRocksDB(parser bchain.BlockChainParser, chain bchain.BlockChain, t *testing.T, extendedIndex bool) (*db.RocksDB, *common.InternalState, string) { tmp, err := ioutil.TempDir("", "testdb") if err != nil { t.Fatal(err) } - d, err := db.NewRocksDB(tmp, 100000, -1, parser, nil) + d, err := db.NewRocksDB(tmp, 100000, -1, parser, nil, extendedIndex) if err != nil { t.Fatal(err) } @@ -95,8 +95,8 @@ func setupRocksDB(parser bchain.BlockChainParser, chain bchain.BlockChain, t *te var metrics *common.Metrics -func setupPublicHTTPServer(parser bchain.BlockChainParser, chain bchain.BlockChain, t *testing.T) (*PublicServer, string) { - d, is, path := setupRocksDB(parser, chain, t) +func setupPublicHTTPServer(parser bchain.BlockChainParser, chain bchain.BlockChain, t *testing.T, extendedIndex bool) (*PublicServer, string) { + d, is, path := setupRocksDB(parser, chain, t, extendedIndex) // setup internal state and match BestHeight to test data is.Coin = "Fakecoin" is.CoinLabel = "Fake Coin" @@ -105,7 +105,7 @@ func setupPublicHTTPServer(parser bchain.BlockChainParser, chain bchain.BlockCha var err error // metrics can be setup only once if metrics == nil { - metrics, err = common.GetMetrics("Fakecoin") + metrics, err = common.GetMetrics("Fakecoin" + strconv.FormatBool(extendedIndex)) if err != nil { glog.Fatal("metrics: ", err) } @@ -1499,7 +1499,7 @@ func fixedTimeNow() time.Time { return time.Date(2022, 9, 15, 12, 43, 56, 0, time.UTC) } -func Test_PublicServer_BitcoinType(t *testing.T) { +func setupChain(t *testing.T) (bchain.BlockChainParser, bchain.BlockChain) { timeNow = fixedTimeNow parser := btc.NewBitcoinParser( btc.GetChainParams("test"), @@ -1515,8 +1515,13 @@ func Test_PublicServer_BitcoinType(t *testing.T) { if err != nil { glog.Fatal("fakechain: ", err) } + return parser, chain +} - s, dbpath := setupPublicHTTPServer(parser, chain, t) +func Test_PublicServer_BitcoinType(t *testing.T) { + parser, chain := setupChain(t) + + s, dbpath := setupPublicHTTPServer(parser, chain, t, false) defer closeAndDestroyPublicServer(t, s, dbpath) s.ConnectFullPublicInterface() // take the handler of the public server and pass it to the test server @@ -1528,6 +1533,83 @@ func Test_PublicServer_BitcoinType(t *testing.T) { websocketTestsBitcoinType(t, ts) } +func httpTestsExtendedIndex(t *testing.T, ts *httptest.Server) { + tests := []struct { + name string + r *http.Request + status int + contentType string + body []string + }{ + { + name: "apiTx v2", + r: newGetRequest(ts.URL + "/api/v2/tx/7c3be24063f268aaa1ed81b64776798f56088757641a34fb156c4f51ed2e9d25"), + status: http.StatusOK, + contentType: "application/json; charset=utf-8", + body: []string{ + `{"txid":"7c3be24063f268aaa1ed81b64776798f56088757641a34fb156c4f51ed2e9d25","vin":[{"txid":"effd9ef509383d536b1c8af5bf434c8efbf521a4f2befd4022bbd68694b4ac75","n":0,"addresses":["mv9uLThosiEnGRbVPS7Vhyw6VssbVRsiAw"],"isAddress":true,"value":"1234567890123"},{"txid":"00b2c06055e5e90e9c82bd4181fde310104391a7fa4f289b1704e5d90caa3840","vout":1,"n":1,"addresses":["mtGXQvBowMkBpnhLckhxhbwYK44Gs9eEtz"],"isAddress":true,"value":"12345"}],"vout":[{"value":"317283951061","n":0,"spent":true,"spentTxId":"3d90d15ed026dc45e19ffb52875ed18fa9e8012ad123d7f7212176e2b0ebdb71","spentHeight":225494,"hex":"76a914ccaaaf374e1b06cb83118453d102587b4273d09588ac","addresses":["mzB8cYrfRwFRFAGTDzV8LkUQy5BQicxGhX"],"isAddress":true},{"value":"917283951061","n":1,"hex":"76a9148d802c045445df49613f6a70ddd2e48526f3701f88ac","addresses":["mtR97eM2HPWVM6c8FGLGcukgaHHQv7THoL"],"isAddress":true},{"value":"0","n":2,"hex":"6a072020f1686f6a20","addresses":["OP_RETURN 2020f1686f6a20"],"isAddress":false}],"blockHash":"00000000eb0443fd7dc4a1ed5c686a8e995057805f9a161d9a5a77a95e72b7b6","blockHeight":225494,"confirmations":1,"blockTime":1521595678,"value":"1234567902122","valueIn":"1234567902468","fees":"346"}`, + }, + }, + { + name: "apiAddress v2 details=txs", + r: newGetRequest(ts.URL + "/api/v2/address/mv9uLThosiEnGRbVPS7Vhyw6VssbVRsiAw?details=txs"), + status: http.StatusOK, + contentType: "application/json; charset=utf-8", + body: []string{ + `{"page":1,"totalPages":1,"itemsOnPage":1000,"address":"mv9uLThosiEnGRbVPS7Vhyw6VssbVRsiAw","balance":"0","totalReceived":"1234567890123","totalSent":"1234567890123","unconfirmedBalance":"0","unconfirmedTxs":0,"txs":2,"transactions":[{"txid":"7c3be24063f268aaa1ed81b64776798f56088757641a34fb156c4f51ed2e9d25","vin":[{"txid":"effd9ef509383d536b1c8af5bf434c8efbf521a4f2befd4022bbd68694b4ac75","n":0,"addresses":["mv9uLThosiEnGRbVPS7Vhyw6VssbVRsiAw"],"isAddress":true,"isOwn":true,"value":"1234567890123"},{"txid":"00b2c06055e5e90e9c82bd4181fde310104391a7fa4f289b1704e5d90caa3840","vout":1,"n":1,"addresses":["mtGXQvBowMkBpnhLckhxhbwYK44Gs9eEtz"],"isAddress":true,"value":"12345"}],"vout":[{"value":"317283951061","n":0,"spent":true,"spentTxId":"3d90d15ed026dc45e19ffb52875ed18fa9e8012ad123d7f7212176e2b0ebdb71","spentHeight":225494,"hex":"76a914ccaaaf374e1b06cb83118453d102587b4273d09588ac","addresses":["mzB8cYrfRwFRFAGTDzV8LkUQy5BQicxGhX"],"isAddress":true},{"value":"917283951061","n":1,"hex":"76a9148d802c045445df49613f6a70ddd2e48526f3701f88ac","addresses":["mtR97eM2HPWVM6c8FGLGcukgaHHQv7THoL"],"isAddress":true},{"value":"0","n":2,"hex":"6a072020f1686f6a20","addresses":["OP_RETURN 2020f1686f6a20"],"isAddress":false}],"blockHash":"00000000eb0443fd7dc4a1ed5c686a8e995057805f9a161d9a5a77a95e72b7b6","blockHeight":225494,"confirmations":1,"blockTime":1521595678,"value":"1234567902122","valueIn":"1234567902468","fees":"346"},{"txid":"effd9ef509383d536b1c8af5bf434c8efbf521a4f2befd4022bbd68694b4ac75","vin":[],"vout":[{"value":"1234567890123","n":0,"spent":true,"spentTxId":"7c3be24063f268aaa1ed81b64776798f56088757641a34fb156c4f51ed2e9d25","spentHeight":225494,"hex":"76a914a08eae93007f22668ab5e4a9c83c8cd1c325e3e088ac","addresses":["mv9uLThosiEnGRbVPS7Vhyw6VssbVRsiAw"],"isAddress":true,"isOwn":true},{"value":"1","n":1,"spent":true,"spentTxId":"3d90d15ed026dc45e19ffb52875ed18fa9e8012ad123d7f7212176e2b0ebdb71","spentIndex":1,"spentHeight":225494,"hex":"a91452724c5178682f70e0ba31c6ec0633755a3b41d987","addresses":["2MzmAKayJmja784jyHvRUW1bXPget1csRRG"],"isAddress":true},{"value":"9876","n":2,"spent":true,"spentTxId":"05e2e48aeabdd9b75def7b48d756ba304713c2aba7b522bf9dbc893fc4231b07","spentHeight":225494,"hex":"a914e921fc4912a315078f370d959f2c4f7b6d2a683c87","addresses":["2NEVv9LJmAnY99W1pFoc5UJjVdypBqdnvu1"],"isAddress":true}],"blockHash":"0000000076fbbed90fd75b0e18856aa35baa984e9c9d444cf746ad85e94e2997","blockHeight":225493,"confirmations":2,"blockTime":1521515026,"value":"1234567900000","valueIn":"0","fees":"0"}]}`, + }, + }, + { + name: "apiGetBlock", + r: newGetRequest(ts.URL + "/api/v2/block/225493"), + status: http.StatusOK, + contentType: "application/json; charset=utf-8", + body: []string{ + `{"page":1,"totalPages":1,"itemsOnPage":1000,"hash":"0000000076fbbed90fd75b0e18856aa35baa984e9c9d444cf746ad85e94e2997","nextBlockHash":"00000000eb0443fd7dc4a1ed5c686a8e995057805f9a161d9a5a77a95e72b7b6","height":225493,"confirmations":2,"size":1234567,"time":1521515026,"version":0,"merkleRoot":"","nonce":"","bits":"","difficulty":"","txCount":2,"txs":[{"txid":"00b2c06055e5e90e9c82bd4181fde310104391a7fa4f289b1704e5d90caa3840","vin":[],"vout":[{"value":"100000000","n":0,"addresses":["mfcWp7DB6NuaZsExybTTXpVgWz559Np4Ti"],"isAddress":true},{"value":"12345","n":1,"spent":true,"spentTxId":"7c3be24063f268aaa1ed81b64776798f56088757641a34fb156c4f51ed2e9d25","spentIndex":1,"spentHeight":225494,"addresses":["mtGXQvBowMkBpnhLckhxhbwYK44Gs9eEtz"],"isAddress":true},{"value":"12345","n":2,"addresses":["mtGXQvBowMkBpnhLckhxhbwYK44Gs9eEtz"],"isAddress":true}],"blockHash":"0000000076fbbed90fd75b0e18856aa35baa984e9c9d444cf746ad85e94e2997","blockHeight":225493,"confirmations":2,"blockTime":1521515026,"value":"100024690","valueIn":"0","fees":"0"},{"txid":"effd9ef509383d536b1c8af5bf434c8efbf521a4f2befd4022bbd68694b4ac75","vin":[],"vout":[{"value":"1234567890123","n":0,"spent":true,"spentTxId":"7c3be24063f268aaa1ed81b64776798f56088757641a34fb156c4f51ed2e9d25","spentHeight":225494,"addresses":["mv9uLThosiEnGRbVPS7Vhyw6VssbVRsiAw"],"isAddress":true},{"value":"1","n":1,"spent":true,"spentTxId":"3d90d15ed026dc45e19ffb52875ed18fa9e8012ad123d7f7212176e2b0ebdb71","spentIndex":1,"spentHeight":225494,"addresses":["2MzmAKayJmja784jyHvRUW1bXPget1csRRG"],"isAddress":true},{"value":"9876","n":2,"spent":true,"spentTxId":"05e2e48aeabdd9b75def7b48d756ba304713c2aba7b522bf9dbc893fc4231b07","spentHeight":225494,"addresses":["2NEVv9LJmAnY99W1pFoc5UJjVdypBqdnvu1"],"isAddress":true}],"blockHash":"0000000076fbbed90fd75b0e18856aa35baa984e9c9d444cf746ad85e94e2997","blockHeight":225493,"confirmations":2,"blockTime":1521515026,"value":"1234567900000","valueIn":"0","fees":"0"}]}`, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + resp, err := http.DefaultClient.Do(tt.r) + if err != nil { + t.Fatal(err) + } + defer resp.Body.Close() + if resp.StatusCode != tt.status { + t.Errorf("StatusCode = %v, want %v", resp.StatusCode, tt.status) + } + if resp.Header["Content-Type"][0] != tt.contentType { + t.Errorf("Content-Type = %v, want %v", resp.Header["Content-Type"][0], tt.contentType) + } + bb, err := ioutil.ReadAll(resp.Body) + if err != nil { + t.Fatal(err) + } + b := string(bb) + for _, c := range tt.body { + if !strings.Contains(b, c) { + t.Errorf("got %v, want to contain %v", b, c) + break + } + } + }) + } +} + +func Test_PublicServer_BitcoinType_ExtendedIndex(t *testing.T) { + parser, chain := setupChain(t) + + s, dbpath := setupPublicHTTPServer(parser, chain, t, true) + defer closeAndDestroyPublicServer(t, s, dbpath) + s.ConnectFullPublicInterface() + // take the handler of the public server and pass it to the test server + ts := httptest.NewServer(s.https.Handler) + defer ts.Close() + + httpTestsExtendedIndex(t, ts) +} + func Test_formatInt64(t *testing.T) { tests := []struct { name string diff --git a/tests/sync/sync.go b/tests/sync/sync.go index e822f194..b3e16421 100644 --- a/tests/sync/sync.go +++ b/tests/sync/sync.go @@ -145,7 +145,7 @@ func makeRocksDB(parser bchain.BlockChainParser, m *common.Metrics, is *common.I return nil, nil, err } - d, err := db.NewRocksDB(p, 1<<17, 1<<14, parser, m) + d, err := db.NewRocksDB(p, 1<<17, 1<<14, parser, m, false) if err != nil { return nil, nil, err }