diff --git a/db/rocksdb.go b/db/rocksdb.go index 3d936882..5c5c0aa3 100644 --- a/db/rocksdb.go +++ b/db/rocksdb.go @@ -263,7 +263,7 @@ func (d *RocksDB) writeAddressRecords(wb *gorocksdb.WriteBatch, block *bchain.Bl func (d *RocksDB) addAddrIDToRecords(op int, wb *gorocksdb.WriteBatch, records map[string][]outpoint, addrID []byte, btxid []byte, vout int32, bh uint32) error { if len(addrID) > 0 { if len(addrID) > 1024 { - glog.Infof("block %d, skipping addrID of length %d", bh, len(addrID)) + glog.Infof("rocksdb: block %d, skipping addrID of length %d", bh, len(addrID)) } else { strAddrID := string(addrID) records[strAddrID] = append(records[strAddrID], outpoint{ @@ -305,18 +305,19 @@ func appendPackedAddrID(txAddrs []byte, addrID []byte, n uint32, remaining int) } func findAndRemoveUnspentAddr(unspentAddrs []byte, vout uint32) ([]byte, uint32, []byte) { + // the addresses are packed as lenaddrID:addrID:vout, where lenaddrID and vout are varints for i := 0; i < len(unspentAddrs); { l, lv1 := unpackVarint(unspentAddrs[i:]) // index of vout of address in unspentAddrs j := i + int(l) + lv1 if j >= len(unspentAddrs) { - glog.Error("Inconsistent data in unspentAddrs") + glog.Error("rocksdb: Inconsistent data in unspentAddrs") return nil, 0, unspentAddrs } n, lv2 := unpackVarint(unspentAddrs[j:]) if uint32(n) == vout { addrID := append([]byte(nil), unspentAddrs[i+lv1:j]...) - unspentAddrs = append(unspentAddrs[:i], unspentAddrs[i+lv2:]...) + unspentAddrs = append(unspentAddrs[:i], unspentAddrs[j+lv2:]...) return addrID, uint32(n), unspentAddrs } i += j + lv2 @@ -325,7 +326,6 @@ func findAndRemoveUnspentAddr(unspentAddrs []byte, vout uint32) ([]byte, uint32, } func (d *RocksDB) writeAddressesUTXO(wb *gorocksdb.WriteBatch, block *bchain.Block, op int) error { - var err error addresses := make(map[string][]outpoint) unspentTxs := make(map[string][]byte) btxIDs := make([][]byte, len(block.Txs)) @@ -357,44 +357,35 @@ func (d *RocksDB) writeAddressesUTXO(wb *gorocksdb.WriteBatch, block *bchain.Blo } // locate unspent addresses and add them to addresses map them in format txid ^index for txi, tx := range block.Txs { - btxID := btxIDs[txi] - // try to find the tx in current block - unspentAddrs, inThisBlock := unspentTxs[string(btxID)] - if !inThisBlock { - unspentAddrs, err = d.getUnspentTx(btxID) + spendingTxid := btxIDs[txi] + for i, input := range tx.Vin { + btxID, err := d.chainParser.PackTxid(input.Txid) if err != nil { return err } - if unspentAddrs == nil { - glog.Warningf("rocksdb: height %d, tx %v in inputs but missing in unspentTxs", block.Height, tx.Txid) - continue + // try to find the tx in current block + unspentAddrs, inThisBlock := unspentTxs[string(btxID)] + if !inThisBlock { + unspentAddrs, err = d.getUnspentTx(btxID) + if err != nil { + return err + } + if unspentAddrs == nil { + glog.Warningf("rocksdb: height %d, tx %v in inputs but missing in unspentTxs", block.Height, tx.Txid) + continue + } } - } - var addrID []byte - var n uint32 - for _, input := range tx.Vin { - addrID, n, unspentAddrs = findAndRemoveUnspentAddr(unspentAddrs, input.Vout) + var addrID []byte + addrID, _, unspentAddrs = findAndRemoveUnspentAddr(unspentAddrs, input.Vout) if addrID == nil { glog.Warningf("rocksdb: height %d, tx %v vout %v in inputs but missing in unspentTxs", block.Height, tx.Txid, input.Vout) continue } - err = d.addAddrIDToRecords(op, wb, addresses, addrID, btxID, int32(^n), block.Height) + err = d.addAddrIDToRecords(op, wb, addresses, addrID, spendingTxid, int32(^i), block.Height) if err != nil { return err } - } - if inThisBlock { - if len(unspentAddrs) == 0 { - delete(unspentTxs, string(btxID)) - } else { - unspentTxs[string(btxID)] = unspentAddrs - } - } else { - if len(unspentAddrs) == 0 { - wb.DeleteCF(d.cfh[cfUnspentTxs], btxID) - } else { - wb.PutCF(d.cfh[cfUnspentTxs], btxID, unspentAddrs) - } + unspentTxs[string(btxID)] = unspentAddrs } } if err := d.writeAddressRecords(wb, block, op, addresses); err != nil { @@ -404,7 +395,11 @@ func (d *RocksDB) writeAddressesUTXO(wb *gorocksdb.WriteBatch, block *bchain.Blo for tx, val := range unspentTxs { switch op { case opInsert: - wb.PutCF(d.cfh[cfUnspentTxs], []byte(tx), val) + if len(val) == 0 { + wb.DeleteCF(d.cfh[cfUnspentTxs], []byte(tx)) + } else { + wb.PutCF(d.cfh[cfUnspentTxs], []byte(tx), val) + } case opDelete: wb.DeleteCF(d.cfh[cfUnspentTxs], []byte(tx)) } @@ -722,6 +717,6 @@ func packVarint(i int32, buf []byte) int { } func unpackVarint(buf []byte) (int32, int) { - i, ofs := vlq.Uint(buf) + i, ofs := vlq.Int(buf) return int32(i), ofs } diff --git a/db/rocksdb_test.go b/db/rocksdb_test.go index 4e4217d2..afc8b065 100644 --- a/db/rocksdb_test.go +++ b/db/rocksdb_test.go @@ -6,6 +6,7 @@ import ( "encoding/hex" "io/ioutil" "os" + "sort" "strconv" "testing" @@ -50,6 +51,9 @@ type keyPair struct { } func checkColumn(d *RocksDB, col int, kp []keyPair) error { + sort.Slice(kp, func(i, j int) bool { + return kp[i].Key < kp[j].Key + }) it := d.db.NewIteratorCF(d.ro, d.cfh[col]) defer it.Close() i := 0 @@ -63,7 +67,7 @@ func checkColumn(d *RocksDB, col int, kp []keyPair) error { } val := hex.EncodeToString(it.Value().Data()) if val != kp[i].Value { - return errors.Errorf("Incorrect key %v found in column %v row %v, expecting %v", val, col, i, kp[i].Value) + return errors.Errorf("Incorrect value %v found in column %v row %v, expecting %v", val, col, i, kp[i].Value) } i++ } @@ -72,6 +76,15 @@ func checkColumn(d *RocksDB, col int, kp []keyPair) error { } return nil } + +// TestRocksDB_Index_UTXO is a composite test testing the whole indexing functionality for UTXO chains +// It does the following: +// 1) Connect two blocks (inputs from 2nd block are spending some outputs from the 1st block) +// 2) GetTransactions for known addresses +// 3) Disconnect block 2 +// 4) GetTransactions for known addresses +// 5) Connect the block 2 back +// After each step, the whole content of DB is examined and any difference against expected state is regarded as failure func TestRocksDB_Index_UTXO(t *testing.T) { d := setupRocksDB(t, &btc.BitcoinParser{Params: btc.GetChainParams("test")}) defer closeAnddestroyRocksDB(t, d) @@ -155,4 +168,113 @@ func TestRocksDB_Index_UTXO(t *testing.T) { } } + // connect 2nd block - use some outputs from the 1st block as the inputs and 1 input uses tx from the same block + block2 := bchain.Block{ + BlockHeader: bchain.BlockHeader{ + Height: 225494, + Hash: "00000000eb0443fd7dc4a1ed5c686a8e995057805f9a161d9a5a77a95e72b7b6", + }, + Txs: []bchain.Tx{ + bchain.Tx{ + Txid: "7c3be24063f268aaa1ed81b64776798f56088757641a34fb156c4f51ed2e9d25", + Vin: []bchain.Vin{ + bchain.Vin{ + Txid: "effd9ef509383d536b1c8af5bf434c8efbf521a4f2befd4022bbd68694b4ac75", + Vout: 0, + }, + bchain.Vin{ + Txid: "00b2c06055e5e90e9c82bd4181fde310104391a7fa4f289b1704e5d90caa3840", + Vout: 1, + }, + }, + Vout: []bchain.Vout{ + bchain.Vout{ + N: 0, + ScriptPubKey: bchain.ScriptPubKey{ + Hex: addressToPubKeyHex("mzB8cYrfRwFRFAGTDzV8LkUQy5BQicxGhX", t, d), + }, + }, + bchain.Vout{ + N: 1, + ScriptPubKey: bchain.ScriptPubKey{ + Hex: addressToPubKeyHex("mtR97eM2HPWVM6c8FGLGcukgaHHQv7THoL", t, d), + }, + }, + }, + }, + bchain.Tx{ + Txid: "3d90d15ed026dc45e19ffb52875ed18fa9e8012ad123d7f7212176e2b0ebdb71", + Vin: []bchain.Vin{ + bchain.Vin{ + Txid: "7c3be24063f268aaa1ed81b64776798f56088757641a34fb156c4f51ed2e9d25", + Vout: 0, + }, + bchain.Vin{ + Txid: "effd9ef509383d536b1c8af5bf434c8efbf521a4f2befd4022bbd68694b4ac75", + Vout: 1, + }, + }, + Vout: []bchain.Vout{ + bchain.Vout{ + N: 0, + ScriptPubKey: bchain.ScriptPubKey{ + Hex: addressToPubKeyHex("mwwoKQE5Lb1G4picHSHDQKg8jw424PF9SC", t, d), + }, + }, + bchain.Vout{ + N: 1, + ScriptPubKey: bchain.ScriptPubKey{ + Hex: addressToPubKeyHex("mmJx9Y8ayz9h14yd9fgCW1bUKoEpkBAquP", t, d), + }, + }, + }, + }, + }, + } + if err := d.ConnectBlock(&block2); err != nil { + t.Fatal(err) + } + if err := checkColumn(d, cfHeight, []keyPair{ + keyPair{"000370d5", "0000000076fbbed90fd75b0e18856aa35baa984e9c9d444cf746ad85e94e2997"}, + keyPair{"000370d6", "00000000eb0443fd7dc4a1ed5c686a8e995057805f9a161d9a5a77a95e72b7b6"}, + }); err != nil { + { + t.Fatal(err) + } + } + if err := checkColumn(d, cfAddresses, []keyPair{ + keyPair{addressToPubKeyHex("mfcWp7DB6NuaZsExybTTXpVgWz559Np4Ti", t, d) + "000370d5", "00b2c06055e5e90e9c82bd4181fde310104391a7fa4f289b1704e5d90caa3840" + "00"}, + keyPair{addressToPubKeyHex("mtGXQvBowMkBpnhLckhxhbwYK44Gs9eEtz", t, d) + "000370d5", "00b2c06055e5e90e9c82bd4181fde310104391a7fa4f289b1704e5d90caa3840" + "02"}, + keyPair{addressToPubKeyHex("mv9uLThosiEnGRbVPS7Vhyw6VssbVRsiAw", t, d) + "000370d5", "effd9ef509383d536b1c8af5bf434c8efbf521a4f2befd4022bbd68694b4ac75" + "00"}, + keyPair{addressToPubKeyHex("2Mz1CYoppGGsLNUGF2YDhTif6J661JitALS", t, d) + "000370d5", "effd9ef509383d536b1c8af5bf434c8efbf521a4f2befd4022bbd68694b4ac75" + "02"}, + keyPair{addressToPubKeyHex("mzB8cYrfRwFRFAGTDzV8LkUQy5BQicxGhX", t, d) + "000370d6", "7c3be24063f268aaa1ed81b64776798f56088757641a34fb156c4f51ed2e9d25" + "00" + "3d90d15ed026dc45e19ffb52875ed18fa9e8012ad123d7f7212176e2b0ebdb71" + "01"}, + keyPair{addressToPubKeyHex("mtR97eM2HPWVM6c8FGLGcukgaHHQv7THoL", t, d) + "000370d6", "7c3be24063f268aaa1ed81b64776798f56088757641a34fb156c4f51ed2e9d25" + "02"}, + keyPair{addressToPubKeyHex("mwwoKQE5Lb1G4picHSHDQKg8jw424PF9SC", t, d) + "000370d6", "3d90d15ed026dc45e19ffb52875ed18fa9e8012ad123d7f7212176e2b0ebdb71" + "00"}, + keyPair{addressToPubKeyHex("mmJx9Y8ayz9h14yd9fgCW1bUKoEpkBAquP", t, d) + "000370d6", "3d90d15ed026dc45e19ffb52875ed18fa9e8012ad123d7f7212176e2b0ebdb71" + "02"}, + keyPair{addressToPubKeyHex("mv9uLThosiEnGRbVPS7Vhyw6VssbVRsiAw", t, d) + "000370d6", "7c3be24063f268aaa1ed81b64776798f56088757641a34fb156c4f51ed2e9d25" + "01"}, + keyPair{addressToPubKeyHex("mtGXQvBowMkBpnhLckhxhbwYK44Gs9eEtz", t, d) + "000370d6", "7c3be24063f268aaa1ed81b64776798f56088757641a34fb156c4f51ed2e9d25" + "03"}, + keyPair{addressToPubKeyHex("2Mz1CYoppGGsLNUGF2YDhTif6J661JitALS", t, d) + "000370d6", "3d90d15ed026dc45e19ffb52875ed18fa9e8012ad123d7f7212176e2b0ebdb71" + "03"}, + }); err != nil { + { + t.Fatal(err) + } + } + if err := checkColumn(d, cfUnspentTxs, []keyPair{ + keyPair{ + "00b2c06055e5e90e9c82bd4181fde310104391a7fa4f289b1704e5d90caa3840", + addressToPubKeyHexWithLenght("mfcWp7DB6NuaZsExybTTXpVgWz559Np4Ti", t, d) + "00", + }, + keyPair{ + "7c3be24063f268aaa1ed81b64776798f56088757641a34fb156c4f51ed2e9d25", + addressToPubKeyHexWithLenght("mtR97eM2HPWVM6c8FGLGcukgaHHQv7THoL", t, d) + "02", + }, + keyPair{ + "3d90d15ed026dc45e19ffb52875ed18fa9e8012ad123d7f7212176e2b0ebdb71", + addressToPubKeyHexWithLenght("mwwoKQE5Lb1G4picHSHDQKg8jw424PF9SC", t, d) + "00" + addressToPubKeyHexWithLenght("mmJx9Y8ayz9h14yd9fgCW1bUKoEpkBAquP", t, d) + "02", + }, + }); err != nil { + { + t.Fatal(err) + } + } }