From 58205ed84a94ad887fa68ad75d19ee79f44dabc6 Mon Sep 17 00:00:00 2001 From: Martin Boehm Date: Mon, 5 Mar 2018 18:14:41 +0100 Subject: [PATCH] Store transactions in RocksDB --- bchain/bitcoinwire.go | 116 ++++++++++++++++++++++++------------------ db/rocksdb.go | 80 ++++++++++++++++++++++++++++- db/rocksdb_test.go | 110 +++++++++++++++++++++++++++++++++++++++ 3 files changed, 255 insertions(+), 51 deletions(-) create mode 100644 db/rocksdb_test.go diff --git a/bchain/bitcoinwire.go b/bchain/bitcoinwire.go index 6ddb5ff9..bf77d591 100644 --- a/bchain/bitcoinwire.go +++ b/bchain/bitcoinwire.go @@ -54,6 +54,72 @@ func OutputScriptToAddresses(script []byte) ([]string, error) { return rv, nil } +// ParseTx parses byte array containing transaction and returns Tx struct +func ParseTx(b []byte) (*Tx, error) { + t := wire.MsgTx{} + r := bytes.NewReader(b) + if err := t.Deserialize(r); err != nil { + return nil, err + } + tx := txFromMsgTx(&t, true) + tx.Hex = hex.EncodeToString(b) + return &tx, nil +} + +func txFromMsgTx(t *wire.MsgTx, parseAddresses bool) Tx { + vin := make([]Vin, len(t.TxIn)) + for i, in := range t.TxIn { + if blockchain.IsCoinBaseTx(t) { + vin[i] = Vin{ + Coinbase: hex.EncodeToString(in.SignatureScript), + Sequence: in.Sequence, + } + break + } + s := ScriptSig{ + Hex: hex.EncodeToString(in.SignatureScript), + // missing: Asm, + } + vin[i] = Vin{ + Txid: in.PreviousOutPoint.Hash.String(), + Vout: in.PreviousOutPoint.Index, + Sequence: in.Sequence, + ScriptSig: s, + } + } + vout := make([]Vout, len(t.TxOut)) + for i, out := range t.TxOut { + addrs := []string{} + if parseAddresses { + addrs, _ = OutputScriptToAddresses(out.PkScript) + } + s := ScriptPubKey{ + Hex: hex.EncodeToString(out.PkScript), + Addresses: addrs, + // missing: Asm, + // missing: Type, + } + vout[i] = Vout{ + Value: float64(out.Value) / 1E8, + N: uint32(i), + ScriptPubKey: s, + } + } + tx := Tx{ + Txid: t.TxHash().String(), + // skip: Version, + LockTime: t.LockTime, + Vin: vin, + Vout: vout, + // skip: BlockHash, + // skip: Confirmations, + // skip: Time, + // skip: Blocktime, + } + return tx +} + +// ParseBlock parses raw block to our Block struct func (p *BitcoinBlockParser) ParseBlock(b []byte) (*Block, error) { w := wire.MsgBlock{} r := bytes.NewReader(b) @@ -64,55 +130,7 @@ func (p *BitcoinBlockParser) ParseBlock(b []byte) (*Block, error) { txs := make([]Tx, len(w.Transactions)) for ti, t := range w.Transactions { - vin := make([]Vin, len(t.TxIn)) - for i, in := range t.TxIn { - if blockchain.IsCoinBaseTx(t) { - vin[i] = Vin{ - Coinbase: hex.EncodeToString(in.SignatureScript), - Sequence: in.Sequence, - } - break - } - s := ScriptSig{ - Hex: hex.EncodeToString(in.SignatureScript), - // missing: Asm, - } - vin[i] = Vin{ - Txid: in.PreviousOutPoint.Hash.String(), - Vout: in.PreviousOutPoint.Index, - Sequence: in.Sequence, - ScriptSig: s, - } - } - vout := make([]Vout, len(t.TxOut)) - for i, out := range t.TxOut { - // addrs, err := OutputScriptToAddresses(out.PkScript) - // if err != nil { - // addrs = []string{} - // } - s := ScriptPubKey{ - Hex: hex.EncodeToString(out.PkScript), - // missing Addresses, - // missing: Asm, - // missing: Type, - } - vout[i] = Vout{ - Value: float64(out.Value), - N: uint32(i), - ScriptPubKey: s, - } - } - txs[ti] = Tx{ - Txid: t.TxHash().String(), - // skip: Version, - LockTime: t.LockTime, - Vin: vin, - Vout: vout, - // skip: BlockHash, - // skip: Confirmations, - // skip: Time, - // skip: Blocktime, - } + txs[ti] = txFromMsgTx(t, false) } return &Block{Txs: txs}, nil diff --git a/db/rocksdb.go b/db/rocksdb.go index fcd178d8..3c3b6be9 100644 --- a/db/rocksdb.go +++ b/db/rocksdb.go @@ -5,6 +5,7 @@ import ( "bytes" "encoding/binary" "encoding/hex" + "math" "os" "path/filepath" @@ -40,9 +41,10 @@ const ( cfHeight cfOutputs cfInputs + cfTransactions ) -var cfNames = []string{"default", "height", "outputs", "inputs"} +var cfNames = []string{"default", "height", "outputs", "inputs", "transactions"} func openDB(path string) (*gorocksdb.DB, []*gorocksdb.ColumnFamilyHandle, error) { c := gorocksdb.NewLRUCache(8 << 30) // 8GB @@ -80,7 +82,7 @@ func openDB(path string) (*gorocksdb.DB, []*gorocksdb.ColumnFamilyHandle, error) optsOutputs.SetMaxOpenFiles(25000) optsOutputs.SetCompression(gorocksdb.NoCompression) - fcOptions := []*gorocksdb.Options{opts, opts, optsOutputs, opts} + fcOptions := []*gorocksdb.Options{opts, opts, optsOutputs, opts, opts} db, cfh, err := gorocksdb.OpenDbColumnFamilies(opts, path, cfNames, fcOptions) if err != nil { @@ -350,6 +352,12 @@ func packOutpoint(txid string, vout uint32) ([]byte, error) { return buf, nil } +func unpackOutpoint(buf []byte) (string, uint32, int) { + txid, _ := unpackTxid(buf[:txIdUnpackedLen]) + vout, o := unpackVaruint(buf[txIdUnpackedLen:]) + return txid, vout, txIdUnpackedLen + o +} + // Block index // GetBestBlock returns the block hash of the block with highest height in the db @@ -520,6 +528,40 @@ func (d *RocksDB) DatabaseSizeOnDisk() (int64, error) { return dirSize(d.path) } +// GetTx returns transaction stored in db and height of the block containing it +func (d *RocksDB) GetTx(txid string) (*bchain.Tx, uint32, error) { + key, err := packTxid(txid) + if err != nil { + return nil, 0, err + } + val, err := d.db.GetCF(d.ro, d.cfh[cfTransactions], key) + if err != nil { + return nil, 0, err + } + defer val.Free() + return unpackTx(val.Data()) +} + +func (d *RocksDB) PutTx(tx *bchain.Tx, height uint32) error { + key, err := packTxid(tx.Txid) + if err != nil { + return nil + } + buf, err := packTx(tx, height) + if err != nil { + return err + } + return d.db.PutCF(d.wo, d.cfh[cfTransactions], key, buf) +} + +func (d *RocksDB) DeleteTx(txid string) error { + key, err := packTxid(txid) + if err != nil { + return nil + } + return d.db.DeleteCF(d.wo, d.cfh[cfTransactions], key) +} + // Helpers const txIdUnpackedLen = 32 @@ -536,6 +578,16 @@ func unpackUint(buf []byte) uint32 { return binary.BigEndian.Uint32(buf) } +func packFloat64(f float64) []byte { + buf := make([]byte, 8) + binary.BigEndian.PutUint64(buf, math.Float64bits(f)) + return buf +} + +func unpackFloat64(buf []byte) float64 { + return math.Float64frombits(binary.BigEndian.Uint64(buf)) +} + func packVaruint(i uint32) []byte { buf := make([]byte, vlq.MaxLen32) ofs := vlq.PutUint(buf, uint64(i)) @@ -547,6 +599,17 @@ func unpackVaruint(buf []byte) (uint32, int) { return uint32(i), ofs } +func packVarint64(i int64) []byte { + buf := make([]byte, vlq.MaxLen64) + ofs := vlq.PutInt(buf, i) + return buf[:ofs] +} + +func unpackVarint64(buf []byte) (int64, int) { + i, ofs := vlq.Int(buf) + return i, ofs +} + func packTxid(txid string) ([]byte, error) { return hex.DecodeString(txid) } @@ -570,3 +633,16 @@ func packOutputScript(script string) ([]byte, error) { func unpackOutputScript(buf []byte) string { return hex.EncodeToString(buf) } + +func packTx(tx *bchain.Tx, height uint32) ([]byte, error) { + buf := make([]byte, 4+len(tx.Hex)/2) + binary.BigEndian.PutUint32(buf[0:4], height) + _, err := hex.Decode(buf[4:], []byte(tx.Hex)) + return buf, err +} + +func unpackTx(buf []byte) (*bchain.Tx, uint32, error) { + height := unpackUint(buf) + tx, err := bchain.ParseTx(buf[4:]) + return tx, height, err +} diff --git a/db/rocksdb_test.go b/db/rocksdb_test.go new file mode 100644 index 00000000..485c25aa --- /dev/null +++ b/db/rocksdb_test.go @@ -0,0 +1,110 @@ +package db + +import ( + "blockbook/bchain" + "encoding/hex" + "reflect" + "testing" +) + +var testTx = bchain.Tx{ + // Blocktime: 1520253021, + Hex: "01000000017f9a22c9cbf54bd902400df746f138f37bcf5b4d93eb755820e974ba43ed5f42040000006a4730440220037f4ed5427cde81d55b9b6a2fd08c8a25090c2c2fff3a75c1a57625ca8a7118022076c702fe55969fa08137f71afd4851c48e31082dd3c40c919c92cdbc826758d30121029f6da5623c9f9b68a9baf9c1bc7511df88fa34c6c2f71f7c62f2f03ff48dca80feffffff019c9700000000000017a9146144d57c8aff48492c9dfb914e120b20bad72d6f8773d00700", + Txid: "056e3d82e5ffd0e915fb9b62797d76263508c34fe3e5dbed30dd3e943930f204", + LockTime: 512115, + // Time: 1520253022, + Vin: []bchain.Vin{ + { + ScriptSig: bchain.ScriptSig{ + Hex: "4730440220037f4ed5427cde81d55b9b6a2fd08c8a25090c2c2fff3a75c1a57625ca8a7118022076c702fe55969fa08137f71afd4851c48e31082dd3c40c919c92cdbc826758d30121029f6da5623c9f9b68a9baf9c1bc7511df88fa34c6c2f71f7c62f2f03ff48dca80", + }, + Txid: "425fed43ba74e9205875eb934d5bcf7bf338f146f70d4002d94bf5cbc9229a7f", + Vout: 4, + Sequence: 4294967294, + }, + }, + Vout: []bchain.Vout{ + { + Value: 0.00038812, + N: 0, + ScriptPubKey: bchain.ScriptPubKey{ + Hex: "a9146144d57c8aff48492c9dfb914e120b20bad72d6f87", + Addresses: []string{ + "3AZKvpKhSh1o8t1QrX3UeXG9d2BhCRnbcK", + }, + }, + }, + }, +} + +var testTxPacked = "0001e24001000000017f9a22c9cbf54bd902400df746f138f37bcf5b4d93eb755820e974ba43ed5f42040000006a4730440220037f4ed5427cde81d55b9b6a2fd08c8a25090c2c2fff3a75c1a57625ca8a7118022076c702fe55969fa08137f71afd4851c48e31082dd3c40c919c92cdbc826758d30121029f6da5623c9f9b68a9baf9c1bc7511df88fa34c6c2f71f7c62f2f03ff48dca80feffffff019c9700000000000017a9146144d57c8aff48492c9dfb914e120b20bad72d6f8773d00700" + +func Test_packTx(t *testing.T) { + type args struct { + tx bchain.Tx + height uint32 + } + tests := []struct { + name string + args args + want string + wantErr bool + }{ + { + name: "1", + args: args{testTx, 123456}, + want: testTxPacked, + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := packTx(&tt.args.tx, tt.args.height) + if (err != nil) != tt.wantErr { + t.Errorf("packTx() error = %v, wantErr %v", err, tt.wantErr) + return + } + h := hex.EncodeToString(got) + if !reflect.DeepEqual(h, tt.want) { + t.Errorf("packTx() = %v, want %v", h, tt.want) + } + }) + } +} + +func Test_unpackTx(t *testing.T) { + type args struct { + packedTx string + } + tests := []struct { + name string + args args + want *bchain.Tx + want1 uint32 + wantErr bool + }{ + { + name: "1", + args: args{packedTx: testTxPacked}, + want: &testTx, + want1: 123456, + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + b, _ := hex.DecodeString(tt.args.packedTx) + got, got1, err := unpackTx(b) + if (err != nil) != tt.wantErr { + t.Errorf("unpackTx() error = %v, wantErr %v", err, tt.wantErr) + return + } + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("unpackTx() got = %v, want %v", got, tt.want) + } + if got1 != tt.want1 { + t.Errorf("unpackTx() got1 = %v, want %v", got1, tt.want1) + } + }) + } +}