Store transactions in RocksDB

bchain/bitcoinwire.go

@@ -54,6 +54,72 @@ func OutputScriptToAddresses(script []byte) ([]string, error) {
 	return rv, nil
 }
 
+// ParseTx parses byte array containing transaction and returns Tx struct
+func ParseTx(b []byte) (*Tx, error) {
+	t := wire.MsgTx{}
+	r := bytes.NewReader(b)
+	if err := t.Deserialize(r); err != nil {
+		return nil, err
+	}
+	tx := txFromMsgTx(&t, true)
+	tx.Hex = hex.EncodeToString(b)
+	return &tx, nil
+}
+
+func txFromMsgTx(t *wire.MsgTx, parseAddresses bool) Tx {
+	vin := make([]Vin, len(t.TxIn))
+	for i, in := range t.TxIn {
+		if blockchain.IsCoinBaseTx(t) {
+			vin[i] = Vin{
+				Coinbase: hex.EncodeToString(in.SignatureScript),
+				Sequence: in.Sequence,
+			}
+			break
+		}
+		s := ScriptSig{
+			Hex: hex.EncodeToString(in.SignatureScript),
+			// missing: Asm,
+		}
+		vin[i] = Vin{
+			Txid:      in.PreviousOutPoint.Hash.String(),
+			Vout:      in.PreviousOutPoint.Index,
+			Sequence:  in.Sequence,
+			ScriptSig: s,
+		}
+	}
+	vout := make([]Vout, len(t.TxOut))
+	for i, out := range t.TxOut {
+		addrs := []string{}
+		if parseAddresses {
+			addrs, _ = OutputScriptToAddresses(out.PkScript)
+		}
+		s := ScriptPubKey{
+			Hex:       hex.EncodeToString(out.PkScript),
+			Addresses: addrs,
+			// missing: Asm,
+			// missing: Type,
+		}
+		vout[i] = Vout{
+			Value:        float64(out.Value) / 1E8,
+			N:            uint32(i),
+			ScriptPubKey: s,
+		}
+	}
+	tx := Tx{
+		Txid: t.TxHash().String(),
+		// skip: Version,
+		LockTime: t.LockTime,
+		Vin:      vin,
+		Vout:     vout,
+		// skip: BlockHash,
+		// skip: Confirmations,
+		// skip: Time,
+		// skip: Blocktime,
+	}
+	return tx
+}
+
+// ParseBlock parses raw block to our Block struct
 func (p *BitcoinBlockParser) ParseBlock(b []byte) (*Block, error) {
 	w := wire.MsgBlock{}
 	r := bytes.NewReader(b)
@@ -64,55 +130,7 @@ func (p *BitcoinBlockParser) ParseBlock(b []byte) (*Block, error) {
 
 	txs := make([]Tx, len(w.Transactions))
 	for ti, t := range w.Transactions {
-		vin := make([]Vin, len(t.TxIn))
+		txs[ti] = txFromMsgTx(t, false)
-		for i, in := range t.TxIn {
-			if blockchain.IsCoinBaseTx(t) {
-				vin[i] = Vin{
-					Coinbase: hex.EncodeToString(in.SignatureScript),
-					Sequence: in.Sequence,
-				}
-				break
-			}
-			s := ScriptSig{
-				Hex: hex.EncodeToString(in.SignatureScript),
-				// missing: Asm,
-			}
-			vin[i] = Vin{
-				Txid:      in.PreviousOutPoint.Hash.String(),
-				Vout:      in.PreviousOutPoint.Index,
-				Sequence:  in.Sequence,
-				ScriptSig: s,
-			}
-		}
-		vout := make([]Vout, len(t.TxOut))
-		for i, out := range t.TxOut {
-			// addrs, err := OutputScriptToAddresses(out.PkScript)
-			// if err != nil {
-			// 	addrs = []string{}
-			// }
-			s := ScriptPubKey{
-				Hex: hex.EncodeToString(out.PkScript),
-				// missing Addresses,
-				// missing: Asm,
-				// missing: Type,
-			}
-			vout[i] = Vout{
-				Value:        float64(out.Value),
-				N:            uint32(i),
-				ScriptPubKey: s,
-			}
-		}
-		txs[ti] = Tx{
-			Txid: t.TxHash().String(),
-			// skip: Version,
-			LockTime: t.LockTime,
-			Vin:      vin,
-			Vout:     vout,
-			// skip: BlockHash,
-			// skip: Confirmations,
-			// skip: Time,
-			// skip: Blocktime,
-		}
 	}
 
 	return &Block{Txs: txs}, nil

db/rocksdb.go

@@ -5,6 +5,7 @@ import (
 	"bytes"
 	"encoding/binary"
 	"encoding/hex"
+	"math"
 	"os"
 	"path/filepath"
 
@@ -40,9 +41,10 @@ const (
 	cfHeight
 	cfOutputs
 	cfInputs
+	cfTransactions
 )
 
-var cfNames = []string{"default", "height", "outputs", "inputs"}
+var cfNames = []string{"default", "height", "outputs", "inputs", "transactions"}
 
 func openDB(path string) (*gorocksdb.DB, []*gorocksdb.ColumnFamilyHandle, error) {
 	c := gorocksdb.NewLRUCache(8 << 30) // 8GB
@@ -80,7 +82,7 @@ func openDB(path string) (*gorocksdb.DB, []*gorocksdb.ColumnFamilyHandle, error)
 	optsOutputs.SetMaxOpenFiles(25000)
 	optsOutputs.SetCompression(gorocksdb.NoCompression)
 
-	fcOptions := []*gorocksdb.Options{opts, opts, optsOutputs, opts}
+	fcOptions := []*gorocksdb.Options{opts, opts, optsOutputs, opts, opts}
 
 	db, cfh, err := gorocksdb.OpenDbColumnFamilies(opts, path, cfNames, fcOptions)
 	if err != nil {
@@ -350,6 +352,12 @@ func packOutpoint(txid string, vout uint32) ([]byte, error) {
 	return buf, nil
 }
 
+func unpackOutpoint(buf []byte) (string, uint32, int) {
+	txid, _ := unpackTxid(buf[:txIdUnpackedLen])
+	vout, o := unpackVaruint(buf[txIdUnpackedLen:])
+	return txid, vout, txIdUnpackedLen + o
+}
+
 // Block index
 
 // GetBestBlock returns the block hash of the block with highest height in the db
@@ -520,6 +528,40 @@ func (d *RocksDB) DatabaseSizeOnDisk() (int64, error) {
 	return dirSize(d.path)
 }
 
+// GetTx returns transaction stored in db and height of the block containing it
+func (d *RocksDB) GetTx(txid string) (*bchain.Tx, uint32, error) {
+	key, err := packTxid(txid)
+	if err != nil {
+		return nil, 0, err
+	}
+	val, err := d.db.GetCF(d.ro, d.cfh[cfTransactions], key)
+	if err != nil {
+		return nil, 0, err
+	}
+	defer val.Free()
+	return unpackTx(val.Data())
+}
+
+func (d *RocksDB) PutTx(tx *bchain.Tx, height uint32) error {
+	key, err := packTxid(tx.Txid)
+	if err != nil {
+		return nil
+	}
+	buf, err := packTx(tx, height)
+	if err != nil {
+		return err
+	}
+	return d.db.PutCF(d.wo, d.cfh[cfTransactions], key, buf)
+}
+
+func (d *RocksDB) DeleteTx(txid string) error {
+	key, err := packTxid(txid)
+	if err != nil {
+		return nil
+	}
+	return d.db.DeleteCF(d.wo, d.cfh[cfTransactions], key)
+}
+
 // Helpers
 
 const txIdUnpackedLen = 32
@@ -536,6 +578,16 @@ func unpackUint(buf []byte) uint32 {
 	return binary.BigEndian.Uint32(buf)
 }
 
+func packFloat64(f float64) []byte {
+	buf := make([]byte, 8)
+	binary.BigEndian.PutUint64(buf, math.Float64bits(f))
+	return buf
+}
+
+func unpackFloat64(buf []byte) float64 {
+	return math.Float64frombits(binary.BigEndian.Uint64(buf))
+}
+
 func packVaruint(i uint32) []byte {
 	buf := make([]byte, vlq.MaxLen32)
 	ofs := vlq.PutUint(buf, uint64(i))
@@ -547,6 +599,17 @@ func unpackVaruint(buf []byte) (uint32, int) {
 	return uint32(i), ofs
 }
 
+func packVarint64(i int64) []byte {
+	buf := make([]byte, vlq.MaxLen64)
+	ofs := vlq.PutInt(buf, i)
+	return buf[:ofs]
+}
+
+func unpackVarint64(buf []byte) (int64, int) {
+	i, ofs := vlq.Int(buf)
+	return i, ofs
+}
+
 func packTxid(txid string) ([]byte, error) {
 	return hex.DecodeString(txid)
 }
@@ -570,3 +633,16 @@ func packOutputScript(script string) ([]byte, error) {
 func unpackOutputScript(buf []byte) string {
 	return hex.EncodeToString(buf)
 }
+
+func packTx(tx *bchain.Tx, height uint32) ([]byte, error) {
+	buf := make([]byte, 4+len(tx.Hex)/2)
+	binary.BigEndian.PutUint32(buf[0:4], height)
+	_, err := hex.Decode(buf[4:], []byte(tx.Hex))
+	return buf, err
+}
+
+func unpackTx(buf []byte) (*bchain.Tx, uint32, error) {
+	height := unpackUint(buf)
+	tx, err := bchain.ParseTx(buf[4:])
+	return tx, height, err
+}

db/rocksdb_test.go

@@ -0,0 +1,110 @@
+package db
+
+import (
+	"blockbook/bchain"
+	"encoding/hex"
+	"reflect"
+	"testing"
+)
+
+var testTx = bchain.Tx{
+	// Blocktime: 1520253021,
+	Hex:      "01000000017f9a22c9cbf54bd902400df746f138f37bcf5b4d93eb755820e974ba43ed5f42040000006a4730440220037f4ed5427cde81d55b9b6a2fd08c8a25090c2c2fff3a75c1a57625ca8a7118022076c702fe55969fa08137f71afd4851c48e31082dd3c40c919c92cdbc826758d30121029f6da5623c9f9b68a9baf9c1bc7511df88fa34c6c2f71f7c62f2f03ff48dca80feffffff019c9700000000000017a9146144d57c8aff48492c9dfb914e120b20bad72d6f8773d00700",
+	Txid:     "056e3d82e5ffd0e915fb9b62797d76263508c34fe3e5dbed30dd3e943930f204",
+	LockTime: 512115,
+	// Time:      1520253022,
+	Vin: []bchain.Vin{
+		{
+			ScriptSig: bchain.ScriptSig{
+				Hex: "4730440220037f4ed5427cde81d55b9b6a2fd08c8a25090c2c2fff3a75c1a57625ca8a7118022076c702fe55969fa08137f71afd4851c48e31082dd3c40c919c92cdbc826758d30121029f6da5623c9f9b68a9baf9c1bc7511df88fa34c6c2f71f7c62f2f03ff48dca80",
+			},
+			Txid:     "425fed43ba74e9205875eb934d5bcf7bf338f146f70d4002d94bf5cbc9229a7f",
+			Vout:     4,
+			Sequence: 4294967294,
+		},
+	},
+	Vout: []bchain.Vout{
+		{
+			Value: 0.00038812,
+			N:     0,
+			ScriptPubKey: bchain.ScriptPubKey{
+				Hex: "a9146144d57c8aff48492c9dfb914e120b20bad72d6f87",
+				Addresses: []string{
+					"3AZKvpKhSh1o8t1QrX3UeXG9d2BhCRnbcK",
+				},
+			},
+		},
+	},
+}
+
+var testTxPacked = "0001e24001000000017f9a22c9cbf54bd902400df746f138f37bcf5b4d93eb755820e974ba43ed5f42040000006a4730440220037f4ed5427cde81d55b9b6a2fd08c8a25090c2c2fff3a75c1a57625ca8a7118022076c702fe55969fa08137f71afd4851c48e31082dd3c40c919c92cdbc826758d30121029f6da5623c9f9b68a9baf9c1bc7511df88fa34c6c2f71f7c62f2f03ff48dca80feffffff019c9700000000000017a9146144d57c8aff48492c9dfb914e120b20bad72d6f8773d00700"
+
+func Test_packTx(t *testing.T) {
+	type args struct {
+		tx     bchain.Tx
+		height uint32
+	}
+	tests := []struct {
+		name    string
+		args    args
+		want    string
+		wantErr bool
+	}{
+		{
+			name:    "1",
+			args:    args{testTx, 123456},
+			want:    testTxPacked,
+			wantErr: false,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got, err := packTx(&tt.args.tx, tt.args.height)
+			if (err != nil) != tt.wantErr {
+				t.Errorf("packTx() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			h := hex.EncodeToString(got)
+			if !reflect.DeepEqual(h, tt.want) {
+				t.Errorf("packTx() = %v, want %v", h, tt.want)
+			}
+		})
+	}
+}
+
+func Test_unpackTx(t *testing.T) {
+	type args struct {
+		packedTx string
+	}
+	tests := []struct {
+		name    string
+		args    args
+		want    *bchain.Tx
+		want1   uint32
+		wantErr bool
+	}{
+		{
+			name:    "1",
+			args:    args{packedTx: testTxPacked},
+			want:    &testTx,
+			want1:   123456,
+			wantErr: false,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			b, _ := hex.DecodeString(tt.args.packedTx)
+			got, got1, err := unpackTx(b)
+			if (err != nil) != tt.wantErr {
+				t.Errorf("unpackTx() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			if !reflect.DeepEqual(got, tt.want) {
+				t.Errorf("unpackTx() got = %v, want %v", got, tt.want)
+			}
+			if got1 != tt.want1 {
+				t.Errorf("unpackTx() got1 = %v, want %v", got1, tt.want1)
+			}
+		})
+	}
+}