RanchiMall/blockbook
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Store transactions in RocksDB

Browse Source
This commit is contained in:
Martin Boehm 2018-03-05 18:14:41 +01:00
parent 54ddfa9e03
commit 58205ed84a
3 changed files with 255 additions and 51 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

116
bchain/bitcoinwire.go
View File

@ -54,6 +54,72 @@ func OutputScriptToAddresses(script []byte) ([]string, error) {
return rv, nil
}
// ParseTx parses byte array containing transaction and returns Tx struct
func ParseTx(b []byte) (*Tx, error) {
t := wire.MsgTx{}
r := bytes.NewReader(b)
if err := t.Deserialize(r); err != nil {
return nil, err
}
tx := txFromMsgTx(&t, true)
tx.Hex = hex.EncodeToString(b)
return &tx, nil
}
func txFromMsgTx(t *wire.MsgTx, parseAddresses bool) Tx {
vin := make([]Vin, len(t.TxIn))
for i, in := range t.TxIn {
if blockchain.IsCoinBaseTx(t) {
vin[i] = Vin{
Coinbase: hex.EncodeToString(in.SignatureScript),
Sequence: in.Sequence,
}
break
}
s := ScriptSig{
Hex: hex.EncodeToString(in.SignatureScript),
// missing: Asm,
}
vin[i] = Vin{
Txid: in.PreviousOutPoint.Hash.String(),
Vout: in.PreviousOutPoint.Index,
Sequence: in.Sequence,
ScriptSig: s,
}
}
vout := make([]Vout, len(t.TxOut))
for i, out := range t.TxOut {
addrs := []string{}
if parseAddresses {
addrs, _ = OutputScriptToAddresses(out.PkScript)
}
s := ScriptPubKey{
Hex: hex.EncodeToString(out.PkScript),
Addresses: addrs,
// missing: Asm,
// missing: Type,
}
vout[i] = Vout{
Value: float64(out.Value) / 1E8,
N: uint32(i),
ScriptPubKey: s,
}
}
tx := Tx{
Txid: t.TxHash().String(),
// skip: Version,
LockTime: t.LockTime,
Vin: vin,
Vout: vout,
// skip: BlockHash,
// skip: Confirmations,
// skip: Time,
// skip: Blocktime,
}
return tx
}
// ParseBlock parses raw block to our Block struct
func (p *BitcoinBlockParser) ParseBlock(b []byte) (*Block, error) {
w := wire.MsgBlock{}
r := bytes.NewReader(b)
@ -64,55 +130,7 @@ func (p *BitcoinBlockParser) ParseBlock(b []byte) (*Block, error) {
txs := make([]Tx, len(w.Transactions))
for ti, t := range w.Transactions {
vin := make([]Vin, len(t.TxIn))
for i, in := range t.TxIn {
if blockchain.IsCoinBaseTx(t) {
vin[i] = Vin{
Coinbase: hex.EncodeToString(in.SignatureScript),
Sequence: in.Sequence,
}
break
}
s := ScriptSig{
Hex: hex.EncodeToString(in.SignatureScript),
// missing: Asm,
}
vin[i] = Vin{
Txid: in.PreviousOutPoint.Hash.String(),
Vout: in.PreviousOutPoint.Index,
Sequence: in.Sequence,
ScriptSig: s,
}
}
vout := make([]Vout, len(t.TxOut))
for i, out := range t.TxOut {
// addrs, err := OutputScriptToAddresses(out.PkScript)
// if err != nil {
// addrs = []string{}
// }
s := ScriptPubKey{
Hex: hex.EncodeToString(out.PkScript),
// missing Addresses,
// missing: Asm,
// missing: Type,
}
vout[i] = Vout{
Value: float64(out.Value),
N: uint32(i),
ScriptPubKey: s,
}
}
txs[ti] = Tx{
Txid: t.TxHash().String(),
// skip: Version,
LockTime: t.LockTime,
Vin: vin,
Vout: vout,
// skip: BlockHash,
// skip: Confirmations,
// skip: Time,
// skip: Blocktime,
}
txs[ti] = txFromMsgTx(t, false)
}
return &Block{Txs: txs}, nil

80
db/rocksdb.go
View File

@ -5,6 +5,7 @@ import (
"bytes"
"encoding/binary"
"encoding/hex"
"math"
"os"
"path/filepath"
@ -40,9 +41,10 @@ const (
cfHeight
cfOutputs
cfInputs
cfTransactions
)
var cfNames = []string{"default", "height", "outputs", "inputs"}
var cfNames = []string{"default", "height", "outputs", "inputs", "transactions"}
func openDB(path string) (*gorocksdb.DB, []*gorocksdb.ColumnFamilyHandle, error) {
c := gorocksdb.NewLRUCache(8 << 30) // 8GB
@ -80,7 +82,7 @@ func openDB(path string) (*gorocksdb.DB, []*gorocksdb.ColumnFamilyHandle, error)
optsOutputs.SetMaxOpenFiles(25000)
optsOutputs.SetCompression(gorocksdb.NoCompression)
fcOptions := []*gorocksdb.Options{opts, opts, optsOutputs, opts}
fcOptions := []*gorocksdb.Options{opts, opts, optsOutputs, opts, opts}
db, cfh, err := gorocksdb.OpenDbColumnFamilies(opts, path, cfNames, fcOptions)
if err != nil {
@ -350,6 +352,12 @@ func packOutpoint(txid string, vout uint32) ([]byte, error) {
return buf, nil
}
func unpackOutpoint(buf []byte) (string, uint32, int) {
txid, _ := unpackTxid(buf[:txIdUnpackedLen])
vout, o := unpackVaruint(buf[txIdUnpackedLen:])
return txid, vout, txIdUnpackedLen + o
}
// Block index
// GetBestBlock returns the block hash of the block with highest height in the db
@ -520,6 +528,40 @@ func (d *RocksDB) DatabaseSizeOnDisk() (int64, error) {
return dirSize(d.path)
}
// GetTx returns transaction stored in db and height of the block containing it
func (d *RocksDB) GetTx(txid string) (*bchain.Tx, uint32, error) {
key, err := packTxid(txid)
if err != nil {
return nil, 0, err
}
val, err := d.db.GetCF(d.ro, d.cfh[cfTransactions], key)
if err != nil {
return nil, 0, err
}
defer val.Free()
return unpackTx(val.Data())
}
func (d *RocksDB) PutTx(tx *bchain.Tx, height uint32) error {
key, err := packTxid(tx.Txid)
if err != nil {
return nil
}
buf, err := packTx(tx, height)
if err != nil {
return err
}
return d.db.PutCF(d.wo, d.cfh[cfTransactions], key, buf)
}
func (d *RocksDB) DeleteTx(txid string) error {
key, err := packTxid(txid)
if err != nil {
return nil
}
return d.db.DeleteCF(d.wo, d.cfh[cfTransactions], key)
}
// Helpers
const txIdUnpackedLen = 32
@ -536,6 +578,16 @@ func unpackUint(buf []byte) uint32 {
return binary.BigEndian.Uint32(buf)
}
func packFloat64(f float64) []byte {
buf := make([]byte, 8)
binary.BigEndian.PutUint64(buf, math.Float64bits(f))
return buf
}
func unpackFloat64(buf []byte) float64 {
return math.Float64frombits(binary.BigEndian.Uint64(buf))
}
func packVaruint(i uint32) []byte {
buf := make([]byte, vlq.MaxLen32)
ofs := vlq.PutUint(buf, uint64(i))
@ -547,6 +599,17 @@ func unpackVaruint(buf []byte) (uint32, int) {
return uint32(i), ofs
}
func packVarint64(i int64) []byte {
buf := make([]byte, vlq.MaxLen64)
ofs := vlq.PutInt(buf, i)
return buf[:ofs]
}
func unpackVarint64(buf []byte) (int64, int) {
i, ofs := vlq.Int(buf)
return i, ofs
}
func packTxid(txid string) ([]byte, error) {
return hex.DecodeString(txid)
}
@ -570,3 +633,16 @@ func packOutputScript(script string) ([]byte, error) {
func unpackOutputScript(buf []byte) string {
return hex.EncodeToString(buf)
}
func packTx(tx *bchain.Tx, height uint32) ([]byte, error) {
buf := make([]byte, 4+len(tx.Hex)/2)
binary.BigEndian.PutUint32(buf[0:4], height)
_, err := hex.Decode(buf[4:], []byte(tx.Hex))
return buf, err
}
func unpackTx(buf []byte) (*bchain.Tx, uint32, error) {
height := unpackUint(buf)
tx, err := bchain.ParseTx(buf[4:])
return tx, height, err
}

110
db/rocksdb_test.go Normal file
View File

@ -0,0 +1,110 @@
package db
import (
"blockbook/bchain"
"encoding/hex"
"reflect"
"testing"
)
var testTx = bchain.Tx{
// Blocktime: 1520253021,
Hex: "01000000017f9a22c9cbf54bd902400df746f138f37bcf5b4d93eb755820e974ba43ed5f42040000006a4730440220037f4ed5427cde81d55b9b6a2fd08c8a25090c2c2fff3a75c1a57625ca8a7118022076c702fe55969fa08137f71afd4851c48e31082dd3c40c919c92cdbc826758d30121029f6da5623c9f9b68a9baf9c1bc7511df88fa34c6c2f71f7c62f2f03ff48dca80feffffff019c9700000000000017a9146144d57c8aff48492c9dfb914e120b20bad72d6f8773d00700",
Txid: "056e3d82e5ffd0e915fb9b62797d76263508c34fe3e5dbed30dd3e943930f204",
LockTime: 512115,
// Time: 1520253022,
Vin: []bchain.Vin{
{
ScriptSig: bchain.ScriptSig{
Hex: "4730440220037f4ed5427cde81d55b9b6a2fd08c8a25090c2c2fff3a75c1a57625ca8a7118022076c702fe55969fa08137f71afd4851c48e31082dd3c40c919c92cdbc826758d30121029f6da5623c9f9b68a9baf9c1bc7511df88fa34c6c2f71f7c62f2f03ff48dca80",
},
Txid: "425fed43ba74e9205875eb934d5bcf7bf338f146f70d4002d94bf5cbc9229a7f",
Vout: 4,
Sequence: 4294967294,
},
},
Vout: []bchain.Vout{
{
Value: 0.00038812,
N: 0,
ScriptPubKey: bchain.ScriptPubKey{
Hex: "a9146144d57c8aff48492c9dfb914e120b20bad72d6f87",
Addresses: []string{
"3AZKvpKhSh1o8t1QrX3UeXG9d2BhCRnbcK",
},
},
},
},
}
var testTxPacked = "0001e24001000000017f9a22c9cbf54bd902400df746f138f37bcf5b4d93eb755820e974ba43ed5f42040000006a4730440220037f4ed5427cde81d55b9b6a2fd08c8a25090c2c2fff3a75c1a57625ca8a7118022076c702fe55969fa08137f71afd4851c48e31082dd3c40c919c92cdbc826758d30121029f6da5623c9f9b68a9baf9c1bc7511df88fa34c6c2f71f7c62f2f03ff48dca80feffffff019c9700000000000017a9146144d57c8aff48492c9dfb914e120b20bad72d6f8773d00700"
func Test_packTx(t *testing.T) {
type args struct {
tx bchain.Tx
height uint32
}
tests := []struct {
name string
args args
want string
wantErr bool
}{
{
name: "1",
args: args{testTx, 123456},
want: testTxPacked,
wantErr: false,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got, err := packTx(&tt.args.tx, tt.args.height)
if (err != nil) != tt.wantErr {
t.Errorf("packTx() error = %v, wantErr %v", err, tt.wantErr)
return
}
h := hex.EncodeToString(got)
if !reflect.DeepEqual(h, tt.want) {
t.Errorf("packTx() = %v, want %v", h, tt.want)
}
})
}
}
func Test_unpackTx(t *testing.T) {
type args struct {
packedTx string
}
tests := []struct {
name string
args args
want *bchain.Tx
want1 uint32
wantErr bool
}{
{
name: "1",
args: args{packedTx: testTxPacked},
want: &testTx,
want1: 123456,
wantErr: false,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
b, _ := hex.DecodeString(tt.args.packedTx)
got, got1, err := unpackTx(b)
if (err != nil) != tt.wantErr {
t.Errorf("unpackTx() error = %v, wantErr %v", err, tt.wantErr)
return
}
if !reflect.DeepEqual(got, tt.want) {
t.Errorf("unpackTx() got = %v, want %v", got, tt.want)
}
if got1 != tt.want1 {
t.Errorf("unpackTx() got1 = %v, want %v", got1, tt.want1)
}
})
}
}