From 33b33af46f907af24db14143f5dbb83699ae17f8 Mon Sep 17 00:00:00 2001 From: wz Date: Mon, 6 May 2019 20:36:34 +0800 Subject: [PATCH] Dev db expandability (#44) * delete leveldb from tm * Add package for sqlite * Add the sqlite framework * delete go-xom * Modify the leveldb of block and state and utxoViewPoints to SQLDB * add mattn vendor * add mattn vendor * Modify test * Modify review proposal * Add Database table dependencies * Add unit test * Modify bug for store * modify update respone --- accesstoken/accesstoken.go | 3 +- accesstoken/accesstoken_test.go | 4 +- account/accounts.go | 2 +- account/accounts_test.go | 8 +- account/utxo_keeper.go | 2 +- account/utxo_keeper_test.go | 4 +- api/api_test.go | 4 +- asset/asset.go | 2 +- asset/asset_test.go | 8 +- blockchain/txfeed/txfeed.go | 2 +- database/{leveldb => }/cache.go | 2 +- database/{leveldb => }/cache_test.go | 2 +- .../tendermint/tmlibs => database}/db/db.go | 37 ++- .../tmlibs/db => database/leveldb}/LICENSE.md | 0 .../tmlibs/db => database/leveldb}/README.md | 0 .../tmlibs/db => database/leveldb}/go_level_db.go | 21 +- .../db => database/leveldb}/go_level_db_test.go | 2 +- .../tmlibs/db => database/leveldb}/mem_db.go | 22 +- .../tmlibs/db => database/leveldb}/mem_db_test.go | 2 +- database/leveldb/utxo_view.go | 71 ----- database/orm/block.go | 58 ++++ database/orm/claim_tx.go | 5 + database/orm/transaction.go | 20 ++ database/orm/utxo.go | 8 + database/sql_store.go | 319 +++++++++++++++++++++ database/sql_store_test.go | 293 +++++++++++++++++++ database/sqlite/sqlite_db.go | 42 +++ database/{leveldb => }/store.go | 7 +- database/{leveldb => }/store_test.go | 6 +- database/utxo_view.go | 149 ++++++++++ database/{leveldb => }/utxo_view_test.go | 6 +- net/http/authn/authn_test.go | 4 +- node/node.go | 25 +- p2p/switch.go | 2 +- protocol/block.go | 4 +- protocol/store.go | 6 +- protocol/txpool_test.go | 2 +- test/bench_blockchain_test.go | 10 +- test/block_test.go | 4 +- test/chain_test_util.go | 9 +- test/integration/standard_transaction_test.go | 4 +- test/protocol_test.go | 4 +- test/protocol_test_util.go | 4 +- test/tx_test.go | 3 +- test/tx_test_util.go | 3 +- test/util.go | 9 +- test/wallet_test_util.go | 4 +- .../tendermint/abci/example/dummy/dummy.go | 3 +- .../abci/example/dummy/persistent_dummy.go | 3 +- .../github.com/tendermint/tmlibs/db/c_level_db.go | 152 ---------- .../tendermint/tmlibs/db/c_level_db_test.go | 86 ------ wallet/annotated.go | 2 +- wallet/indexer.go | 2 +- wallet/recovery.go | 3 +- wallet/recovery_test.go | 4 +- wallet/unconfirmed_test.go | 9 +- wallet/utxo.go | 2 +- wallet/utxo_test.go | 3 +- wallet/wallet.go | 2 +- wallet/wallet_test.go | 6 +- 60 files changed, 1065 insertions(+), 420 deletions(-) rename database/{leveldb => }/cache.go (98%) rename database/{leveldb => }/cache_test.go (98%) rename {vendor/github.com/tendermint/tmlibs => database}/db/db.go (58%) rename {vendor/github.com/tendermint/tmlibs/db => database/leveldb}/LICENSE.md (100%) rename {vendor/github.com/tendermint/tmlibs/db => database/leveldb}/README.md (100%) rename {vendor/github.com/tendermint/tmlibs/db => database/leveldb}/go_level_db.go (87%) rename {vendor/github.com/tendermint/tmlibs/db => database/leveldb}/go_level_db_test.go (99%) rename {vendor/github.com/tendermint/tmlibs/db => database/leveldb}/mem_db.go (87%) rename {vendor/github.com/tendermint/tmlibs/db => database/leveldb}/mem_db_test.go (98%) delete mode 100644 database/leveldb/utxo_view.go create mode 100644 database/orm/block.go create mode 100644 database/orm/claim_tx.go create mode 100644 database/orm/transaction.go create mode 100644 database/orm/utxo.go create mode 100644 database/sql_store.go create mode 100644 database/sql_store_test.go create mode 100644 database/sqlite/sqlite_db.go rename database/{leveldb => }/store.go (98%) rename database/{leveldb => }/store_test.go (97%) create mode 100644 database/utxo_view.go rename database/{leveldb => }/utxo_view_test.go (97%) delete mode 100644 vendor/github.com/tendermint/tmlibs/db/c_level_db.go delete mode 100644 vendor/github.com/tendermint/tmlibs/db/c_level_db_test.go diff --git a/accesstoken/accesstoken.go b/accesstoken/accesstoken.go index 7f87270f..7b32b0e7 100644 --- a/accesstoken/accesstoken.go +++ b/accesstoken/accesstoken.go @@ -10,9 +10,8 @@ import ( "strings" "time" - dbm "github.com/tendermint/tmlibs/db" - "github.com/vapor/crypto/sha3pool" + dbm "github.com/vapor/database/db" "github.com/vapor/errors" ) diff --git a/accesstoken/accesstoken_test.go b/accesstoken/accesstoken_test.go index 1679a5f2..e5639cdf 100644 --- a/accesstoken/accesstoken_test.go +++ b/accesstoken/accesstoken_test.go @@ -6,8 +6,8 @@ import ( "strings" "testing" - dbm "github.com/tendermint/tmlibs/db" - + dbm "github.com/vapor/database/db" + _ "github.com/vapor/database/leveldb" "github.com/vapor/errors" ) diff --git a/account/accounts.go b/account/accounts.go index 427b0421..48076a85 100644 --- a/account/accounts.go +++ b/account/accounts.go @@ -11,7 +11,6 @@ import ( "github.com/golang/groupcache/lru" log "github.com/sirupsen/logrus" - dbm "github.com/tendermint/tmlibs/db" "github.com/vapor/blockchain/signers" "github.com/vapor/blockchain/txbuilder" @@ -21,6 +20,7 @@ import ( "github.com/vapor/crypto" "github.com/vapor/crypto/ed25519/chainkd" "github.com/vapor/crypto/sha3pool" + dbm "github.com/vapor/database/db" "github.com/vapor/equity/pegin_contract" "github.com/vapor/errors" "github.com/vapor/protocol" diff --git a/account/accounts_test.go b/account/accounts_test.go index 8ef5ea32..c0d21fea 100644 --- a/account/accounts_test.go +++ b/account/accounts_test.go @@ -7,15 +7,15 @@ import ( "strings" "testing" - dbm "github.com/tendermint/tmlibs/db" - "github.com/vapor/blockchain/pseudohsm" "github.com/vapor/blockchain/signers" "github.com/vapor/common" "github.com/vapor/config" "github.com/vapor/consensus" "github.com/vapor/crypto/ed25519/chainkd" - "github.com/vapor/database/leveldb" + "github.com/vapor/database" + dbm "github.com/vapor/database/db" + _ "github.com/vapor/database/leveldb" "github.com/vapor/errors" "github.com/vapor/protocol" "github.com/vapor/testutil" @@ -229,7 +229,7 @@ func mockAccountManager(t *testing.T) *Manager { testDB := dbm.NewDB("testdb", "memdb", "temp") defer os.RemoveAll("temp") - store := leveldb.NewStore(testDB) + store := database.NewStore(testDB) txPool := protocol.NewTxPool(store) chain, err := protocol.NewChain(store, txPool) if err != nil { diff --git a/account/utxo_keeper.go b/account/utxo_keeper.go index d00d0f39..eb63b6a8 100644 --- a/account/utxo_keeper.go +++ b/account/utxo_keeper.go @@ -9,8 +9,8 @@ import ( "time" log "github.com/sirupsen/logrus" - dbm "github.com/tendermint/tmlibs/db" + dbm "github.com/vapor/database/db" "github.com/vapor/errors" "github.com/vapor/protocol/bc" ) diff --git a/account/utxo_keeper_test.go b/account/utxo_keeper_test.go index c452e0c5..2fd0df46 100644 --- a/account/utxo_keeper_test.go +++ b/account/utxo_keeper_test.go @@ -6,8 +6,8 @@ import ( "testing" "time" - dbm "github.com/tendermint/tmlibs/db" - + dbm "github.com/vapor/database/db" + _ "github.com/vapor/database/leveldb" "github.com/vapor/protocol/bc" "github.com/vapor/testutil" ) diff --git a/api/api_test.go b/api/api_test.go index bc7627c9..d82b26c3 100644 --- a/api/api_test.go +++ b/api/api_test.go @@ -8,12 +8,12 @@ import ( "os" "testing" - dbm "github.com/tendermint/tmlibs/db" - "github.com/vapor/accesstoken" "github.com/vapor/blockchain/rpc" "github.com/vapor/blockchain/txbuilder" "github.com/vapor/consensus" + dbm "github.com/vapor/database/db" + _ "github.com/vapor/database/leveldb" "github.com/vapor/testutil" ) diff --git a/asset/asset.go b/asset/asset.go index e1de5250..d1eb18f0 100644 --- a/asset/asset.go +++ b/asset/asset.go @@ -7,7 +7,6 @@ import ( "sync" "github.com/golang/groupcache/lru" - dbm "github.com/tendermint/tmlibs/db" "golang.org/x/crypto/sha3" "github.com/vapor/blockchain/signers" @@ -15,6 +14,7 @@ import ( "github.com/vapor/consensus" "github.com/vapor/crypto/ed25519" "github.com/vapor/crypto/ed25519/chainkd" + dbm "github.com/vapor/database/db" chainjson "github.com/vapor/encoding/json" "github.com/vapor/errors" "github.com/vapor/protocol" diff --git a/asset/asset_test.go b/asset/asset_test.go index 33b1445e..94e6b2b7 100644 --- a/asset/asset_test.go +++ b/asset/asset_test.go @@ -9,13 +9,13 @@ import ( "strings" "testing" - dbm "github.com/tendermint/tmlibs/db" - "github.com/vapor/common" "github.com/vapor/config" "github.com/vapor/consensus" "github.com/vapor/crypto/ed25519/chainkd" - "github.com/vapor/database/leveldb" + "github.com/vapor/database" + dbm "github.com/vapor/database/db" + _ "github.com/vapor/database/leveldb" "github.com/vapor/protocol" "github.com/vapor/testutil" ) @@ -153,7 +153,7 @@ func TestListAssets(t *testing.T) { } func mockChain(testDB dbm.DB) (*protocol.Chain, error) { - store := leveldb.NewStore(testDB) + store := database.NewStore(testDB) txPool := protocol.NewTxPool(store) chain, err := protocol.NewChain(store, txPool) if err != nil { diff --git a/blockchain/txfeed/txfeed.go b/blockchain/txfeed/txfeed.go index 824069cc..c93ba1a1 100644 --- a/blockchain/txfeed/txfeed.go +++ b/blockchain/txfeed/txfeed.go @@ -7,9 +7,9 @@ import ( "strings" log "github.com/sirupsen/logrus" - dbm "github.com/tendermint/tmlibs/db" "github.com/vapor/blockchain/query" + dbm "github.com/vapor/database/db" "github.com/vapor/errors" "github.com/vapor/protocol" "github.com/vapor/protocol/bc" diff --git a/database/leveldb/cache.go b/database/cache.go similarity index 98% rename from database/leveldb/cache.go rename to database/cache.go index 45346673..7c100bf5 100644 --- a/database/leveldb/cache.go +++ b/database/cache.go @@ -1,4 +1,4 @@ -package leveldb +package database import ( "fmt" diff --git a/database/leveldb/cache_test.go b/database/cache_test.go similarity index 98% rename from database/leveldb/cache_test.go rename to database/cache_test.go index 7dae1b91..9aa915e5 100644 --- a/database/leveldb/cache_test.go +++ b/database/cache_test.go @@ -1,4 +1,4 @@ -package leveldb +package database import ( "testing" diff --git a/vendor/github.com/tendermint/tmlibs/db/db.go b/database/db/db.go similarity index 58% rename from vendor/github.com/tendermint/tmlibs/db/db.go rename to database/db/db.go index 8156c1e9..22b71a9b 100644 --- a/vendor/github.com/tendermint/tmlibs/db/db.go +++ b/database/db/db.go @@ -1,6 +1,14 @@ package db -import . "github.com/tendermint/tmlibs/common" +import ( + "github.com/jinzhu/gorm" + . "github.com/tendermint/tmlibs/common" +) + +type SQLDB interface { + Name() string + Db() *gorm.DB +} type DB interface { Get([]byte) []byte @@ -29,6 +37,7 @@ type Iterator interface { Key() []byte Value() []byte + Seek([]byte) bool Release() Error() error @@ -41,15 +50,15 @@ const ( CLevelDBBackendStr = "cleveldb" GoLevelDBBackendStr = "goleveldb" MemDBBackendStr = "memdb" + SqliteDBBackendStr = "sqlitedb" ) type dbCreator func(name string, dir string) (DB, error) var backends = map[string]dbCreator{} -func registerDBCreator(backend string, creator dbCreator, force bool) { - _, ok := backends[backend] - if !force && ok { +func RegisterDBCreator(backend string, creator dbCreator, force bool) { + if _, ok := backends[backend]; !force && ok { return } backends[backend] = creator @@ -62,3 +71,23 @@ func NewDB(name string, backend string, dir string) DB { } return db } + +type sqlDbCreator func(name string, dir string) (SQLDB, error) + +var sqlBackends = map[string]sqlDbCreator{} + +func RegisterSqlDBCreator(backend string, creator sqlDbCreator, force bool) { + _, ok := sqlBackends[backend] + if !force && ok { + return + } + sqlBackends[backend] = creator +} + +func NewSqlDB(name string, backend string, dir string) SQLDB { + db, err := sqlBackends[backend](name, dir) + if err != nil { + PanicSanity(Fmt("Error initializing DB: %v", err)) + } + return db +} diff --git a/vendor/github.com/tendermint/tmlibs/db/LICENSE.md b/database/leveldb/LICENSE.md similarity index 100% rename from vendor/github.com/tendermint/tmlibs/db/LICENSE.md rename to database/leveldb/LICENSE.md diff --git a/vendor/github.com/tendermint/tmlibs/db/README.md b/database/leveldb/README.md similarity index 100% rename from vendor/github.com/tendermint/tmlibs/db/README.md rename to database/leveldb/README.md diff --git a/vendor/github.com/tendermint/tmlibs/db/go_level_db.go b/database/leveldb/go_level_db.go similarity index 87% rename from vendor/github.com/tendermint/tmlibs/db/go_level_db.go rename to database/leveldb/go_level_db.go index 4abd7611..b641d619 100644 --- a/vendor/github.com/tendermint/tmlibs/db/go_level_db.go +++ b/database/leveldb/go_level_db.go @@ -1,4 +1,4 @@ -package db +package leveldb import ( "fmt" @@ -9,16 +9,17 @@ import ( "github.com/syndtr/goleveldb/leveldb/iterator" "github.com/syndtr/goleveldb/leveldb/opt" "github.com/syndtr/goleveldb/leveldb/util" - . "github.com/tendermint/tmlibs/common" + + "github.com/vapor/database/db" ) func init() { - dbCreator := func(name string, dir string) (DB, error) { + dbCreator := func(name string, dir string) (db.DB, error) { return NewGoLevelDB(name, dir) } - registerDBCreator(LevelDBBackendStr, dbCreator, false) - registerDBCreator(GoLevelDBBackendStr, dbCreator, false) + db.RegisterDBCreator(db.LevelDBBackendStr, dbCreator, false) + db.RegisterDBCreator(db.GoLevelDBBackendStr, dbCreator, false) } type GoLevelDB struct { @@ -139,6 +140,10 @@ func (it *goLevelDBIterator) Value() []byte { return v } +func (it *goLevelDBIterator) Seek(point []byte) bool { + return it.source.Seek(point) +} + func (it *goLevelDBIterator) Error() error { return it.source.Error() } @@ -151,15 +156,15 @@ func (it *goLevelDBIterator) Release() { it.source.Release() } -func (db *GoLevelDB) Iterator() Iterator { +func (db *GoLevelDB) Iterator() db.Iterator { return &goLevelDBIterator{db.db.NewIterator(nil, nil)} } -func (db *GoLevelDB) IteratorPrefix(prefix []byte) Iterator { +func (db *GoLevelDB) IteratorPrefix(prefix []byte) db.Iterator { return &goLevelDBIterator{db.db.NewIterator(util.BytesPrefix(prefix), nil)} } -func (db *GoLevelDB) NewBatch() Batch { +func (db *GoLevelDB) NewBatch() db.Batch { batch := new(leveldb.Batch) return &goLevelDBBatch{db, batch} } diff --git a/vendor/github.com/tendermint/tmlibs/db/go_level_db_test.go b/database/leveldb/go_level_db_test.go similarity index 99% rename from vendor/github.com/tendermint/tmlibs/db/go_level_db_test.go rename to database/leveldb/go_level_db_test.go index 2cd3192c..ff541679 100644 --- a/vendor/github.com/tendermint/tmlibs/db/go_level_db_test.go +++ b/database/leveldb/go_level_db_test.go @@ -1,4 +1,4 @@ -package db +package leveldb import ( "bytes" diff --git a/vendor/github.com/tendermint/tmlibs/db/mem_db.go b/database/leveldb/mem_db.go similarity index 87% rename from vendor/github.com/tendermint/tmlibs/db/mem_db.go rename to database/leveldb/mem_db.go index 2f507321..bcb0be93 100644 --- a/vendor/github.com/tendermint/tmlibs/db/mem_db.go +++ b/database/leveldb/mem_db.go @@ -1,14 +1,16 @@ -package db +package leveldb import ( "fmt" "sort" "strings" "sync" + + "github.com/vapor/database/db" ) func init() { - registerDBCreator(MemDBBackendStr, func(name string, dir string) (DB, error) { + db.RegisterDBCreator(db.MemDBBackendStr, func(name string, dir string) (db.DB, error) { return NewMemDB(), nil }, false) } @@ -101,6 +103,16 @@ func (it *memDBIterator) Value() []byte { return it.db.Get(it.Key()) } +func (it *memDBIterator) Seek(point []byte) bool { + for i, key := range it.keys { + if key >= string(point) { + it.last = i + return true + } + } + return false +} + func (it *memDBIterator) Release() { it.db = nil it.keys = nil @@ -110,11 +122,11 @@ func (it *memDBIterator) Error() error { return nil } -func (db *MemDB) Iterator() Iterator { +func (db *MemDB) Iterator() db.Iterator { return db.IteratorPrefix([]byte{}) } -func (db *MemDB) IteratorPrefix(prefix []byte) Iterator { +func (db *MemDB) IteratorPrefix(prefix []byte) db.Iterator { it := newMemDBIterator() it.db = db it.last = -1 @@ -133,7 +145,7 @@ func (db *MemDB) IteratorPrefix(prefix []byte) Iterator { return it } -func (db *MemDB) NewBatch() Batch { +func (db *MemDB) NewBatch() db.Batch { return &memDBBatch{db, nil} } diff --git a/vendor/github.com/tendermint/tmlibs/db/mem_db_test.go b/database/leveldb/mem_db_test.go similarity index 98% rename from vendor/github.com/tendermint/tmlibs/db/mem_db_test.go rename to database/leveldb/mem_db_test.go index 503e361f..043984b4 100644 --- a/vendor/github.com/tendermint/tmlibs/db/mem_db_test.go +++ b/database/leveldb/mem_db_test.go @@ -1,4 +1,4 @@ -package db +package leveldb import ( "testing" diff --git a/database/leveldb/utxo_view.go b/database/leveldb/utxo_view.go deleted file mode 100644 index 2bb9473d..00000000 --- a/database/leveldb/utxo_view.go +++ /dev/null @@ -1,71 +0,0 @@ -package leveldb - -import ( - dbm "github.com/tendermint/tmlibs/db" - - "github.com/golang/protobuf/proto" - "github.com/vapor/database/storage" - "github.com/vapor/errors" - "github.com/vapor/protocol/bc" - "github.com/vapor/protocol/state" -) - -const utxoPreFix = "UT:" - -func calcUtxoKey(hash *bc.Hash) []byte { - return []byte(utxoPreFix + hash.String()) -} - -func getTransactionsUtxo(db dbm.DB, view *state.UtxoViewpoint, txs []*bc.Tx) error { - for _, tx := range txs { - for _, prevout := range tx.SpentOutputIDs { - if view.HasUtxo(&prevout) { - continue - } - data := db.Get(calcUtxoKey(&prevout)) - if data == nil { - continue - } - var utxo storage.UtxoEntry - if err := proto.Unmarshal(data, &utxo); err != nil { - return errors.Wrap(err, "unmarshaling utxo entry") - } - - view.Entries[prevout] = &utxo - } - } - - return nil -} - -func getUtxo(db dbm.DB, hash *bc.Hash) (*storage.UtxoEntry, error) { - var utxo storage.UtxoEntry - data := db.Get(calcUtxoKey(hash)) - if data == nil { - return nil, errors.New("can't find utxo in db") - } - if err := proto.Unmarshal(data, &utxo); err != nil { - return nil, errors.Wrap(err, "unmarshaling utxo entry") - } - return &utxo, nil -} - -func saveUtxoView(batch dbm.Batch, view *state.UtxoViewpoint) error { - for key, entry := range view.Entries { - if entry.Spent && !entry.IsCoinBase && !entry.IsCliam { - batch.Delete(calcUtxoKey(&key)) - continue - } - - b, err := proto.Marshal(entry) - if err != nil { - return errors.Wrap(err, "marshaling utxo entry") - } - batch.Set(calcUtxoKey(&key), b) - } - return nil -} - -func SaveUtxoView(batch dbm.Batch, view *state.UtxoViewpoint) error { - return saveUtxoView(batch, view) -} diff --git a/database/orm/block.go b/database/orm/block.go new file mode 100644 index 00000000..75a738e2 --- /dev/null +++ b/database/orm/block.go @@ -0,0 +1,58 @@ +package orm + +import ( + "github.com/vapor/protocol/bc" + "github.com/vapor/protocol/bc/types" +) + +type BlockStoreState struct { + StoreKey string `gorm:"primary_key"` + Height uint64 + Hash string +} + +type BlockHeader struct { + ID uint `gorm:"AUTO_INCREMENT"` + BlockHash string `gorm:"primary_key"` + Height uint64 `sql:"index"` + Version uint64 + PreviousBlockHash string + Timestamp uint64 + TransactionsMerkleRoot string + TransactionStatusHash string +} + +func stringToHash(str string) (*bc.Hash, error) { + hash := &bc.Hash{} + if err := hash.UnmarshalText([]byte(str)); err != nil { + return nil, err + } + return hash, nil +} + +func (bh *BlockHeader) ToTypesBlockHeader() (*types.BlockHeader, error) { + previousBlockHash, err := stringToHash(bh.PreviousBlockHash) + if err != nil { + return nil, err + } + + transactionsMerkleRoot, err := stringToHash(bh.TransactionsMerkleRoot) + if err != nil { + return nil, err + } + transactionStatusHash, err := stringToHash(bh.TransactionStatusHash) + if err != nil { + return nil, err + } + + return &types.BlockHeader{ + Version: bh.Version, + Height: bh.Height, + PreviousBlockHash: *previousBlockHash, + Timestamp: bh.Timestamp, + BlockCommitment: types.BlockCommitment{ + TransactionsMerkleRoot: *transactionsMerkleRoot, + TransactionStatusHash: *transactionStatusHash, + }, + }, nil +} diff --git a/database/orm/claim_tx.go b/database/orm/claim_tx.go new file mode 100644 index 00000000..b62f94c4 --- /dev/null +++ b/database/orm/claim_tx.go @@ -0,0 +1,5 @@ +package orm + +type ClaimTxState struct { + TxHash string +} diff --git a/database/orm/transaction.go b/database/orm/transaction.go new file mode 100644 index 00000000..1c6fa436 --- /dev/null +++ b/database/orm/transaction.go @@ -0,0 +1,20 @@ +package orm + +import "github.com/vapor/protocol/bc/types" + +type Transaction struct { + BlockHeaderID uint + TxIndex uint64 + RawData string + StatusFail bool + + BlockHeader *BlockHeader `gorm:"FOREIGNKEY:BlockHeaderID;AssociationForeignKey:ID"` +} + +func (t *Transaction) UnmarshalText() (*types.Tx, error) { + tx := &types.Tx{} + if err := tx.UnmarshalText([]byte(t.RawData)); err != nil { + return nil, err + } + return tx, nil +} diff --git a/database/orm/utxo.go b/database/orm/utxo.go new file mode 100644 index 00000000..e989196c --- /dev/null +++ b/database/orm/utxo.go @@ -0,0 +1,8 @@ +package orm + +type Utxo struct { + OutputID string + IsCoinBase bool + BlockHeight uint64 + Spent bool +} diff --git a/database/sql_store.go b/database/sql_store.go new file mode 100644 index 00000000..ad7fa344 --- /dev/null +++ b/database/sql_store.go @@ -0,0 +1,319 @@ +package database + +import ( + "time" + + log "github.com/sirupsen/logrus" + "github.com/tendermint/tmlibs/common" + + dbm "github.com/vapor/database/db" + "github.com/vapor/database/orm" + "github.com/vapor/database/storage" + "github.com/vapor/errors" + "github.com/vapor/protocol" + "github.com/vapor/protocol/bc" + "github.com/vapor/protocol/bc/types" + "github.com/vapor/protocol/state" +) + +const logModuleSQL = "SQLdb" + +func loadBlockSQLStoreStateJSON(db dbm.SQLDB) *protocol.BlockStoreState { + bsj := orm.BlockStoreState{ + StoreKey: string(blockStoreKey), + } + + SQLDB := db.Db() + if err := SQLDB.Where(&bsj).First(&bsj).Error; err != nil { + return nil + } + + hash := &bc.Hash{} + if err := hash.UnmarshalText([]byte(bsj.Hash)); err != nil { + common.PanicCrisis(common.Fmt("Could not unmarshalText bytes: %s", bsj.Hash)) + } + + return &protocol.BlockStoreState{Height: bsj.Height, Hash: hash} +} + +// A SQLStore encapsulates storage for blockchain validation. +// It satisfies the interface protocol.Store, and provides additional +// methods for querying current data. +type SQLStore struct { + db dbm.SQLDB + cache blockCache +} + +// GetBlockFromSQLDB return the block by given hash +func GetBlockFromSQLDB(db dbm.SQLDB, hash *bc.Hash) *types.Block { + blockHeader := &orm.BlockHeader{BlockHash: hash.String()} + if err := db.Db().Where(blockHeader).Find(blockHeader).Error; err != nil { + return nil + } + + txs := []*orm.Transaction{} + if err := db.Db().Where(&orm.Transaction{BlockHeaderID: blockHeader.ID}).Order("tx_index asc").Find(&txs).Error; err != nil { + return nil + } + + block, err := toBlock(blockHeader, txs) + if err != nil { + return nil + } + + return block +} + +func toBlock(header *orm.BlockHeader, txs []*orm.Transaction) (*types.Block, error) { + + blockHeader, err := header.ToTypesBlockHeader() + if err != nil { + return nil, err + } + + var transactions []*types.Tx + + for _, tx := range txs { + transaction, err := tx.UnmarshalText() + if err != nil { + return nil, err + } + transactions = append(transactions, transaction) + } + + block := &types.Block{ + BlockHeader: *blockHeader, + Transactions: transactions, + } + + return block, nil +} + +// NewSQLStore creates and returns a new Store object. +func NewSQLStore(db dbm.SQLDB) *SQLStore { + cache := newBlockCache(func(hash *bc.Hash) *types.Block { + return GetBlockFromSQLDB(db, hash) + }) + return &SQLStore{ + db: db, + cache: cache, + } +} + +// GetUtxo will search the utxo in db +func (s *SQLStore) GetUtxo(hash *bc.Hash) (*storage.UtxoEntry, error) { + return getUtxoFromSQLDB(s.db, hash) +} + +// BlockExist check if the block is stored in disk +func (s *SQLStore) BlockExist(hash *bc.Hash) bool { + block, err := s.cache.lookup(hash) + return err == nil && block != nil +} + +// GetBlock return the block by given hash +func (s *SQLStore) GetBlock(hash *bc.Hash) (*types.Block, error) { + return s.cache.lookup(hash) +} + +// GetTransactionsUtxo will return all the utxo that related to the input txs +func (s *SQLStore) GetTransactionsUtxo(view *state.UtxoViewpoint, txs []*bc.Tx) error { + return getTransactionsUtxoFromSQLDB(s.db, view, txs) +} + +// GetTransactionStatus will return the utxo that related to the block hash +func (s *SQLStore) GetTransactionStatus(hash *bc.Hash) (*bc.TransactionStatus, error) { + ts := &bc.TransactionStatus{} + query := s.db.Db().Model(&orm.Transaction{}).Joins("join block_headers on block_headers.id = transactions.block_header_id").Where("block_headers.block_hash = ?", hash.String()) + rows, err := query.Select("transactions.status_fail, block_headers.version").Order("transactions.tx_index asc").Rows() + if err != nil { + return nil, err + } + + for rows.Next() { + var ( + statusFail bool + version uint64 + ) + if err := rows.Scan(&statusFail, &version); err != nil { + return nil, err + } + + ts.Version = version + ts.VerifyStatus = append(ts.VerifyStatus, &bc.TxVerifyResult{StatusFail: statusFail}) + + } + return ts, nil +} + +// GetStoreStatus return the BlockStoreStateJSON +func (s *SQLStore) GetStoreStatus() *protocol.BlockStoreState { + return loadBlockSQLStoreStateJSON(s.db) +} + +func (s *SQLStore) LoadBlockIndex(stateBestHeight uint64) (*state.BlockIndex, error) { + startTime := time.Now() + blockIndex := state.NewBlockIndex() + + var lastNode *state.BlockNode + rows, err := s.db.Db().Model(&orm.BlockHeader{}).Order("height").Rows() + if err != nil { + return nil, err + } + defer rows.Close() + + for rows.Next() { + header := orm.BlockHeader{} + if err := rows.Scan(&header.ID, &header.BlockHash, &header.Height, &header.Version, &header.PreviousBlockHash, &header.Timestamp, &header.TransactionsMerkleRoot, &header.TransactionStatusHash); err != nil { + return nil, err + } + if header.Height > stateBestHeight { + break + } + + typesBlockHeader, err := header.ToTypesBlockHeader() + if err != nil { + return nil, err + } + + previousBlockHash := typesBlockHeader.PreviousBlockHash + + var parent *state.BlockNode + if lastNode == nil || lastNode.Hash == previousBlockHash { + parent = lastNode + } else { + parent = blockIndex.GetNode(&previousBlockHash) + } + + node, err := state.NewBlockNode(typesBlockHeader, parent) + if err != nil { + return nil, err + } + + blockIndex.AddNode(node) + lastNode = node + } + + log.WithFields(log.Fields{ + "module": logModule, + "height": stateBestHeight, + "duration": time.Since(startTime), + }).Debug("initialize load history block index from database") + return blockIndex, nil +} + +// SaveBlock persists a new block in the protocol. +func (s *SQLStore) SaveBlock(block *types.Block, ts *bc.TransactionStatus) error { + startTime := time.Now() + + blockHash := block.Hash() + SQLDB := s.db.Db() + tx := SQLDB.Begin() + + // Save block header details + blockHeader := &orm.BlockHeader{ + Height: block.Height, + BlockHash: blockHash.String(), + Version: block.Version, + PreviousBlockHash: block.PreviousBlockHash.String(), + Timestamp: block.Timestamp, + TransactionsMerkleRoot: block.TransactionsMerkleRoot.String(), + TransactionStatusHash: block.TransactionStatusHash.String(), + } + + if err := tx.Create(blockHeader).Error; err != nil { + tx.Rollback() + return err + } + + // Save tx + for index, transaction := range block.Transactions { + rawTx, err := transaction.MarshalText() + if err != nil { + return err + } + ormTransaction := &orm.Transaction{ + BlockHeaderID: blockHeader.ID, + TxIndex: uint64(index), + RawData: string(rawTx), + StatusFail: ts.VerifyStatus[index].StatusFail, + } + if err := tx.Create(ormTransaction).Error; err != nil { + tx.Rollback() + return err + } + } + + if err := tx.Commit().Error; err != nil { + tx.Rollback() + return errors.Wrap(err, "commit transaction") + } + + log.WithFields(log.Fields{ + "module": logModule, + "height": block.Height, + "hash": blockHash.String(), + "duration": time.Since(startTime), + }).Info("block saved on disk") + return nil +} + +// SaveChainStatus save the core's newest status && delete old status +func (s *SQLStore) SaveChainStatus(node *state.BlockNode, view *state.UtxoViewpoint) error { + SQLDB := s.db.Db() + tx := SQLDB.Begin() + + if err := saveUtxoViewToSQLDB(tx, view); err != nil { + tx.Rollback() + return err + } + + state := &orm.BlockStoreState{ + StoreKey: string(blockStoreKey), + Height: node.Height, + Hash: node.Hash.String(), + } + + db := tx.Model(&orm.BlockStoreState{}).Update(state) + + if err := db.Error; err != nil { + tx.Rollback() + return err + } + + if db.RowsAffected == 0 { + if err := tx.Save(state).Error; err != nil { + tx.Rollback() + return err + } + } + + if err := tx.Commit().Error; err != nil { + tx.Rollback() + return errors.Wrap(err, "commit transaction") + } + + return nil +} + +func (s *SQLStore) IsWithdrawSpent(hash *bc.Hash) bool { + data := &orm.ClaimTxState{ + TxHash: hash.String(), + } + count := 0 + if err := s.db.Db().Where(data).First(data).Count(&count).Error; err != nil { + return false + } + + return count > 0 +} + +func (s *SQLStore) SetWithdrawSpent(hash *bc.Hash) error { + data := &orm.ClaimTxState{ + TxHash: hash.String(), + } + if err := s.db.Db().Create(data).Error; err != nil { + return err + } + return nil +} diff --git a/database/sql_store_test.go b/database/sql_store_test.go new file mode 100644 index 00000000..2c54680e --- /dev/null +++ b/database/sql_store_test.go @@ -0,0 +1,293 @@ +package database + +import ( + "os" + "testing" + + "github.com/vapor/common" + "github.com/vapor/config" + "github.com/vapor/consensus" + dbm "github.com/vapor/database/db" + _ "github.com/vapor/database/leveldb" + "github.com/vapor/database/orm" + _ "github.com/vapor/database/sqlite" + "github.com/vapor/database/storage" + "github.com/vapor/protocol" + "github.com/vapor/protocol/bc" + "github.com/vapor/protocol/bc/types" + "github.com/vapor/protocol/state" + "github.com/vapor/testutil" +) + +func TestLoadBlockIndexFromSQLDB(t *testing.T) { + sqlDB := dbm.NewSqlDB("sql", "sqlitedb", "temp") + defer func() { + sqlDB.Db().Close() + os.RemoveAll("temp") + }() + + sqlDB.Db().AutoMigrate(&orm.BlockHeader{}, &orm.Transaction{}, &orm.BlockStoreState{}, &orm.ClaimTxState{}, &orm.Utxo{}) + + sqlStore := NewSQLStore(sqlDB) + + config.CommonConfig = config.DefaultConfig() + config.CommonConfig.Consensus.SelfVoteSigners = append(config.CommonConfig.Consensus.SelfVoteSigners, "vsm1qkm743xmgnvh84pmjchq2s4tnfpgu9ae2f9slep") + config.CommonConfig.Consensus.XPrv = "a8e281b615809046698fb0b0f2804a36d824d48fa443350f10f1b80649d39e5f1e85cf9855548915e36137345910606cbc8e7dd8497c831dce899ee6ac112445" + for _, v := range config.CommonConfig.Consensus.SelfVoteSigners { + address, err := common.DecodeAddress(v, &consensus.SoloNetParams) + if err != nil { + t.Fatal(err) + } + config.CommonConfig.Consensus.Signers = append(config.CommonConfig.Consensus.Signers, address) + } + block := config.GenesisBlock() + txStatus := bc.NewTransactionStatus() + txStatus.SetStatus(0, false) + + if err := sqlStore.SaveBlock(block, txStatus); err != nil { + t.Fatal(err) + } + + for block.Height <= 128 { + preHash := block.Hash() + block.PreviousBlockHash = preHash + block.Height += 1 + if err := sqlStore.SaveBlock(block, txStatus); err != nil { + t.Fatal(err) + } + + if block.Height%32 != 0 { + continue + } + + for i := uint64(0); i < block.Height/32; i++ { + if err := sqlStore.SaveBlock(block, txStatus); err != nil { + t.Fatal(err) + } + } + } + + if _, err := sqlStore.LoadBlockIndex(128); err != nil { + t.Fatal(err) + } +} + +func TestLoadBlockIndexBestHeightFromSQLDB(t *testing.T) { + cases := []struct { + blockBestHeight uint64 + stateBestHeight uint64 + }{ + { + blockBestHeight: 100, + stateBestHeight: 90, + }, + { + blockBestHeight: 100, + stateBestHeight: 0, + }, + { + blockBestHeight: 100, + stateBestHeight: 100, + }, + } + + sqlDB := dbm.NewSqlDB("sql", "sqlitedb", "temp") + defer func() { + sqlDB.Db().Close() + os.RemoveAll("temp") + }() + + sqlDB.Db().AutoMigrate(&orm.BlockHeader{}, &orm.Transaction{}, &orm.BlockStoreState{}, &orm.ClaimTxState{}, &orm.Utxo{}) + + sqlStore := NewSQLStore(sqlDB) + var savedBlocks []types.Block + config.CommonConfig = config.DefaultConfig() + config.CommonConfig.Consensus.SelfVoteSigners = append(config.CommonConfig.Consensus.SelfVoteSigners, "vsm1qkm743xmgnvh84pmjchq2s4tnfpgu9ae2f9slep") + config.CommonConfig.Consensus.XPrv = "a8e281b615809046698fb0b0f2804a36d824d48fa443350f10f1b80649d39e5f1e85cf9855548915e36137345910606cbc8e7dd8497c831dce899ee6ac112445" + for _, v := range config.CommonConfig.Consensus.SelfVoteSigners { + address, err := common.DecodeAddress(v, &consensus.SoloNetParams) + if err != nil { + t.Fatal(err) + } + config.CommonConfig.Consensus.Signers = append(config.CommonConfig.Consensus.Signers, address) + } + + for _, c := range cases { + block := config.GenesisBlock() + txStatus := bc.NewTransactionStatus() + txStatus.SetStatus(0, false) + + for i := uint64(0); i < c.blockBestHeight; i++ { + if err := sqlStore.SaveBlock(block, txStatus); err != nil { + t.Fatal(err) + } + + savedBlocks = append(savedBlocks, *block) + block.PreviousBlockHash = block.Hash() + block.Height++ + } + + index, err := sqlStore.LoadBlockIndex(c.stateBestHeight) + if err != nil { + t.Fatal(err) + } + + for _, block := range savedBlocks { + blockHash := block.Hash() + if block.Height <= c.stateBestHeight != index.BlockExist(&blockHash) { + t.Errorf("Error in load block index") + } + } + } +} + +func TestLoadBlockIndexEqualsFromSQLDB(t *testing.T) { + sqlDB := dbm.NewSqlDB("sql", "sqlitedb", "temp") + defer func() { + sqlDB.Db().Close() + os.RemoveAll("temp") + }() + + sqlDB.Db().AutoMigrate(&orm.BlockHeader{}, &orm.Transaction{}, &orm.BlockStoreState{}, &orm.ClaimTxState{}, &orm.Utxo{}) + + sqlStore := NewSQLStore(sqlDB) + config.CommonConfig = config.DefaultConfig() + config.CommonConfig.Consensus.SelfVoteSigners = append(config.CommonConfig.Consensus.SelfVoteSigners, "vsm1qkm743xmgnvh84pmjchq2s4tnfpgu9ae2f9slep") + config.CommonConfig.Consensus.XPrv = "a8e281b615809046698fb0b0f2804a36d824d48fa443350f10f1b80649d39e5f1e85cf9855548915e36137345910606cbc8e7dd8497c831dce899ee6ac112445" + for _, v := range config.CommonConfig.Consensus.SelfVoteSigners { + address, err := common.DecodeAddress(v, &consensus.SoloNetParams) + if err != nil { + t.Fatal(err) + } + config.CommonConfig.Consensus.Signers = append(config.CommonConfig.Consensus.Signers, address) + } + + block := config.GenesisBlock() + txStatus := bc.NewTransactionStatus() + txStatus.SetStatus(0, false) + + expectBlockIndex := state.NewBlockIndex() + var parent *state.BlockNode + + for block.Height <= 100 { + if err := sqlStore.SaveBlock(block, txStatus); err != nil { + t.Fatal(err) + } + + if block.Height != 0 { + parent = expectBlockIndex.GetNode(&block.PreviousBlockHash) + } + + node, err := state.NewBlockNode(&block.BlockHeader, parent) + if err != nil { + t.Fatal(err) + } + + expectBlockIndex.AddNode(node) + block.PreviousBlockHash = block.Hash() + block.Height++ + } + + index, err := sqlStore.LoadBlockIndex(100) + if err != nil { + t.Fatal(err) + } + + if !testutil.DeepEqual(expectBlockIndex, index) { + t.Errorf("got block index:%v, expect block index:%v", index, expectBlockIndex) + } +} +func TestSaveChainStatusToSQLDB(t *testing.T) { + sqlDB := dbm.NewSqlDB("sql", "sqlitedb", "temp") + defer func() { + sqlDB.Db().Close() + os.RemoveAll("temp") + }() + + sqlDB.Db().AutoMigrate(&orm.BlockHeader{}, &orm.Transaction{}, &orm.BlockStoreState{}, &orm.ClaimTxState{}, &orm.Utxo{}) + + sqlStore := NewSQLStore(sqlDB) + + node := &state.BlockNode{Height: 100, Hash: bc.Hash{V0: 0, V1: 1, V2: 2, V3: 3}} + view := &state.UtxoViewpoint{ + Entries: map[bc.Hash]*storage.UtxoEntry{ + bc.Hash{V0: 1, V1: 2, V2: 3, V3: 4}: &storage.UtxoEntry{IsCoinBase: false, BlockHeight: 100, Spent: false}, + bc.Hash{V0: 1, V1: 2, V2: 3, V3: 4}: &storage.UtxoEntry{IsCoinBase: true, BlockHeight: 100, Spent: true}, + bc.Hash{V0: 1, V1: 1, V2: 3, V3: 4}: &storage.UtxoEntry{IsCoinBase: false, BlockHeight: 100, Spent: true}, + }, + } + + if err := sqlStore.SaveChainStatus(node, view); err != nil { + t.Fatal(err) + } + + expectStatus := &protocol.BlockStoreState{Height: node.Height, Hash: &node.Hash} + if !testutil.DeepEqual(sqlStore.GetStoreStatus(), expectStatus) { + t.Errorf("got block status:%v, expect block status:%v", sqlStore.GetStoreStatus(), expectStatus) + } + + for hash, utxo := range view.Entries { + if utxo.Spent && !utxo.IsCoinBase { + continue + } + + gotUtxo, err := sqlStore.GetUtxo(&hash) + if err != nil { + t.Fatal(err) + } + + if !testutil.DeepEqual(utxo, gotUtxo) { + t.Errorf("got utxo entry:%v, expect utxo entry:%v", gotUtxo, utxo) + } + } +} + +func TestSaveBlockToSQLDB(t *testing.T) { + sqlDB := dbm.NewSqlDB("sql", "sqlitedb", "temp") + defer func() { + sqlDB.Db().Close() + os.RemoveAll("temp") + }() + + sqlDB.Db().AutoMigrate(&orm.BlockHeader{}, &orm.Transaction{}, &orm.BlockStoreState{}, &orm.ClaimTxState{}, &orm.Utxo{}) + + sqlStore := NewSQLStore(sqlDB) + config.CommonConfig = config.DefaultConfig() + config.CommonConfig.Consensus.SelfVoteSigners = append(config.CommonConfig.Consensus.SelfVoteSigners, "vsm1qkm743xmgnvh84pmjchq2s4tnfpgu9ae2f9slep") + config.CommonConfig.Consensus.XPrv = "a8e281b615809046698fb0b0f2804a36d824d48fa443350f10f1b80649d39e5f1e85cf9855548915e36137345910606cbc8e7dd8497c831dce899ee6ac112445" + for _, v := range config.CommonConfig.Consensus.SelfVoteSigners { + address, err := common.DecodeAddress(v, &consensus.SoloNetParams) + if err != nil { + t.Fatal(err) + } + config.CommonConfig.Consensus.Signers = append(config.CommonConfig.Consensus.Signers, address) + } + + block := config.GenesisBlock() + status := &bc.TransactionStatus{Version: block.Version, VerifyStatus: []*bc.TxVerifyResult{{StatusFail: true}}} + if err := sqlStore.SaveBlock(block, status); err != nil { + t.Fatal(err) + } + + blockHash := block.Hash() + gotBlock, err := sqlStore.GetBlock(&blockHash) + if err != nil { + t.Fatal(err) + } + + gotBlock.Transactions[0].Tx.SerializedSize = 0 + gotBlock.Transactions[0].SerializedSize = 0 + if !testutil.DeepEqual(block, gotBlock) { + t.Errorf("got block:%v, expect block:%v", gotBlock, block) + } + + gotStatus, err := sqlStore.GetTransactionStatus(&blockHash) + if err != nil { + t.Fatal(err) + } + + if !testutil.DeepEqual(status, gotStatus) { + t.Errorf("got status:%v, expect status:%v", gotStatus, status) + } + +} diff --git a/database/sqlite/sqlite_db.go b/database/sqlite/sqlite_db.go new file mode 100644 index 00000000..6719d2a1 --- /dev/null +++ b/database/sqlite/sqlite_db.go @@ -0,0 +1,42 @@ +package sqlite + +import ( + "path" + + "github.com/jinzhu/gorm" + _ "github.com/jinzhu/gorm/dialects/sqlite" + cmn "github.com/tendermint/tmlibs/common" + + "github.com/vapor/database/db" +) + +func init() { + dbCreator := func(name string, dir string) (db.SQLDB, error) { + return NewSqliteDB(name, dir) + } + db.RegisterSqlDBCreator(db.SqliteDBBackendStr, dbCreator, false) +} + +type SqliteDB struct { + db *gorm.DB +} + +func NewSqliteDB(name string, dir string) (*SqliteDB, error) { + dbPath := path.Join(dir, name) + cmn.EnsureDir(dbPath, 0700) + dbFilePath := path.Join(dbPath, name+".db") + db, err := gorm.Open("sqlite3", dbFilePath) + if err != nil { + return nil, err + } + database := &SqliteDB{db: db} + return database, nil +} + +func (s *SqliteDB) Name() string { + return "sqlite3" +} + +func (s *SqliteDB) Db() *gorm.DB { + return s.db +} diff --git a/database/leveldb/store.go b/database/store.go similarity index 98% rename from database/leveldb/store.go rename to database/store.go index 3e5d4cb6..91eacdaf 100644 --- a/database/leveldb/store.go +++ b/database/store.go @@ -1,4 +1,4 @@ -package leveldb +package database import ( "encoding/binary" @@ -8,8 +8,8 @@ import ( "github.com/golang/protobuf/proto" log "github.com/sirupsen/logrus" "github.com/tendermint/tmlibs/common" - dbm "github.com/tendermint/tmlibs/db" + dbm "github.com/vapor/database/db" "github.com/vapor/database/storage" "github.com/vapor/errors" "github.com/vapor/protocol" @@ -237,10 +237,11 @@ func (s *Store) IsWithdrawSpent(hash *bc.Hash) bool { return false } -func (s *Store) SetWithdrawSpent(hash *bc.Hash) { +func (s *Store) SetWithdrawSpent(hash *bc.Hash) error { batch := s.db.NewBatch() batch.Set(calcClaimTxKey(hash), []byte("1")) batch.Write() + return nil } func (s *Store) Set(hash *bc.Hash, data []byte) error { diff --git a/database/leveldb/store_test.go b/database/store_test.go similarity index 97% rename from database/leveldb/store_test.go rename to database/store_test.go index a126447a..534ec9ed 100644 --- a/database/leveldb/store_test.go +++ b/database/store_test.go @@ -1,4 +1,4 @@ -package leveldb +package database import ( "os" @@ -7,10 +7,10 @@ import ( "github.com/vapor/common" "github.com/vapor/config" "github.com/vapor/consensus" + dbm "github.com/vapor/database/db" + _ "github.com/vapor/database/leveldb" "github.com/vapor/protocol/bc" "github.com/vapor/protocol/bc/types" - - dbm "github.com/tendermint/tmlibs/db" ) func TestLoadBlockIndex(t *testing.T) { diff --git a/database/utxo_view.go b/database/utxo_view.go new file mode 100644 index 00000000..80e4244a --- /dev/null +++ b/database/utxo_view.go @@ -0,0 +1,149 @@ +package database + +import ( + "github.com/golang/protobuf/proto" + "github.com/jinzhu/gorm" + + dbm "github.com/vapor/database/db" + "github.com/vapor/database/orm" + "github.com/vapor/database/storage" + "github.com/vapor/errors" + "github.com/vapor/protocol/bc" + "github.com/vapor/protocol/state" +) + +const utxoPreFix = "UT:" + +func calcUtxoKey(hash *bc.Hash) []byte { + return []byte(utxoPreFix + hash.String()) +} + +func getTransactionsUtxo(db dbm.DB, view *state.UtxoViewpoint, txs []*bc.Tx) error { + for _, tx := range txs { + for _, prevout := range tx.SpentOutputIDs { + if view.HasUtxo(&prevout) { + continue + } + data := db.Get(calcUtxoKey(&prevout)) + if data == nil { + continue + } + var utxo storage.UtxoEntry + if err := proto.Unmarshal(data, &utxo); err != nil { + return errors.Wrap(err, "unmarshaling utxo entry") + } + + view.Entries[prevout] = &utxo + } + } + + return nil +} + +func getUtxo(db dbm.DB, hash *bc.Hash) (*storage.UtxoEntry, error) { + var utxo storage.UtxoEntry + data := db.Get(calcUtxoKey(hash)) + if data == nil { + return nil, errors.New("can't find utxo in db") + } + if err := proto.Unmarshal(data, &utxo); err != nil { + return nil, errors.Wrap(err, "unmarshaling utxo entry") + } + return &utxo, nil +} + +func saveUtxoView(batch dbm.Batch, view *state.UtxoViewpoint) error { + for key, entry := range view.Entries { + if entry.Spent && !entry.IsCoinBase && !entry.IsCliam { + batch.Delete(calcUtxoKey(&key)) + continue + } + + b, err := proto.Marshal(entry) + if err != nil { + return errors.Wrap(err, "marshaling utxo entry") + } + batch.Set(calcUtxoKey(&key), b) + } + return nil +} + +func getTransactionsUtxoFromSQLDB(db dbm.SQLDB, view *state.UtxoViewpoint, txs []*bc.Tx) error { + for _, tx := range txs { + for _, prevout := range tx.SpentOutputIDs { + if view.HasUtxo(&prevout) { + continue + } + data := &orm.Utxo{ + OutputID: prevout.String(), + } + + if err := db.Db().Where(data).Find(data).Error; err != nil { + if err != gorm.ErrRecordNotFound { + return err + } + continue + } + + view.Entries[prevout] = &storage.UtxoEntry{ + IsCoinBase: data.IsCoinBase, + BlockHeight: data.BlockHeight, + Spent: data.Spent, + } + } + } + + return nil +} + +func getUtxoFromSQLDB(db dbm.SQLDB, hash *bc.Hash) (*storage.UtxoEntry, error) { + utxoViewpoint := &orm.Utxo{ + OutputID: hash.String(), + } + + if err := db.Db().Where(utxoViewpoint).Find(utxoViewpoint).Error; err != nil { + return nil, err + } + + return &storage.UtxoEntry{ + IsCoinBase: utxoViewpoint.IsCoinBase, + BlockHeight: utxoViewpoint.BlockHeight, + Spent: utxoViewpoint.Spent, + }, nil +} + +func saveUtxoViewToSQLDB(tx *gorm.DB, view *state.UtxoViewpoint) error { + for key, entry := range view.Entries { + if entry.Spent && !entry.IsCoinBase && !entry.IsCliam { + if err := tx.Where("output_id = ?", key.String()).Delete(&orm.Utxo{}).Error; err != nil { + return err + } + continue + } + utxoViewpoint := &orm.Utxo{ + OutputID: key.String(), + IsCoinBase: entry.IsCoinBase, + BlockHeight: entry.BlockHeight, + Spent: entry.Spent, + } + + if entry.IsCoinBase { + db := tx.Model(&orm.Utxo{}).Where(utxoViewpoint).Update("spent", entry.Spent) + if err := db.Error; err != nil { + return err + } + if db.RowsAffected != 0 { + continue + } + } + + if err := tx.Create(utxoViewpoint).Error; err != nil { + return err + } + } + return nil +} + +func SaveUtxoView(batch dbm.Batch, view *state.UtxoViewpoint) error { + return saveUtxoView(batch, view) +} diff --git a/database/leveldb/utxo_view_test.go b/database/utxo_view_test.go similarity index 97% rename from database/leveldb/utxo_view_test.go rename to database/utxo_view_test.go index 56361ce9..0fb5b08c 100644 --- a/database/leveldb/utxo_view_test.go +++ b/database/utxo_view_test.go @@ -1,11 +1,11 @@ -package leveldb +package database import ( "os" "testing" - dbm "github.com/tendermint/tmlibs/db" - + dbm "github.com/vapor/database/db" + _ "github.com/vapor/database/leveldb" "github.com/vapor/database/storage" "github.com/vapor/protocol/bc" "github.com/vapor/protocol/state" diff --git a/net/http/authn/authn_test.go b/net/http/authn/authn_test.go index c9736cca..71476cce 100644 --- a/net/http/authn/authn_test.go +++ b/net/http/authn/authn_test.go @@ -6,9 +6,9 @@ import ( "strings" "testing" - dbm "github.com/tendermint/tmlibs/db" - "github.com/vapor/accesstoken" + dbm "github.com/vapor/database/db" + _ "github.com/vapor/database/leveldb" "github.com/vapor/errors" ) diff --git a/node/node.go b/node/node.go index c30ce928..7354485b 100644 --- a/node/node.go +++ b/node/node.go @@ -15,7 +15,6 @@ import ( "github.com/prometheus/prometheus/util/flock" log "github.com/sirupsen/logrus" cmn "github.com/tendermint/tmlibs/common" - dbm "github.com/tendermint/tmlibs/db" browser "github.com/toqueteos/webbrowser" "github.com/vapor/accesstoken" @@ -28,7 +27,11 @@ import ( cfg "github.com/vapor/config" "github.com/vapor/consensus" "github.com/vapor/crypto/ed25519/chainkd" - "github.com/vapor/database/leveldb" + "github.com/vapor/database" + dbm "github.com/vapor/database/db" + _ "github.com/vapor/database/leveldb" + "github.com/vapor/database/orm" + _ "github.com/vapor/database/sqlite" "github.com/vapor/env" "github.com/vapor/mining/miner" "github.com/vapor/net/websocket" @@ -47,6 +50,7 @@ const ( type Node struct { cmn.BaseService + db dbm.SQLDB // config config *cfg.Config @@ -83,14 +87,16 @@ func NewNode(config *cfg.Config) *Node { if config.DBBackend != "memdb" && config.DBBackend != "leveldb" { cmn.Exit(cmn.Fmt("Param db_backend [%v] is invalid, use leveldb or memdb", config.DBBackend)) } - coreDB := dbm.NewDB("core", config.DBBackend, config.DBDir()) - store := leveldb.NewStore(coreDB) + + sqlDB := dbm.NewSqlDB("sql", "sqlitedb", config.DBDir()) + initDatabaseTable(sqlDB) + sqlStore := database.NewSQLStore(sqlDB) tokenDB := dbm.NewDB("accesstoken", config.DBBackend, config.DBDir()) accessTokens := accesstoken.NewStore(tokenDB) - txPool := protocol.NewTxPool(store) - chain, err := protocol.NewChain(store, txPool) + txPool := protocol.NewTxPool(sqlStore) + chain, err := protocol.NewChain(sqlStore, txPool) if err != nil { cmn.Exit(cmn.Fmt("Failed to create chain structure: %v", err)) } @@ -157,6 +163,7 @@ func NewNode(config *cfg.Config) *Node { }() } node := &Node{ + db: sqlDB, config: config, syncManager: syncManager, accessTokens: accessTokens, @@ -307,6 +314,8 @@ func (n *Node) OnStop() { if !n.config.VaultMode { n.syncManager.Stop() } + + n.db.Db().Close() } func (n *Node) RunForever() { @@ -395,3 +404,7 @@ func initDpos(chain *protocol.Chain, config *cfg.Config) { } } } + +func initDatabaseTable(db dbm.SQLDB) { + db.Db().AutoMigrate(&orm.BlockHeader{}, &orm.Transaction{}, &orm.BlockStoreState{}, &orm.ClaimTxState{}, &orm.Utxo{}) +} diff --git a/p2p/switch.go b/p2p/switch.go index d00c9aa0..ab6b1386 100644 --- a/p2p/switch.go +++ b/p2p/switch.go @@ -10,10 +10,10 @@ import ( log "github.com/sirupsen/logrus" "github.com/tendermint/go-crypto" cmn "github.com/tendermint/tmlibs/common" - dbm "github.com/tendermint/tmlibs/db" cfg "github.com/vapor/config" "github.com/vapor/consensus" + dbm "github.com/vapor/database/db" "github.com/vapor/errors" "github.com/vapor/p2p/connection" "github.com/vapor/p2p/discover" diff --git a/protocol/block.go b/protocol/block.go index 1e119a4c..d709dac3 100644 --- a/protocol/block.go +++ b/protocol/block.go @@ -102,7 +102,9 @@ func (c *Chain) connectBlock(block *types.Block) (err error) { for key, value := range tx.Entries { switch value.(type) { case *bc.Claim: - c.store.SetWithdrawSpent(&key) + if err := c.store.SetWithdrawSpent(&key); err != nil { + return err + } default: continue } diff --git a/protocol/store.go b/protocol/store.go index 915d0675..482c2de9 100644 --- a/protocol/store.go +++ b/protocol/store.go @@ -22,10 +22,10 @@ type Store interface { SaveChainStatus(*state.BlockNode, *state.UtxoViewpoint) error IsWithdrawSpent(hash *bc.Hash) bool - SetWithdrawSpent(hash *bc.Hash) + SetWithdrawSpent(hash *bc.Hash) error - Set(hash *bc.Hash, data []byte) error - Get(hash *bc.Hash) ([]byte, error) + //Set(hash *bc.Hash, data []byte) error + //Get(hash *bc.Hash) ([]byte, error) } // BlockStoreState represents the core's db status diff --git a/protocol/txpool_test.go b/protocol/txpool_test.go index 0bca2d71..0fb82a8d 100644 --- a/protocol/txpool_test.go +++ b/protocol/txpool_test.go @@ -77,7 +77,7 @@ func (s *mockStore) LoadBlockIndex(uint64) (*state.BlockIndex, error) func (s *mockStore) SaveBlock(*types.Block, *bc.TransactionStatus) error { return nil } func (s *mockStore) SaveChainStatus(*state.BlockNode, *state.UtxoViewpoint) error { return nil } func (s *mockStore) IsWithdrawSpent(hash *bc.Hash) bool { return true } -func (s *mockStore) SetWithdrawSpent(hash *bc.Hash) {} +func (s *mockStore) SetWithdrawSpent(hash *bc.Hash) error { return nil } func (s *mockStore) Set(hash *bc.Hash, data []byte) error { return nil } func (s *mockStore) Get(hash *bc.Hash) ([]byte, error) { return nil, nil } diff --git a/test/bench_blockchain_test.go b/test/bench_blockchain_test.go index d3827f29..234bad7a 100644 --- a/test/bench_blockchain_test.go +++ b/test/bench_blockchain_test.go @@ -7,15 +7,15 @@ import ( "testing" "time" - dbm "github.com/tendermint/tmlibs/db" - "github.com/vapor/account" "github.com/vapor/blockchain/pseudohsm" "github.com/vapor/blockchain/signers" "github.com/vapor/blockchain/txbuilder" "github.com/vapor/consensus" "github.com/vapor/crypto/ed25519/chainkd" - "github.com/vapor/database/leveldb" + "github.com/vapor/database" + dbm "github.com/vapor/database/db" + _ "github.com/vapor/database/leveldb" "github.com/vapor/database/storage" "github.com/vapor/mining" "github.com/vapor/protocol" @@ -137,7 +137,7 @@ func GenerateChainData(dirPath string, testDB dbm.DB, txNumber, otherAssetNum in return nil, nil, nil, err } - store := leveldb.NewStore(testDB) + store := database.NewStore(testDB) txPool := protocol.NewTxPool(store) chain, err := protocol.NewChain(store, txPool) if err != nil { @@ -361,7 +361,7 @@ func CreateTxbyNum(txNumber, otherAssetNum int) ([]*types.Tx, error) { func SetUtxoView(db dbm.DB, view *state.UtxoViewpoint) error { batch := db.NewBatch() - if err := leveldb.SaveUtxoView(batch, view); err != nil { + if err := database.SaveUtxoView(batch, view); err != nil { return err } batch.Write() diff --git a/test/block_test.go b/test/block_test.go index c31d44c3..919bed05 100644 --- a/test/block_test.go +++ b/test/block_test.go @@ -7,9 +7,9 @@ import ( "testing" "time" - dbm "github.com/tendermint/tmlibs/db" - "github.com/vapor/consensus" + dbm "github.com/vapor/database/db" + _ "github.com/vapor/database/leveldb" "github.com/vapor/protocol/bc" "github.com/vapor/protocol/bc/types" "github.com/vapor/protocol/vm" diff --git a/test/chain_test_util.go b/test/chain_test_util.go index c358ddea..c147984d 100644 --- a/test/chain_test_util.go +++ b/test/chain_test_util.go @@ -5,12 +5,13 @@ import ( "os" "time" - dbm "github.com/tendermint/tmlibs/db" - "github.com/golang/protobuf/proto" + "github.com/vapor/blockchain/txbuilder" "github.com/vapor/consensus" - "github.com/vapor/database/leveldb" + "github.com/vapor/database" + dbm "github.com/vapor/database/db" + _ "github.com/vapor/database/leveldb" "github.com/vapor/database/storage" "github.com/vapor/protocol" "github.com/vapor/protocol/bc" @@ -23,7 +24,7 @@ const utxoPrefix = "UT:" type chainTestContext struct { Chain *protocol.Chain DB dbm.DB - Store *leveldb.Store + Store *database.Store } func (ctx *chainTestContext) validateStatus(block *types.Block) error { diff --git a/test/integration/standard_transaction_test.go b/test/integration/standard_transaction_test.go index 250c41a1..b0a4d858 100644 --- a/test/integration/standard_transaction_test.go +++ b/test/integration/standard_transaction_test.go @@ -6,12 +6,12 @@ import ( "os" "testing" - dbm "github.com/tendermint/tmlibs/db" - "github.com/vapor/account" "github.com/vapor/blockchain/pseudohsm" "github.com/vapor/blockchain/signers" "github.com/vapor/crypto/ed25519/chainkd" + dbm "github.com/vapor/database/db" + _ "github.com/vapor/database/leveldb" "github.com/vapor/protocol/bc/types" "github.com/vapor/protocol/validation" "github.com/vapor/test" diff --git a/test/protocol_test.go b/test/protocol_test.go index b02a5116..094fa0f0 100644 --- a/test/protocol_test.go +++ b/test/protocol_test.go @@ -6,9 +6,9 @@ import ( "os" "testing" - dbm "github.com/tendermint/tmlibs/db" - "github.com/vapor/consensus" + dbm "github.com/vapor/database/db" + _ "github.com/vapor/database/leveldb" "github.com/vapor/protocol/bc/types" "github.com/vapor/protocol/vm" ) diff --git a/test/protocol_test_util.go b/test/protocol_test_util.go index fa2c7c4f..55ff90e1 100644 --- a/test/protocol_test_util.go +++ b/test/protocol_test_util.go @@ -3,8 +3,8 @@ package test import ( "fmt" - dbm "github.com/tendermint/tmlibs/db" - + dbm "github.com/vapor/database/db" + _ "github.com/vapor/database/leveldb" "github.com/vapor/protocol" "github.com/vapor/protocol/bc/types" ) diff --git a/test/tx_test.go b/test/tx_test.go index 7bd5d618..80ea6e45 100644 --- a/test/tx_test.go +++ b/test/tx_test.go @@ -10,12 +10,13 @@ import ( "testing" log "github.com/sirupsen/logrus" - dbm "github.com/tendermint/tmlibs/db" "github.com/vapor/account" "github.com/vapor/asset" "github.com/vapor/blockchain/pseudohsm" "github.com/vapor/consensus" + dbm "github.com/vapor/database/db" + _ "github.com/vapor/database/leveldb" "github.com/vapor/protocol/bc" "github.com/vapor/protocol/bc/types" "github.com/vapor/protocol/validation" diff --git a/test/tx_test_util.go b/test/tx_test_util.go index f113dd39..941f440a 100644 --- a/test/tx_test_util.go +++ b/test/tx_test_util.go @@ -6,8 +6,6 @@ import ( "fmt" "time" - "github.com/tendermint/tmlibs/db" - "github.com/vapor/account" "github.com/vapor/asset" "github.com/vapor/blockchain/pseudohsm" @@ -17,6 +15,7 @@ import ( "github.com/vapor/consensus" "github.com/vapor/crypto/ed25519/chainkd" "github.com/vapor/crypto/sha3pool" + db "github.com/vapor/database/db" "github.com/vapor/errors" "github.com/vapor/protocol/bc" "github.com/vapor/protocol/bc/types" diff --git a/test/util.go b/test/util.go index aa9ee7b2..739664b5 100644 --- a/test/util.go +++ b/test/util.go @@ -4,8 +4,6 @@ import ( "context" "time" - dbm "github.com/tendermint/tmlibs/db" - "github.com/vapor/account" "github.com/vapor/blockchain/pseudohsm" "github.com/vapor/blockchain/txbuilder" @@ -13,7 +11,8 @@ import ( "github.com/vapor/config" "github.com/vapor/consensus" "github.com/vapor/crypto/ed25519/chainkd" - "github.com/vapor/database/leveldb" + "github.com/vapor/database" + dbm "github.com/vapor/database/db" "github.com/vapor/protocol" "github.com/vapor/protocol/bc" "github.com/vapor/protocol/bc/types" @@ -26,7 +25,7 @@ const ( ) // MockChain mock chain with genesis block -func MockChain(testDB dbm.DB) (*protocol.Chain, *leveldb.Store, *protocol.TxPool, error) { +func MockChain(testDB dbm.DB) (*protocol.Chain, *database.Store, *protocol.TxPool, error) { config.CommonConfig = config.DefaultConfig() consensus.SoloNetParams.Signer = "78673764e0ba91a4c5ba9ec0c8c23c69e3d73bf27970e05e0a977e81e13bde475264d3b177a96646bc0ce517ae7fd63504c183ab6d330dea184331a4cf5912d5" config.CommonConfig.Consensus.SelfVoteSigners = append(config.CommonConfig.Consensus.SelfVoteSigners, "vsm1qkm743xmgnvh84pmjchq2s4tnfpgu9ae2f9slep") @@ -39,7 +38,7 @@ func MockChain(testDB dbm.DB) (*protocol.Chain, *leveldb.Store, *protocol.TxPool config.CommonConfig.Consensus.Signers = append(config.CommonConfig.Consensus.Signers, address) } - store := leveldb.NewStore(testDB) + store := database.NewStore(testDB) txPool := protocol.NewTxPool(store) chain, err := protocol.NewChain(store, txPool) diff --git a/test/wallet_test_util.go b/test/wallet_test_util.go index 1ef1dc2d..2247d432 100644 --- a/test/wallet_test_util.go +++ b/test/wallet_test_util.go @@ -7,8 +7,6 @@ import ( "path" "reflect" - dbm "github.com/tendermint/tmlibs/db" - "github.com/vapor/account" "github.com/vapor/asset" "github.com/vapor/blockchain/pseudohsm" @@ -17,6 +15,8 @@ import ( "github.com/vapor/config" "github.com/vapor/consensus" "github.com/vapor/crypto/ed25519/chainkd" + dbm "github.com/vapor/database/db" + _ "github.com/vapor/database/leveldb" "github.com/vapor/protocol" "github.com/vapor/protocol/bc/types" w "github.com/vapor/wallet" diff --git a/vendor/github.com/tendermint/abci/example/dummy/dummy.go b/vendor/github.com/tendermint/abci/example/dummy/dummy.go index 7e95c859..bfeaf019 100644 --- a/vendor/github.com/tendermint/abci/example/dummy/dummy.go +++ b/vendor/github.com/tendermint/abci/example/dummy/dummy.go @@ -7,7 +7,8 @@ import ( wire "github.com/tendermint/go-wire" "github.com/tendermint/iavl" cmn "github.com/tendermint/tmlibs/common" - dbm "github.com/tendermint/tmlibs/db" + + dbm "github.com/vapor/database/db" ) type DummyApplication struct { diff --git a/vendor/github.com/tendermint/abci/example/dummy/persistent_dummy.go b/vendor/github.com/tendermint/abci/example/dummy/persistent_dummy.go index 4c480175..93bb4306 100644 --- a/vendor/github.com/tendermint/abci/example/dummy/persistent_dummy.go +++ b/vendor/github.com/tendermint/abci/example/dummy/persistent_dummy.go @@ -10,8 +10,9 @@ import ( crypto "github.com/tendermint/go-crypto" "github.com/tendermint/iavl" cmn "github.com/tendermint/tmlibs/common" - dbm "github.com/tendermint/tmlibs/db" "github.com/tendermint/tmlibs/log" + + dbm "github.com/vapor/database/db" ) const ( diff --git a/vendor/github.com/tendermint/tmlibs/db/c_level_db.go b/vendor/github.com/tendermint/tmlibs/db/c_level_db.go deleted file mode 100644 index b1ae49a1..00000000 --- a/vendor/github.com/tendermint/tmlibs/db/c_level_db.go +++ /dev/null @@ -1,152 +0,0 @@ -// +build gcc - -package db - -import ( - "fmt" - "path" - - "github.com/jmhodges/levigo" - - . "github.com/tendermint/tmlibs/common" -) - -func init() { - dbCreator := func(name string, dir string) (DB, error) { - return NewCLevelDB(name, dir) - } - registerDBCreator(LevelDBBackendStr, dbCreator, true) - registerDBCreator(CLevelDBBackendStr, dbCreator, false) -} - -type CLevelDB struct { - db *levigo.DB - ro *levigo.ReadOptions - wo *levigo.WriteOptions - woSync *levigo.WriteOptions -} - -func NewCLevelDB(name string, dir string) (*CLevelDB, error) { - dbPath := path.Join(dir, name+".db") - - opts := levigo.NewOptions() - opts.SetCache(levigo.NewLRUCache(1 << 30)) - opts.SetCreateIfMissing(true) - db, err := levigo.Open(dbPath, opts) - if err != nil { - return nil, err - } - ro := levigo.NewReadOptions() - wo := levigo.NewWriteOptions() - woSync := levigo.NewWriteOptions() - woSync.SetSync(true) - database := &CLevelDB{ - db: db, - ro: ro, - wo: wo, - woSync: woSync, - } - return database, nil -} - -func (db *CLevelDB) Get(key []byte) []byte { - res, err := db.db.Get(db.ro, key) - if err != nil { - PanicCrisis(err) - } - return res -} - -func (db *CLevelDB) Set(key []byte, value []byte) { - err := db.db.Put(db.wo, key, value) - if err != nil { - PanicCrisis(err) - } -} - -func (db *CLevelDB) SetSync(key []byte, value []byte) { - err := db.db.Put(db.woSync, key, value) - if err != nil { - PanicCrisis(err) - } -} - -func (db *CLevelDB) Delete(key []byte) { - err := db.db.Delete(db.wo, key) - if err != nil { - PanicCrisis(err) - } -} - -func (db *CLevelDB) DeleteSync(key []byte) { - err := db.db.Delete(db.woSync, key) - if err != nil { - PanicCrisis(err) - } -} - -func (db *CLevelDB) DB() *levigo.DB { - return db.db -} - -func (db *CLevelDB) Close() { - db.db.Close() - db.ro.Close() - db.wo.Close() - db.woSync.Close() -} - -func (db *CLevelDB) Print() { - iter := db.db.NewIterator(db.ro) - defer iter.Close() - for iter.Seek(nil); iter.Valid(); iter.Next() { - key := iter.Key() - value := iter.Value() - fmt.Printf("[%X]:\t[%X]\n", key, value) - } -} - -func (db *CLevelDB) Stats() map[string]string { - // TODO: Find the available properties for the C LevelDB implementation - keys := []string{} - - stats := make(map[string]string) - for _, key := range keys { - str, err := db.db.GetProperty(key) - if err == nil { - stats[key] = str - } - } - return stats -} - -func (db *CLevelDB) Iterator() Iterator { - return db.db.NewIterator(nil, nil) -} - -func (db *CLevelDB) NewBatch() Batch { - batch := levigo.NewWriteBatch() - return &cLevelDBBatch{db, batch} -} - -//-------------------------------------------------------------------------------- - -type cLevelDBBatch struct { - db *CLevelDB - batch *levigo.WriteBatch -} - -func (mBatch *cLevelDBBatch) Set(key, value []byte) { - mBatch.batch.Put(key, value) -} - -func (mBatch *cLevelDBBatch) Delete(key []byte) { - mBatch.batch.Delete(key) -} - -func (mBatch *cLevelDBBatch) Write() { - err := mBatch.db.db.Write(mBatch.db.wo, mBatch.batch) - if err != nil { - PanicCrisis(err) - } -} diff --git a/vendor/github.com/tendermint/tmlibs/db/c_level_db_test.go b/vendor/github.com/tendermint/tmlibs/db/c_level_db_test.go deleted file mode 100644 index e7336cc5..00000000 --- a/vendor/github.com/tendermint/tmlibs/db/c_level_db_test.go +++ /dev/null @@ -1,86 +0,0 @@ -// +build gcc - -package db - -import ( - "bytes" - "fmt" - "testing" - - . "github.com/tendermint/tmlibs/common" -) - -func BenchmarkRandomReadsWrites2(b *testing.B) { - b.StopTimer() - - numItems := int64(1000000) - internal := map[int64]int64{} - for i := 0; i < int(numItems); i++ { - internal[int64(i)] = int64(0) - } - db, err := NewCLevelDB(Fmt("test_%x", RandStr(12)), "") - if err != nil { - b.Fatal(err.Error()) - return - } - - fmt.Println("ok, starting") - b.StartTimer() - - for i := 0; i < b.N; i++ { - // Write something - { - idx := (int64(RandInt()) % numItems) - internal[idx] += 1 - val := internal[idx] - idxBytes := int642Bytes(int64(idx)) - valBytes := int642Bytes(int64(val)) - //fmt.Printf("Set %X -> %X\n", idxBytes, valBytes) - db.Set( - idxBytes, - valBytes, - ) - } - // Read something - { - idx := (int64(RandInt()) % numItems) - val := internal[idx] - idxBytes := int642Bytes(int64(idx)) - valBytes := db.Get(idxBytes) - //fmt.Printf("Get %X -> %X\n", idxBytes, valBytes) - if val == 0 { - if !bytes.Equal(valBytes, nil) { - b.Errorf("Expected %v for %v, got %X", - nil, idx, valBytes) - break - } - } else { - if len(valBytes) != 8 { - b.Errorf("Expected length 8 for %v, got %X", - idx, valBytes) - break - } - valGot := bytes2Int64(valBytes) - if val != valGot { - b.Errorf("Expected %v for %v, got %v", - val, idx, valGot) - break - } - } - } - } - - db.Close() -} - -/* -func int642Bytes(i int64) []byte { - buf := make([]byte, 8) - binary.BigEndian.PutUint64(buf, uint64(i)) - return buf -} - -func bytes2Int64(buf []byte) int64 { - return int64(binary.BigEndian.Uint64(buf)) -} -*/ diff --git a/wallet/annotated.go b/wallet/annotated.go index 5f327df2..4925e54b 100644 --- a/wallet/annotated.go +++ b/wallet/annotated.go @@ -5,7 +5,6 @@ import ( "fmt" log "github.com/sirupsen/logrus" - "github.com/tendermint/tmlibs/db" "github.com/vapor/account" "github.com/vapor/asset" @@ -15,6 +14,7 @@ import ( "github.com/vapor/consensus" "github.com/vapor/consensus/segwit" "github.com/vapor/crypto/sha3pool" + db "github.com/vapor/database/db" "github.com/vapor/protocol/bc" "github.com/vapor/protocol/bc/types" "github.com/vapor/protocol/vm/vmutil" diff --git a/wallet/indexer.go b/wallet/indexer.go index 09e74bd1..93643e10 100644 --- a/wallet/indexer.go +++ b/wallet/indexer.go @@ -7,12 +7,12 @@ import ( "sort" log "github.com/sirupsen/logrus" - "github.com/tendermint/tmlibs/db" "github.com/vapor/account" "github.com/vapor/asset" "github.com/vapor/blockchain/query" "github.com/vapor/crypto/sha3pool" + db "github.com/vapor/database/db" chainjson "github.com/vapor/encoding/json" "github.com/vapor/protocol/bc" "github.com/vapor/protocol/bc/types" diff --git a/wallet/recovery.go b/wallet/recovery.go index ad2ca400..17b211ae 100644 --- a/wallet/recovery.go +++ b/wallet/recovery.go @@ -8,12 +8,11 @@ import ( "sync/atomic" "time" - "github.com/tendermint/tmlibs/db" - "github.com/vapor/account" "github.com/vapor/blockchain/signers" "github.com/vapor/crypto/ed25519/chainkd" "github.com/vapor/crypto/sha3pool" + db "github.com/vapor/database/db" "github.com/vapor/errors" "github.com/vapor/protocol/bc" "github.com/vapor/protocol/bc/types" diff --git a/wallet/recovery_test.go b/wallet/recovery_test.go index 1aa034cf..4d933a7d 100644 --- a/wallet/recovery_test.go +++ b/wallet/recovery_test.go @@ -9,14 +9,14 @@ import ( "testing" "time" - dbm "github.com/tendermint/tmlibs/db" - "github.com/vapor/account" "github.com/vapor/blockchain/pseudohsm" "github.com/vapor/blockchain/signers" "github.com/vapor/blockchain/txbuilder" "github.com/vapor/consensus" "github.com/vapor/crypto/ed25519/chainkd" + dbm "github.com/vapor/database/db" + _ "github.com/vapor/database/leveldb" "github.com/vapor/protocol/bc" "github.com/vapor/protocol/bc/types" ) diff --git a/wallet/unconfirmed_test.go b/wallet/unconfirmed_test.go index 310b4058..73a51cec 100644 --- a/wallet/unconfirmed_test.go +++ b/wallet/unconfirmed_test.go @@ -5,17 +5,16 @@ import ( "os" "testing" - dbm "github.com/tendermint/tmlibs/db" - "github.com/vapor/blockchain/signers" - "github.com/vapor/common" - "github.com/vapor/config" - "github.com/vapor/account" "github.com/vapor/asset" "github.com/vapor/blockchain/pseudohsm" "github.com/vapor/blockchain/query" + "github.com/vapor/blockchain/signers" + "github.com/vapor/common" + "github.com/vapor/config" "github.com/vapor/consensus" "github.com/vapor/crypto/ed25519/chainkd" + dbm "github.com/vapor/database/db" "github.com/vapor/protocol/bc/types" "github.com/vapor/testutil" ) diff --git a/wallet/utxo.go b/wallet/utxo.go index 39635931..6acd09a8 100644 --- a/wallet/utxo.go +++ b/wallet/utxo.go @@ -5,12 +5,12 @@ import ( "fmt" log "github.com/sirupsen/logrus" - "github.com/tendermint/tmlibs/db" "github.com/vapor/account" "github.com/vapor/consensus" "github.com/vapor/consensus/segwit" "github.com/vapor/crypto/sha3pool" + db "github.com/vapor/database/db" "github.com/vapor/errors" "github.com/vapor/protocol/bc" "github.com/vapor/protocol/bc/types" diff --git a/wallet/utxo_test.go b/wallet/utxo_test.go index 52013c58..fc02defd 100644 --- a/wallet/utxo_test.go +++ b/wallet/utxo_test.go @@ -8,10 +8,9 @@ import ( "sort" "testing" - dbm "github.com/tendermint/tmlibs/db" - "github.com/vapor/account" "github.com/vapor/consensus" + dbm "github.com/vapor/database/db" "github.com/vapor/protocol/bc" "github.com/vapor/protocol/bc/types" "github.com/vapor/testutil" diff --git a/wallet/wallet.go b/wallet/wallet.go index 3f764185..802b6565 100644 --- a/wallet/wallet.go +++ b/wallet/wallet.go @@ -5,12 +5,12 @@ import ( "sync" log "github.com/sirupsen/logrus" - "github.com/tendermint/tmlibs/db" "github.com/vapor/account" "github.com/vapor/asset" "github.com/vapor/blockchain/pseudohsm" "github.com/vapor/common" + db "github.com/vapor/database/db" "github.com/vapor/protocol" "github.com/vapor/protocol/bc" "github.com/vapor/protocol/bc/types" diff --git a/wallet/wallet_test.go b/wallet/wallet_test.go index 8f325f3a..0f2a9881 100644 --- a/wallet/wallet_test.go +++ b/wallet/wallet_test.go @@ -6,7 +6,6 @@ import ( "testing" "time" - dbm "github.com/tendermint/tmlibs/db" "github.com/vapor/account" "github.com/vapor/asset" "github.com/vapor/blockchain/pseudohsm" @@ -16,7 +15,8 @@ import ( "github.com/vapor/config" "github.com/vapor/consensus" "github.com/vapor/crypto/ed25519/chainkd" - "github.com/vapor/database/leveldb" + "github.com/vapor/database" + dbm "github.com/vapor/database/db" "github.com/vapor/protocol" "github.com/vapor/protocol/bc" "github.com/vapor/protocol/bc/types" @@ -49,7 +49,7 @@ func TestWalletUpdate(t *testing.T) { testDB := dbm.NewDB("testdb", "leveldb", "temp") defer os.RemoveAll("temp") - store := leveldb.NewStore(testDB) + store := database.NewStore(testDB) txPool := protocol.NewTxPool(store) chain, err := protocol.NewChain(store, txPool) -- 2.11.0