X-Git-Url: http://git.osdn.net/view?a=blobdiff_plain;f=database%2Fstore.go;h=70b6fed5823a01b922dc223533b9fffcac61cf7a;hb=98c3baac82cb06ce60cb06b146f6375af2d9d141;hp=9fbad6c0a7fadb25d1a09abbed0976e7142f4723;hpb=8fe4018eeb53db8deacc075fbe7f8567335854c0;p=bytom%2Fbytom.git diff --git a/database/store.go b/database/store.go index 9fbad6c0..70b6fed5 100644 --- a/database/store.go +++ b/database/store.go @@ -5,26 +5,28 @@ import ( "encoding/json" "time" - "github.com/golang/protobuf/proto" log "github.com/sirupsen/logrus" "github.com/tendermint/tmlibs/common" - "github.com/bytom/database/storage" - "github.com/bytom/errors" - "github.com/bytom/protocol" - "github.com/bytom/protocol/bc" - "github.com/bytom/protocol/bc/types" - "github.com/bytom/protocol/state" - dbm "github.com/bytom/database/leveldb" + "github.com/bytom/bytom/consensus" + dbm "github.com/bytom/bytom/database/leveldb" + "github.com/bytom/bytom/database/storage" + "github.com/bytom/bytom/errors" + "github.com/bytom/bytom/protocol" + "github.com/bytom/bytom/protocol/bc" + "github.com/bytom/bytom/protocol/bc/types" + "github.com/bytom/bytom/protocol/state" ) const logModule = "leveldb" var ( - BlockStoreKey = []byte("blockStore") - BlockPrefix = []byte("B:") - BlockHeaderPrefix = []byte("BH:") - TxStatusPrefix = []byte("BTS:") + // CheckpointPrefix represent the namespace of checkpoints in db + CheckpointPrefix = []byte("CP:") + // BlockStoreKey block store key + BlockStoreKey = []byte("blockStore") + // BlockHeaderIndexPrefix block header index with height + BlockHeaderIndexPrefix = []byte("BH:") ) func loadBlockStoreStateJSON(db dbm.DB) *protocol.BlockStoreState { @@ -44,155 +46,112 @@ func loadBlockStoreStateJSON(db dbm.DB) *protocol.BlockStoreState { // methods for querying current data. type Store struct { db dbm.DB - cache blockCache + cache cache } -func CalcBlockKey(hash *bc.Hash) []byte { - return append(BlockPrefix, hash.Bytes()...) -} - -func CalcBlockHeaderKey(height uint64, hash *bc.Hash) []byte { - buf := [8]byte{} - binary.BigEndian.PutUint64(buf[:], height) - key := append(BlockHeaderPrefix, buf[:]...) - return append(key, hash.Bytes()...) -} +// NewStore creates and returns a new Store object. +func NewStore(db dbm.DB) *Store { + fillBlockHeaderFn := func(hash *bc.Hash) (*types.BlockHeader, error) { + return GetBlockHeader(db, hash) + } -func CalcTxStatusKey(hash *bc.Hash) []byte { - return append(TxStatusPrefix, hash.Bytes()...) -} + fillBlockTxsFn := func(hash *bc.Hash) ([]*types.Tx, error) { + return GetBlockTransactions(db, hash) + } -// GetBlock return the block by given hash -func GetBlock(db dbm.DB, hash *bc.Hash) (*types.Block, error) { - bytez := db.Get(CalcBlockKey(hash)) - if bytez == nil { - return nil, nil + fillBlockHashesFn := func(height uint64) ([]*bc.Hash, error) { + return GetBlockHashesByHeight(db, height) } - block := &types.Block{} - err := block.UnmarshalText(bytez) - return block, err -} + fillMainChainHashFn := func(height uint64) (*bc.Hash, error) { + return GetMainChainHash(db, height) + } -// NewStore creates and returns a new Store object. -func NewStore(db dbm.DB) *Store { - cache := newBlockCache(func(hash *bc.Hash) (*types.Block, error) { - return GetBlock(db, hash) - }) + cache := newCache(fillBlockHeaderFn, fillBlockTxsFn, fillBlockHashesFn, fillMainChainHashFn) return &Store{ db: db, cache: cache, } } +// GetBlockHeader return the BlockHeader by given hash +func (s *Store) GetBlockHeader(hash *bc.Hash) (*types.BlockHeader, error) { + return s.cache.lookupBlockHeader(hash) +} + // GetUtxo will search the utxo in db func (s *Store) GetUtxo(hash *bc.Hash) (*storage.UtxoEntry, error) { return getUtxo(s.db, hash) } -// BlockExist check if the block is stored in disk -func (s *Store) BlockExist(hash *bc.Hash) bool { - block, err := s.cache.lookup(hash) - return err == nil && block != nil +func (s *Store) GetContract(hash [32]byte) ([]byte, error) { + return getContract(s.db, hash) } -// GetBlock return the block by given hash -func (s *Store) GetBlock(hash *bc.Hash) (*types.Block, error) { - return s.cache.lookup(hash) -} - -// GetTransactionsUtxo will return all the utxo that related to the input txs -func (s *Store) GetTransactionsUtxo(view *state.UtxoViewpoint, txs []*bc.Tx) error { - return getTransactionsUtxo(s.db, view, txs) +// BlockExist check if the block is stored in disk +func (s *Store) BlockExist(hash *bc.Hash) bool { + _, err := s.cache.lookupBlockHeader(hash) + return err == nil } -// GetTransactionStatus will return the utxo that related to the block hash -func (s *Store) GetTransactionStatus(hash *bc.Hash) (*bc.TransactionStatus, error) { - data := s.db.Get(CalcTxStatusKey(hash)) - if data == nil { - return nil, errors.New("can't find the transaction status by given hash") +// SaveBlockHeader persists a new block header in the protocol. +func (s *Store) SaveBlockHeader(blockHeader *types.BlockHeader) error { + binaryBlockHeader, err := blockHeader.MarshalText() + if err != nil { + return errors.Wrap(err, "Marshal block header") } - ts := &bc.TransactionStatus{} - if err := proto.Unmarshal(data, ts); err != nil { - return nil, errors.Wrap(err, "unmarshaling transaction status") - } - return ts, nil + blockHash := blockHeader.Hash() + s.db.Set(CalcBlockHeaderKey(&blockHash), binaryBlockHeader) + s.cache.removeBlockHeader(blockHeader) + return nil } -// GetStoreStatus return the BlockStoreStateJSON -func (s *Store) GetStoreStatus() *protocol.BlockStoreState { - return loadBlockStoreStateJSON(s.db) +// GetBlockHashesByHeight return the block hash by the specified height +func (s *Store) GetBlockHashesByHeight(height uint64) ([]*bc.Hash, error) { + return s.cache.lookupBlockHashesByHeight(height) } -func (s *Store) LoadBlockIndex(stateBestHeight uint64) (*state.BlockIndex, error) { - startTime := time.Now() - blockIndex := state.NewBlockIndex() - bhIter := s.db.IteratorPrefix(BlockHeaderPrefix) - defer bhIter.Release() - - var lastNode *state.BlockNode - for bhIter.Next() { - bh := &types.BlockHeader{} - if err := bh.UnmarshalText(bhIter.Value()); err != nil { - return nil, err - } - - // If a block with a height greater than the best height of state is added to the index, - // It may cause a bug that the new block cant not be process properly. - if bh.Height > stateBestHeight { - break - } - - var parent *state.BlockNode - if lastNode == nil || lastNode.Hash == bh.PreviousBlockHash { - parent = lastNode - } else { - parent = blockIndex.GetNode(&bh.PreviousBlockHash) - } - - node, err := state.NewBlockNode(bh, parent) - if err != nil { - return nil, err - } - - blockIndex.AddNode(node) - lastNode = node - } - - log.WithFields(log.Fields{ - "module": logModule, - "height": stateBestHeight, - "duration": time.Since(startTime), - }).Debug("initialize load history block index from database") - return blockIndex, nil +// GetMainChainHash return the block hash by the specified height +func (s *Store) GetMainChainHash(height uint64) (*bc.Hash, error) { + return s.cache.lookupMainChainHash(height) } // SaveBlock persists a new block in the protocol. -func (s *Store) SaveBlock(block *types.Block, ts *bc.TransactionStatus) error { +func (s *Store) SaveBlock(block *types.Block) error { startTime := time.Now() - binaryBlock, err := block.MarshalText() + binaryBlockHeader, err := block.MarshalTextForBlockHeader() if err != nil { - return errors.Wrap(err, "Marshal block meta") + return errors.Wrap(err, "Marshal block header") } - binaryBlockHeader, err := block.BlockHeader.MarshalText() + binaryBlockTxs, err := block.MarshalTextForTransactions() if err != nil { - return errors.Wrap(err, "Marshal block header") + return errors.Wrap(err, "Marshal block transactions") } - binaryTxStatus, err := proto.Marshal(ts) + blockHashes := []*bc.Hash{} + hashes, err := s.GetBlockHashesByHeight(block.Height) if err != nil { - return errors.Wrap(err, "marshal block transaction status") + return err } + blockHashes = append(blockHashes, hashes...) blockHash := block.Hash() + blockHashes = append(blockHashes, &blockHash) + binaryBlockHashes, err := json.Marshal(blockHashes) + if err != nil { + return errors.Wrap(err, "Marshal block hashes") + } + batch := s.db.NewBatch() - batch.Set(CalcBlockKey(&blockHash), binaryBlock) - batch.Set(CalcBlockHeaderKey(block.Height, &blockHash), binaryBlockHeader) - batch.Set(CalcTxStatusKey(&blockHash), binaryTxStatus) + batch.Set(CalcBlockHashesKey(block.Height), binaryBlockHashes) + batch.Set(CalcBlockHeaderKey(&blockHash), binaryBlockHeader) + batch.Set(CalcBlockTransactionsKey(&blockHash), binaryBlockTxs) + batch.Set(CalcBlockHeaderIndexKey(block.Height, &blockHash), binaryBlockHeader) batch.Write() + s.cache.removeBlockHashes(block.Height) log.WithFields(log.Fields{ "module": logModule, "height": block.Height, @@ -202,19 +161,201 @@ func (s *Store) SaveBlock(block *types.Block, ts *bc.TransactionStatus) error { return nil } +// GetBlockTransactions return the Block transactions by given hash +func (s *Store) GetBlockTransactions(hash *bc.Hash) ([]*types.Tx, error) { + return s.cache.lookupBlockTxs(hash) +} + +// GetBlock return the block by given hash +func (s *Store) GetBlock(hash *bc.Hash) (*types.Block, error) { + blockHeader, err := s.GetBlockHeader(hash) + if err != nil { + return nil, err + } + + txs, err := s.GetBlockTransactions(hash) + if err != nil { + return nil, err + } + + return &types.Block{ + BlockHeader: *blockHeader, + Transactions: txs, + }, nil +} + +// GetTransactionsUtxo will return all the utxo that related to the input txs +func (s *Store) GetTransactionsUtxo(view *state.UtxoViewpoint, txs []*bc.Tx) error { + return getTransactionsUtxo(s.db, view, txs) +} + +// GetStoreStatus return the BlockStoreStateJSON +func (s *Store) GetStoreStatus() *protocol.BlockStoreState { + return loadBlockStoreStateJSON(s.db) +} + // SaveChainStatus save the core's newest status && delete old status -func (s *Store) SaveChainStatus(node *state.BlockNode, view *state.UtxoViewpoint) error { +func (s *Store) SaveChainStatus(blockHeader *types.BlockHeader, mainBlockHeaders []*types.BlockHeader, view *state.UtxoViewpoint, contractView *state.ContractViewpoint, finalizedHeight uint64, finalizedHash *bc.Hash) error { batch := s.db.NewBatch() if err := saveUtxoView(batch, view); err != nil { return err } - bytes, err := json.Marshal(protocol.BlockStoreState{Height: node.Height, Hash: &node.Hash}) + if err := deleteContractView(s.db, batch, contractView); err != nil { + return err + } + + if err := saveContractView(s.db, batch, contractView); err != nil { + return err + } + + blockHeaderHash := blockHeader.Hash() + bytes, err := json.Marshal( + protocol.BlockStoreState{ + Height: blockHeader.Height, + Hash: &blockHeaderHash, + FinalizedHeight: finalizedHeight, + FinalizedHash: finalizedHash, + }) if err != nil { return err } batch.Set(BlockStoreKey, bytes) + + var clearCacheFuncs []func() + // save main chain blockHeaders + for _, blockHeader := range mainBlockHeaders { + bh := blockHeader + blockHash := bh.Hash() + binaryBlockHash, err := blockHash.MarshalText() + if err != nil { + return errors.Wrap(err, "Marshal block hash") + } + + batch.Set(calcMainChainIndexPrefix(bh.Height), binaryBlockHash) + clearCacheFuncs = append(clearCacheFuncs, func() { + s.cache.removeMainChainHash(bh.Height) + }) + } + batch.Write() + for _, clearCacheFunc := range clearCacheFuncs { + clearCacheFunc() + } + + return nil +} + +func calcCheckpointKey(height uint64, hash *bc.Hash) []byte { + buf := make([]byte, 8) + binary.BigEndian.PutUint64(buf, height) + key := append(CheckpointPrefix, buf...) + if hash != nil { + key = append(key, hash.Bytes()...) + } + return key +} + +func (s *Store) GetCheckpoint(hash *bc.Hash) (*state.Checkpoint, error) { + header, err := s.GetBlockHeader(hash) + if err != nil { + return nil, err + } + + data := s.db.Get(calcCheckpointKey(header.Height, hash)) + checkpoint := &state.Checkpoint{} + if err := json.Unmarshal(data, checkpoint); err != nil { + return nil, err + } + + checkpoint.SupLinks = append(checkpoint.SupLinks, header.SupLinks...) + return checkpoint, nil +} + +// GetCheckpointsByHeight return all checkpoints of specified block height +func (s *Store) GetCheckpointsByHeight(height uint64) ([]*state.Checkpoint, error) { + iter := s.db.IteratorPrefix(calcCheckpointKey(height, nil)) + defer iter.Release() + return s.loadCheckpointsFromIter(iter) +} + +// CheckpointsFromNode return all checkpoints from specified block height and hash +func (s *Store) CheckpointsFromNode(height uint64, hash *bc.Hash) ([]*state.Checkpoint, error) { + startKey := calcCheckpointKey(height, hash) + iter := s.db.IteratorPrefixWithStart(CheckpointPrefix, startKey, false) + + firstCheckpoint := &state.Checkpoint{} + if err := json.Unmarshal(iter.Value(), firstCheckpoint); err != nil { + return nil, err + } + + checkpoints := []*state.Checkpoint{firstCheckpoint} + subs, err := s.loadCheckpointsFromIter(iter) + if err != nil { + return nil, err + } + + checkpoints = append(checkpoints, subs...) + return checkpoints, nil +} + +func (s *Store) loadCheckpointsFromIter(iter dbm.Iterator) ([]*state.Checkpoint, error) { + var checkpoints []*state.Checkpoint + defer iter.Release() + for iter.Next() { + checkpoint := &state.Checkpoint{} + if err := json.Unmarshal(iter.Value(), checkpoint); err != nil { + return nil, err + } + + header, err := s.GetBlockHeader(&checkpoint.Hash) + if err != nil { + return nil, err + } + + checkpoint.SupLinks = append(checkpoint.SupLinks, header.SupLinks...) + checkpoints = append(checkpoints, checkpoint) + } + return checkpoints, nil +} + +// SaveCheckpoints bulk save multiple checkpoint +func (s *Store) SaveCheckpoints(checkpoints []*state.Checkpoint) error { + batch := s.db.NewBatch() + + if err := s.saveCheckpoints(batch, checkpoints); err != nil { + return err + } + batch.Write() return nil } + +func (s *Store) saveCheckpoints(batch dbm.Batch, checkpoints []*state.Checkpoint) error { + for _, checkpoint := range checkpoints { + startTime := time.Now() + data, err := json.Marshal(checkpoint) + if err != nil { + return err + } + + if checkpoint.Height%consensus.ActiveNetParams.BlocksOfEpoch != 1 { + header, err := s.GetBlockHeader(&checkpoint.Hash) + if err != nil { + return err + } + + batch.Delete(calcCheckpointKey(header.Height-1, &header.PreviousBlockHash)) + } + + batch.Set(calcCheckpointKey(checkpoint.Height, &checkpoint.Hash), data) + log.WithFields(log.Fields{ + "module": logModule, + "height": checkpoint.Height, + "hash": checkpoint.Hash.String(), + "status": checkpoint.Status, + "duration": time.Since(startTime), + }).Info("checkpoint saved on disk") + } + return nil +}