From f9e8455ed02759378767d4dc3222a40e33a024c9 Mon Sep 17 00:00:00 2001 From: HAOYUatHZ <37070449+HAOYUatHZ@users.noreply.github.com> Date: Mon, 7 Jan 2019 16:48:54 +0800 Subject: [PATCH] Recommit (#1547) * Fix nil pointer error * Init commitMap * Init blockSnapshot * Combine recover block & submitWork * Add commitMap cleaning * Clean up * Refine code * Add recommitTicker * Add comments * Clean * Add config * Save recommitInterval as uint64 in miningpool * Fix typo * Refine code * Remake a block freshly * Refine code * Refine code * Clean up * Fix deleteMap logic * Refine code * Update Readme * Fix test --- README.md | 3 +- api/miner.go | 10 ++++++ cmd/bytomd/commands/run_node.go | 4 ++- config/config.go | 20 +++++++++--- mining/miningpool/miningpool.go | 70 +++++++++++++++++++++++++---------------- node/node.go | 4 +-- test/integration/run_test.go | 2 +- 7 files changed, 77 insertions(+), 36 deletions(-) diff --git a/README.md b/README.md index 98107b32..85280a50 100644 --- a/README.md +++ b/README.md @@ -95,7 +95,8 @@ available flags for `bytomd node`: --auth.disable Disable rpc access authenticate --chain_id string Select network type -h, --help help for node - --mining Enable mining + --mining.enable Enable mining + --mining.recommit_interval Set mining pool recomit interval in seconds (default 15) --p2p.dial_timeout int Set dial timeout (default 3) --p2p.handshake_timeout int Set handshake timeout (default 30) --p2p.laddr string Node listen address. diff --git a/api/miner.go b/api/miner.go index 4fd2ec6d..9d393653 100644 --- a/api/miner.go +++ b/api/miner.go @@ -11,6 +11,8 @@ import ( "github.com/bytom/protocol/bc/types" ) +var ErrEmptyWorkSubmission = errors.New("empty work submission") + // BlockHeaderJSON struct provides support for get work in json format, when it also follows // BlockHeader structure type BlockHeaderJSON struct { @@ -99,6 +101,10 @@ type SubmitWorkReq struct { // submitWork submits work in compressed protobuf format func (a *API) submitWork(ctx context.Context, req *SubmitWorkReq) Response { + if req.BlockHeader == nil { + return NewErrorResponse(ErrEmptyWorkSubmission) + } + if err := a.SubmitWork(req.BlockHeader); err != nil { return NewErrorResponse(err) } @@ -112,6 +118,10 @@ type SubmitWorkJSONReq struct { // submitWorkJSON submits work in json format func (a *API) submitWorkJSON(ctx context.Context, req *SubmitWorkJSONReq) Response { + if req.BlockHeader == nil { + return NewErrorResponse(ErrEmptyWorkSubmission) + } + bh := &types.BlockHeader{ Version: req.BlockHeader.Version, Height: req.BlockHeader.Height, diff --git a/cmd/bytomd/commands/run_node.go b/cmd/bytomd/commands/run_node.go index 48c7dc63..ed4e03c3 100644 --- a/cmd/bytomd/commands/run_node.go +++ b/cmd/bytomd/commands/run_node.go @@ -18,7 +18,9 @@ var runNodeCmd = &cobra.Command{ func init() { runNodeCmd.Flags().String("prof_laddr", config.ProfListenAddress, "Use http to profile bytomd programs") - runNodeCmd.Flags().Bool("mining", config.Mining, "Enable mining") + + runNodeCmd.Flags().Bool("mining.enable", config.Mining.Enable, "Enable mining") + runNodeCmd.Flags().Uint64("mining.recommit_interval", config.Mining.RecommitInterval, "Set mining pool recomit interval in seconds") runNodeCmd.Flags().Bool("simd.enable", config.Simd.Enable, "Enable SIMD mechan for tensority") diff --git a/config/config.go b/config/config.go index 4e8f29ff..c5500c21 100644 --- a/config/config.go +++ b/config/config.go @@ -23,6 +23,7 @@ type Config struct { Auth *RPCAuthConfig `mapstructure:"auth"` Web *WebConfig `mapstructure:"web"` Simd *SimdConfig `mapstructure:"simd"` + Mining *MiningConfig `mapstructure:"mining"` Websocket *WebsocketConfig `mapstructure:"ws"` } @@ -35,6 +36,7 @@ func DefaultConfig() *Config { Auth: DefaultRPCAuthConfig(), Web: DefaultWebConfig(), Simd: DefaultSimdConfig(), + Mining: DefaultMiningConfig(), Websocket: DefaultWebsocketConfig(), } } @@ -64,8 +66,6 @@ type BaseConfig struct { // TCP or UNIX socket address for the profiling server to listen on ProfListenAddress string `mapstructure:"prof_laddr"` - Mining bool `mapstructure:"mining"` - // Database backend: leveldb | memdb DBBackend string `mapstructure:"db_backend"` @@ -88,7 +88,6 @@ func DefaultBaseConfig() BaseConfig { return BaseConfig{ Moniker: "anonymous", ProfListenAddress: "", - Mining: false, DBBackend: "leveldb", DBPath: "data", KeysPath: "keystore", @@ -149,6 +148,11 @@ type SimdConfig struct { Enable bool `mapstructure:"enable"` } +type MiningConfig struct { + Enable bool `mapstructure:"enable"` + RecommitInterval uint64 `mapstructure:"recommit_interval"` +} + type WebsocketConfig struct { MaxNumWebsockets int `mapstructure:"max_num_websockets"` MaxNumConcurrentReqs int `mapstructure:"max_num_concurrent_reqs"` @@ -177,13 +181,21 @@ func DefaultWalletConfig() *WalletConfig { } } -// Default configurable web parameters. +// Default configurable blockheader verification parameters. func DefaultSimdConfig() *SimdConfig { return &SimdConfig{ Enable: false, } } +// Default configurable mining parameters. +func DefaultMiningConfig() *MiningConfig { + return &MiningConfig{ + Enable: false, + RecommitInterval: 15, + } +} + func DefaultWebsocketConfig() *WebsocketConfig { return &WebsocketConfig{ MaxNumWebsockets: 25, diff --git a/mining/miningpool/miningpool.go b/mining/miningpool/miningpool.go index 33520d56..0d140040 100644 --- a/mining/miningpool/miningpool.go +++ b/mining/miningpool/miningpool.go @@ -11,6 +11,7 @@ import ( "github.com/bytom/event" "github.com/bytom/mining" "github.com/bytom/protocol" + "github.com/bytom/protocol/bc" "github.com/bytom/protocol/bc/types" ) @@ -25,9 +26,11 @@ type submitBlockMsg struct { // MiningPool is the support struct for p2p mine pool type MiningPool struct { - mutex sync.RWMutex - block *types.Block - submitCh chan *submitBlockMsg + mutex sync.RWMutex + blockHeader *types.BlockHeader + submitCh chan *submitBlockMsg + commitMap map[bc.Hash]([]*types.Tx) + recommitInterval time.Duration chain *protocol.Chain accountManager *account.Manager @@ -36,30 +39,36 @@ type MiningPool struct { } // NewMiningPool will create a new MiningPool -func NewMiningPool(c *protocol.Chain, accountManager *account.Manager, txPool *protocol.TxPool, dispatcher *event.Dispatcher) *MiningPool { +func NewMiningPool(c *protocol.Chain, accountManager *account.Manager, txPool *protocol.TxPool, dispatcher *event.Dispatcher, recommitInterval uint64) *MiningPool { m := &MiningPool{ - submitCh: make(chan *submitBlockMsg, maxSubmitChSize), - chain: c, - accountManager: accountManager, - txPool: txPool, - eventDispatcher: dispatcher, + submitCh: make(chan *submitBlockMsg, maxSubmitChSize), + commitMap: make(map[bc.Hash]([]*types.Tx)), + recommitInterval: time.Duration(recommitInterval) * time.Second, + chain: c, + accountManager: accountManager, + txPool: txPool, + eventDispatcher: dispatcher, } - m.generateBlock() + m.generateBlock(true) go m.blockUpdater() return m } // blockUpdater is the goroutine for keep update mining block func (m *MiningPool) blockUpdater() { + recommitTicker := time.NewTicker(m.recommitInterval) for { select { + case <-recommitTicker.C: + m.generateBlock(false) + case <-m.chain.BlockWaiter(m.chain.BestBlockHeight() + 1): - m.generateBlock() + m.generateBlock(true) case submitMsg := <-m.submitCh: err := m.submitWork(submitMsg.blockHeader) if err == nil { - m.generateBlock() + m.generateBlock(true) } submitMsg.reply <- err } @@ -67,27 +76,34 @@ func (m *MiningPool) blockUpdater() { } // generateBlock generates a block template to mine -func (m *MiningPool) generateBlock() { +func (m *MiningPool) generateBlock(isNextHeight bool) { m.mutex.Lock() defer m.mutex.Unlock() + if isNextHeight { + // make a new commitMap, so that the expired map will be deleted(garbage-collected) + m.commitMap = make(map[bc.Hash]([]*types.Tx)) + } + block, err := mining.NewBlockTemplate(m.chain, m.txPool, m.accountManager) if err != nil { log.Errorf("miningpool: failed on create NewBlockTemplate: %v", err) return } - m.block = block + + // The previous memory will be reclaimed by gc + m.blockHeader = &block.BlockHeader + m.commitMap[block.TransactionsMerkleRoot] = block.Transactions } // GetWork will return a block header for p2p mining func (m *MiningPool) GetWork() (*types.BlockHeader, error) { - if m.block != nil { + if m.blockHeader != nil { m.mutex.RLock() defer m.mutex.RUnlock() - m.block.BlockHeader.Timestamp = uint64(time.Now().Unix()) - bh := m.block.BlockHeader - return &bh, nil + m.blockHeader.Timestamp = uint64(time.Now().Unix()) + return m.blockHeader, nil } return nil, errors.New("no block is ready for mining") } @@ -107,13 +123,17 @@ func (m *MiningPool) submitWork(bh *types.BlockHeader) error { m.mutex.Lock() defer m.mutex.Unlock() - if m.block == nil || bh.PreviousBlockHash != m.block.PreviousBlockHash { + if m.blockHeader == nil || bh.PreviousBlockHash != m.blockHeader.PreviousBlockHash { return errors.New("pending mining block has been changed") } - m.block.Nonce = bh.Nonce - m.block.Timestamp = bh.Timestamp - isOrphan, err := m.chain.ProcessBlock(m.block) + txs, ok := m.commitMap[bh.TransactionsMerkleRoot] + if !ok { + return errors.New("TransactionsMerkleRoot not found in history") + } + + block := &types.Block{*bh, txs} + isOrphan, err := m.chain.ProcessBlock(block) if err != nil { return err } @@ -121,9 +141,5 @@ func (m *MiningPool) submitWork(bh *types.BlockHeader) error { return errors.New("submit result is orphan") } - if err := m.eventDispatcher.Post(event.NewMinedBlockEvent{Block: m.block}); err != nil { - return err - } - - return nil + return m.eventDispatcher.Post(event.NewMinedBlockEvent{Block: block}) } diff --git a/node/node.go b/node/node.go index 2399fd78..ccef22bf 100644 --- a/node/node.go +++ b/node/node.go @@ -147,13 +147,13 @@ func NewNode(config *cfg.Config) *Node { wallet: wallet, chain: chain, txfeed: txFeed, - miningEnable: config.Mining, + miningEnable: config.Mining.Enable, notificationMgr: notificationMgr, } node.cpuMiner = cpuminer.NewCPUMiner(chain, accounts, txPool, dispatcher) - node.miningPool = miningpool.NewMiningPool(chain, accounts, txPool, dispatcher) + node.miningPool = miningpool.NewMiningPool(chain, accounts, txPool, dispatcher, config.Mining.RecommitInterval) node.BaseService = *cmn.NewBaseService(nil, "Node", node) diff --git a/test/integration/run_test.go b/test/integration/run_test.go index a8e8c215..0078b99a 100644 --- a/test/integration/run_test.go +++ b/test/integration/run_test.go @@ -12,7 +12,7 @@ import ( func mockConfig() *cfg.Config { var config = cfg.DefaultConfig() config.Wallet.Disable = false - config.Mining = true + config.Mining.Enable = true config.ApiAddress = "127.0.0.1:9888" return config } -- 2.11.0