feat: support global tx in api.listTransactions and api.getTransaction (#1694)

* feat: add wallet.SaveGlobalTxIdxFlag

* feat: add 	blockHash, position, err := parseGlobalTxIdx(string(globalTxIdx))

* feat: implement parseGlobalTxIdx

* wip

* fix: fix calcGlobalTxIndex()

* feat: support getGlobalTxByTxID() in GetTransactionByTxID()

* refactor: rename save_global_tx_index

* refactor: use encoding/binary for position in txIndex

* feat: use wallet.TxIndexFlag to control GetTransactionByTxID flow

* refactor: wrap txIndexFlag inside NewWallet()

* fix: fix wallet_test_util

* refactor: reduce globalTxIdx space by using bytes directly

diff --git a/cmd/bytomd/commands/run_node.go b/cmd/bytomd/commands/run_node.go
index da24e8b..c61ed34 100644 (file)
--- a/cmd/bytomd/commands/run_node.go
+++ b/cmd/bytomd/commands/run_node.go
@@ -28,6 +28,7 @@ func init() {
 
        runNodeCmd.Flags().Bool("wallet.disable", config.Wallet.Disable, "Disable wallet")
        runNodeCmd.Flags().Bool("wallet.rescan", config.Wallet.Rescan, "Rescan wallet")
+       runNodeCmd.Flags().Bool("wallet.txindex", config.Wallet.TxIndex, "Save global tx index")
        runNodeCmd.Flags().Bool("vault_mode", config.VaultMode, "Run in the offline enviroment")
        runNodeCmd.Flags().Bool("web.closed", config.Web.Closed, "Lanch web browser or not")
        runNodeCmd.Flags().String("chain_id", config.ChainID, "Select network type")

diff --git a/config/config.go b/config/config.go
index 03829b8..f1de23c 100644 (file)
--- a/config/config.go
+++ b/config/config.go
@@ -177,6 +177,7 @@ func DefaultP2PConfig() *P2PConfig {
 type WalletConfig struct {
        Disable  bool   `mapstructure:"disable"`
        Rescan   bool   `mapstructure:"rescan"`
+       TxIndex  bool   `mapstructure:"txindex"`
        MaxTxFee uint64 `mapstructure:"max_tx_fee"`
 }
 
@@ -216,6 +217,7 @@ func DefaultWalletConfig() *WalletConfig {
        return &WalletConfig{
                Disable:  false,
                Rescan:   false,
+               TxIndex:  false,
                MaxTxFee: uint64(1000000000),
        }
 }

diff --git a/node/node.go b/node/node.go
index fd319f9..05d1b69 100644 (file)
--- a/node/node.go
+++ b/node/node.go
@@ -22,6 +22,8 @@ import (
        "github.com/bytom/blockchain/txfeed"
        cfg "github.com/bytom/config"
        "github.com/bytom/consensus"
+       "github.com/bytom/database"
+       dbm "github.com/bytom/database/leveldb"
        "github.com/bytom/env"
        "github.com/bytom/event"
        "github.com/bytom/mining/cpuminer"
@@ -32,8 +34,6 @@ import (
        "github.com/bytom/p2p"
        "github.com/bytom/protocol"
        w "github.com/bytom/wallet"
-       dbm "github.com/bytom/database/leveldb"
-       "github.com/bytom/database"
 )
 
 const (
@@ -109,7 +109,7 @@ func NewNode(config *cfg.Config) *Node {
                walletDB := dbm.NewDB("wallet", config.DBBackend, config.DBDir())
                accounts = account.NewManager(walletDB, chain)
                assets = asset.NewRegistry(walletDB, chain)
-               wallet, err = w.NewWallet(walletDB, accounts, assets, hsm, chain, dispatcher)
+               wallet, err = w.NewWallet(walletDB, accounts, assets, hsm, chain, dispatcher, config.Wallet.TxIndex)
                if err != nil {
                        log.WithFields(log.Fields{"module": logModule, "error": err}).Error("init NewWallet")
                }

diff --git a/test/wallet_test_util.go b/test/wallet_test_util.go
index 92e8e37..07e81aa 100644 (file)
--- a/test/wallet_test_util.go
+++ b/test/wallet_test_util.go
@@ -260,7 +260,7 @@ func (cfg *walletTestConfig) Run() error {
        accountManager := account.NewManager(walletDB, chain)
        assets := asset.NewRegistry(walletDB, chain)
        dispatcher := event.NewDispatcher()
-       wallet, err := w.NewWallet(walletDB, accountManager, assets, hsm, chain, dispatcher)
+       wallet, err := w.NewWallet(walletDB, accountManager, assets, hsm, chain, dispatcher, false)
        if err != nil {
                return err
        }

diff --git a/wallet/indexer.go b/wallet/indexer.go
index f7907d3..0951a45 100644 (file)
--- a/wallet/indexer.go
+++ b/wallet/indexer.go
@@ -1,6 +1,7 @@
 package wallet
 
 import (
+       "encoding/binary"
        "encoding/json"
        "fmt"
        "sort"
@@ -11,10 +12,11 @@ import (
        "github.com/bytom/asset"
        "github.com/bytom/blockchain/query"
        "github.com/bytom/crypto/sha3pool"
+       dbm "github.com/bytom/database/leveldb"
        chainjson "github.com/bytom/encoding/json"
+       "github.com/bytom/errors"
        "github.com/bytom/protocol/bc"
        "github.com/bytom/protocol/bc/types"
-       dbm "github.com/bytom/database/leveldb"
 )
 
 const (
@@ -26,6 +28,8 @@ const (
        GlobalTxIndexPrefix = "GTID:"
 )
 
+var errAccntTxIDNotFound = errors.New("account TXID not found")
+
 func formatKey(blockHeight uint64, position uint32) string {
        return fmt.Sprintf("%016x%08x", blockHeight, position)
 }
@@ -46,8 +50,19 @@ func calcGlobalTxIndexKey(txID string) []byte {
        return []byte(GlobalTxIndexPrefix + txID)
 }
 
-func calcGlobalTxIndex(blockHash *bc.Hash, position int) []byte {
-       return []byte(fmt.Sprintf("%064x%08x", blockHash.String(), position))
+func calcGlobalTxIndex(blockHash *bc.Hash, position uint64) []byte {
+       txIdx := make([]byte, 40)
+       copy(txIdx[:32], blockHash.Bytes())
+       binary.BigEndian.PutUint64(txIdx[32:], position)
+       return txIdx
+}
+
+func parseGlobalTxIdx(globalTxIdx []byte) (*bc.Hash, uint64) {
+       var hashBytes [32]byte
+       copy(hashBytes[:], globalTxIdx[:32])
+       hash := bc.NewHash(hashBytes)
+       position := binary.BigEndian.Uint64(globalTxIdx[32:])
+       return &hash, position
 }
 
 // deleteTransaction delete transactions when orphan block rollback
@@ -124,9 +139,13 @@ func (w *Wallet) indexTransactions(batch dbm.Batch, b *types.Block, txStatus *bc
                batch.Delete(calcUnconfirmedTxKey(tx.ID.String()))
        }
 
+       if !w.TxIndexFlag {
+               return nil
+       }
+
        for position, globalTx := range b.Transactions {
                blockHash := b.BlockHeader.Hash()
-               batch.Set(calcGlobalTxIndexKey(globalTx.ID.String()), calcGlobalTxIndex(&blockHash, position))
+               batch.Set(calcGlobalTxIndexKey(globalTx.ID.String()), calcGlobalTxIndex(&blockHash, uint64(position)))
        }
 
        return nil
@@ -166,21 +185,57 @@ transactionLoop:
 
 // GetTransactionByTxID get transaction by txID
 func (w *Wallet) GetTransactionByTxID(txID string) (*query.AnnotatedTx, error) {
+       if annotatedTx, err := w.getAccountTxByTxID(txID); err == nil {
+               return annotatedTx, nil
+       } else if !w.TxIndexFlag {
+               return nil, err
+       }
+
+       return w.getGlobalTxByTxID(txID)
+}
+
+func (w *Wallet) getAccountTxByTxID(txID string) (*query.AnnotatedTx, error) {
+       annotatedTx := &query.AnnotatedTx{}
        formatKey := w.DB.Get(calcTxIndexKey(txID))
        if formatKey == nil {
-               return nil, fmt.Errorf("No transaction(tx_id=%s) ", txID)
+               return nil, errAccntTxIDNotFound
        }
 
-       annotatedTx := &query.AnnotatedTx{}
        txInfo := w.DB.Get(calcAnnotatedKey(string(formatKey)))
        if err := json.Unmarshal(txInfo, annotatedTx); err != nil {
                return nil, err
        }
-       annotateTxsAsset(w, []*query.AnnotatedTx{annotatedTx})
 
+       annotateTxsAsset(w, []*query.AnnotatedTx{annotatedTx})
        return annotatedTx, nil
 }
 
+func (w *Wallet) getGlobalTxByTxID(txID string) (*query.AnnotatedTx, error) {
+       globalTxIdx := w.DB.Get(calcGlobalTxIndexKey(txID))
+       if globalTxIdx == nil {
+               return nil, fmt.Errorf("No transaction(tx_id=%s) ", txID)
+       }
+
+       blockHash, pos := parseGlobalTxIdx(globalTxIdx)
+       block, err := w.chain.GetBlockByHash(blockHash)
+       if err != nil {
+               return nil, err
+       }
+
+       txStatus, err := w.chain.GetTransactionStatus(blockHash)
+       if err != nil {
+               return nil, err
+       }
+
+       statusFail, err := txStatus.GetStatus(int(pos))
+       if err != nil {
+               return nil, err
+       }
+
+       tx := block.Transactions[int(pos)]
+       return w.buildAnnotatedTransaction(tx, block, statusFail, int(pos)), nil
+}
+
 // GetTransactionsSummary get transactions summary
 func (w *Wallet) GetTransactionsSummary(transactions []*query.AnnotatedTx) []TxSummary {
        Txs := []TxSummary{}

diff --git a/wallet/unconfirmed_test.go b/wallet/unconfirmed_test.go
index 6cc22f4..b0d6602 100644 (file)
--- a/wallet/unconfirmed_test.go
+++ b/wallet/unconfirmed_test.go
@@ -58,7 +58,7 @@ func TestWalletUnconfirmedTxs(t *testing.T) {
        }
 
        dispatcher := event.NewDispatcher()
-       w := mockWallet(testDB, accountManager, reg, nil, dispatcher)
+       w := mockWallet(testDB, accountManager, reg, nil, dispatcher, false)
        utxos := []*account.UTXO{}
        btmUtxo := mockUTXO(controlProg, consensus.BTMAssetID)
        utxos = append(utxos, btmUtxo)

diff --git a/wallet/wallet.go b/wallet/wallet.go
index 7ee2f34..73cf84b 100644 (file)
--- a/wallet/wallet.go
+++ b/wallet/wallet.go
@@ -9,12 +9,12 @@ import (
        "github.com/bytom/account"
        "github.com/bytom/asset"
        "github.com/bytom/blockchain/pseudohsm"
+       dbm "github.com/bytom/database/leveldb"
        "github.com/bytom/errors"
        "github.com/bytom/event"
        "github.com/bytom/protocol"
        "github.com/bytom/protocol/bc"
        "github.com/bytom/protocol/bc/types"
-       dbm "github.com/bytom/database/leveldb"
 )
 
 const (
@@ -45,6 +45,7 @@ type Wallet struct {
        DB              dbm.DB
        rw              sync.RWMutex
        status          StatusInfo
+       TxIndexFlag     bool
        AccountMgr      *account.Manager
        AssetReg        *asset.Registry
        Hsm             *pseudohsm.HSM
@@ -57,7 +58,7 @@ type Wallet struct {
 }
 
 //NewWallet return a new wallet instance
-func NewWallet(walletDB dbm.DB, account *account.Manager, asset *asset.Registry, hsm *pseudohsm.HSM, chain *protocol.Chain, dispatcher *event.Dispatcher) (*Wallet, error) {
+func NewWallet(walletDB dbm.DB, account *account.Manager, asset *asset.Registry, hsm *pseudohsm.HSM, chain *protocol.Chain, dispatcher *event.Dispatcher, txIndexFlag bool) (*Wallet, error) {
        w := &Wallet{
                DB:              walletDB,
                AccountMgr:      account,
@@ -67,6 +68,7 @@ func NewWallet(walletDB dbm.DB, account *account.Manager, asset *asset.Registry,
                RecoveryMgr:     newRecoveryManager(walletDB, account),
                eventDispatcher: dispatcher,
                rescanCh:        make(chan struct{}, 1),
+               TxIndexFlag:     txIndexFlag,
        }
 
        if err := w.loadWalletInfo(); err != nil {

diff --git a/wallet/wallet_test.go b/wallet/wallet_test.go
index 5d239b0..15d53ec 100644 (file)
--- a/wallet/wallet_test.go
+++ b/wallet/wallet_test.go
@@ -24,6 +24,26 @@ import (
        "github.com/bytom/protocol/bc/types"
 )
 
+func TestEncodeDecodeGlobalTxIndex(t *testing.T) {
+       want := &struct {
+               BlockHash bc.Hash
+               Position  uint64
+       }{
+               BlockHash: bc.NewHash([32]byte{0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, 0x20}),
+               Position:  1,
+       }
+
+       globalTxIdx := calcGlobalTxIndex(&want.BlockHash, want.Position)
+       blockHashGot, positionGot := parseGlobalTxIdx(globalTxIdx)
+       if *blockHashGot != want.BlockHash {
+               t.Errorf("blockHash mismatch. Get: %v. Expect: %v", *blockHashGot, want.BlockHash)
+       }
+
+       if positionGot != want.Position {
+               t.Errorf("position mismatch. Get: %v. Expect: %v", positionGot, want.Position)
+       }
+}
+
 func TestWalletVersion(t *testing.T) {
        // prepare wallet
        dirPath, err := ioutil.TempDir(".", "")
@@ -36,7 +56,7 @@ func TestWalletVersion(t *testing.T) {
        defer os.RemoveAll("temp")
 
        dispatcher := event.NewDispatcher()
-       w := mockWallet(testDB, nil, nil, nil, dispatcher)
+       w := mockWallet(testDB, nil, nil, nil, dispatcher, false)
 
        // legacy status test case
        type legacyStatusInfo struct {
@@ -152,7 +172,7 @@ func TestWalletUpdate(t *testing.T) {
        txStatus.SetStatus(1, false)
        store.SaveBlock(block, txStatus)
 
-       w := mockWallet(testDB, accountManager, reg, chain, dispatcher)
+       w := mockWallet(testDB, accountManager, reg, chain, dispatcher, true)
        err = w.AttachBlock(block)
        if err != nil {
                t.Fatal(err)
@@ -174,7 +194,7 @@ func TestWalletUpdate(t *testing.T) {
        for position, tx := range block.Transactions {
                get := w.DB.Get(calcGlobalTxIndexKey(tx.ID.String()))
                bh := block.BlockHeader.Hash()
-               expect := calcGlobalTxIndex(&bh, position)
+               expect := calcGlobalTxIndex(&bh, uint64(position))
                if !reflect.DeepEqual(get, expect) {
                        t.Fatalf("position#%d: compare retrieved globalTxIdx err", position)
                }
@@ -243,7 +263,7 @@ func TestMemPoolTxQueryLoop(t *testing.T) {
        //block := mockSingleBlock(tx)
        txStatus := bc.NewTransactionStatus()
        txStatus.SetStatus(0, false)
-       w, err := NewWallet(testDB, accountManager, reg, hsm, chain, dispatcher)
+       w, err := NewWallet(testDB, accountManager, reg, hsm, chain, dispatcher, false)
        go w.memPoolTxQueryLoop()
        w.eventDispatcher.Post(protocol.TxMsgEvent{TxMsg: &protocol.TxPoolMsg{TxDesc: &protocol.TxDesc{Tx: tx}, MsgType: protocol.MsgNewTx}})
        time.Sleep(time.Millisecond * 10)
@@ -300,7 +320,7 @@ func mockTxData(utxos []*account.UTXO, testAccount *account.Account) (*txbuilder
        return tplBuilder.Build()
 }
 
-func mockWallet(walletDB dbm.DB, account *account.Manager, asset *asset.Registry, chain *protocol.Chain, dispatcher *event.Dispatcher) *Wallet {
+func mockWallet(walletDB dbm.DB, account *account.Manager, asset *asset.Registry, chain *protocol.Chain, dispatcher *event.Dispatcher, txIndexFlag bool) *Wallet {
        wallet := &Wallet{
                DB:              walletDB,
                AccountMgr:      account,
@@ -308,6 +328,7 @@ func mockWallet(walletDB dbm.DB, account *account.Manager, asset *asset.Registry
                chain:           chain,
                RecoveryMgr:     newRecoveryManager(walletDB, account),
                eventDispatcher: dispatcher,
+               TxIndexFlag:     txIndexFlag,
        }
        wallet.txMsgSub, _ = wallet.eventDispatcher.Subscribe(protocol.TxMsgEvent{})
        return wallet