From: wz Date: Mon, 1 Jul 2019 07:52:48 +0000 (+0800) Subject: Paging (#234) X-Git-Tag: v1.0.5~184 X-Git-Url: http://git.osdn.net/view?p=bytom%2Fvapor.git;a=commitdiff_plain;h=e2294cec495077e4d76d8ff22ad4cae63c4eb2d2 Paging (#234) * add paging * fix review * fix review * remove space * fix review * add AccountAlias * fix test --- diff --git a/account/accounts.go b/account/accounts.go index 0526efe7..befbe509 100644 --- a/account/accounts.go +++ b/account/accounts.go @@ -57,6 +57,7 @@ var ( ErrContractIndex = errors.New("Exceeded maximum addresses per account") ErrAccountIndex = errors.New("Exceeded maximum accounts per xpub") ErrFindTransaction = errors.New("No transaction") + ErrAccountIDEmpty = errors.New("account_id is empty") ) // ContractKey account control promgram store prefix diff --git a/api/query.go b/api/query.go index 938b3d33..abbda524 100644 --- a/api/query.go +++ b/api/query.go @@ -149,49 +149,37 @@ func (a *API) getTransaction(ctx context.Context, txInfo struct { // POST /list-transactions func (a *API) listTransactions(ctx context.Context, filter struct { - ID string `json:"id"` - AccountID string `json:"account_id"` - Detail bool `json:"detail"` - Unconfirmed bool `json:"unconfirmed"` - From uint `json:"from"` - Count uint `json:"count"` + AccountID string `json:"account_id"` + AccountAlias string `json:"account_alias"` + StartTxID string `json:"start_tx_id"` + Detail bool `json:"detail"` + Unconfirmed bool `json:"unconfirmed"` + Count uint `json:"count"` }) Response { - transactions := []*query.AnnotatedTx{} - var err error - var transaction *query.AnnotatedTx - - if filter.ID != "" { - transaction, err = a.wallet.GetTransactionByTxID(filter.ID) - if err != nil && filter.Unconfirmed { - transaction, err = a.wallet.GetUnconfirmedTxByTxID(filter.ID) - } - - if err != nil { - return NewErrorResponse(err) - } - transactions = []*query.AnnotatedTx{transaction} - } else { - transactions, err = a.wallet.GetTransactions(filter.AccountID) + accountID := filter.AccountID + if filter.AccountAlias != "" { + acc, err := a.wallet.AccountMgr.FindByAlias(filter.AccountAlias) if err != nil { return NewErrorResponse(err) } + accountID = acc.ID + } - if filter.Unconfirmed { - unconfirmedTxs, err := a.wallet.GetUnconfirmedTxs(filter.AccountID) - if err != nil { - return NewErrorResponse(err) - } - transactions = append(unconfirmedTxs, transactions...) - } + if accountID == "" { + return NewErrorResponse(account.ErrAccountIDEmpty) + } + + transactions, err := a.wallet.GetTransactions(accountID, filter.StartTxID, filter.Count, filter.Unconfirmed) + if err != nil { + return NewErrorResponse(err) } if filter.Detail == false { txSummary := a.wallet.GetTransactionsSummary(transactions) - start, end := getPageRange(len(txSummary), filter.From, filter.Count) - return NewSuccessResponse(txSummary[start:end]) + return NewSuccessResponse(txSummary) } - start, end := getPageRange(len(transactions), filter.From, filter.Count) - return NewSuccessResponse(transactions[start:end]) + + return NewSuccessResponse(transactions) } // POST /get-unconfirmed-transaction diff --git a/database/leveldb/db.go b/database/leveldb/db.go index 38cab5bd..39f0772f 100644 --- a/database/leveldb/db.go +++ b/database/leveldb/db.go @@ -1,4 +1,4 @@ -package db +package leveldb import . "github.com/tendermint/tmlibs/common" @@ -12,6 +12,7 @@ type DB interface { NewBatch() Batch Iterator() Iterator IteratorPrefix([]byte) Iterator + IteratorPrefixWithStart(Prefix, start []byte) Iterator // For debugging Print() diff --git a/database/leveldb/db_test.go b/database/leveldb/db_test.go new file mode 100644 index 00000000..ccd37ed9 --- /dev/null +++ b/database/leveldb/db_test.go @@ -0,0 +1,92 @@ +package leveldb + +import ( + "fmt" + "io/ioutil" + "os" + "testing" + + "github.com/stretchr/testify/require" +) + +func newTempDB(t *testing.T, backend string) (db DB, dbDir string) { + dirname, err := ioutil.TempDir("", "db_common_test") + require.Nil(t, err) + return NewDB("testdb", backend, dirname), dirname +} + +func TestDBIteratorSingleKey(t *testing.T) { + for backend := range backends { + t.Run(fmt.Sprintf("Backend %s", backend), func(t *testing.T) { + db, dir := newTempDB(t, backend) + defer os.RemoveAll(dir) + + db.Set([]byte("1"), []byte("value_1")) + itr := db.IteratorPrefixWithStart(nil, nil) + require.Equal(t, []byte(""), itr.Key()) + require.Equal(t, true, itr.Next()) + require.Equal(t, []byte("1"), itr.Key()) + }) + } +} + +func TestDBIteratorTwoKeys(t *testing.T) { + for backend := range backends { + t.Run(fmt.Sprintf("Backend %s", backend), func(t *testing.T) { + db, dir := newTempDB(t, backend) + defer os.RemoveAll(dir) + + db.SetSync([]byte("1"), []byte("value_1")) + db.SetSync([]byte("2"), []byte("value_1")) + + itr := db.IteratorPrefixWithStart(nil, []byte("1")) + + require.Equal(t, []byte("1"), itr.Key()) + + require.Equal(t, true, itr.Next()) + itr = db.IteratorPrefixWithStart(nil, []byte("2")) + + require.Equal(t, false, itr.Next()) + }) + } +} + +func TestDBIterator(t *testing.T) { + dirname, err := ioutil.TempDir("", "db_common_test") + require.Nil(t, err) + + db, err := NewGoLevelDB("testdb", dirname) + if err != nil { + t.Fatal(err) + } + + defer func() { + db.Close() + os.RemoveAll(dirname) + }() + + db.SetSync([]byte("aaa1"), []byte("value_1")) + db.SetSync([]byte("aaa22"), []byte("value_2")) + db.SetSync([]byte("bbb22"), []byte("value_3")) + + itr := db.IteratorPrefixWithStart([]byte("aaa"), []byte("aaa1")) + defer itr.Release() + + require.Equal(t, true, itr.Next()) + require.Equal(t, []byte("aaa22"), itr.Key()) + + require.Equal(t, false, itr.Next()) + + itr = db.IteratorPrefixWithStart([]byte("aaa"), nil) + + require.Equal(t, true, itr.Next()) + require.Equal(t, []byte("aaa1"), itr.Key()) + + require.Equal(t, true, itr.Next()) + require.Equal(t, []byte("aaa22"), itr.Key()) + + require.Equal(t, false, itr.Next()) + + itr = db.IteratorPrefixWithStart([]byte("bbb"), []byte("aaa1")) + require.Equal(t, false, itr.Next()) +} diff --git a/database/leveldb/go_level_db.go b/database/leveldb/go_level_db.go index e9e8d3dd..7a2b8927 100644 --- a/database/leveldb/go_level_db.go +++ b/database/leveldb/go_level_db.go @@ -1,4 +1,4 @@ -package db +package leveldb import ( "fmt" @@ -119,6 +119,18 @@ func (db *GoLevelDB) Stats() map[string]string { type goLevelDBIterator struct { source iterator.Iterator + start []byte +} + +func newGoLevelDBIterator(source iterator.Iterator, start []byte) *goLevelDBIterator { + if start != nil { + source.Seek(start) + } + + return &goLevelDBIterator{ + source: source, + start: start, + } } // Key returns a copy of the current key. @@ -148,6 +160,7 @@ func (it *goLevelDBIterator) Error() error { } func (it *goLevelDBIterator) Next() bool { + it.assertNoError() return it.source.Next() } @@ -155,12 +168,23 @@ func (it *goLevelDBIterator) Release() { it.source.Release() } +func (it *goLevelDBIterator) assertNoError() { + if err := it.source.Error(); err != nil { + panic(err) + } +} + func (db *GoLevelDB) Iterator() Iterator { - return &goLevelDBIterator{db.db.NewIterator(nil, nil)} + return &goLevelDBIterator{source: db.db.NewIterator(nil, nil)} } func (db *GoLevelDB) IteratorPrefix(prefix []byte) Iterator { - return &goLevelDBIterator{db.db.NewIterator(util.BytesPrefix(prefix), nil)} + return &goLevelDBIterator{source: db.db.NewIterator(util.BytesPrefix(prefix), nil)} +} + +func (db *GoLevelDB) IteratorPrefixWithStart(Prefix, start []byte) Iterator { + itr := db.db.NewIterator(util.BytesPrefix(Prefix), nil) + return newGoLevelDBIterator(itr, start) } func (db *GoLevelDB) NewBatch() Batch { diff --git a/database/leveldb/go_level_db_test.go b/database/leveldb/go_level_db_test.go index 2cd3192c..fc40bfbc 100644 --- a/database/leveldb/go_level_db_test.go +++ b/database/leveldb/go_level_db_test.go @@ -1,4 +1,4 @@ -package db +package leveldb import ( "bytes" @@ -30,7 +30,7 @@ func BenchmarkRandomReadsWrites(b *testing.B) { // Write something { idx := (int64(RandInt()) % numItems) - internal[idx] += 1 + internal[idx]++ val := internal[idx] idxBytes := int642Bytes(int64(idx)) valBytes := int642Bytes(int64(val)) diff --git a/database/leveldb/mem_db.go b/database/leveldb/mem_db.go index 62f40fc6..63e2b76c 100644 --- a/database/leveldb/mem_db.go +++ b/database/leveldb/mem_db.go @@ -1,6 +1,7 @@ -package db +package leveldb import ( + "bytes" "fmt" "sort" "strings" @@ -78,13 +79,29 @@ func (db *MemDB) Stats() map[string]string { type memDBIterator struct { last int keys []string - db *MemDB + db DB + + start []byte } func newMemDBIterator() *memDBIterator { return &memDBIterator{} } +// Keys is expected to be in reverse order for reverse iterators. +func newMemDBIteratorWithArgs(db DB, keys []string, start []byte) *memDBIterator { + itr := &memDBIterator{ + db: db, + keys: keys, + start: start, + last: -1, + } + if start != nil { + itr.Seek(start) + } + return itr +} + func (it *memDBIterator) Next() bool { if it.last >= len(it.keys)-1 { return false @@ -94,6 +111,9 @@ func (it *memDBIterator) Next() bool { } func (it *memDBIterator) Key() []byte { + if it.last < 0 { + return []byte("") + } return []byte(it.keys[it.last]) } @@ -143,10 +163,30 @@ func (db *MemDB) IteratorPrefix(prefix []byte) Iterator { return it } +func (db *MemDB) IteratorPrefixWithStart(Prefix, start []byte) Iterator { + db.mtx.Lock() + defer db.mtx.Unlock() + + keys := db.getSortedKeys(start) + return newMemDBIteratorWithArgs(db, keys, start) +} + func (db *MemDB) NewBatch() Batch { return &memDBBatch{db, nil} } +func (db *MemDB) getSortedKeys(start []byte) []string { + keys := []string{} + for key := range db.db { + if bytes.Compare([]byte(key), start) < 0 { + continue + } + keys = append(keys, key) + } + sort.Strings(keys) + return keys +} + //-------------------------------------------------------------------------------- type memDBBatch struct { diff --git a/database/leveldb/mem_db_test.go b/database/leveldb/mem_db_test.go index 503e361f..459f8722 100644 --- a/database/leveldb/mem_db_test.go +++ b/database/leveldb/mem_db_test.go @@ -1,4 +1,4 @@ -package db +package leveldb import ( "testing" @@ -23,7 +23,7 @@ func TestMemDbIterator(t *testing.T) { i := 0 for iter.Next() { assert.Equal(t, db.Get(iter.Key()), iter.Value(), "values dont match for key") - i += 1 + i++ } assert.Equal(t, i, len(db.db), "iterator didnt cover whole db") } diff --git a/wallet/indexer.go b/wallet/indexer.go index bc61dc33..46017791 100644 --- a/wallet/indexer.go +++ b/wallet/indexer.go @@ -30,7 +30,7 @@ const ( GlobalTxIndexPrefix = "GTID:" ) -var errAccntTxIDNotFound = errors.New("account TXID not found") +var ErrAccntTxIDNotFound = errors.New("account TXID not found") func formatKey(blockHeight uint64, position uint32) string { return fmt.Sprintf("%016x%08x", blockHeight, position) @@ -198,7 +198,7 @@ func (w *Wallet) getAccountTxByTxID(txID string) (*query.AnnotatedTx, error) { annotatedTx := &query.AnnotatedTx{} formatKey := w.DB.Get(calcTxIndexKey(txID)) if formatKey == nil { - return nil, errAccntTxIDNotFound + return nil, ErrAccntTxIDNotFound } txInfo := w.DB.Get(calcAnnotatedKey(string(formatKey))) @@ -288,15 +288,34 @@ func findTransactionsByAccount(annotatedTx *query.AnnotatedTx, accountID string) return false } -// GetTransactions get all walletDB transactions, and filter transactions by accountID optional -func (w *Wallet) GetTransactions(accountID string) ([]*query.AnnotatedTx, error) { +// GetTransactions get all walletDB transactions or unconfirmed transactions, and filter transactions by accountID and StartTxID optional +func (w *Wallet) GetTransactions(accountID string, StartTxID string, count uint, unconfirmed bool) ([]*query.AnnotatedTx, error) { annotatedTxs := []*query.AnnotatedTx{} + var startKey []byte + preFix := TxPrefix - txIter := w.DB.IteratorPrefix([]byte(TxPrefix)) - defer txIter.Release() - for txIter.Next() { + if StartTxID != "" { + if unconfirmed { + startKey = calcUnconfirmedTxKey(StartTxID) + } else { + formatKey := w.DB.Get(calcTxIndexKey(StartTxID)) + if formatKey == nil { + return nil, ErrAccntTxIDNotFound + } + startKey = calcAnnotatedKey(string(formatKey)) + } + } + + if unconfirmed { + preFix = UnconfirmedTxPrefix + } + + itr := w.DB.IteratorPrefixWithStart([]byte(preFix), startKey) + defer itr.Release() + + for txNum := count; itr.Next() && txNum > 0; txNum-- { annotatedTx := &query.AnnotatedTx{} - if err := json.Unmarshal(txIter.Value(), &annotatedTx); err != nil { + if err := json.Unmarshal(itr.Value(), &annotatedTx); err != nil { return nil, err } @@ -306,6 +325,10 @@ func (w *Wallet) GetTransactions(accountID string) ([]*query.AnnotatedTx, error) } } + if unconfirmed { + sort.Sort(SortByTimestamp(annotatedTxs)) + } + return annotatedTxs, nil } diff --git a/wallet/wallet_test.go b/wallet/wallet_test.go index 823eab9e..ac966f48 100644 --- a/wallet/wallet_test.go +++ b/wallet/wallet_test.go @@ -186,7 +186,7 @@ func TestWalletUpdate(t *testing.T) { t.Fatal(err) } - wants, err := w.GetTransactions("") + wants, err := w.GetTransactions(testAccount.ID, "", 1, false) if len(wants) != 1 { t.Fatal(err) }