1 // Copyright (c) 2015-2016 The btcsuite developers
2 // Use of this source code is governed by an ISC
3 // license that can be found in the LICENSE file.
5 // This file is part of the ffldb package rather than the ffldb_test package as
6 // it provides whitebox testing.
20 "github.com/btcsuite/btcd/chaincfg"
21 "github.com/btcsuite/btcd/database"
22 "github.com/btcsuite/btcd/wire"
23 "github.com/btcsuite/btcutil"
24 "github.com/btcsuite/goleveldb/leveldb"
25 ldberrors "github.com/btcsuite/goleveldb/leveldb/errors"
29 // blockDataNet is the expected network in the test block data.
30 blockDataNet = wire.MainNet
32 // blockDataFile is the path to a file containing the first 256 blocks
33 // of the block chain.
34 blockDataFile = filepath.Join("..", "testdata", "blocks1-256.bz2")
36 // errSubTestFail is used to signal that a sub test returned false.
37 errSubTestFail = fmt.Errorf("sub test failure")
40 // loadBlocks loads the blocks contained in the testdata directory and returns
42 func loadBlocks(t *testing.T, dataFile string, network wire.BitcoinNet) ([]*btcutil.Block, error) {
43 // Open the file that contains the blocks for reading.
44 fi, err := os.Open(dataFile)
46 t.Errorf("failed to open file %v, err %v", dataFile, err)
50 if err := fi.Close(); err != nil {
51 t.Errorf("failed to close file %v %v", dataFile,
55 dr := bzip2.NewReader(fi)
57 // Set the first block as the genesis block.
58 blocks := make([]*btcutil.Block, 0, 256)
59 genesis := btcutil.NewBlock(chaincfg.MainNetParams.GenesisBlock)
60 blocks = append(blocks, genesis)
62 // Load the remaining blocks.
63 for height := 1; ; height++ {
65 err := binary.Read(dr, binary.LittleEndian, &net)
67 // Hit end of file at the expected offset. No error.
71 t.Errorf("Failed to load network type for block %d: %v",
75 if net != uint32(network) {
76 t.Errorf("Block doesn't match network: %v expects %v",
82 err = binary.Read(dr, binary.LittleEndian, &blockLen)
84 t.Errorf("Failed to load block size for block %d: %v",
90 blockBytes := make([]byte, blockLen)
91 _, err = io.ReadFull(dr, blockBytes)
93 t.Errorf("Failed to load block %d: %v", height, err)
97 // Deserialize and store the block.
98 block, err := btcutil.NewBlockFromBytes(blockBytes)
100 t.Errorf("Failed to parse block %v: %v", height, err)
103 blocks = append(blocks, block)
109 // checkDbError ensures the passed error is a database.Error with an error code
110 // that matches the passed error code.
111 func checkDbError(t *testing.T, testName string, gotErr error, wantErrCode database.ErrorCode) bool {
112 dbErr, ok := gotErr.(database.Error)
114 t.Errorf("%s: unexpected error type - got %T, want %T",
115 testName, gotErr, database.Error{})
118 if dbErr.ErrorCode != wantErrCode {
119 t.Errorf("%s: unexpected error code - got %s (%s), want %s",
120 testName, dbErr.ErrorCode, dbErr.Description,
128 // testContext is used to store context information about a running test which
129 // is passed into helper functions.
130 type testContext struct {
133 files map[uint32]*lockableFile
134 maxFileSizes map[uint32]int64
135 blocks []*btcutil.Block
138 // TestConvertErr ensures the leveldb error to database error conversion works
140 func TestConvertErr(t *testing.T) {
145 wantErrCode database.ErrorCode
147 {&ldberrors.ErrCorrupted{}, database.ErrCorruption},
148 {leveldb.ErrClosed, database.ErrDbNotOpen},
149 {leveldb.ErrSnapshotReleased, database.ErrTxClosed},
150 {leveldb.ErrIterReleased, database.ErrTxClosed},
153 for i, test := range tests {
154 gotErr := convertErr("test", test.err)
155 if gotErr.ErrorCode != test.wantErrCode {
156 t.Errorf("convertErr #%d unexpected error - got %v, "+
157 "want %v", i, gotErr.ErrorCode, test.wantErrCode)
163 // TestCornerCases ensures several corner cases which can happen when opening
164 // a database and/or block files work as expected.
165 func TestCornerCases(t *testing.T) {
168 // Create a file at the datapase path to force the open below to fail.
169 dbPath := filepath.Join(os.TempDir(), "ffldb-errors")
170 _ = os.RemoveAll(dbPath)
171 fi, err := os.Create(dbPath)
173 t.Errorf("os.Create: unexpected error: %v", err)
178 // Ensure creating a new database fails when a file exists where a
179 // directory is needed.
180 testName := "openDB: fail due to file at target location"
181 wantErrCode := database.ErrDriverSpecific
182 idb, err := openDB(dbPath, blockDataNet, true)
183 if !checkDbError(t, testName, err, wantErrCode) {
187 _ = os.RemoveAll(dbPath)
191 // Remove the file and create the database to run tests against. It
192 // should be successful this time.
193 _ = os.RemoveAll(dbPath)
194 idb, err = openDB(dbPath, blockDataNet, true)
196 t.Errorf("openDB: unexpected error: %v", err)
199 defer os.RemoveAll(dbPath)
202 // Ensure attempting to write to a file that can't be created returns
203 // the expected error.
204 testName = "writeBlock: open file failure"
205 filePath := blockFilePath(dbPath, 0)
206 if err := os.Mkdir(filePath, 0755); err != nil {
207 t.Errorf("os.Mkdir: unexpected error: %v", err)
210 store := idb.(*db).store
211 _, err = store.writeBlock([]byte{0x00})
212 if !checkDbError(t, testName, err, database.ErrDriverSpecific) {
215 _ = os.RemoveAll(filePath)
217 // Close the underlying leveldb database out from under the database.
218 ldb := idb.(*db).cache.ldb
221 // Ensure initilization errors in the underlying database work as
223 testName = "initDB: reinitialization"
224 wantErrCode = database.ErrDbNotOpen
226 if !checkDbError(t, testName, err, wantErrCode) {
230 // Ensure the View handles errors in the underlying leveldb database
232 testName = "View: underlying leveldb error"
233 wantErrCode = database.ErrDbNotOpen
234 err = idb.View(func(tx database.Tx) error {
237 if !checkDbError(t, testName, err, wantErrCode) {
241 // Ensure the Update handles errors in the underlying leveldb database
243 testName = "Update: underlying leveldb error"
244 err = idb.Update(func(tx database.Tx) error {
247 if !checkDbError(t, testName, err, wantErrCode) {
252 // resetDatabase removes everything from the opened database associated with the
253 // test context including all metadata and the mock files.
254 func resetDatabase(tc *testContext) bool {
255 // Reset the metadata.
256 err := tc.db.Update(func(tx database.Tx) error {
257 // Remove all the keys using a cursor while also generating a
258 // list of buckets. It's not safe to remove keys during ForEach
259 // iteration nor is it safe to remove buckets during cursor
260 // iteration, so this dual approach is needed.
261 var bucketNames [][]byte
262 cursor := tx.Metadata().Cursor()
263 for ok := cursor.First(); ok; ok = cursor.Next() {
264 if cursor.Value() != nil {
265 if err := cursor.Delete(); err != nil {
269 bucketNames = append(bucketNames, cursor.Key())
273 // Remove the buckets.
274 for _, k := range bucketNames {
275 if err := tx.Metadata().DeleteBucket(k); err != nil {
280 _, err := tx.Metadata().CreateBucket(blockIdxBucketName)
284 tc.t.Errorf("Update: unexpected error: %v", err)
288 // Reset the mock files.
289 store := tc.db.(*db).store
290 wc := store.writeCursor
292 if wc.curFile.file != nil {
293 wc.curFile.file.Close()
294 wc.curFile.file = nil
301 tc.files = make(map[uint32]*lockableFile)
302 tc.maxFileSizes = make(map[uint32]int64)
306 // testWriteFailures tests various failures paths when writing to the block
308 func testWriteFailures(tc *testContext) bool {
309 if !resetDatabase(tc) {
313 // Ensure file sync errors during flush return the expected error.
314 store := tc.db.(*db).store
315 testName := "flush: file sync failure"
316 store.writeCursor.Lock()
317 oldFile := store.writeCursor.curFile
318 store.writeCursor.curFile = &lockableFile{
319 file: &mockFile{forceSyncErr: true, maxSize: -1},
321 store.writeCursor.Unlock()
322 err := tc.db.(*db).cache.flush()
323 if !checkDbError(tc.t, testName, err, database.ErrDriverSpecific) {
326 store.writeCursor.Lock()
327 store.writeCursor.curFile = oldFile
328 store.writeCursor.Unlock()
330 // Force errors in the various error paths when writing data by using
331 // mock files with a limited max size.
332 block0Bytes, _ := tc.blocks[0].Bytes()
337 // Force an error when writing the network bytes.
338 {fileNum: 0, maxSize: 2},
340 // Force an error when writing the block size.
341 {fileNum: 0, maxSize: 6},
343 // Force an error when writing the block.
344 {fileNum: 0, maxSize: 17},
346 // Force an error when writing the checksum.
347 {fileNum: 0, maxSize: int64(len(block0Bytes)) + 10},
349 // Force an error after writing enough blocks for force multiple
351 {fileNum: 15, maxSize: 1},
354 for i, test := range tests {
355 if !resetDatabase(tc) {
359 // Ensure storing the specified number of blocks using a mock
360 // file that fails the write fails when the transaction is
361 // committed, not when the block is stored.
362 tc.maxFileSizes = map[uint32]int64{test.fileNum: test.maxSize}
363 err := tc.db.Update(func(tx database.Tx) error {
364 for i, block := range tc.blocks {
365 err := tx.StoreBlock(block)
367 tc.t.Errorf("StoreBlock (%d): unexpected "+
369 return errSubTestFail
375 testName := fmt.Sprintf("Force update commit failure - test "+
376 "%d, fileNum %d, maxsize %d", i, test.fileNum,
378 if !checkDbError(tc.t, testName, err, database.ErrDriverSpecific) {
379 tc.t.Errorf("%v", err)
383 // Ensure the commit rollback removed all extra files and data.
384 if len(tc.files) != 1 {
385 tc.t.Errorf("Update rollback: new not removed - want "+
386 "1 file, got %d", len(tc.files))
389 if _, ok := tc.files[0]; !ok {
390 tc.t.Error("Update rollback: file 0 does not exist")
393 file := tc.files[0].file.(*mockFile)
394 if len(file.data) != 0 {
395 tc.t.Errorf("Update rollback: file did not truncate - "+
396 "want len 0, got len %d", len(file.data))
404 // testBlockFileErrors ensures the database returns expected errors with various
405 // file-related issues such as closed and missing files.
406 func testBlockFileErrors(tc *testContext) bool {
407 if !resetDatabase(tc) {
411 // Ensure errors in blockFile and openFile when requesting invalid file
413 store := tc.db.(*db).store
414 testName := "blockFile invalid file open"
415 _, err := store.blockFile(^uint32(0))
416 if !checkDbError(tc.t, testName, err, database.ErrDriverSpecific) {
419 testName = "openFile invalid file open"
420 _, err = store.openFile(^uint32(0))
421 if !checkDbError(tc.t, testName, err, database.ErrDriverSpecific) {
425 // Insert the first block into the mock file.
426 err = tc.db.Update(func(tx database.Tx) error {
427 err := tx.StoreBlock(tc.blocks[0])
429 tc.t.Errorf("StoreBlock: unexpected error: %v", err)
430 return errSubTestFail
436 if err != errSubTestFail {
437 tc.t.Errorf("Update: unexpected error: %v", err)
442 // Ensure errors in readBlock and readBlockRegion when requesting a file
443 // number that doesn't exist.
444 block0Hash := tc.blocks[0].Hash()
445 testName = "readBlock invalid file number"
446 invalidLoc := blockLocation{
447 blockFileNum: ^uint32(0),
450 _, err = store.readBlock(block0Hash, invalidLoc)
451 if !checkDbError(tc.t, testName, err, database.ErrDriverSpecific) {
454 testName = "readBlockRegion invalid file number"
455 _, err = store.readBlockRegion(invalidLoc, 0, 80)
456 if !checkDbError(tc.t, testName, err, database.ErrDriverSpecific) {
460 // Close the block file out from under the database.
461 store.writeCursor.curFile.Lock()
462 store.writeCursor.curFile.file.Close()
463 store.writeCursor.curFile.Unlock()
465 // Ensure failures in FetchBlock and FetchBlockRegion(s) since the
466 // underlying file they need to read from has been closed.
467 err = tc.db.View(func(tx database.Tx) error {
468 testName = "FetchBlock closed file"
469 wantErrCode := database.ErrDriverSpecific
470 _, err := tx.FetchBlock(block0Hash)
471 if !checkDbError(tc.t, testName, err, wantErrCode) {
472 return errSubTestFail
475 testName = "FetchBlockRegion closed file"
476 regions := []database.BlockRegion{
483 _, err = tx.FetchBlockRegion(®ions[0])
484 if !checkDbError(tc.t, testName, err, wantErrCode) {
485 return errSubTestFail
488 testName = "FetchBlockRegions closed file"
489 _, err = tx.FetchBlockRegions(regions)
490 if !checkDbError(tc.t, testName, err, wantErrCode) {
491 return errSubTestFail
497 if err != errSubTestFail {
498 tc.t.Errorf("View: unexpected error: %v", err)
506 // testCorruption ensures the database returns expected errors under various
507 // corruption scenarios.
508 func testCorruption(tc *testContext) bool {
509 if !resetDatabase(tc) {
513 // Insert the first block into the mock file.
514 err := tc.db.Update(func(tx database.Tx) error {
515 err := tx.StoreBlock(tc.blocks[0])
517 tc.t.Errorf("StoreBlock: unexpected error: %v", err)
518 return errSubTestFail
524 if err != errSubTestFail {
525 tc.t.Errorf("Update: unexpected error: %v", err)
530 // Ensure corruption is detected by intentionally modifying the bytes
531 // stored to the mock file and reading the block.
532 block0Bytes, _ := tc.blocks[0].Bytes()
533 block0Hash := tc.blocks[0].Hash()
537 wantErrCode database.ErrorCode
539 // One of the network bytes. The checksum needs to be fixed so
540 // the invalid network is detected.
541 {2, true, database.ErrDriverSpecific},
543 // The same network byte, but this time don't fix the checksum
544 // to ensure the corruption is detected.
545 {2, false, database.ErrCorruption},
547 // One of the block length bytes.
548 {6, false, database.ErrCorruption},
550 // Random header byte.
551 {17, false, database.ErrCorruption},
553 // Random transaction byte.
554 {90, false, database.ErrCorruption},
556 // Random checksum byte.
557 {uint32(len(block0Bytes)) + 10, false, database.ErrCorruption},
559 err = tc.db.View(func(tx database.Tx) error {
560 data := tc.files[0].file.(*mockFile).data
561 for i, test := range tests {
562 // Corrupt the byte at the offset by a single bit.
563 data[test.offset] ^= 0x10
565 // Fix the checksum if requested to force other errors.
567 var oldChecksumBytes [4]byte
568 copy(oldChecksumBytes[:], data[fileLen-4:])
569 if test.fixChecksum {
570 toSum := data[:fileLen-4]
571 cksum := crc32.Checksum(toSum, castagnoli)
572 binary.BigEndian.PutUint32(data[fileLen-4:], cksum)
575 testName := fmt.Sprintf("FetchBlock (test #%d): "+
577 _, err := tx.FetchBlock(block0Hash)
578 if !checkDbError(tc.t, testName, err, test.wantErrCode) {
579 return errSubTestFail
582 // Reset the corrupted data back to the original.
583 data[test.offset] ^= 0x10
584 if test.fixChecksum {
585 copy(data[fileLen-4:], oldChecksumBytes[:])
592 if err != errSubTestFail {
593 tc.t.Errorf("View: unexpected error: %v", err)
601 // TestFailureScenarios ensures several failure scenarios such as database
602 // corruption, block file write failures, and rollback failures are handled
604 func TestFailureScenarios(t *testing.T) {
605 // Create a new database to run tests against.
606 dbPath := filepath.Join(os.TempDir(), "ffldb-failurescenarios")
607 _ = os.RemoveAll(dbPath)
608 idb, err := database.Create(dbType, dbPath, blockDataNet)
610 t.Errorf("Failed to create test database (%s) %v", dbType, err)
613 defer os.RemoveAll(dbPath)
616 // Create a test context to pass around.
620 files: make(map[uint32]*lockableFile),
621 maxFileSizes: make(map[uint32]int64),
624 // Change the maximum file size to a small value to force multiple flat
625 // files with the test data set and replace the file-related functions
626 // to make use of mock files in memory. This allows injection of
627 // various file-related errors.
628 store := idb.(*db).store
629 store.maxBlockFileSize = 1024 // 1KiB
630 store.openWriteFileFunc = func(fileNum uint32) (filer, error) {
631 if file, ok := tc.files[fileNum]; ok {
632 // "Reopen" the file.
634 mock := file.file.(*mockFile)
642 // Limit the max size of the mock file as specified in the test
645 if maxFileSize, ok := tc.maxFileSizes[fileNum]; ok {
646 maxSize = int64(maxFileSize)
648 file := &mockFile{maxSize: int64(maxSize)}
649 tc.files[fileNum] = &lockableFile{file: file}
652 store.openFileFunc = func(fileNum uint32) (*lockableFile, error) {
653 // Force error when trying to open max file num.
654 if fileNum == ^uint32(0) {
655 return nil, makeDbErr(database.ErrDriverSpecific,
658 if file, ok := tc.files[fileNum]; ok {
659 // "Reopen" the file.
661 mock := file.file.(*mockFile)
668 file := &lockableFile{file: &mockFile{}}
669 tc.files[fileNum] = file
672 store.deleteFileFunc = func(fileNum uint32) error {
673 if file, ok := tc.files[fileNum]; ok {
677 delete(tc.files, fileNum)
681 str := fmt.Sprintf("file %d does not exist", fileNum)
682 return makeDbErr(database.ErrDriverSpecific, str, nil)
685 // Load the test blocks and save in the test context for use throughout
687 blocks, err := loadBlocks(t, blockDataFile, blockDataNet)
689 t.Errorf("loadBlocks: Unexpected error: %v", err)
694 // Test various failures paths when writing to the block files.
695 if !testWriteFailures(tc) {
699 // Test various file-related issues such as closed and missing files.
700 if !testBlockFileErrors(tc) {
704 // Test various corruption scenarios.