fs/btrfs/disk-io.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * Copyright (C) 2007 Oracle.  All rights reserved.
   4  */
   5
   6 #include <linux/fs.h>
   7 #include <linux/blkdev.h>
   8 #include <linux/radix-tree.h>
   9 #include <linux/writeback.h>
  10 #include <linux/buffer_head.h>
  11 #include <linux/workqueue.h>
  12 #include <linux/kthread.h>
  13 #include <linux/slab.h>
  14 #include <linux/migrate.h>
  15 #include <linux/ratelimit.h>
  16 #include <linux/uuid.h>
  17 #include <linux/semaphore.h>
  18 #include <linux/error-injection.h>
  19 #include <linux/crc32c.h>
  20 #include <linux/sched/mm.h>
  21 #include <asm/unaligned.h>
  22 #include <crypto/hash.h>
  23 #include "ctree.h"
  24 #include "disk-io.h"
  25 #include "transaction.h"
  26 #include "btrfs_inode.h"
  27 #include "volumes.h"
  28 #include "print-tree.h"
  29 #include "locking.h"
  30 #include "tree-log.h"
  31 #include "free-space-cache.h"
  32 #include "free-space-tree.h"
  33 #include "inode-map.h"
  34 #include "check-integrity.h"
  35 #include "rcu-string.h"
  36 #include "dev-replace.h"
  37 #include "raid56.h"
  38 #include "sysfs.h"
  39 #include "qgroup.h"
  40 #include "compression.h"
  41 #include "tree-checker.h"
  42 #include "ref-verify.h"
  43 #include "block-group.h"
  44 #include "discard.h"
  45
  46 #define BTRFS_SUPER_FLAG_SUPP   (BTRFS_HEADER_FLAG_WRITTEN |\
  47                                  BTRFS_HEADER_FLAG_RELOC |\
  48                                  BTRFS_SUPER_FLAG_ERROR |\
  49                                  BTRFS_SUPER_FLAG_SEEDING |\
  50                                  BTRFS_SUPER_FLAG_METADUMP |\
  51                                  BTRFS_SUPER_FLAG_METADUMP_V2)
  52
  53 static const struct extent_io_ops btree_extent_io_ops;
  54 static void end_workqueue_fn(struct btrfs_work *work);
  55 static void btrfs_destroy_ordered_extents(struct btrfs_root *root);
  56 static int btrfs_destroy_delayed_refs(struct btrfs_transaction *trans,
  57                                       struct btrfs_fs_info *fs_info);
  58 static void btrfs_destroy_delalloc_inodes(struct btrfs_root *root);
  59 static int btrfs_destroy_marked_extents(struct btrfs_fs_info *fs_info,
  60                                         struct extent_io_tree *dirty_pages,
  61                                         int mark);
  62 static int btrfs_destroy_pinned_extent(struct btrfs_fs_info *fs_info,
  63                                        struct extent_io_tree *pinned_extents);
  64 static int btrfs_cleanup_transaction(struct btrfs_fs_info *fs_info);
  65 static void btrfs_error_commit_super(struct btrfs_fs_info *fs_info);
  66
  67 /*
  68  * btrfs_end_io_wq structs are used to do processing in task context when an IO
  69  * is complete.  This is used during reads to verify checksums, and it is used
  70  * by writes to insert metadata for new file extents after IO is complete.
  71  */
  72 struct btrfs_end_io_wq {
  73         struct bio *bio;
  74         bio_end_io_t *end_io;
  75         void *private;
  76         struct btrfs_fs_info *info;
  77         blk_status_t status;
  78         enum btrfs_wq_endio_type metadata;
  79         struct btrfs_work work;
  80 };
  81
  82 static struct kmem_cache *btrfs_end_io_wq_cache;
  83
  84 int __init btrfs_end_io_wq_init(void)
  85 {
  86         btrfs_end_io_wq_cache = kmem_cache_create("btrfs_end_io_wq",
  87                                         sizeof(struct btrfs_end_io_wq),
  88                                         0,
  89                                         SLAB_MEM_SPREAD,
  90                                         NULL);
  91         if (!btrfs_end_io_wq_cache)
  92                 return -ENOMEM;
  93         return 0;
  94 }
  95
  96 void __cold btrfs_end_io_wq_exit(void)
  97 {
  98         kmem_cache_destroy(btrfs_end_io_wq_cache);
  99 }
 100
 101 /*
 102  * async submit bios are used to offload expensive checksumming
 103  * onto the worker threads.  They checksum file and metadata bios
 104  * just before they are sent down the IO stack.
 105  */
 106 struct async_submit_bio {
 107         void *private_data;
 108         struct bio *bio;
 109         extent_submit_bio_start_t *submit_bio_start;
 110         int mirror_num;
 111         /*
 112          * bio_offset is optional, can be used if the pages in the bio
 113          * can't tell us where in the file the bio should go
 114          */
 115         u64 bio_offset;
 116         struct btrfs_work work;
 117         blk_status_t status;
 118 };
 119
 120 /*
 121  * Lockdep class keys for extent_buffer->lock's in this root.  For a given
 122  * eb, the lockdep key is determined by the btrfs_root it belongs to and
 123  * the level the eb occupies in the tree.
 124  *
 125  * Different roots are used for different purposes and may nest inside each
 126  * other and they require separate keysets.  As lockdep keys should be
 127  * static, assign keysets according to the purpose of the root as indicated
 128  * by btrfs_root->root_key.objectid.  This ensures that all special purpose
 129  * roots have separate keysets.
 130  *
 131  * Lock-nesting across peer nodes is always done with the immediate parent
 132  * node locked thus preventing deadlock.  As lockdep doesn't know this, use
 133  * subclass to avoid triggering lockdep warning in such cases.
 134  *
 135  * The key is set by the readpage_end_io_hook after the buffer has passed
 136  * csum validation but before the pages are unlocked.  It is also set by
 137  * btrfs_init_new_buffer on freshly allocated blocks.
 138  *
 139  * We also add a check to make sure the highest level of the tree is the
 140  * same as our lockdep setup here.  If BTRFS_MAX_LEVEL changes, this code
 141  * needs update as well.
 142  */
 143 #ifdef CONFIG_DEBUG_LOCK_ALLOC
 144 # if BTRFS_MAX_LEVEL != 8
 145 #  error
 146 # endif
 147
 148 static struct btrfs_lockdep_keyset {
 149         u64                     id;             /* root objectid */
 150         const char              *name_stem;     /* lock name stem */
 151         char                    names[BTRFS_MAX_LEVEL + 1][20];
 152         struct lock_class_key   keys[BTRFS_MAX_LEVEL + 1];
 153 } btrfs_lockdep_keysets[] = {
 154         { .id = BTRFS_ROOT_TREE_OBJECTID,       .name_stem = "root"     },
 155         { .id = BTRFS_EXTENT_TREE_OBJECTID,     .name_stem = "extent"   },
 156         { .id = BTRFS_CHUNK_TREE_OBJECTID,      .name_stem = "chunk"    },
 157         { .id = BTRFS_DEV_TREE_OBJECTID,        .name_stem = "dev"      },
 158         { .id = BTRFS_FS_TREE_OBJECTID,         .name_stem = "fs"       },
 159         { .id = BTRFS_CSUM_TREE_OBJECTID,       .name_stem = "csum"     },
 160         { .id = BTRFS_QUOTA_TREE_OBJECTID,      .name_stem = "quota"    },
 161         { .id = BTRFS_TREE_LOG_OBJECTID,        .name_stem = "log"      },
 162         { .id = BTRFS_TREE_RELOC_OBJECTID,      .name_stem = "treloc"   },
 163         { .id = BTRFS_DATA_RELOC_TREE_OBJECTID, .name_stem = "dreloc"   },
 164         { .id = BTRFS_UUID_TREE_OBJECTID,       .name_stem = "uuid"     },
 165         { .id = BTRFS_FREE_SPACE_TREE_OBJECTID, .name_stem = "free-space" },
 166         { .id = 0,                              .name_stem = "tree"     },
 167 };
 168
 169 void __init btrfs_init_lockdep(void)
 170 {
 171         int i, j;
 172
 173         /* initialize lockdep class names */
 174         for (i = 0; i < ARRAY_SIZE(btrfs_lockdep_keysets); i++) {
 175                 struct btrfs_lockdep_keyset *ks = &btrfs_lockdep_keysets[i];
 176
 177                 for (j = 0; j < ARRAY_SIZE(ks->names); j++)
 178                         snprintf(ks->names[j], sizeof(ks->names[j]),
 179                                  "btrfs-%s-%02d", ks->name_stem, j);
 180         }
 181 }
 182
 183 void btrfs_set_buffer_lockdep_class(u64 objectid, struct extent_buffer *eb,
 184                                     int level)
 185 {
 186         struct btrfs_lockdep_keyset *ks;
 187
 188         BUG_ON(level >= ARRAY_SIZE(ks->keys));
 189
 190         /* find the matching keyset, id 0 is the default entry */
 191         for (ks = btrfs_lockdep_keysets; ks->id; ks++)
 192                 if (ks->id == objectid)
 193                         break;
 194
 195         lockdep_set_class_and_name(&eb->lock,
 196                                    &ks->keys[level], ks->names[level]);
 197 }
 198
 199 #endif
 200
 201 /*
 202  * extents on the btree inode are pretty simple, there's one extent
 203  * that covers the entire device
 204  */
 205 struct extent_map *btree_get_extent(struct btrfs_inode *inode,
 206                                     struct page *page, size_t pg_offset,
 207                                     u64 start, u64 len)
 208 {
 209         struct extent_map_tree *em_tree = &inode->extent_tree;
 210         struct extent_map *em;
 211         int ret;
 212
 213         read_lock(&em_tree->lock);
 214         em = lookup_extent_mapping(em_tree, start, len);
 215         if (em) {
 216                 read_unlock(&em_tree->lock);
 217                 goto out;
 218         }
 219         read_unlock(&em_tree->lock);
 220
 221         em = alloc_extent_map();
 222         if (!em) {
 223                 em = ERR_PTR(-ENOMEM);
 224                 goto out;
 225         }
 226         em->start = 0;
 227         em->len = (u64)-1;
 228         em->block_len = (u64)-1;
 229         em->block_start = 0;
 230
 231         write_lock(&em_tree->lock);
 232         ret = add_extent_mapping(em_tree, em, 0);
 233         if (ret == -EEXIST) {
 234                 free_extent_map(em);
 235                 em = lookup_extent_mapping(em_tree, start, len);
 236                 if (!em)
 237                         em = ERR_PTR(-EIO);
 238         } else if (ret) {
 239                 free_extent_map(em);
 240                 em = ERR_PTR(ret);
 241         }
 242         write_unlock(&em_tree->lock);
 243
 244 out:
 245         return em;
 246 }
 247
 248 /*
 249  * Compute the csum of a btree block and store the result to provided buffer.
 250  *
 251  * Returns error if the extent buffer cannot be mapped.
 252  */
 253 static int csum_tree_block(struct extent_buffer *buf, u8 *result)
 254 {
 255         struct btrfs_fs_info *fs_info = buf->fs_info;
 256         SHASH_DESC_ON_STACK(shash, fs_info->csum_shash);
 257         unsigned long len;
 258         unsigned long cur_len;
 259         unsigned long offset = BTRFS_CSUM_SIZE;
 260         char *kaddr;
 261         unsigned long map_start;
 262         unsigned long map_len;
 263         int err;
 264
 265         shash->tfm = fs_info->csum_shash;
 266         crypto_shash_init(shash);
 267
 268         len = buf->len - offset;
 269
 270         while (len > 0) {
 271                 /*
 272                  * Note: we don't need to check for the err == 1 case here, as
 273                  * with the given combination of 'start = BTRFS_CSUM_SIZE (32)'
 274                  * and 'min_len = 32' and the currently implemented mapping
 275                  * algorithm we cannot cross a page boundary.
 276                  */
 277                 err = map_private_extent_buffer(buf, offset, 32,
 278                                         &kaddr, &map_start, &map_len);
 279                 if (WARN_ON(err))
 280                         return err;
 281                 cur_len = min(len, map_len - (offset - map_start));
 282                 crypto_shash_update(shash, kaddr + offset - map_start, cur_len);
 283                 len -= cur_len;
 284                 offset += cur_len;
 285         }
 286         memset(result, 0, BTRFS_CSUM_SIZE);
 287
 288         crypto_shash_final(shash, result);
 289
 290         return 0;
 291 }
 292
 293 /*
 294  * we can't consider a given block up to date unless the transid of the
 295  * block matches the transid in the parent node's pointer.  This is how we
 296  * detect blocks that either didn't get written at all or got written
 297  * in the wrong place.
 298  */
 299 static int verify_parent_transid(struct extent_io_tree *io_tree,
 300                                  struct extent_buffer *eb, u64 parent_transid,
 301                                  int atomic)
 302 {
 303         struct extent_state *cached_state = NULL;
 304         int ret;
 305         bool need_lock = (current->journal_info == BTRFS_SEND_TRANS_STUB);
 306
 307         if (!parent_transid || btrfs_header_generation(eb) == parent_transid)
 308                 return 0;
 309
 310         if (atomic)
 311                 return -EAGAIN;
 312
 313         if (need_lock) {
 314                 btrfs_tree_read_lock(eb);
 315                 btrfs_set_lock_blocking_read(eb);
 316         }
 317
 318         lock_extent_bits(io_tree, eb->start, eb->start + eb->len - 1,
 319                          &cached_state);
 320         if (extent_buffer_uptodate(eb) &&
 321             btrfs_header_generation(eb) == parent_transid) {
 322                 ret = 0;
 323                 goto out;
 324         }
 325         btrfs_err_rl(eb->fs_info,
 326                 "parent transid verify failed on %llu wanted %llu found %llu",
 327                         eb->start,
 328                         parent_transid, btrfs_header_generation(eb));
 329         ret = 1;
 330
 331         /*
 332          * Things reading via commit roots that don't have normal protection,
 333          * like send, can have a really old block in cache that may point at a
 334          * block that has been freed and re-allocated.  So don't clear uptodate
 335          * if we find an eb that is under IO (dirty/writeback) because we could
 336          * end up reading in the stale data and then writing it back out and
 337          * making everybody very sad.
 338          */
 339         if (!extent_buffer_under_io(eb))
 340                 clear_extent_buffer_uptodate(eb);
 341 out:
 342         unlock_extent_cached(io_tree, eb->start, eb->start + eb->len - 1,
 343                              &cached_state);
 344         if (need_lock)
 345                 btrfs_tree_read_unlock_blocking(eb);
 346         return ret;
 347 }
 348
 349 static bool btrfs_supported_super_csum(u16 csum_type)
 350 {
 351         switch (csum_type) {
 352         case BTRFS_CSUM_TYPE_CRC32:
 353         case BTRFS_CSUM_TYPE_XXHASH:
 354         case BTRFS_CSUM_TYPE_SHA256:
 355         case BTRFS_CSUM_TYPE_BLAKE2:
 356                 return true;
 357         default:
 358                 return false;
 359         }
 360 }
 361
 362 /*
 363  * Return 0 if the superblock checksum type matches the checksum value of that
 364  * algorithm. Pass the raw disk superblock data.
 365  */
 366 static int btrfs_check_super_csum(struct btrfs_fs_info *fs_info,
 367                                   char *raw_disk_sb)
 368 {
 369         struct btrfs_super_block *disk_sb =
 370                 (struct btrfs_super_block *)raw_disk_sb;
 371         char result[BTRFS_CSUM_SIZE];
 372         SHASH_DESC_ON_STACK(shash, fs_info->csum_shash);
 373
 374         shash->tfm = fs_info->csum_shash;
 375         crypto_shash_init(shash);
 376
 377         /*
 378          * The super_block structure does not span the whole
 379          * BTRFS_SUPER_INFO_SIZE range, we expect that the unused space is
 380          * filled with zeros and is included in the checksum.
 381          */
 382         crypto_shash_update(shash, raw_disk_sb + BTRFS_CSUM_SIZE,
 383                             BTRFS_SUPER_INFO_SIZE - BTRFS_CSUM_SIZE);
 384         crypto_shash_final(shash, result);
 385
 386         if (memcmp(disk_sb->csum, result, btrfs_super_csum_size(disk_sb)))
 387                 return 1;
 388
 389         return 0;
 390 }
 391
 392 int btrfs_verify_level_key(struct extent_buffer *eb, int level,
 393                            struct btrfs_key *first_key, u64 parent_transid)
 394 {
 395         struct btrfs_fs_info *fs_info = eb->fs_info;
 396         int found_level;
 397         struct btrfs_key found_key;
 398         int ret;
 399
 400         found_level = btrfs_header_level(eb);
 401         if (found_level != level) {
 402                 WARN(IS_ENABLED(CONFIG_BTRFS_DEBUG),
 403                      KERN_ERR "BTRFS: tree level check failed\n");
 404                 btrfs_err(fs_info,
 405 "tree level mismatch detected, bytenr=%llu level expected=%u has=%u",
 406                           eb->start, level, found_level);
 407                 return -EIO;
 408         }
 409
 410         if (!first_key)
 411                 return 0;
 412
 413         /*
 414          * For live tree block (new tree blocks in current transaction),
 415          * we need proper lock context to avoid race, which is impossible here.
 416          * So we only checks tree blocks which is read from disk, whose
 417          * generation <= fs_info->last_trans_committed.
 418          */
 419         if (btrfs_header_generation(eb) > fs_info->last_trans_committed)
 420                 return 0;
 421
 422         /* We have @first_key, so this @eb must have at least one item */
 423         if (btrfs_header_nritems(eb) == 0) {
 424                 btrfs_err(fs_info,
 425                 "invalid tree nritems, bytenr=%llu nritems=0 expect >0",
 426                           eb->start);
 427                 WARN_ON(IS_ENABLED(CONFIG_BTRFS_DEBUG));
 428                 return -EUCLEAN;
 429         }
 430
 431         if (found_level)
 432                 btrfs_node_key_to_cpu(eb, &found_key, 0);
 433         else
 434                 btrfs_item_key_to_cpu(eb, &found_key, 0);
 435         ret = btrfs_comp_cpu_keys(first_key, &found_key);
 436
 437         if (ret) {
 438                 WARN(IS_ENABLED(CONFIG_BTRFS_DEBUG),
 439                      KERN_ERR "BTRFS: tree first key check failed\n");
 440                 btrfs_err(fs_info,
 441 "tree first key mismatch detected, bytenr=%llu parent_transid=%llu key expected=(%llu,%u,%llu) has=(%llu,%u,%llu)",
 442                           eb->start, parent_transid, first_key->objectid,
 443                           first_key->type, first_key->offset,
 444                           found_key.objectid, found_key.type,
 445                           found_key.offset);
 446         }
 447         return ret;
 448 }
 449
 450 /*
 451  * helper to read a given tree block, doing retries as required when
 452  * the checksums don't match and we have alternate mirrors to try.
 453  *
 454  * @parent_transid:     expected transid, skip check if 0
 455  * @level:              expected level, mandatory check
 456  * @first_key:          expected key of first slot, skip check if NULL
 457  */
 458 static int btree_read_extent_buffer_pages(struct extent_buffer *eb,
 459                                           u64 parent_transid, int level,
 460                                           struct btrfs_key *first_key)
 461 {
 462         struct btrfs_fs_info *fs_info = eb->fs_info;
 463         struct extent_io_tree *io_tree;
 464         int failed = 0;
 465         int ret;
 466         int num_copies = 0;
 467         int mirror_num = 0;
 468         int failed_mirror = 0;
 469
 470         io_tree = &BTRFS_I(fs_info->btree_inode)->io_tree;
 471         while (1) {
 472                 clear_bit(EXTENT_BUFFER_CORRUPT, &eb->bflags);
 473                 ret = read_extent_buffer_pages(eb, WAIT_COMPLETE, mirror_num);
 474                 if (!ret) {
 475                         if (verify_parent_transid(io_tree, eb,
 476                                                    parent_transid, 0))
 477                                 ret = -EIO;
 478                         else if (btrfs_verify_level_key(eb, level,
 479                                                 first_key, parent_transid))
 480                                 ret = -EUCLEAN;
 481                         else
 482                                 break;
 483                 }
 484
 485                 num_copies = btrfs_num_copies(fs_info,
 486                                               eb->start, eb->len);
 487                 if (num_copies == 1)
 488                         break;
 489
 490                 if (!failed_mirror) {
 491                         failed = 1;
 492                         failed_mirror = eb->read_mirror;
 493                 }
 494
 495                 mirror_num++;
 496                 if (mirror_num == failed_mirror)
 497                         mirror_num++;
 498
 499                 if (mirror_num > num_copies)
 500                         break;
 501         }
 502
 503         if (failed && !ret && failed_mirror)
 504                 btrfs_repair_eb_io_failure(eb, failed_mirror);
 505
 506         return ret;
 507 }
 508
 509 /*
 510  * checksum a dirty tree block before IO.  This has extra checks to make sure
 511  * we only fill in the checksum field in the first page of a multi-page block
 512  */
 513
 514 static int csum_dirty_buffer(struct btrfs_fs_info *fs_info, struct page *page)
 515 {
 516         u64 start = page_offset(page);
 517         u64 found_start;
 518         u8 result[BTRFS_CSUM_SIZE];
 519         u16 csum_size = btrfs_super_csum_size(fs_info->super_copy);
 520         struct extent_buffer *eb;
 521         int ret;
 522
 523         eb = (struct extent_buffer *)page->private;
 524         if (page != eb->pages[0])
 525                 return 0;
 526
 527         found_start = btrfs_header_bytenr(eb);
 528         /*
 529          * Please do not consolidate these warnings into a single if.
 530          * It is useful to know what went wrong.
 531          */
 532         if (WARN_ON(found_start != start))
 533                 return -EUCLEAN;
 534         if (WARN_ON(!PageUptodate(page)))
 535                 return -EUCLEAN;
 536
 537         ASSERT(memcmp_extent_buffer(eb, fs_info->fs_devices->metadata_uuid,
 538                         btrfs_header_fsid(), BTRFS_FSID_SIZE) == 0);
 539
 540         if (csum_tree_block(eb, result))
 541                 return -EINVAL;
 542
 543         if (btrfs_header_level(eb))
 544                 ret = btrfs_check_node(eb);
 545         else
 546                 ret = btrfs_check_leaf_full(eb);
 547
 548         if (ret < 0) {
 549                 btrfs_print_tree(eb, 0);
 550                 btrfs_err(fs_info,
 551                 "block=%llu write time tree block corruption detected",
 552                           eb->start);
 553                 WARN_ON(IS_ENABLED(CONFIG_BTRFS_DEBUG));
 554                 return ret;
 555         }
 556         write_extent_buffer(eb, result, 0, csum_size);
 557
 558         return 0;
 559 }
 560
 561 static int check_tree_block_fsid(struct extent_buffer *eb)
 562 {
 563         struct btrfs_fs_info *fs_info = eb->fs_info;
 564         struct btrfs_fs_devices *fs_devices = fs_info->fs_devices;
 565         u8 fsid[BTRFS_FSID_SIZE];
 566         int ret = 1;
 567
 568         read_extent_buffer(eb, fsid, btrfs_header_fsid(), BTRFS_FSID_SIZE);
 569         while (fs_devices) {
 570                 u8 *metadata_uuid;
 571
 572                 /*
 573                  * Checking the incompat flag is only valid for the current
 574                  * fs. For seed devices it's forbidden to have their uuid
 575                  * changed so reading ->fsid in this case is fine
 576                  */
 577                 if (fs_devices == fs_info->fs_devices &&
 578                     btrfs_fs_incompat(fs_info, METADATA_UUID))
 579                         metadata_uuid = fs_devices->metadata_uuid;
 580                 else
 581                         metadata_uuid = fs_devices->fsid;
 582
 583                 if (!memcmp(fsid, metadata_uuid, BTRFS_FSID_SIZE)) {
 584                         ret = 0;
 585                         break;
 586                 }
 587                 fs_devices = fs_devices->seed;
 588         }
 589         return ret;
 590 }
 591
 592 static int btree_readpage_end_io_hook(struct btrfs_io_bio *io_bio,
 593                                       u64 phy_offset, struct page *page,
 594                                       u64 start, u64 end, int mirror)
 595 {
 596         u64 found_start;
 597         int found_level;
 598         struct extent_buffer *eb;
 599         struct btrfs_root *root = BTRFS_I(page->mapping->host)->root;
 600         struct btrfs_fs_info *fs_info = root->fs_info;
 601         u16 csum_size = btrfs_super_csum_size(fs_info->super_copy);
 602         int ret = 0;
 603         u8 result[BTRFS_CSUM_SIZE];
 604         int reads_done;
 605
 606         if (!page->private)
 607                 goto out;
 608
 609         eb = (struct extent_buffer *)page->private;
 610
 611         /* the pending IO might have been the only thing that kept this buffer
 612          * in memory.  Make sure we have a ref for all this other checks
 613          */
 614         atomic_inc(&eb->refs);
 615
 616         reads_done = atomic_dec_and_test(&eb->io_pages);
 617         if (!reads_done)
 618                 goto err;
 619
 620         eb->read_mirror = mirror;
 621         if (test_bit(EXTENT_BUFFER_READ_ERR, &eb->bflags)) {
 622                 ret = -EIO;
 623                 goto err;
 624         }
 625
 626         found_start = btrfs_header_bytenr(eb);
 627         if (found_start != eb->start) {
 628                 btrfs_err_rl(fs_info, "bad tree block start, want %llu have %llu",
 629                              eb->start, found_start);
 630                 ret = -EIO;
 631                 goto err;
 632         }
 633         if (check_tree_block_fsid(eb)) {
 634                 btrfs_err_rl(fs_info, "bad fsid on block %llu",
 635                              eb->start);
 636                 ret = -EIO;
 637                 goto err;
 638         }
 639         found_level = btrfs_header_level(eb);
 640         if (found_level >= BTRFS_MAX_LEVEL) {
 641                 btrfs_err(fs_info, "bad tree block level %d on %llu",
 642                           (int)btrfs_header_level(eb), eb->start);
 643                 ret = -EIO;
 644                 goto err;
 645         }
 646
 647         btrfs_set_buffer_lockdep_class(btrfs_header_owner(eb),
 648                                        eb, found_level);
 649
 650         ret = csum_tree_block(eb, result);
 651         if (ret)
 652                 goto err;
 653
 654         if (memcmp_extent_buffer(eb, result, 0, csum_size)) {
 655                 u32 val;
 656                 u32 found = 0;
 657
 658                 memcpy(&found, result, csum_size);
 659
 660                 read_extent_buffer(eb, &val, 0, csum_size);
 661                 btrfs_warn_rl(fs_info,
 662                 "%s checksum verify failed on %llu wanted %x found %x level %d",
 663                               fs_info->sb->s_id, eb->start,
 664                               val, found, btrfs_header_level(eb));
 665                 ret = -EUCLEAN;
 666                 goto err;
 667         }
 668
 669         /*
 670          * If this is a leaf block and it is corrupt, set the corrupt bit so
 671          * that we don't try and read the other copies of this block, just
 672          * return -EIO.
 673          */
 674         if (found_level == 0 && btrfs_check_leaf_full(eb)) {
 675                 set_bit(EXTENT_BUFFER_CORRUPT, &eb->bflags);
 676                 ret = -EIO;
 677         }
 678
 679         if (found_level > 0 && btrfs_check_node(eb))
 680                 ret = -EIO;
 681
 682         if (!ret)
 683                 set_extent_buffer_uptodate(eb);
 684         else
 685                 btrfs_err(fs_info,
 686                           "block=%llu read time tree block corruption detected",
 687                           eb->start);
 688 err:
 689         if (reads_done &&
 690             test_and_clear_bit(EXTENT_BUFFER_READAHEAD, &eb->bflags))
 691                 btree_readahead_hook(eb, ret);
 692
 693         if (ret) {
 694                 /*
 695                  * our io error hook is going to dec the io pages
 696                  * again, we have to make sure it has something
 697                  * to decrement
 698                  */
 699                 atomic_inc(&eb->io_pages);
 700                 clear_extent_buffer_uptodate(eb);
 701         }
 702         free_extent_buffer(eb);
 703 out:
 704         return ret;
 705 }
 706
 707 static void end_workqueue_bio(struct bio *bio)
 708 {
 709         struct btrfs_end_io_wq *end_io_wq = bio->bi_private;
 710         struct btrfs_fs_info *fs_info;
 711         struct btrfs_workqueue *wq;
 712
 713         fs_info = end_io_wq->info;
 714         end_io_wq->status = bio->bi_status;
 715
 716         if (bio_op(bio) == REQ_OP_WRITE) {
 717                 if (end_io_wq->metadata == BTRFS_WQ_ENDIO_METADATA)
 718                         wq = fs_info->endio_meta_write_workers;
 719                 else if (end_io_wq->metadata == BTRFS_WQ_ENDIO_FREE_SPACE)
 720                         wq = fs_info->endio_freespace_worker;
 721                 else if (end_io_wq->metadata == BTRFS_WQ_ENDIO_RAID56)
 722                         wq = fs_info->endio_raid56_workers;
 723                 else
 724                         wq = fs_info->endio_write_workers;
 725         } else {
 726                 if (unlikely(end_io_wq->metadata == BTRFS_WQ_ENDIO_DIO_REPAIR))
 727                         wq = fs_info->endio_repair_workers;
 728                 else if (end_io_wq->metadata == BTRFS_WQ_ENDIO_RAID56)
 729                         wq = fs_info->endio_raid56_workers;
 730                 else if (end_io_wq->metadata)
 731                         wq = fs_info->endio_meta_workers;
 732                 else
 733                         wq = fs_info->endio_workers;
 734         }
 735
 736         btrfs_init_work(&end_io_wq->work, end_workqueue_fn, NULL, NULL);
 737         btrfs_queue_work(wq, &end_io_wq->work);
 738 }
 739
 740 blk_status_t btrfs_bio_wq_end_io(struct btrfs_fs_info *info, struct bio *bio,
 741                         enum btrfs_wq_endio_type metadata)
 742 {
 743         struct btrfs_end_io_wq *end_io_wq;
 744
 745         end_io_wq = kmem_cache_alloc(btrfs_end_io_wq_cache, GFP_NOFS);
 746         if (!end_io_wq)
 747                 return BLK_STS_RESOURCE;
 748
 749         end_io_wq->private = bio->bi_private;
 750         end_io_wq->end_io = bio->bi_end_io;
 751         end_io_wq->info = info;
 752         end_io_wq->status = 0;
 753         end_io_wq->bio = bio;
 754         end_io_wq->metadata = metadata;
 755
 756         bio->bi_private = end_io_wq;
 757         bio->bi_end_io = end_workqueue_bio;
 758         return 0;
 759 }
 760
 761 static void run_one_async_start(struct btrfs_work *work)
 762 {
 763         struct async_submit_bio *async;
 764         blk_status_t ret;
 765
 766         async = container_of(work, struct  async_submit_bio, work);
 767         ret = async->submit_bio_start(async->private_data, async->bio,
 768                                       async->bio_offset);
 769         if (ret)
 770                 async->status = ret;
 771 }
 772
 773 /*
 774  * In order to insert checksums into the metadata in large chunks, we wait
 775  * until bio submission time.   All the pages in the bio are checksummed and
 776  * sums are attached onto the ordered extent record.
 777  *
 778  * At IO completion time the csums attached on the ordered extent record are
 779  * inserted into the tree.
 780  */
 781 static void run_one_async_done(struct btrfs_work *work)
 782 {
 783         struct async_submit_bio *async;
 784         struct inode *inode;
 785         blk_status_t ret;
 786
 787         async = container_of(work, struct  async_submit_bio, work);
 788         inode = async->private_data;
 789
 790         /* If an error occurred we just want to clean up the bio and move on */
 791         if (async->status) {
 792                 async->bio->bi_status = async->status;
 793                 bio_endio(async->bio);
 794                 return;
 795         }
 796
 797         /*
 798          * All of the bios that pass through here are from async helpers.
 799          * Use REQ_CGROUP_PUNT to issue them from the owning cgroup's context.
 800          * This changes nothing when cgroups aren't in use.
 801          */
 802         async->bio->bi_opf |= REQ_CGROUP_PUNT;
 803         ret = btrfs_map_bio(btrfs_sb(inode->i_sb), async->bio, async->mirror_num);
 804         if (ret) {
 805                 async->bio->bi_status = ret;
 806                 bio_endio(async->bio);
 807         }
 808 }
 809
 810 static void run_one_async_free(struct btrfs_work *work)
 811 {
 812         struct async_submit_bio *async;
 813
 814         async = container_of(work, struct  async_submit_bio, work);
 815         kfree(async);
 816 }
 817
 818 blk_status_t btrfs_wq_submit_bio(struct btrfs_fs_info *fs_info, struct bio *bio,
 819                                  int mirror_num, unsigned long bio_flags,
 820                                  u64 bio_offset, void *private_data,
 821                                  extent_submit_bio_start_t *submit_bio_start)
 822 {
 823         struct async_submit_bio *async;
 824
 825         async = kmalloc(sizeof(*async), GFP_NOFS);
 826         if (!async)
 827                 return BLK_STS_RESOURCE;
 828
 829         async->private_data = private_data;
 830         async->bio = bio;
 831         async->mirror_num = mirror_num;
 832         async->submit_bio_start = submit_bio_start;
 833
 834         btrfs_init_work(&async->work, run_one_async_start, run_one_async_done,
 835                         run_one_async_free);
 836
 837         async->bio_offset = bio_offset;
 838
 839         async->status = 0;
 840
 841         if (op_is_sync(bio->bi_opf))
 842                 btrfs_set_work_high_priority(&async->work);
 843
 844         btrfs_queue_work(fs_info->workers, &async->work);
 845         return 0;
 846 }
 847
 848 static blk_status_t btree_csum_one_bio(struct bio *bio)
 849 {
 850         struct bio_vec *bvec;
 851         struct btrfs_root *root;
 852         int ret = 0;
 853         struct bvec_iter_all iter_all;
 854
 855         ASSERT(!bio_flagged(bio, BIO_CLONED));
 856         bio_for_each_segment_all(bvec, bio, iter_all) {
 857                 root = BTRFS_I(bvec->bv_page->mapping->host)->root;
 858                 ret = csum_dirty_buffer(root->fs_info, bvec->bv_page);
 859                 if (ret)
 860                         break;
 861         }
 862
 863         return errno_to_blk_status(ret);
 864 }
 865
 866 static blk_status_t btree_submit_bio_start(void *private_data, struct bio *bio,
 867                                              u64 bio_offset)
 868 {
 869         /*
 870          * when we're called for a write, we're already in the async
 871          * submission context.  Just jump into btrfs_map_bio
 872          */
 873         return btree_csum_one_bio(bio);
 874 }
 875
 876 static int check_async_write(struct btrfs_fs_info *fs_info,
 877                              struct btrfs_inode *bi)
 878 {
 879         if (atomic_read(&bi->sync_writers))
 880                 return 0;
 881         if (test_bit(BTRFS_FS_CSUM_IMPL_FAST, &fs_info->flags))
 882                 return 0;
 883         return 1;
 884 }
 885
 886 static blk_status_t btree_submit_bio_hook(struct inode *inode, struct bio *bio,
 887                                           int mirror_num,
 888                                           unsigned long bio_flags)
 889 {
 890         struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
 891         int async = check_async_write(fs_info, BTRFS_I(inode));
 892         blk_status_t ret;
 893
 894         if (bio_op(bio) != REQ_OP_WRITE) {
 895                 /*
 896                  * called for a read, do the setup so that checksum validation
 897                  * can happen in the async kernel threads
 898                  */
 899                 ret = btrfs_bio_wq_end_io(fs_info, bio,
 900                                           BTRFS_WQ_ENDIO_METADATA);
 901                 if (ret)
 902                         goto out_w_error;
 903                 ret = btrfs_map_bio(fs_info, bio, mirror_num);
 904         } else if (!async) {
 905                 ret = btree_csum_one_bio(bio);
 906                 if (ret)
 907                         goto out_w_error;
 908                 ret = btrfs_map_bio(fs_info, bio, mirror_num);
 909         } else {
 910                 /*
 911                  * kthread helpers are used to submit writes so that
 912                  * checksumming can happen in parallel across all CPUs
 913                  */
 914                 ret = btrfs_wq_submit_bio(fs_info, bio, mirror_num, 0,
 915                                           0, inode, btree_submit_bio_start);
 916         }
 917
 918         if (ret)
 919                 goto out_w_error;
 920         return 0;
 921
 922 out_w_error:
 923         bio->bi_status = ret;
 924         bio_endio(bio);
 925         return ret;
 926 }
 927
 928 #ifdef CONFIG_MIGRATION
 929 static int btree_migratepage(struct address_space *mapping,
 930                         struct page *newpage, struct page *page,
 931                         enum migrate_mode mode)
 932 {
 933         /*
 934          * we can't safely write a btree page from here,
 935          * we haven't done the locking hook
 936          */
 937         if (PageDirty(page))
 938                 return -EAGAIN;
 939         /*
 940          * Buffers may be managed in a filesystem specific way.
 941          * We must have no buffers or drop them.
 942          */
 943         if (page_has_private(page) &&
 944             !try_to_release_page(page, GFP_KERNEL))
 945                 return -EAGAIN;
 946         return migrate_page(mapping, newpage, page, mode);
 947 }
 948 #endif
 949
 950
 951 static int btree_writepages(struct address_space *mapping,
 952                             struct writeback_control *wbc)
 953 {
 954         struct btrfs_fs_info *fs_info;
 955         int ret;
 956
 957         if (wbc->sync_mode == WB_SYNC_NONE) {
 958
 959                 if (wbc->for_kupdate)
 960                         return 0;
 961
 962                 fs_info = BTRFS_I(mapping->host)->root->fs_info;
 963                 /* this is a bit racy, but that's ok */
 964                 ret = __percpu_counter_compare(&fs_info->dirty_metadata_bytes,
 965                                              BTRFS_DIRTY_METADATA_THRESH,
 966                                              fs_info->dirty_metadata_batch);
 967                 if (ret < 0)
 968                         return 0;
 969         }
 970         return btree_write_cache_pages(mapping, wbc);
 971 }
 972
 973 static int btree_readpage(struct file *file, struct page *page)
 974 {
 975         struct extent_io_tree *tree;
 976         tree = &BTRFS_I(page->mapping->host)->io_tree;
 977         return extent_read_full_page(tree, page, btree_get_extent, 0);
 978 }
 979
 980 static int btree_releasepage(struct page *page, gfp_t gfp_flags)
 981 {
 982         if (PageWriteback(page) || PageDirty(page))
 983                 return 0;
 984
 985         return try_release_extent_buffer(page);
 986 }
 987
 988 static void btree_invalidatepage(struct page *page, unsigned int offset,
 989                                  unsigned int length)
 990 {
 991         struct extent_io_tree *tree;
 992         tree = &BTRFS_I(page->mapping->host)->io_tree;
 993         extent_invalidatepage(tree, page, offset);
 994         btree_releasepage(page, GFP_NOFS);
 995         if (PagePrivate(page)) {
 996                 btrfs_warn(BTRFS_I(page->mapping->host)->root->fs_info,
 997                            "page private not zero on page %llu",
 998                            (unsigned long long)page_offset(page));
 999                 ClearPagePrivate(page);
1000                 set_page_private(page, 0);
1001                 put_page(page);
1002         }
1003 }
1004
1005 static int btree_set_page_dirty(struct page *page)
1006 {
1007 #ifdef DEBUG
1008         struct extent_buffer *eb;
1009
1010         BUG_ON(!PagePrivate(page));
1011         eb = (struct extent_buffer *)page->private;
1012         BUG_ON(!eb);
1013         BUG_ON(!test_bit(EXTENT_BUFFER_DIRTY, &eb->bflags));
1014         BUG_ON(!atomic_read(&eb->refs));
1015         btrfs_assert_tree_locked(eb);
1016 #endif
1017         return __set_page_dirty_nobuffers(page);
1018 }
1019
1020 static const struct address_space_operations btree_aops = {
1021         .readpage       = btree_readpage,
1022         .writepages     = btree_writepages,
1023         .releasepage    = btree_releasepage,
1024         .invalidatepage = btree_invalidatepage,
1025 #ifdef CONFIG_MIGRATION
1026         .migratepage    = btree_migratepage,
1027 #endif
1028         .set_page_dirty = btree_set_page_dirty,
1029 };
1030
1031 void readahead_tree_block(struct btrfs_fs_info *fs_info, u64 bytenr)
1032 {
1033         struct extent_buffer *buf = NULL;
1034         int ret;
1035
1036         buf = btrfs_find_create_tree_block(fs_info, bytenr);
1037         if (IS_ERR(buf))
1038                 return;
1039
1040         ret = read_extent_buffer_pages(buf, WAIT_NONE, 0);
1041         if (ret < 0)
1042                 free_extent_buffer_stale(buf);
1043         else
1044                 free_extent_buffer(buf);
1045 }
1046
1047 struct extent_buffer *btrfs_find_create_tree_block(
1048                                                 struct btrfs_fs_info *fs_info,
1049                                                 u64 bytenr)
1050 {
1051         if (btrfs_is_testing(fs_info))
1052                 return alloc_test_extent_buffer(fs_info, bytenr);
1053         return alloc_extent_buffer(fs_info, bytenr);
1054 }
1055
1056 /*
1057  * Read tree block at logical address @bytenr and do variant basic but critical
1058  * verification.
1059  *
1060  * @parent_transid:     expected transid of this tree block, skip check if 0
1061  * @level:              expected level, mandatory check
1062  * @first_key:          expected key in slot 0, skip check if NULL
1063  */
1064 struct extent_buffer *read_tree_block(struct btrfs_fs_info *fs_info, u64 bytenr,
1065                                       u64 parent_transid, int level,
1066                                       struct btrfs_key *first_key)
1067 {
1068         struct extent_buffer *buf = NULL;
1069         int ret;
1070
1071         buf = btrfs_find_create_tree_block(fs_info, bytenr);
1072         if (IS_ERR(buf))
1073                 return buf;
1074
1075         ret = btree_read_extent_buffer_pages(buf, parent_transid,
1076                                              level, first_key);
1077         if (ret) {
1078                 free_extent_buffer_stale(buf);
1079                 return ERR_PTR(ret);
1080         }
1081         return buf;
1082
1083 }
1084
1085 void btrfs_clean_tree_block(struct extent_buffer *buf)
1086 {
1087         struct btrfs_fs_info *fs_info = buf->fs_info;
1088         if (btrfs_header_generation(buf) ==
1089             fs_info->running_transaction->transid) {
1090                 btrfs_assert_tree_locked(buf);
1091
1092                 if (test_and_clear_bit(EXTENT_BUFFER_DIRTY, &buf->bflags)) {
1093                         percpu_counter_add_batch(&fs_info->dirty_metadata_bytes,
1094                                                  -buf->len,
1095                                                  fs_info->dirty_metadata_batch);
1096                         /* ugh, clear_extent_buffer_dirty needs to lock the page */
1097                         btrfs_set_lock_blocking_write(buf);
1098                         clear_extent_buffer_dirty(buf);
1099                 }
1100         }
1101 }
1102
1103 static struct btrfs_subvolume_writers *btrfs_alloc_subvolume_writers(void)
1104 {
1105         struct btrfs_subvolume_writers *writers;
1106         int ret;
1107
1108         writers = kmalloc(sizeof(*writers), GFP_NOFS);
1109         if (!writers)
1110                 return ERR_PTR(-ENOMEM);
1111
1112         ret = percpu_counter_init(&writers->counter, 0, GFP_NOFS);
1113         if (ret < 0) {
1114                 kfree(writers);
1115                 return ERR_PTR(ret);
1116         }
1117
1118         init_waitqueue_head(&writers->wait);
1119         return writers;
1120 }
1121
1122 static void
1123 btrfs_free_subvolume_writers(struct btrfs_subvolume_writers *writers)
1124 {
1125         percpu_counter_destroy(&writers->counter);
1126         kfree(writers);
1127 }
1128
1129 static void __setup_root(struct btrfs_root *root, struct btrfs_fs_info *fs_info,
1130                          u64 objectid)
1131 {
1132         bool dummy = test_bit(BTRFS_FS_STATE_DUMMY_FS_INFO, &fs_info->fs_state);
1133         root->fs_info = fs_info;
1134         root->node = NULL;
1135         root->commit_root = NULL;
1136         root->state = 0;
1137         root->orphan_cleanup_state = 0;
1138
1139         root->last_trans = 0;
1140         root->highest_objectid = 0;
1141         root->nr_delalloc_inodes = 0;
1142         root->nr_ordered_extents = 0;
1143         root->inode_tree = RB_ROOT;
1144         INIT_RADIX_TREE(&root->delayed_nodes_tree, GFP_ATOMIC);
1145         root->block_rsv = NULL;
1146
1147         INIT_LIST_HEAD(&root->dirty_list);
1148         INIT_LIST_HEAD(&root->root_list);
1149         INIT_LIST_HEAD(&root->delalloc_inodes);
1150         INIT_LIST_HEAD(&root->delalloc_root);
1151         INIT_LIST_HEAD(&root->ordered_extents);
1152         INIT_LIST_HEAD(&root->ordered_root);
1153         INIT_LIST_HEAD(&root->reloc_dirty_list);
1154         INIT_LIST_HEAD(&root->logged_list[0]);
1155         INIT_LIST_HEAD(&root->logged_list[1]);
1156         spin_lock_init(&root->inode_lock);
1157         spin_lock_init(&root->delalloc_lock);
1158         spin_lock_init(&root->ordered_extent_lock);
1159         spin_lock_init(&root->accounting_lock);
1160         spin_lock_init(&root->log_extents_lock[0]);
1161         spin_lock_init(&root->log_extents_lock[1]);
1162         spin_lock_init(&root->qgroup_meta_rsv_lock);
1163         mutex_init(&root->objectid_mutex);
1164         mutex_init(&root->log_mutex);
1165         mutex_init(&root->ordered_extent_mutex);
1166         mutex_init(&root->delalloc_mutex);
1167         init_waitqueue_head(&root->log_writer_wait);
1168         init_waitqueue_head(&root->log_commit_wait[0]);
1169         init_waitqueue_head(&root->log_commit_wait[1]);
1170         INIT_LIST_HEAD(&root->log_ctxs[0]);
1171         INIT_LIST_HEAD(&root->log_ctxs[1]);
1172         atomic_set(&root->log_commit[0], 0);
1173         atomic_set(&root->log_commit[1], 0);
1174         atomic_set(&root->log_writers, 0);
1175         atomic_set(&root->log_batch, 0);
1176         refcount_set(&root->refs, 1);
1177         atomic_set(&root->will_be_snapshotted, 0);
1178         atomic_set(&root->snapshot_force_cow, 0);
1179         atomic_set(&root->nr_swapfiles, 0);
1180         root->log_transid = 0;
1181         root->log_transid_committed = -1;
1182         root->last_log_commit = 0;
1183         if (!dummy)
1184                 extent_io_tree_init(fs_info, &root->dirty_log_pages,
1185                                     IO_TREE_ROOT_DIRTY_LOG_PAGES, NULL);
1186
1187         memset(&root->root_key, 0, sizeof(root->root_key));
1188         memset(&root->root_item, 0, sizeof(root->root_item));
1189         memset(&root->defrag_progress, 0, sizeof(root->defrag_progress));
1190         if (!dummy)
1191                 root->defrag_trans_start = fs_info->generation;
1192         else
1193                 root->defrag_trans_start = 0;
1194         root->root_key.objectid = objectid;
1195         root->anon_dev = 0;
1196
1197         spin_lock_init(&root->root_item_lock);
1198         btrfs_qgroup_init_swapped_blocks(&root->swapped_blocks);
1199 }
1200
1201 static struct btrfs_root *btrfs_alloc_root(struct btrfs_fs_info *fs_info,
1202                                            u64 objectid, gfp_t flags)
1203 {
1204         struct btrfs_root *root = kzalloc(sizeof(*root), flags);
1205         if (root)
1206                 __setup_root(root, fs_info, objectid);
1207         return root;
1208 }
1209
1210 #ifdef CONFIG_BTRFS_FS_RUN_SANITY_TESTS
1211 /* Should only be used by the testing infrastructure */
1212 struct btrfs_root *btrfs_alloc_dummy_root(struct btrfs_fs_info *fs_info)
1213 {
1214         struct btrfs_root *root;
1215
1216         if (!fs_info)
1217                 return ERR_PTR(-EINVAL);
1218
1219         root = btrfs_alloc_root(fs_info, BTRFS_ROOT_TREE_OBJECTID, GFP_KERNEL);
1220         if (!root)
1221                 return ERR_PTR(-ENOMEM);
1222
1223         /* We don't use the stripesize in selftest, set it as sectorsize */
1224         root->alloc_bytenr = 0;
1225
1226         return root;
1227 }
1228 #endif
1229
1230 struct btrfs_root *btrfs_create_tree(struct btrfs_trans_handle *trans,
1231                                      u64 objectid)
1232 {
1233         struct btrfs_fs_info *fs_info = trans->fs_info;
1234         struct extent_buffer *leaf;
1235         struct btrfs_root *tree_root = fs_info->tree_root;
1236         struct btrfs_root *root;
1237         struct btrfs_key key;
1238         unsigned int nofs_flag;
1239         int ret = 0;
1240         uuid_le uuid = NULL_UUID_LE;
1241
1242         /*
1243          * We're holding a transaction handle, so use a NOFS memory allocation
1244          * context to avoid deadlock if reclaim happens.
1245          */
1246         nofs_flag = memalloc_nofs_save();
1247         root = btrfs_alloc_root(fs_info, objectid, GFP_KERNEL);
1248         memalloc_nofs_restore(nofs_flag);
1249         if (!root)
1250                 return ERR_PTR(-ENOMEM);
1251
1252         root->root_key.objectid = objectid;
1253         root->root_key.type = BTRFS_ROOT_ITEM_KEY;
1254         root->root_key.offset = 0;
1255
1256         leaf = btrfs_alloc_tree_block(trans, root, 0, objectid, NULL, 0, 0, 0);
1257         if (IS_ERR(leaf)) {
1258                 ret = PTR_ERR(leaf);
1259                 leaf = NULL;
1260                 goto fail;
1261         }
1262
1263         root->node = leaf;
1264         btrfs_mark_buffer_dirty(leaf);
1265
1266         root->commit_root = btrfs_root_node(root);
1267         set_bit(BTRFS_ROOT_TRACK_DIRTY, &root->state);
1268
1269         root->root_item.flags = 0;
1270         root->root_item.byte_limit = 0;
1271         btrfs_set_root_bytenr(&root->root_item, leaf->start);
1272         btrfs_set_root_generation(&root->root_item, trans->transid);
1273         btrfs_set_root_level(&root->root_item, 0);
1274         btrfs_set_root_refs(&root->root_item, 1);
1275         btrfs_set_root_used(&root->root_item, leaf->len);
1276         btrfs_set_root_last_snapshot(&root->root_item, 0);
1277         btrfs_set_root_dirid(&root->root_item, 0);
1278         if (is_fstree(objectid))
1279                 uuid_le_gen(&uuid);
1280         memcpy(root->root_item.uuid, uuid.b, BTRFS_UUID_SIZE);
1281         root->root_item.drop_level = 0;
1282
1283         key.objectid = objectid;
1284         key.type = BTRFS_ROOT_ITEM_KEY;
1285         key.offset = 0;
1286         ret = btrfs_insert_root(trans, tree_root, &key, &root->root_item);
1287         if (ret)
1288                 goto fail;
1289
1290         btrfs_tree_unlock(leaf);
1291
1292         return root;
1293
1294 fail:
1295         if (leaf) {
1296                 btrfs_tree_unlock(leaf);
1297                 free_extent_buffer(root->commit_root);
1298                 free_extent_buffer(leaf);
1299         }
1300         kfree(root);
1301
1302         return ERR_PTR(ret);
1303 }
1304
1305 static struct btrfs_root *alloc_log_tree(struct btrfs_trans_handle *trans,
1306                                          struct btrfs_fs_info *fs_info)
1307 {
1308         struct btrfs_root *root;
1309         struct extent_buffer *leaf;
1310
1311         root = btrfs_alloc_root(fs_info, BTRFS_TREE_LOG_OBJECTID, GFP_NOFS);
1312         if (!root)
1313                 return ERR_PTR(-ENOMEM);
1314
1315         root->root_key.objectid = BTRFS_TREE_LOG_OBJECTID;
1316         root->root_key.type = BTRFS_ROOT_ITEM_KEY;
1317         root->root_key.offset = BTRFS_TREE_LOG_OBJECTID;
1318
1319         /*
1320          * DON'T set REF_COWS for log trees
1321          *
1322          * log trees do not get reference counted because they go away
1323          * before a real commit is actually done.  They do store pointers
1324          * to file data extents, and those reference counts still get
1325          * updated (along with back refs to the log tree).
1326          */
1327
1328         leaf = btrfs_alloc_tree_block(trans, root, 0, BTRFS_TREE_LOG_OBJECTID,
1329                         NULL, 0, 0, 0);
1330         if (IS_ERR(leaf)) {
1331                 kfree(root);
1332                 return ERR_CAST(leaf);
1333         }
1334
1335         root->node = leaf;
1336
1337         btrfs_mark_buffer_dirty(root->node);
1338         btrfs_tree_unlock(root->node);
1339         return root;
1340 }
1341
1342 int btrfs_init_log_root_tree(struct btrfs_trans_handle *trans,
1343                              struct btrfs_fs_info *fs_info)
1344 {
1345         struct btrfs_root *log_root;
1346
1347         log_root = alloc_log_tree(trans, fs_info);
1348         if (IS_ERR(log_root))
1349                 return PTR_ERR(log_root);
1350         WARN_ON(fs_info->log_root_tree);
1351         fs_info->log_root_tree = log_root;
1352         return 0;
1353 }
1354
1355 int btrfs_add_log_tree(struct btrfs_trans_handle *trans,
1356                        struct btrfs_root *root)
1357 {
1358         struct btrfs_fs_info *fs_info = root->fs_info;
1359         struct btrfs_root *log_root;
1360         struct btrfs_inode_item *inode_item;
1361
1362         log_root = alloc_log_tree(trans, fs_info);
1363         if (IS_ERR(log_root))
1364                 return PTR_ERR(log_root);
1365
1366         log_root->last_trans = trans->transid;
1367         log_root->root_key.offset = root->root_key.objectid;
1368
1369         inode_item = &log_root->root_item.inode;
1370         btrfs_set_stack_inode_generation(inode_item, 1);
1371         btrfs_set_stack_inode_size(inode_item, 3);
1372         btrfs_set_stack_inode_nlink(inode_item, 1);
1373         btrfs_set_stack_inode_nbytes(inode_item,
1374                                      fs_info->nodesize);
1375         btrfs_set_stack_inode_mode(inode_item, S_IFDIR | 0755);
1376
1377         btrfs_set_root_node(&log_root->root_item, log_root->node);
1378
1379         WARN_ON(root->log_root);
1380         root->log_root = log_root;
1381         root->log_transid = 0;
1382         root->log_transid_committed = -1;
1383         root->last_log_commit = 0;
1384         return 0;
1385 }
1386
1387 struct btrfs_root *btrfs_read_tree_root(struct btrfs_root *tree_root,
1388                                         struct btrfs_key *key)
1389 {
1390         struct btrfs_root *root;
1391         struct btrfs_fs_info *fs_info = tree_root->fs_info;
1392         struct btrfs_path *path;
1393         u64 generation;
1394         int ret;
1395         int level;
1396
1397         path = btrfs_alloc_path();
1398         if (!path)
1399                 return ERR_PTR(-ENOMEM);
1400
1401         root = btrfs_alloc_root(fs_info, key->objectid, GFP_NOFS);
1402         if (!root) {
1403                 ret = -ENOMEM;
1404                 goto alloc_fail;
1405         }
1406
1407         ret = btrfs_find_root(tree_root, key, path,
1408                               &root->root_item, &root->root_key);
1409         if (ret) {
1410                 if (ret > 0)
1411                         ret = -ENOENT;
1412                 goto find_fail;
1413         }
1414
1415         generation = btrfs_root_generation(&root->root_item);
1416         level = btrfs_root_level(&root->root_item);
1417         root->node = read_tree_block(fs_info,
1418                                      btrfs_root_bytenr(&root->root_item),
1419                                      generation, level, NULL);
1420         if (IS_ERR(root->node)) {
1421                 ret = PTR_ERR(root->node);
1422                 goto find_fail;
1423         } else if (!btrfs_buffer_uptodate(root->node, generation, 0)) {
1424                 ret = -EIO;
1425                 free_extent_buffer(root->node);
1426                 goto find_fail;
1427         }
1428         root->commit_root = btrfs_root_node(root);
1429 out:
1430         btrfs_free_path(path);
1431         return root;
1432
1433 find_fail:
1434         kfree(root);
1435 alloc_fail:
1436         root = ERR_PTR(ret);
1437         goto out;
1438 }
1439
1440 int btrfs_init_fs_root(struct btrfs_root *root)
1441 {
1442         int ret;
1443         struct btrfs_subvolume_writers *writers;
1444
1445         root->free_ino_ctl = kzalloc(sizeof(*root->free_ino_ctl), GFP_NOFS);
1446         root->free_ino_pinned = kzalloc(sizeof(*root->free_ino_pinned),
1447                                         GFP_NOFS);
1448         if (!root->free_ino_pinned || !root->free_ino_ctl) {
1449                 ret = -ENOMEM;
1450                 goto fail;
1451         }
1452
1453         writers = btrfs_alloc_subvolume_writers();
1454         if (IS_ERR(writers)) {
1455                 ret = PTR_ERR(writers);
1456                 goto fail;
1457         }
1458         root->subv_writers = writers;
1459
1460         if (root->root_key.objectid != BTRFS_TREE_LOG_OBJECTID) {
1461                 set_bit(BTRFS_ROOT_REF_COWS, &root->state);
1462                 btrfs_check_and_init_root_item(&root->root_item);
1463         }
1464
1465         btrfs_init_free_ino_ctl(root);
1466         spin_lock_init(&root->ino_cache_lock);
1467         init_waitqueue_head(&root->ino_cache_wait);
1468
1469         ret = get_anon_bdev(&root->anon_dev);
1470         if (ret)
1471                 goto fail;
1472
1473         mutex_lock(&root->objectid_mutex);
1474         ret = btrfs_find_highest_objectid(root,
1475                                         &root->highest_objectid);
1476         if (ret) {
1477                 mutex_unlock(&root->objectid_mutex);
1478                 goto fail;
1479         }
1480
1481         ASSERT(root->highest_objectid <= BTRFS_LAST_FREE_OBJECTID);
1482
1483         mutex_unlock(&root->objectid_mutex);
1484
1485         return 0;
1486 fail:
1487         /* The caller is responsible to call btrfs_free_fs_root */
1488         return ret;
1489 }
1490
1491 struct btrfs_root *btrfs_lookup_fs_root(struct btrfs_fs_info *fs_info,
1492                                         u64 root_id)
1493 {
1494         struct btrfs_root *root;
1495
1496         spin_lock(&fs_info->fs_roots_radix_lock);
1497         root = radix_tree_lookup(&fs_info->fs_roots_radix,
1498                                  (unsigned long)root_id);
1499         spin_unlock(&fs_info->fs_roots_radix_lock);
1500         return root;
1501 }
1502
1503 int btrfs_insert_fs_root(struct btrfs_fs_info *fs_info,
1504                          struct btrfs_root *root)
1505 {
1506         int ret;
1507
1508         ret = radix_tree_preload(GFP_NOFS);
1509         if (ret)
1510                 return ret;
1511
1512         spin_lock(&fs_info->fs_roots_radix_lock);
1513         ret = radix_tree_insert(&fs_info->fs_roots_radix,
1514                                 (unsigned long)root->root_key.objectid,
1515                                 root);
1516         if (ret == 0)
1517                 set_bit(BTRFS_ROOT_IN_RADIX, &root->state);
1518         spin_unlock(&fs_info->fs_roots_radix_lock);
1519         radix_tree_preload_end();
1520
1521         return ret;
1522 }
1523
1524 struct btrfs_root *btrfs_get_fs_root(struct btrfs_fs_info *fs_info,
1525                                      struct btrfs_key *location,
1526                                      bool check_ref)
1527 {
1528         struct btrfs_root *root;
1529         struct btrfs_path *path;
1530         struct btrfs_key key;
1531         int ret;
1532
1533         if (location->objectid == BTRFS_ROOT_TREE_OBJECTID)
1534                 return fs_info->tree_root;
1535         if (location->objectid == BTRFS_EXTENT_TREE_OBJECTID)
1536                 return fs_info->extent_root;
1537         if (location->objectid == BTRFS_CHUNK_TREE_OBJECTID)
1538                 return fs_info->chunk_root;
1539         if (location->objectid == BTRFS_DEV_TREE_OBJECTID)
1540                 return fs_info->dev_root;
1541         if (location->objectid == BTRFS_CSUM_TREE_OBJECTID)
1542                 return fs_info->csum_root;
1543         if (location->objectid == BTRFS_QUOTA_TREE_OBJECTID)
1544                 return fs_info->quota_root ? fs_info->quota_root :
1545                                              ERR_PTR(-ENOENT);
1546         if (location->objectid == BTRFS_UUID_TREE_OBJECTID)
1547                 return fs_info->uuid_root ? fs_info->uuid_root :
1548                                             ERR_PTR(-ENOENT);
1549         if (location->objectid == BTRFS_FREE_SPACE_TREE_OBJECTID)
1550                 return fs_info->free_space_root ? fs_info->free_space_root :
1551                                                   ERR_PTR(-ENOENT);
1552 again:
1553         root = btrfs_lookup_fs_root(fs_info, location->objectid);
1554         if (root) {
1555                 if (check_ref && btrfs_root_refs(&root->root_item) == 0)
1556                         return ERR_PTR(-ENOENT);
1557                 return root;
1558         }
1559
1560         root = btrfs_read_tree_root(fs_info->tree_root, location);
1561         if (IS_ERR(root))
1562                 return root;
1563
1564         if (check_ref && btrfs_root_refs(&root->root_item) == 0) {
1565                 ret = -ENOENT;
1566                 goto fail;
1567         }
1568
1569         ret = btrfs_init_fs_root(root);
1570         if (ret)
1571                 goto fail;
1572
1573         path = btrfs_alloc_path();
1574         if (!path) {
1575                 ret = -ENOMEM;
1576                 goto fail;
1577         }
1578         key.objectid = BTRFS_ORPHAN_OBJECTID;
1579         key.type = BTRFS_ORPHAN_ITEM_KEY;
1580         key.offset = location->objectid;
1581
1582         ret = btrfs_search_slot(NULL, fs_info->tree_root, &key, path, 0, 0);
1583         btrfs_free_path(path);
1584         if (ret < 0)
1585                 goto fail;
1586         if (ret == 0)
1587                 set_bit(BTRFS_ROOT_ORPHAN_ITEM_INSERTED, &root->state);
1588
1589         ret = btrfs_insert_fs_root(fs_info, root);
1590         if (ret) {
1591                 if (ret == -EEXIST) {
1592                         btrfs_free_fs_root(root);
1593                         goto again;
1594                 }
1595                 goto fail;
1596         }
1597         return root;
1598 fail:
1599         btrfs_free_fs_root(root);
1600         return ERR_PTR(ret);
1601 }
1602
1603 static int btrfs_congested_fn(void *congested_data, int bdi_bits)
1604 {
1605         struct btrfs_fs_info *info = (struct btrfs_fs_info *)congested_data;
1606         int ret = 0;
1607         struct btrfs_device *device;
1608         struct backing_dev_info *bdi;
1609
1610         rcu_read_lock();
1611         list_for_each_entry_rcu(device, &info->fs_devices->devices, dev_list) {
1612                 if (!device->bdev)
1613                         continue;
1614                 bdi = device->bdev->bd_bdi;
1615                 if (bdi_congested(bdi, bdi_bits)) {
1616                         ret = 1;
1617                         break;
1618                 }
1619         }
1620         rcu_read_unlock();
1621         return ret;
1622 }
1623
1624 /*
1625  * called by the kthread helper functions to finally call the bio end_io
1626  * functions.  This is where read checksum verification actually happens
1627  */
1628 static void end_workqueue_fn(struct btrfs_work *work)
1629 {
1630         struct bio *bio;
1631         struct btrfs_end_io_wq *end_io_wq;
1632
1633         end_io_wq = container_of(work, struct btrfs_end_io_wq, work);
1634         bio = end_io_wq->bio;
1635
1636         bio->bi_status = end_io_wq->status;
1637         bio->bi_private = end_io_wq->private;
1638         bio->bi_end_io = end_io_wq->end_io;
1639         bio_endio(bio);
1640         kmem_cache_free(btrfs_end_io_wq_cache, end_io_wq);
1641 }
1642
1643 static int cleaner_kthread(void *arg)
1644 {
1645         struct btrfs_root *root = arg;
1646         struct btrfs_fs_info *fs_info = root->fs_info;
1647         int again;
1648
1649         while (1) {
1650                 again = 0;
1651
1652                 set_bit(BTRFS_FS_CLEANER_RUNNING, &fs_info->flags);
1653
1654                 /* Make the cleaner go to sleep early. */
1655                 if (btrfs_need_cleaner_sleep(fs_info))
1656                         goto sleep;
1657
1658                 /*
1659                  * Do not do anything if we might cause open_ctree() to block
1660                  * before we have finished mounting the filesystem.
1661                  */
1662                 if (!test_bit(BTRFS_FS_OPEN, &fs_info->flags))
1663                         goto sleep;
1664
1665                 if (!mutex_trylock(&fs_info->cleaner_mutex))
1666                         goto sleep;
1667
1668                 /*
1669                  * Avoid the problem that we change the status of the fs
1670                  * during the above check and trylock.
1671                  */
1672                 if (btrfs_need_cleaner_sleep(fs_info)) {
1673                         mutex_unlock(&fs_info->cleaner_mutex);
1674                         goto sleep;
1675                 }
1676
1677                 btrfs_run_delayed_iputs(fs_info);
1678
1679                 again = btrfs_clean_one_deleted_snapshot(root);
1680                 mutex_unlock(&fs_info->cleaner_mutex);
1681
1682                 /*
1683                  * The defragger has dealt with the R/O remount and umount,
1684                  * needn't do anything special here.
1685                  */
1686                 btrfs_run_defrag_inodes(fs_info);
1687
1688                 /*
1689                  * Acquires fs_info->delete_unused_bgs_mutex to avoid racing
1690                  * with relocation (btrfs_relocate_chunk) and relocation
1691                  * acquires fs_info->cleaner_mutex (btrfs_relocate_block_group)
1692                  * after acquiring fs_info->delete_unused_bgs_mutex. So we
1693                  * can't hold, nor need to, fs_info->cleaner_mutex when deleting
1694                  * unused block groups.
1695                  */
1696                 btrfs_delete_unused_bgs(fs_info);
1697 sleep:
1698                 clear_bit(BTRFS_FS_CLEANER_RUNNING, &fs_info->flags);
1699                 if (kthread_should_park())
1700                         kthread_parkme();
1701                 if (kthread_should_stop())
1702                         return 0;
1703                 if (!again) {
1704                         set_current_state(TASK_INTERRUPTIBLE);
1705                         schedule();
1706                         __set_current_state(TASK_RUNNING);
1707                 }
1708         }
1709 }
1710
1711 static int transaction_kthread(void *arg)
1712 {
1713         struct btrfs_root *root = arg;
1714         struct btrfs_fs_info *fs_info = root->fs_info;
1715         struct btrfs_trans_handle *trans;
1716         struct btrfs_transaction *cur;
1717         u64 transid;
1718         time64_t now;
1719         unsigned long delay;
1720         bool cannot_commit;
1721
1722         do {
1723                 cannot_commit = false;
1724                 delay = HZ * fs_info->commit_interval;
1725                 mutex_lock(&fs_info->transaction_kthread_mutex);
1726
1727                 spin_lock(&fs_info->trans_lock);
1728                 cur = fs_info->running_transaction;
1729                 if (!cur) {
1730                         spin_unlock(&fs_info->trans_lock);
1731                         goto sleep;
1732                 }
1733
1734                 now = ktime_get_seconds();
1735                 if (cur->state < TRANS_STATE_COMMIT_START &&
1736                     !test_bit(BTRFS_FS_NEED_ASYNC_COMMIT, &fs_info->flags) &&
1737                     (now < cur->start_time ||
1738                      now - cur->start_time < fs_info->commit_interval)) {
1739                         spin_unlock(&fs_info->trans_lock);
1740                         delay = HZ * 5;
1741                         goto sleep;
1742                 }
1743                 transid = cur->transid;
1744                 spin_unlock(&fs_info->trans_lock);
1745
1746                 /* If the file system is aborted, this will always fail. */
1747                 trans = btrfs_attach_transaction(root);
1748                 if (IS_ERR(trans)) {
1749                         if (PTR_ERR(trans) != -ENOENT)
1750                                 cannot_commit = true;
1751                         goto sleep;
1752                 }
1753                 if (transid == trans->transid) {
1754                         btrfs_commit_transaction(trans);
1755                 } else {
1756                         btrfs_end_transaction(trans);
1757                 }
1758 sleep:
1759                 wake_up_process(fs_info->cleaner_kthread);
1760                 mutex_unlock(&fs_info->transaction_kthread_mutex);
1761
1762                 if (unlikely(test_bit(BTRFS_FS_STATE_ERROR,
1763                                       &fs_info->fs_state)))
1764                         btrfs_cleanup_transaction(fs_info);
1765                 if (!kthread_should_stop() &&
1766                                 (!btrfs_transaction_blocked(fs_info) ||
1767                                  cannot_commit))
1768                         schedule_timeout_interruptible(delay);
1769         } while (!kthread_should_stop());
1770         return 0;
1771 }
1772
1773 /*
1774  * This will find the highest generation in the array of root backups.  The
1775  * index of the highest array is returned, or -EINVAL if we can't find
1776  * anything.
1777  *
1778  * We check to make sure the array is valid by comparing the
1779  * generation of the latest  root in the array with the generation
1780  * in the super block.  If they don't match we pitch it.
1781  */
1782 static int find_newest_super_backup(struct btrfs_fs_info *info)
1783 {
1784         const u64 newest_gen = btrfs_super_generation(info->super_copy);
1785         u64 cur;
1786         struct btrfs_root_backup *root_backup;
1787         int i;
1788
1789         for (i = 0; i < BTRFS_NUM_BACKUP_ROOTS; i++) {
1790                 root_backup = info->super_copy->super_roots + i;
1791                 cur = btrfs_backup_tree_root_gen(root_backup);
1792                 if (cur == newest_gen)
1793                         return i;
1794         }
1795
1796         return -EINVAL;
1797 }
1798
1799 /*
1800  * copy all the root pointers into the super backup array.
1801  * this will bump the backup pointer by one when it is
1802  * done
1803  */
1804 static void backup_super_roots(struct btrfs_fs_info *info)
1805 {
1806         const int next_backup = info->backup_root_index;
1807         struct btrfs_root_backup *root_backup;
1808
1809         root_backup = info->super_for_commit->super_roots + next_backup;
1810
1811         /*
1812          * make sure all of our padding and empty slots get zero filled
1813          * regardless of which ones we use today
1814          */
1815         memset(root_backup, 0, sizeof(*root_backup));
1816
1817         info->backup_root_index = (next_backup + 1) % BTRFS_NUM_BACKUP_ROOTS;
1818
1819         btrfs_set_backup_tree_root(root_backup, info->tree_root->node->start);
1820         btrfs_set_backup_tree_root_gen(root_backup,
1821                                btrfs_header_generation(info->tree_root->node));
1822
1823         btrfs_set_backup_tree_root_level(root_backup,
1824                                btrfs_header_level(info->tree_root->node));
1825
1826         btrfs_set_backup_chunk_root(root_backup, info->chunk_root->node->start);
1827         btrfs_set_backup_chunk_root_gen(root_backup,
1828                                btrfs_header_generation(info->chunk_root->node));
1829         btrfs_set_backup_chunk_root_level(root_backup,
1830                                btrfs_header_level(info->chunk_root->node));
1831
1832         btrfs_set_backup_extent_root(root_backup, info->extent_root->node->start);
1833         btrfs_set_backup_extent_root_gen(root_backup,
1834                                btrfs_header_generation(info->extent_root->node));
1835         btrfs_set_backup_extent_root_level(root_backup,
1836                                btrfs_header_level(info->extent_root->node));
1837
1838         /*
1839          * we might commit during log recovery, which happens before we set
1840          * the fs_root.  Make sure it is valid before we fill it in.
1841          */
1842         if (info->fs_root && info->fs_root->node) {
1843                 btrfs_set_backup_fs_root(root_backup,
1844                                          info->fs_root->node->start);
1845                 btrfs_set_backup_fs_root_gen(root_backup,
1846                                btrfs_header_generation(info->fs_root->node));
1847                 btrfs_set_backup_fs_root_level(root_backup,
1848                                btrfs_header_level(info->fs_root->node));
1849         }
1850
1851         btrfs_set_backup_dev_root(root_backup, info->dev_root->node->start);
1852         btrfs_set_backup_dev_root_gen(root_backup,
1853                                btrfs_header_generation(info->dev_root->node));
1854         btrfs_set_backup_dev_root_level(root_backup,
1855                                        btrfs_header_level(info->dev_root->node));
1856
1857         btrfs_set_backup_csum_root(root_backup, info->csum_root->node->start);
1858         btrfs_set_backup_csum_root_gen(root_backup,
1859                                btrfs_header_generation(info->csum_root->node));
1860         btrfs_set_backup_csum_root_level(root_backup,
1861                                btrfs_header_level(info->csum_root->node));
1862
1863         btrfs_set_backup_total_bytes(root_backup,
1864                              btrfs_super_total_bytes(info->super_copy));
1865         btrfs_set_backup_bytes_used(root_backup,
1866                              btrfs_super_bytes_used(info->super_copy));
1867         btrfs_set_backup_num_devices(root_backup,
1868                              btrfs_super_num_devices(info->super_copy));
1869
1870         /*
1871          * if we don't copy this out to the super_copy, it won't get remembered
1872          * for the next commit
1873          */
1874         memcpy(&info->super_copy->super_roots,
1875                &info->super_for_commit->super_roots,
1876                sizeof(*root_backup) * BTRFS_NUM_BACKUP_ROOTS);
1877 }
1878
1879 /*
1880  * read_backup_root - Reads a backup root based on the passed priority. Prio 0
1881  * is the newest, prio 1/2/3 are 2nd newest/3rd newest/4th (oldest) backup roots
1882  *
1883  * fs_info - filesystem whose backup roots need to be read
1884  * priority - priority of backup root required
1885  *
1886  * Returns backup root index on success and -EINVAL otherwise.
1887  */
1888 static int read_backup_root(struct btrfs_fs_info *fs_info, u8 priority)
1889 {
1890         int backup_index = find_newest_super_backup(fs_info);
1891         struct btrfs_super_block *super = fs_info->super_copy;
1892         struct btrfs_root_backup *root_backup;
1893
1894         if (priority < BTRFS_NUM_BACKUP_ROOTS && backup_index >= 0) {
1895                 if (priority == 0)
1896                         return backup_index;
1897
1898                 backup_index = backup_index + BTRFS_NUM_BACKUP_ROOTS - priority;
1899                 backup_index %= BTRFS_NUM_BACKUP_ROOTS;
1900         } else {
1901                 return -EINVAL;
1902         }
1903
1904         root_backup = super->super_roots + backup_index;
1905
1906         btrfs_set_super_generation(super,
1907                                    btrfs_backup_tree_root_gen(root_backup));
1908         btrfs_set_super_root(super, btrfs_backup_tree_root(root_backup));
1909         btrfs_set_super_root_level(super,
1910                                    btrfs_backup_tree_root_level(root_backup));
1911         btrfs_set_super_bytes_used(super, btrfs_backup_bytes_used(root_backup));
1912
1913         /*
1914          * Fixme: the total bytes and num_devices need to match or we should
1915          * need a fsck
1916          */
1917         btrfs_set_super_total_bytes(super, btrfs_backup_total_bytes(root_backup));
1918         btrfs_set_super_num_devices(super, btrfs_backup_num_devices(root_backup));
1919
1920         return backup_index;
1921 }
1922
1923 /* helper to cleanup workers */
1924 static void btrfs_stop_all_workers(struct btrfs_fs_info *fs_info)
1925 {
1926         btrfs_destroy_workqueue(fs_info->fixup_workers);
1927         btrfs_destroy_workqueue(fs_info->delalloc_workers);
1928         btrfs_destroy_workqueue(fs_info->workers);
1929         btrfs_destroy_workqueue(fs_info->endio_workers);
1930         btrfs_destroy_workqueue(fs_info->endio_raid56_workers);
1931         btrfs_destroy_workqueue(fs_info->endio_repair_workers);
1932         btrfs_destroy_workqueue(fs_info->rmw_workers);
1933         btrfs_destroy_workqueue(fs_info->endio_write_workers);
1934         btrfs_destroy_workqueue(fs_info->endio_freespace_worker);
1935         btrfs_destroy_workqueue(fs_info->delayed_workers);
1936         btrfs_destroy_workqueue(fs_info->caching_workers);
1937         btrfs_destroy_workqueue(fs_info->readahead_workers);
1938         btrfs_destroy_workqueue(fs_info->flush_workers);
1939         btrfs_destroy_workqueue(fs_info->qgroup_rescan_workers);
1940         if (fs_info->discard_ctl.discard_workers)
1941                 destroy_workqueue(fs_info->discard_ctl.discard_workers);
1942         /*
1943          * Now that all other work queues are destroyed, we can safely destroy
1944          * the queues used for metadata I/O, since tasks from those other work
1945          * queues can do metadata I/O operations.
1946          */
1947         btrfs_destroy_workqueue(fs_info->endio_meta_workers);
1948         btrfs_destroy_workqueue(fs_info->endio_meta_write_workers);
1949 }
1950
1951 static void free_root_extent_buffers(struct btrfs_root *root)
1952 {
1953         if (root) {
1954                 free_extent_buffer(root->node);
1955                 free_extent_buffer(root->commit_root);
1956                 root->node = NULL;
1957                 root->commit_root = NULL;
1958         }
1959 }
1960
1961 /* helper to cleanup tree roots */
1962 static void free_root_pointers(struct btrfs_fs_info *info, bool free_chunk_root)
1963 {
1964         free_root_extent_buffers(info->tree_root);
1965
1966         free_root_extent_buffers(info->dev_root);
1967         free_root_extent_buffers(info->extent_root);
1968         free_root_extent_buffers(info->csum_root);
1969         free_root_extent_buffers(info->quota_root);
1970         free_root_extent_buffers(info->uuid_root);
1971         if (free_chunk_root)
1972                 free_root_extent_buffers(info->chunk_root);
1973         free_root_extent_buffers(info->free_space_root);
1974 }
1975
1976 void btrfs_free_fs_roots(struct btrfs_fs_info *fs_info)
1977 {
1978         int ret;
1979         struct btrfs_root *gang[8];
1980         int i;
1981
1982         while (!list_empty(&fs_info->dead_roots)) {
1983                 gang[0] = list_entry(fs_info->dead_roots.next,
1984                                      struct btrfs_root, root_list);
1985                 list_del(&gang[0]->root_list);
1986
1987                 if (test_bit(BTRFS_ROOT_IN_RADIX, &gang[0]->state)) {
1988                         btrfs_drop_and_free_fs_root(fs_info, gang[0]);
1989                 } else {
1990                         free_extent_buffer(gang[0]->node);
1991                         free_extent_buffer(gang[0]->commit_root);
1992                         btrfs_put_fs_root(gang[0]);
1993                 }
1994         }
1995
1996         while (1) {
1997                 ret = radix_tree_gang_lookup(&fs_info->fs_roots_radix,
1998                                              (void **)gang, 0,
1999                                              ARRAY_SIZE(gang));
2000                 if (!ret)
2001                         break;
2002                 for (i = 0; i < ret; i++)
2003                         btrfs_drop_and_free_fs_root(fs_info, gang[i]);
2004         }
2005
2006         if (test_bit(BTRFS_FS_STATE_ERROR, &fs_info->fs_state)) {
2007                 btrfs_free_log_root_tree(NULL, fs_info);
2008                 btrfs_destroy_pinned_extent(fs_info, fs_info->pinned_extents);
2009         }
2010 }
2011
2012 static void btrfs_init_scrub(struct btrfs_fs_info *fs_info)
2013 {
2014         mutex_init(&fs_info->scrub_lock);
2015         atomic_set(&fs_info->scrubs_running, 0);
2016         atomic_set(&fs_info->scrub_pause_req, 0);
2017         atomic_set(&fs_info->scrubs_paused, 0);
2018         atomic_set(&fs_info->scrub_cancel_req, 0);
2019         init_waitqueue_head(&fs_info->scrub_pause_wait);
2020         refcount_set(&fs_info->scrub_workers_refcnt, 0);
2021 }
2022
2023 static void btrfs_init_balance(struct btrfs_fs_info *fs_info)
2024 {
2025         spin_lock_init(&fs_info->balance_lock);
2026         mutex_init(&fs_info->balance_mutex);
2027         atomic_set(&fs_info->balance_pause_req, 0);
2028         atomic_set(&fs_info->balance_cancel_req, 0);
2029         fs_info->balance_ctl = NULL;
2030         init_waitqueue_head(&fs_info->balance_wait_q);
2031 }
2032
2033 static void btrfs_init_btree_inode(struct btrfs_fs_info *fs_info)
2034 {
2035         struct inode *inode = fs_info->btree_inode;
2036
2037         inode->i_ino = BTRFS_BTREE_INODE_OBJECTID;
2038         set_nlink(inode, 1);
2039         /*
2040          * we set the i_size on the btree inode to the max possible int.
2041          * the real end of the address space is determined by all of
2042          * the devices in the system
2043          */
2044         inode->i_size = OFFSET_MAX;
2045         inode->i_mapping->a_ops = &btree_aops;
2046
2047         RB_CLEAR_NODE(&BTRFS_I(inode)->rb_node);
2048         extent_io_tree_init(fs_info, &BTRFS_I(inode)->io_tree,
2049                             IO_TREE_INODE_IO, inode);
2050         BTRFS_I(inode)->io_tree.track_uptodate = false;
2051         extent_map_tree_init(&BTRFS_I(inode)->extent_tree);
2052
2053         BTRFS_I(inode)->io_tree.ops = &btree_extent_io_ops;
2054
2055         BTRFS_I(inode)->root = fs_info->tree_root;
2056         memset(&BTRFS_I(inode)->location, 0, sizeof(struct btrfs_key));
2057         set_bit(BTRFS_INODE_DUMMY, &BTRFS_I(inode)->runtime_flags);
2058         btrfs_insert_inode_hash(inode);
2059 }
2060
2061 static void btrfs_init_dev_replace_locks(struct btrfs_fs_info *fs_info)
2062 {
2063         mutex_init(&fs_info->dev_replace.lock_finishing_cancel_unmount);
2064         init_rwsem(&fs_info->dev_replace.rwsem);
2065         init_waitqueue_head(&fs_info->dev_replace.replace_wait);
2066 }
2067
2068 static void btrfs_init_qgroup(struct btrfs_fs_info *fs_info)
2069 {
2070         spin_lock_init(&fs_info->qgroup_lock);
2071         mutex_init(&fs_info->qgroup_ioctl_lock);
2072         fs_info->qgroup_tree = RB_ROOT;
2073         INIT_LIST_HEAD(&fs_info->dirty_qgroups);
2074         fs_info->qgroup_seq = 1;
2075         fs_info->qgroup_ulist = NULL;
2076         fs_info->qgroup_rescan_running = false;
2077         mutex_init(&fs_info->qgroup_rescan_lock);
2078 }
2079
2080 static int btrfs_init_workqueues(struct btrfs_fs_info *fs_info,
2081                 struct btrfs_fs_devices *fs_devices)
2082 {
2083         u32 max_active = fs_info->thread_pool_size;
2084         unsigned int flags = WQ_MEM_RECLAIM | WQ_FREEZABLE | WQ_UNBOUND;
2085
2086         fs_info->workers =
2087                 btrfs_alloc_workqueue(fs_info, "worker",
2088                                       flags | WQ_HIGHPRI, max_active, 16);
2089
2090         fs_info->delalloc_workers =
2091                 btrfs_alloc_workqueue(fs_info, "delalloc",
2092                                       flags, max_active, 2);
2093
2094         fs_info->flush_workers =
2095                 btrfs_alloc_workqueue(fs_info, "flush_delalloc",
2096                                       flags, max_active, 0);
2097
2098         fs_info->caching_workers =
2099                 btrfs_alloc_workqueue(fs_info, "cache", flags, max_active, 0);
2100
2101         fs_info->fixup_workers =
2102                 btrfs_alloc_workqueue(fs_info, "fixup", flags, 1, 0);
2103
2104         /*
2105          * endios are largely parallel and should have a very
2106          * low idle thresh
2107          */
2108         fs_info->endio_workers =
2109                 btrfs_alloc_workqueue(fs_info, "endio", flags, max_active, 4);
2110         fs_info->endio_meta_workers =
2111                 btrfs_alloc_workqueue(fs_info, "endio-meta", flags,
2112                                       max_active, 4);
2113         fs_info->endio_meta_write_workers =
2114                 btrfs_alloc_workqueue(fs_info, "endio-meta-write", flags,
2115                                       max_active, 2);
2116         fs_info->endio_raid56_workers =
2117                 btrfs_alloc_workqueue(fs_info, "endio-raid56", flags,
2118                                       max_active, 4);
2119         fs_info->endio_repair_workers =
2120                 btrfs_alloc_workqueue(fs_info, "endio-repair", flags, 1, 0);
2121         fs_info->rmw_workers =
2122                 btrfs_alloc_workqueue(fs_info, "rmw", flags, max_active, 2);
2123         fs_info->endio_write_workers =
2124                 btrfs_alloc_workqueue(fs_info, "endio-write", flags,
2125                                       max_active, 2);
2126         fs_info->endio_freespace_worker =
2127                 btrfs_alloc_workqueue(fs_info, "freespace-write", flags,
2128                                       max_active, 0);
2129         fs_info->delayed_workers =
2130                 btrfs_alloc_workqueue(fs_info, "delayed-meta", flags,
2131                                       max_active, 0);
2132         fs_info->readahead_workers =
2133                 btrfs_alloc_workqueue(fs_info, "readahead", flags,
2134                                       max_active, 2);
2135         fs_info->qgroup_rescan_workers =
2136                 btrfs_alloc_workqueue(fs_info, "qgroup-rescan", flags, 1, 0);
2137         fs_info->discard_ctl.discard_workers =
2138                 alloc_workqueue("btrfs_discard", WQ_UNBOUND | WQ_FREEZABLE, 1);
2139
2140         if (!(fs_info->workers && fs_info->delalloc_workers &&
2141               fs_info->flush_workers &&
2142               fs_info->endio_workers && fs_info->endio_meta_workers &&
2143               fs_info->endio_meta_write_workers &&
2144               fs_info->endio_repair_workers &&
2145               fs_info->endio_write_workers && fs_info->endio_raid56_workers &&
2146               fs_info->endio_freespace_worker && fs_info->rmw_workers &&
2147               fs_info->caching_workers && fs_info->readahead_workers &&
2148               fs_info->fixup_workers && fs_info->delayed_workers &&
2149               fs_info->qgroup_rescan_workers &&
2150               fs_info->discard_ctl.discard_workers)) {
2151                 return -ENOMEM;
2152         }
2153
2154         return 0;
2155 }
2156
2157 static int btrfs_init_csum_hash(struct btrfs_fs_info *fs_info, u16 csum_type)
2158 {
2159         struct crypto_shash *csum_shash;
2160         const char *csum_driver = btrfs_super_csum_driver(csum_type);
2161
2162         csum_shash = crypto_alloc_shash(csum_driver, 0, 0);
2163
2164         if (IS_ERR(csum_shash)) {
2165                 btrfs_err(fs_info, "error allocating %s hash for checksum",
2166                           csum_driver);
2167                 return PTR_ERR(csum_shash);
2168         }
2169
2170         fs_info->csum_shash = csum_shash;
2171
2172         return 0;
2173 }
2174
2175 static void btrfs_free_csum_hash(struct btrfs_fs_info *fs_info)
2176 {
2177         crypto_free_shash(fs_info->csum_shash);
2178 }
2179
2180 static int btrfs_replay_log(struct btrfs_fs_info *fs_info,
2181                             struct btrfs_fs_devices *fs_devices)
2182 {
2183         int ret;
2184         struct btrfs_root *log_tree_root;
2185         struct btrfs_super_block *disk_super = fs_info->super_copy;
2186         u64 bytenr = btrfs_super_log_root(disk_super);
2187         int level = btrfs_super_log_root_level(disk_super);
2188
2189         if (fs_devices->rw_devices == 0) {
2190                 btrfs_warn(fs_info, "log replay required on RO media");
2191                 return -EIO;
2192         }
2193
2194         log_tree_root = btrfs_alloc_root(fs_info, BTRFS_TREE_LOG_OBJECTID,
2195                                          GFP_KERNEL);
2196         if (!log_tree_root)
2197                 return -ENOMEM;
2198
2199         log_tree_root->node = read_tree_block(fs_info, bytenr,
2200                                               fs_info->generation + 1,
2201                                               level, NULL);
2202         if (IS_ERR(log_tree_root->node)) {
2203                 btrfs_warn(fs_info, "failed to read log tree");
2204                 ret = PTR_ERR(log_tree_root->node);
2205                 kfree(log_tree_root);
2206                 return ret;
2207         } else if (!extent_buffer_uptodate(log_tree_root->node)) {
2208                 btrfs_err(fs_info, "failed to read log tree");
2209                 free_extent_buffer(log_tree_root->node);
2210                 kfree(log_tree_root);
2211                 return -EIO;
2212         }
2213         /* returns with log_tree_root freed on success */
2214         ret = btrfs_recover_log_trees(log_tree_root);
2215         if (ret) {
2216                 btrfs_handle_fs_error(fs_info, ret,
2217                                       "Failed to recover log tree");
2218                 free_extent_buffer(log_tree_root->node);
2219                 kfree(log_tree_root);
2220                 return ret;
2221         }
2222
2223         if (sb_rdonly(fs_info->sb)) {
2224                 ret = btrfs_commit_super(fs_info);
2225                 if (ret)
2226                         return ret;
2227         }
2228
2229         return 0;
2230 }
2231
2232 static int btrfs_read_roots(struct btrfs_fs_info *fs_info)
2233 {
2234         struct btrfs_root *tree_root = fs_info->tree_root;
2235         struct btrfs_root *root;
2236         struct btrfs_key location;
2237         int ret;
2238
2239         BUG_ON(!fs_info->tree_root);
2240
2241         location.objectid = BTRFS_EXTENT_TREE_OBJECTID;
2242         location.type = BTRFS_ROOT_ITEM_KEY;
2243         location.offset = 0;
2244
2245         root = btrfs_read_tree_root(tree_root, &location);
2246         if (IS_ERR(root)) {
2247                 ret = PTR_ERR(root);
2248                 goto out;
2249         }
2250         set_bit(BTRFS_ROOT_TRACK_DIRTY, &root->state);
2251         fs_info->extent_root = root;
2252
2253         location.objectid = BTRFS_DEV_TREE_OBJECTID;
2254         root = btrfs_read_tree_root(tree_root, &location);
2255         if (IS_ERR(root)) {
2256                 ret = PTR_ERR(root);
2257                 goto out;
2258         }
2259         set_bit(BTRFS_ROOT_TRACK_DIRTY, &root->state);
2260         fs_info->dev_root = root;
2261         btrfs_init_devices_late(fs_info);
2262
2263         location.objectid = BTRFS_CSUM_TREE_OBJECTID;
2264         root = btrfs_read_tree_root(tree_root, &location);
2265         if (IS_ERR(root)) {
2266                 ret = PTR_ERR(root);
2267                 goto out;
2268         }
2269         set_bit(BTRFS_ROOT_TRACK_DIRTY, &root->state);
2270         fs_info->csum_root = root;
2271
2272         location.objectid = BTRFS_QUOTA_TREE_OBJECTID;
2273         root = btrfs_read_tree_root(tree_root, &location);
2274         if (!IS_ERR(root)) {
2275                 set_bit(BTRFS_ROOT_TRACK_DIRTY, &root->state);
2276                 set_bit(BTRFS_FS_QUOTA_ENABLED, &fs_info->flags);
2277                 fs_info->quota_root = root;
2278         }
2279
2280         location.objectid = BTRFS_UUID_TREE_OBJECTID;
2281         root = btrfs_read_tree_root(tree_root, &location);
2282         if (IS_ERR(root)) {
2283                 ret = PTR_ERR(root);
2284                 if (ret != -ENOENT)
2285                         goto out;
2286         } else {
2287                 set_bit(BTRFS_ROOT_TRACK_DIRTY, &root->state);
2288                 fs_info->uuid_root = root;
2289         }
2290
2291         if (btrfs_fs_compat_ro(fs_info, FREE_SPACE_TREE)) {
2292                 location.objectid = BTRFS_FREE_SPACE_TREE_OBJECTID;
2293                 root = btrfs_read_tree_root(tree_root, &location);
2294                 if (IS_ERR(root)) {
2295                         ret = PTR_ERR(root);
2296                         goto out;
2297                 }
2298                 set_bit(BTRFS_ROOT_TRACK_DIRTY, &root->state);
2299                 fs_info->free_space_root = root;
2300         }
2301
2302         return 0;
2303 out:
2304         btrfs_warn(fs_info, "failed to read root (objectid=%llu): %d",
2305                    location.objectid, ret);
2306         return ret;
2307 }
2308
2309 /*
2310  * Real super block validation
2311  * NOTE: super csum type and incompat features will not be checked here.
2312  *
2313  * @sb:         super block to check
2314  * @mirror_num: the super block number to check its bytenr:
2315  *              0       the primary (1st) sb
2316  *              1, 2    2nd and 3rd backup copy
2317  *             -1       skip bytenr check
2318  */
2319 static int validate_super(struct btrfs_fs_info *fs_info,
2320                             struct btrfs_super_block *sb, int mirror_num)
2321 {
2322         u64 nodesize = btrfs_super_nodesize(sb);
2323         u64 sectorsize = btrfs_super_sectorsize(sb);
2324         int ret = 0;
2325
2326         if (btrfs_super_magic(sb) != BTRFS_MAGIC) {
2327                 btrfs_err(fs_info, "no valid FS found");
2328                 ret = -EINVAL;
2329         }
2330         if (btrfs_super_flags(sb) & ~BTRFS_SUPER_FLAG_SUPP) {
2331                 btrfs_err(fs_info, "unrecognized or unsupported super flag: %llu",
2332                                 btrfs_super_flags(sb) & ~BTRFS_SUPER_FLAG_SUPP);
2333                 ret = -EINVAL;
2334         }
2335         if (btrfs_super_root_level(sb) >= BTRFS_MAX_LEVEL) {
2336                 btrfs_err(fs_info, "tree_root level too big: %d >= %d",
2337                                 btrfs_super_root_level(sb), BTRFS_MAX_LEVEL);
2338                 ret = -EINVAL;
2339         }
2340         if (btrfs_super_chunk_root_level(sb) >= BTRFS_MAX_LEVEL) {
2341                 btrfs_err(fs_info, "chunk_root level too big: %d >= %d",
2342                                 btrfs_super_chunk_root_level(sb), BTRFS_MAX_LEVEL);
2343                 ret = -EINVAL;
2344         }
2345         if (btrfs_super_log_root_level(sb) >= BTRFS_MAX_LEVEL) {
2346                 btrfs_err(fs_info, "log_root level too big: %d >= %d",
2347                                 btrfs_super_log_root_level(sb), BTRFS_MAX_LEVEL);
2348                 ret = -EINVAL;
2349         }
2350
2351         /*
2352          * Check sectorsize and nodesize first, other check will need it.
2353          * Check all possible sectorsize(4K, 8K, 16K, 32K, 64K) here.
2354          */
2355         if (!is_power_of_2(sectorsize) || sectorsize < 4096 ||
2356             sectorsize > BTRFS_MAX_METADATA_BLOCKSIZE) {
2357                 btrfs_err(fs_info, "invalid sectorsize %llu", sectorsize);
2358                 ret = -EINVAL;
2359         }
2360         /* Only PAGE SIZE is supported yet */
2361         if (sectorsize != PAGE_SIZE) {
2362                 btrfs_err(fs_info,
2363                         "sectorsize %llu not supported yet, only support %lu",
2364                         sectorsize, PAGE_SIZE);
2365                 ret = -EINVAL;
2366         }
2367         if (!is_power_of_2(nodesize) || nodesize < sectorsize ||
2368             nodesize > BTRFS_MAX_METADATA_BLOCKSIZE) {
2369                 btrfs_err(fs_info, "invalid nodesize %llu", nodesize);
2370                 ret = -EINVAL;
2371         }
2372         if (nodesize != le32_to_cpu(sb->__unused_leafsize)) {
2373                 btrfs_err(fs_info, "invalid leafsize %u, should be %llu",
2374                           le32_to_cpu(sb->__unused_leafsize), nodesize);
2375                 ret = -EINVAL;
2376         }
2377
2378         /* Root alignment check */
2379         if (!IS_ALIGNED(btrfs_super_root(sb), sectorsize)) {
2380                 btrfs_warn(fs_info, "tree_root block unaligned: %llu",
2381                            btrfs_super_root(sb));
2382                 ret = -EINVAL;
2383         }
2384         if (!IS_ALIGNED(btrfs_super_chunk_root(sb), sectorsize)) {
2385                 btrfs_warn(fs_info, "chunk_root block unaligned: %llu",
2386                            btrfs_super_chunk_root(sb));
2387                 ret = -EINVAL;
2388         }
2389         if (!IS_ALIGNED(btrfs_super_log_root(sb), sectorsize)) {
2390                 btrfs_warn(fs_info, "log_root block unaligned: %llu",
2391                            btrfs_super_log_root(sb));
2392                 ret = -EINVAL;
2393         }
2394
2395         if (memcmp(fs_info->fs_devices->metadata_uuid, sb->dev_item.fsid,
2396                    BTRFS_FSID_SIZE) != 0) {
2397                 btrfs_err(fs_info,
2398                         "dev_item UUID does not match metadata fsid: %pU != %pU",
2399                         fs_info->fs_devices->metadata_uuid, sb->dev_item.fsid);
2400                 ret = -EINVAL;
2401         }
2402
2403         /*
2404          * Hint to catch really bogus numbers, bitflips or so, more exact checks are
2405          * done later
2406          */
2407         if (btrfs_super_bytes_used(sb) < 6 * btrfs_super_nodesize(sb)) {
2408                 btrfs_err(fs_info, "bytes_used is too small %llu",
2409                           btrfs_super_bytes_used(sb));
2410                 ret = -EINVAL;
2411         }
2412         if (!is_power_of_2(btrfs_super_stripesize(sb))) {
2413                 btrfs_err(fs_info, "invalid stripesize %u",
2414                           btrfs_super_stripesize(sb));
2415                 ret = -EINVAL;
2416         }
2417         if (btrfs_super_num_devices(sb) > (1UL << 31))
2418                 btrfs_warn(fs_info, "suspicious number of devices: %llu",
2419                            btrfs_super_num_devices(sb));
2420         if (btrfs_super_num_devices(sb) == 0) {
2421                 btrfs_err(fs_info, "number of devices is 0");
2422                 ret = -EINVAL;
2423         }
2424
2425         if (mirror_num >= 0 &&
2426             btrfs_super_bytenr(sb) != btrfs_sb_offset(mirror_num)) {
2427                 btrfs_err(fs_info, "super offset mismatch %llu != %u",
2428                           btrfs_super_bytenr(sb), BTRFS_SUPER_INFO_OFFSET);
2429                 ret = -EINVAL;
2430         }
2431
2432         /*
2433          * Obvious sys_chunk_array corruptions, it must hold at least one key
2434          * and one chunk
2435          */
2436         if (btrfs_super_sys_array_size(sb) > BTRFS_SYSTEM_CHUNK_ARRAY_SIZE) {
2437                 btrfs_err(fs_info, "system chunk array too big %u > %u",
2438                           btrfs_super_sys_array_size(sb),
2439                           BTRFS_SYSTEM_CHUNK_ARRAY_SIZE);
2440                 ret = -EINVAL;
2441         }
2442         if (btrfs_super_sys_array_size(sb) < sizeof(struct btrfs_disk_key)
2443                         + sizeof(struct btrfs_chunk)) {
2444                 btrfs_err(fs_info, "system chunk array too small %u < %zu",
2445                           btrfs_super_sys_array_size(sb),
2446                           sizeof(struct btrfs_disk_key)
2447                           + sizeof(struct btrfs_chunk));
2448                 ret = -EINVAL;
2449         }
2450
2451         /*
2452          * The generation is a global counter, we'll trust it more than the others
2453          * but it's still possible that it's the one that's wrong.
2454          */
2455         if (btrfs_super_generation(sb) < btrfs_super_chunk_root_generation(sb))
2456                 btrfs_warn(fs_info,
2457                         "suspicious: generation < chunk_root_generation: %llu < %llu",
2458                         btrfs_super_generation(sb),
2459                         btrfs_super_chunk_root_generation(sb));
2460         if (btrfs_super_generation(sb) < btrfs_super_cache_generation(sb)
2461             && btrfs_super_cache_generation(sb) != (u64)-1)
2462                 btrfs_warn(fs_info,
2463                         "suspicious: generation < cache_generation: %llu < %llu",
2464                         btrfs_super_generation(sb),
2465                         btrfs_super_cache_generation(sb));
2466
2467         return ret;
2468 }
2469
2470 /*
2471  * Validation of super block at mount time.
2472  * Some checks already done early at mount time, like csum type and incompat
2473  * flags will be skipped.
2474  */
2475 static int btrfs_validate_mount_super(struct btrfs_fs_info *fs_info)
2476 {
2477         return validate_super(fs_info, fs_info->super_copy, 0);
2478 }
2479
2480 /*
2481  * Validation of super block at write time.
2482  * Some checks like bytenr check will be skipped as their values will be
2483  * overwritten soon.
2484  * Extra checks like csum type and incompat flags will be done here.
2485  */
2486 static int btrfs_validate_write_super(struct btrfs_fs_info *fs_info,
2487                                       struct btrfs_super_block *sb)
2488 {
2489         int ret;
2490
2491         ret = validate_super(fs_info, sb, -1);
2492         if (ret < 0)
2493                 goto out;
2494         if (!btrfs_supported_super_csum(btrfs_super_csum_type(sb))) {
2495                 ret = -EUCLEAN;
2496                 btrfs_err(fs_info, "invalid csum type, has %u want %u",
2497                           btrfs_super_csum_type(sb), BTRFS_CSUM_TYPE_CRC32);
2498                 goto out;
2499         }
2500         if (btrfs_super_incompat_flags(sb) & ~BTRFS_FEATURE_INCOMPAT_SUPP) {
2501                 ret = -EUCLEAN;
2502                 btrfs_err(fs_info,
2503                 "invalid incompat flags, has 0x%llx valid mask 0x%llx",
2504                           btrfs_super_incompat_flags(sb),
2505                           (unsigned long long)BTRFS_FEATURE_INCOMPAT_SUPP);
2506                 goto out;
2507         }
2508 out:
2509         if (ret < 0)
2510                 btrfs_err(fs_info,
2511                 "super block corruption detected before writing it to disk");
2512         return ret;
2513 }
2514
2515 static int __cold init_tree_roots(struct btrfs_fs_info *fs_info)
2516 {
2517         int backup_index = find_newest_super_backup(fs_info);
2518         struct btrfs_super_block *sb = fs_info->super_copy;
2519         struct btrfs_root *tree_root = fs_info->tree_root;
2520         bool handle_error = false;
2521         int ret = 0;
2522         int i;
2523
2524         for (i = 0; i < BTRFS_NUM_BACKUP_ROOTS; i++) {
2525                 u64 generation;
2526                 int level;
2527
2528                 if (handle_error) {
2529                         if (!IS_ERR(tree_root->node))
2530                                 free_extent_buffer(tree_root->node);
2531                         tree_root->node = NULL;
2532
2533                         if (!btrfs_test_opt(fs_info, USEBACKUPROOT))
2534                                 break;
2535
2536                         free_root_pointers(fs_info, 0);
2537
2538                         /*
2539                          * Don't use the log in recovery mode, it won't be
2540                          * valid
2541                          */
2542                         btrfs_set_super_log_root(sb, 0);
2543
2544                         /* We can't trust the free space cache either */
2545                         btrfs_set_opt(fs_info->mount_opt, CLEAR_CACHE);
2546
2547                         ret = read_backup_root(fs_info, i);
2548                         backup_index = ret;
2549                         if (ret < 0)
2550                                 return ret;
2551                 }
2552                 generation = btrfs_super_generation(sb);
2553                 level = btrfs_super_root_level(sb);
2554                 tree_root->node = read_tree_block(fs_info, btrfs_super_root(sb),
2555                                                   generation, level, NULL);
2556                 if (IS_ERR(tree_root->node) ||
2557                     !extent_buffer_uptodate(tree_root->node)) {
2558                         handle_error = true;
2559
2560                         if (IS_ERR(tree_root->node))
2561                                 ret = PTR_ERR(tree_root->node);
2562                         else if (!extent_buffer_uptodate(tree_root->node))
2563                                 ret = -EUCLEAN;
2564
2565                         btrfs_warn(fs_info, "failed to read tree root");
2566                         continue;
2567                 }
2568
2569                 btrfs_set_root_node(&tree_root->root_item, tree_root->node);
2570                 tree_root->commit_root = btrfs_root_node(tree_root);
2571                 btrfs_set_root_refs(&tree_root->root_item, 1);
2572
2573                 /*
2574                  * No need to hold btrfs_root::objectid_mutex since the fs
2575                  * hasn't been fully initialised and we are the only user
2576                  */
2577                 ret = btrfs_find_highest_objectid(tree_root,
2578                                                 &tree_root->highest_objectid);
2579                 if (ret < 0) {
2580                         handle_error = true;
2581                         continue;
2582                 }
2583
2584                 ASSERT(tree_root->highest_objectid <= BTRFS_LAST_FREE_OBJECTID);
2585
2586                 ret = btrfs_read_roots(fs_info);
2587                 if (ret < 0) {
2588                         handle_error = true;
2589                         continue;
2590                 }
2591
2592                 /* All successful */
2593                 fs_info->generation = generation;
2594                 fs_info->last_trans_committed = generation;
2595
2596                 /* Always begin writing backup roots after the one being used */
2597                 if (backup_index < 0) {
2598                         fs_info->backup_root_index = 0;
2599                 } else {
2600                         fs_info->backup_root_index = backup_index + 1;
2601                         fs_info->backup_root_index %= BTRFS_NUM_BACKUP_ROOTS;
2602                 }
2603                 break;
2604         }
2605
2606         return ret;
2607 }
2608
2609 int __cold open_ctree(struct super_block *sb,
2610                struct btrfs_fs_devices *fs_devices,
2611                char *options)
2612 {
2613         u32 sectorsize;
2614         u32 nodesize;
2615         u32 stripesize;
2616         u64 generation;
2617         u64 features;
2618         u16 csum_type;
2619         struct btrfs_key location;
2620         struct buffer_head *bh;
2621         struct btrfs_super_block *disk_super;
2622         struct btrfs_fs_info *fs_info = btrfs_sb(sb);
2623         struct btrfs_root *tree_root;
2624         struct btrfs_root *chunk_root;
2625         int ret;
2626         int err = -EINVAL;
2627         int clear_free_space_tree = 0;
2628         int level;
2629
2630         tree_root = btrfs_alloc_root(fs_info, BTRFS_ROOT_TREE_OBJECTID,
2631                                      GFP_KERNEL);
2632         fs_info->tree_root = tree_root;
2633         chunk_root = btrfs_alloc_root(fs_info, BTRFS_CHUNK_TREE_OBJECTID,
2634                                       GFP_KERNEL);
2635         fs_info->chunk_root = chunk_root;
2636         if (!tree_root || !chunk_root) {
2637                 err = -ENOMEM;
2638                 goto fail;
2639         }
2640
2641         ret = init_srcu_struct(&fs_info->subvol_srcu);
2642         if (ret) {
2643                 err = ret;
2644                 goto fail;
2645         }
2646
2647         ret = percpu_counter_init(&fs_info->dio_bytes, 0, GFP_KERNEL);
2648         if (ret) {
2649                 err = ret;
2650                 goto fail_srcu;
2651         }
2652
2653         ret = percpu_counter_init(&fs_info->dirty_metadata_bytes, 0, GFP_KERNEL);
2654         if (ret) {
2655                 err = ret;
2656                 goto fail_dio_bytes;
2657         }
2658         fs_info->dirty_metadata_batch = PAGE_SIZE *
2659                                         (1 + ilog2(nr_cpu_ids));
2660
2661         ret = percpu_counter_init(&fs_info->delalloc_bytes, 0, GFP_KERNEL);
2662         if (ret) {
2663                 err = ret;
2664                 goto fail_dirty_metadata_bytes;
2665         }
2666
2667         ret = percpu_counter_init(&fs_info->dev_replace.bio_counter, 0,
2668                         GFP_KERNEL);
2669         if (ret) {
2670                 err = ret;
2671                 goto fail_delalloc_bytes;
2672         }
2673
2674         INIT_RADIX_TREE(&fs_info->fs_roots_radix, GFP_ATOMIC);
2675         INIT_RADIX_TREE(&fs_info->buffer_radix, GFP_ATOMIC);
2676         INIT_LIST_HEAD(&fs_info->trans_list);
2677         INIT_LIST_HEAD(&fs_info->dead_roots);
2678         INIT_LIST_HEAD(&fs_info->delayed_iputs);
2679         INIT_LIST_HEAD(&fs_info->delalloc_roots);
2680         INIT_LIST_HEAD(&fs_info->caching_block_groups);
2681         spin_lock_init(&fs_info->delalloc_root_lock);
2682         spin_lock_init(&fs_info->trans_lock);
2683         spin_lock_init(&fs_info->fs_roots_radix_lock);
2684         spin_lock_init(&fs_info->delayed_iput_lock);
2685         spin_lock_init(&fs_info->defrag_inodes_lock);
2686         spin_lock_init(&fs_info->super_lock);
2687         spin_lock_init(&fs_info->buffer_lock);
2688         spin_lock_init(&fs_info->unused_bgs_lock);
2689         rwlock_init(&fs_info->tree_mod_log_lock);
2690         mutex_init(&fs_info->unused_bg_unpin_mutex);
2691         mutex_init(&fs_info->delete_unused_bgs_mutex);
2692         mutex_init(&fs_info->reloc_mutex);
2693         mutex_init(&fs_info->delalloc_root_mutex);
2694         seqlock_init(&fs_info->profiles_lock);
2695
2696         INIT_LIST_HEAD(&fs_info->dirty_cowonly_roots);
2697         INIT_LIST_HEAD(&fs_info->space_info);
2698         INIT_LIST_HEAD(&fs_info->tree_mod_seq_list);
2699         INIT_LIST_HEAD(&fs_info->unused_bgs);
2700         extent_map_tree_init(&fs_info->mapping_tree);
2701         btrfs_init_block_rsv(&fs_info->global_block_rsv,
2702                              BTRFS_BLOCK_RSV_GLOBAL);
2703         btrfs_init_block_rsv(&fs_info->trans_block_rsv, BTRFS_BLOCK_RSV_TRANS);
2704         btrfs_init_block_rsv(&fs_info->chunk_block_rsv, BTRFS_BLOCK_RSV_CHUNK);
2705         btrfs_init_block_rsv(&fs_info->empty_block_rsv, BTRFS_BLOCK_RSV_EMPTY);
2706         btrfs_init_block_rsv(&fs_info->delayed_block_rsv,
2707                              BTRFS_BLOCK_RSV_DELOPS);
2708         btrfs_init_block_rsv(&fs_info->delayed_refs_rsv,
2709                              BTRFS_BLOCK_RSV_DELREFS);
2710
2711         atomic_set(&fs_info->async_delalloc_pages, 0);
2712         atomic_set(&fs_info->defrag_running, 0);
2713         atomic_set(&fs_info->reada_works_cnt, 0);
2714         atomic_set(&fs_info->nr_delayed_iputs, 0);
2715         atomic64_set(&fs_info->tree_mod_seq, 0);
2716         fs_info->sb = sb;
2717         fs_info->max_inline = BTRFS_DEFAULT_MAX_INLINE;
2718         fs_info->metadata_ratio = 0;
2719         fs_info->defrag_inodes = RB_ROOT;
2720         atomic64_set(&fs_info->free_chunk_space, 0);
2721         fs_info->tree_mod_log = RB_ROOT;
2722         fs_info->commit_interval = BTRFS_DEFAULT_COMMIT_INTERVAL;
2723         fs_info->avg_delayed_ref_runtime = NSEC_PER_SEC >> 6; /* div by 64 */
2724         /* readahead state */
2725         INIT_RADIX_TREE(&fs_info->reada_tree, GFP_NOFS & ~__GFP_DIRECT_RECLAIM);
2726         spin_lock_init(&fs_info->reada_lock);
2727         btrfs_init_ref_verify(fs_info);
2728
2729         fs_info->thread_pool_size = min_t(unsigned long,
2730                                           num_online_cpus() + 2, 8);
2731
2732         INIT_LIST_HEAD(&fs_info->ordered_roots);
2733         spin_lock_init(&fs_info->ordered_root_lock);
2734
2735         fs_info->btree_inode = new_inode(sb);
2736         if (!fs_info->btree_inode) {
2737                 err = -ENOMEM;
2738                 goto fail_bio_counter;
2739         }
2740         mapping_set_gfp_mask(fs_info->btree_inode->i_mapping, GFP_NOFS);
2741
2742         fs_info->delayed_root = kmalloc(sizeof(struct btrfs_delayed_root),
2743                                         GFP_KERNEL);
2744         if (!fs_info->delayed_root) {
2745                 err = -ENOMEM;
2746                 goto fail_iput;
2747         }
2748         btrfs_init_delayed_root(fs_info->delayed_root);
2749
2750         btrfs_init_scrub(fs_info);
2751 #ifdef CONFIG_BTRFS_FS_CHECK_INTEGRITY
2752         fs_info->check_integrity_print_mask = 0;
2753 #endif
2754         btrfs_init_balance(fs_info);
2755         btrfs_init_async_reclaim_work(&fs_info->async_reclaim_work);
2756
2757         sb->s_blocksize = BTRFS_BDEV_BLOCKSIZE;
2758         sb->s_blocksize_bits = blksize_bits(BTRFS_BDEV_BLOCKSIZE);
2759
2760         btrfs_init_btree_inode(fs_info);
2761
2762         spin_lock_init(&fs_info->block_group_cache_lock);
2763         fs_info->block_group_cache_tree = RB_ROOT;
2764         fs_info->first_logical_byte = (u64)-1;
2765
2766         extent_io_tree_init(fs_info, &fs_info->freed_extents[0],
2767                             IO_TREE_FS_INFO_FREED_EXTENTS0, NULL);
2768         extent_io_tree_init(fs_info, &fs_info->freed_extents[1],
2769                             IO_TREE_FS_INFO_FREED_EXTENTS1, NULL);
2770         fs_info->pinned_extents = &fs_info->freed_extents[0];
2771         set_bit(BTRFS_FS_BARRIER, &fs_info->flags);
2772
2773         mutex_init(&fs_info->ordered_operations_mutex);
2774         mutex_init(&fs_info->tree_log_mutex);
2775         mutex_init(&fs_info->chunk_mutex);
2776         mutex_init(&fs_info->transaction_kthread_mutex);
2777         mutex_init(&fs_info->cleaner_mutex);
2778         mutex_init(&fs_info->ro_block_group_mutex);
2779         init_rwsem(&fs_info->commit_root_sem);
2780         init_rwsem(&fs_info->cleanup_work_sem);
2781         init_rwsem(&fs_info->subvol_sem);
2782         sema_init(&fs_info->uuid_tree_rescan_sem, 1);
2783
2784         btrfs_init_dev_replace_locks(fs_info);
2785         btrfs_init_qgroup(fs_info);
2786         btrfs_discard_init(fs_info);
2787
2788         btrfs_init_free_cluster(&fs_info->meta_alloc_cluster);
2789         btrfs_init_free_cluster(&fs_info->data_alloc_cluster);
2790
2791         init_waitqueue_head(&fs_info->transaction_throttle);
2792         init_waitqueue_head(&fs_info->transaction_wait);
2793         init_waitqueue_head(&fs_info->transaction_blocked_wait);
2794         init_waitqueue_head(&fs_info->async_submit_wait);
2795         init_waitqueue_head(&fs_info->delayed_iputs_wait);
2796
2797         /* Usable values until the real ones are cached from the superblock */
2798         fs_info->nodesize = 4096;
2799         fs_info->sectorsize = 4096;
2800         fs_info->stripesize = 4096;
2801
2802         spin_lock_init(&fs_info->swapfile_pins_lock);
2803         fs_info->swapfile_pins = RB_ROOT;
2804
2805         fs_info->send_in_progress = 0;
2806
2807         ret = btrfs_alloc_stripe_hash_table(fs_info);
2808         if (ret) {
2809                 err = ret;
2810                 goto fail_alloc;
2811         }
2812
2813         invalidate_bdev(fs_devices->latest_bdev);
2814
2815         /*
2816          * Read super block and check the signature bytes only
2817          */
2818         bh = btrfs_read_dev_super(fs_devices->latest_bdev);
2819         if (IS_ERR(bh)) {
2820                 err = PTR_ERR(bh);
2821                 goto fail_alloc;
2822         }
2823
2824         /*
2825          * Verify the type first, if that or the the checksum value are
2826          * corrupted, we'll find out
2827          */
2828         csum_type = btrfs_super_csum_type((struct btrfs_super_block *)bh->b_data);
2829         if (!btrfs_supported_super_csum(csum_type)) {
2830                 btrfs_err(fs_info, "unsupported checksum algorithm: %u",
2831                           csum_type);
2832                 err = -EINVAL;
2833                 brelse(bh);
2834                 goto fail_alloc;
2835         }
2836
2837         ret = btrfs_init_csum_hash(fs_info, csum_type);
2838         if (ret) {
2839                 err = ret;
2840                 goto fail_alloc;
2841         }
2842
2843         /*
2844          * We want to check superblock checksum, the type is stored inside.
2845          * Pass the whole disk block of size BTRFS_SUPER_INFO_SIZE (4k).
2846          */
2847         if (btrfs_check_super_csum(fs_info, bh->b_data)) {
2848                 btrfs_err(fs_info, "superblock checksum mismatch");
2849                 err = -EINVAL;
2850                 brelse(bh);
2851                 goto fail_csum;
2852         }
2853
2854         /*
2855          * super_copy is zeroed at allocation time and we never touch the
2856          * following bytes up to INFO_SIZE, the checksum is calculated from
2857          * the whole block of INFO_SIZE
2858          */
2859         memcpy(fs_info->super_copy, bh->b_data, sizeof(*fs_info->super_copy));
2860         brelse(bh);
2861
2862         disk_super = fs_info->super_copy;
2863
2864         ASSERT(!memcmp(fs_info->fs_devices->fsid, fs_info->super_copy->fsid,
2865                        BTRFS_FSID_SIZE));
2866
2867         if (btrfs_fs_incompat(fs_info, METADATA_UUID)) {
2868                 ASSERT(!memcmp(fs_info->fs_devices->metadata_uuid,
2869                                 fs_info->super_copy->metadata_uuid,
2870                                 BTRFS_FSID_SIZE));
2871         }
2872
2873         features = btrfs_super_flags(disk_super);
2874         if (features & BTRFS_SUPER_FLAG_CHANGING_FSID_V2) {
2875                 features &= ~BTRFS_SUPER_FLAG_CHANGING_FSID_V2;
2876                 btrfs_set_super_flags(disk_super, features);
2877                 btrfs_info(fs_info,
2878                         "found metadata UUID change in progress flag, clearing");
2879         }
2880
2881         memcpy(fs_info->super_for_commit, fs_info->super_copy,
2882                sizeof(*fs_info->super_for_commit));
2883
2884         ret = btrfs_validate_mount_super(fs_info);
2885         if (ret) {
2886                 btrfs_err(fs_info, "superblock contains fatal errors");
2887                 err = -EINVAL;
2888                 goto fail_csum;
2889         }
2890
2891         if (!btrfs_super_root(disk_super))
2892                 goto fail_csum;
2893
2894         /* check FS state, whether FS is broken. */
2895         if (btrfs_super_flags(disk_super) & BTRFS_SUPER_FLAG_ERROR)
2896                 set_bit(BTRFS_FS_STATE_ERROR, &fs_info->fs_state);
2897
2898         /*
2899          * In the long term, we'll store the compression type in the super
2900          * block, and it'll be used for per file compression control.
2901          */
2902         fs_info->compress_type = BTRFS_COMPRESS_ZLIB;
2903
2904         ret = btrfs_parse_options(fs_info, options, sb->s_flags);
2905         if (ret) {
2906                 err = ret;
2907                 goto fail_csum;
2908         }
2909
2910         features = btrfs_super_incompat_flags(disk_super) &
2911                 ~BTRFS_FEATURE_INCOMPAT_SUPP;
2912         if (features) {
2913                 btrfs_err(fs_info,
2914                     "cannot mount because of unsupported optional features (%llx)",
2915                     features);
2916                 err = -EINVAL;
2917                 goto fail_csum;
2918         }
2919
2920         features = btrfs_super_incompat_flags(disk_super);
2921         features |= BTRFS_FEATURE_INCOMPAT_MIXED_BACKREF;
2922         if (fs_info->compress_type == BTRFS_COMPRESS_LZO)
2923                 features |= BTRFS_FEATURE_INCOMPAT_COMPRESS_LZO;
2924         else if (fs_info->compress_type == BTRFS_COMPRESS_ZSTD)
2925                 features |= BTRFS_FEATURE_INCOMPAT_COMPRESS_ZSTD;
2926
2927         if (features & BTRFS_FEATURE_INCOMPAT_SKINNY_METADATA)
2928                 btrfs_info(fs_info, "has skinny extents");
2929
2930         /*
2931          * flag our filesystem as having big metadata blocks if
2932          * they are bigger than the page size
2933          */
2934         if (btrfs_super_nodesize(disk_super) > PAGE_SIZE) {
2935                 if (!(features & BTRFS_FEATURE_INCOMPAT_BIG_METADATA))
2936                         btrfs_info(fs_info,
2937                                 "flagging fs with big metadata feature");
2938                 features |= BTRFS_FEATURE_INCOMPAT_BIG_METADATA;
2939         }
2940
2941         nodesize = btrfs_super_nodesize(disk_super);
2942         sectorsize = btrfs_super_sectorsize(disk_super);
2943         stripesize = sectorsize;
2944         fs_info->dirty_metadata_batch = nodesize * (1 + ilog2(nr_cpu_ids));
2945         fs_info->delalloc_batch = sectorsize * 512 * (1 + ilog2(nr_cpu_ids));
2946
2947         /* Cache block sizes */
2948         fs_info->nodesize = nodesize;
2949         fs_info->sectorsize = sectorsize;
2950         fs_info->stripesize = stripesize;
2951
2952         /*
2953          * mixed block groups end up with duplicate but slightly offset
2954          * extent buffers for the same range.  It leads to corruptions
2955          */
2956         if ((features & BTRFS_FEATURE_INCOMPAT_MIXED_GROUPS) &&
2957             (sectorsize != nodesize)) {
2958                 btrfs_err(fs_info,
2959 "unequal nodesize/sectorsize (%u != %u) are not allowed for mixed block groups",
2960                         nodesize, sectorsize);
2961                 goto fail_csum;
2962         }
2963
2964         /*
2965          * Needn't use the lock because there is no other task which will
2966          * update the flag.
2967          */
2968         btrfs_set_super_incompat_flags(disk_super, features);
2969
2970         features = btrfs_super_compat_ro_flags(disk_super) &
2971                 ~BTRFS_FEATURE_COMPAT_RO_SUPP;
2972         if (!sb_rdonly(sb) && features) {
2973                 btrfs_err(fs_info,
2974         "cannot mount read-write because of unsupported optional features (%llx)",
2975                        features);
2976                 err = -EINVAL;
2977                 goto fail_csum;
2978         }
2979
2980         ret = btrfs_init_workqueues(fs_info, fs_devices);
2981         if (ret) {
2982                 err = ret;
2983                 goto fail_sb_buffer;
2984         }
2985
2986         sb->s_bdi->congested_fn = btrfs_congested_fn;
2987         sb->s_bdi->congested_data = fs_info;
2988         sb->s_bdi->capabilities |= BDI_CAP_CGROUP_WRITEBACK;
2989         sb->s_bdi->ra_pages = VM_READAHEAD_PAGES;
2990         sb->s_bdi->ra_pages *= btrfs_super_num_devices(disk_super);
2991         sb->s_bdi->ra_pages = max(sb->s_bdi->ra_pages, SZ_4M / PAGE_SIZE);
2992
2993         sb->s_blocksize = sectorsize;
2994         sb->s_blocksize_bits = blksize_bits(sectorsize);
2995         memcpy(&sb->s_uuid, fs_info->fs_devices->fsid, BTRFS_FSID_SIZE);
2996
2997         mutex_lock(&fs_info->chunk_mutex);
2998         ret = btrfs_read_sys_array(fs_info);
2999         mutex_unlock(&fs_info->chunk_mutex);
3000         if (ret) {
3001                 btrfs_err(fs_info, "failed to read the system array: %d", ret);
3002                 goto fail_sb_buffer;
3003         }
3004
3005         generation = btrfs_super_chunk_root_generation(disk_super);
3006         level = btrfs_super_chunk_root_level(disk_super);
3007
3008         chunk_root->node = read_tree_block(fs_info,
3009                                            btrfs_super_chunk_root(disk_super),
3010                                            generation, level, NULL);
3011         if (IS_ERR(chunk_root->node) ||
3012             !extent_buffer_uptodate(chunk_root->node)) {
3013                 btrfs_err(fs_info, "failed to read chunk root");
3014                 if (!IS_ERR(chunk_root->node))
3015                         free_extent_buffer(chunk_root->node);
3016                 chunk_root->node = NULL;
3017                 goto fail_tree_roots;
3018         }
3019         btrfs_set_root_node(&chunk_root->root_item, chunk_root->node);
3020         chunk_root->commit_root = btrfs_root_node(chunk_root);
3021
3022         read_extent_buffer(chunk_root->node, fs_info->chunk_tree_uuid,
3023            btrfs_header_chunk_tree_uuid(chunk_root->node), BTRFS_UUID_SIZE);
3024
3025         ret = btrfs_read_chunk_tree(fs_info);
3026         if (ret) {
3027                 btrfs_err(fs_info, "failed to read chunk tree: %d", ret);
3028                 goto fail_tree_roots;
3029         }
3030
3031         /*
3032          * Keep the devid that is marked to be the target device for the
3033          * device replace procedure
3034          */
3035         btrfs_free_extra_devids(fs_devices, 0);
3036
3037         if (!fs_devices->latest_bdev) {
3038                 btrfs_err(fs_info, "failed to read devices");
3039                 goto fail_tree_roots;
3040         }
3041
3042         ret = init_tree_roots(fs_info);
3043         if (ret)
3044                 goto fail_tree_roots;
3045
3046         ret = btrfs_verify_dev_extents(fs_info);
3047         if (ret) {
3048                 btrfs_err(fs_info,
3049                           "failed to verify dev extents against chunks: %d",
3050                           ret);
3051                 goto fail_block_groups;
3052         }
3053         ret = btrfs_recover_balance(fs_info);
3054         if (ret) {
3055                 btrfs_err(fs_info, "failed to recover balance: %d", ret);
3056                 goto fail_block_groups;
3057         }
3058
3059         ret = btrfs_init_dev_stats(fs_info);
3060         if (ret) {
3061                 btrfs_err(fs_info, "failed to init dev_stats: %d", ret);
3062                 goto fail_block_groups;
3063         }
3064
3065         ret = btrfs_init_dev_replace(fs_info);
3066         if (ret) {
3067                 btrfs_err(fs_info, "failed to init dev_replace: %d", ret);
3068                 goto fail_block_groups;
3069         }
3070
3071         btrfs_free_extra_devids(fs_devices, 1);
3072
3073         ret = btrfs_sysfs_add_fsid(fs_devices);
3074         if (ret) {
3075                 btrfs_err(fs_info, "failed to init sysfs fsid interface: %d",
3076                                 ret);
3077                 goto fail_block_groups;
3078         }
3079
3080         ret = btrfs_sysfs_add_mounted(fs_info);
3081         if (ret) {
3082                 btrfs_err(fs_info, "failed to init sysfs interface: %d", ret);
3083                 goto fail_fsdev_sysfs;
3084         }
3085
3086         ret = btrfs_init_space_info(fs_info);
3087         if (ret) {
3088                 btrfs_err(fs_info, "failed to initialize space info: %d", ret);
3089                 goto fail_sysfs;
3090         }
3091
3092         ret = btrfs_read_block_groups(fs_info);
3093         if (ret) {
3094                 btrfs_err(fs_info, "failed to read block groups: %d", ret);
3095                 goto fail_sysfs;
3096         }
3097
3098         if (!sb_rdonly(sb) && !btrfs_check_rw_degradable(fs_info, NULL)) {
3099                 btrfs_warn(fs_info,
3100                 "writable mount is not allowed due to too many missing devices");
3101                 goto fail_sysfs;
3102         }
3103
3104         fs_info->cleaner_kthread = kthread_run(cleaner_kthread, tree_root,
3105                                                "btrfs-cleaner");
3106         if (IS_ERR(fs_info->cleaner_kthread))
3107                 goto fail_sysfs;
3108
3109         fs_info->transaction_kthread = kthread_run(transaction_kthread,
3110                                                    tree_root,
3111                                                    "btrfs-transaction");
3112         if (IS_ERR(fs_info->transaction_kthread))
3113                 goto fail_cleaner;
3114
3115         if (!btrfs_test_opt(fs_info, NOSSD) &&
3116             !fs_info->fs_devices->rotating) {
3117                 btrfs_set_and_info(fs_info, SSD, "enabling ssd optimizations");
3118         }
3119
3120         /*
3121          * Mount does not set all options immediately, we can do it now and do
3122          * not have to wait for transaction commit
3123          */
3124         btrfs_apply_pending_changes(fs_info);
3125
3126 #ifdef CONFIG_BTRFS_FS_CHECK_INTEGRITY
3127         if (btrfs_test_opt(fs_info, CHECK_INTEGRITY)) {
3128                 ret = btrfsic_mount(fs_info, fs_devices,
3129                                     btrfs_test_opt(fs_info,
3130                                         CHECK_INTEGRITY_INCLUDING_EXTENT_DATA) ?
3131                                     1 : 0,
3132                                     fs_info->check_integrity_print_mask);
3133                 if (ret)
3134                         btrfs_warn(fs_info,
3135                                 "failed to initialize integrity check module: %d",
3136                                 ret);
3137         }
3138 #endif
3139         ret = btrfs_read_qgroup_config(fs_info);
3140         if (ret)
3141                 goto fail_trans_kthread;
3142
3143         if (btrfs_build_ref_tree(fs_info))
3144                 btrfs_err(fs_info, "couldn't build ref tree");
3145
3146         /* do not make disk changes in broken FS or nologreplay is given */
3147         if (btrfs_super_log_root(disk_super) != 0 &&
3148             !btrfs_test_opt(fs_info, NOLOGREPLAY)) {
3149                 btrfs_info(fs_info, "start tree-log replay");
3150                 ret = btrfs_replay_log(fs_info, fs_devices);
3151                 if (ret) {
3152                         err = ret;
3153                         goto fail_qgroup;
3154                 }
3155         }
3156
3157         ret = btrfs_find_orphan_roots(fs_info);
3158         if (ret)
3159                 goto fail_qgroup;
3160
3161         if (!sb_rdonly(sb)) {
3162                 ret = btrfs_cleanup_fs_roots(fs_info);
3163                 if (ret)
3164                         goto fail_qgroup;
3165
3166                 mutex_lock(&fs_info->cleaner_mutex);
3167                 ret = btrfs_recover_relocation(tree_root);
3168                 mutex_unlock(&fs_info->cleaner_mutex);
3169                 if (ret < 0) {
3170                         btrfs_warn(fs_info, "failed to recover relocation: %d",
3171                                         ret);
3172                         err = -EINVAL;
3173                         goto fail_qgroup;
3174                 }
3175         }
3176
3177         location.objectid = BTRFS_FS_TREE_OBJECTID;
3178         location.type = BTRFS_ROOT_ITEM_KEY;
3179         location.offset = 0;
3180
3181         fs_info->fs_root = btrfs_get_fs_root(fs_info, &location, true);
3182         if (IS_ERR(fs_info->fs_root)) {
3183                 err = PTR_ERR(fs_info->fs_root);
3184                 btrfs_warn(fs_info, "failed to read fs tree: %d", err);
3185                 fs_info->fs_root = NULL;
3186                 goto fail_qgroup;
3187         }
3188
3189         if (sb_rdonly(sb))
3190                 return 0;
3191
3192         if (btrfs_test_opt(fs_info, CLEAR_CACHE) &&
3193             btrfs_fs_compat_ro(fs_info, FREE_SPACE_TREE)) {
3194                 clear_free_space_tree = 1;
3195         } else if (btrfs_fs_compat_ro(fs_info, FREE_SPACE_TREE) &&
3196                    !btrfs_fs_compat_ro(fs_info, FREE_SPACE_TREE_VALID)) {
3197                 btrfs_warn(fs_info, "free space tree is invalid");
3198                 clear_free_space_tree = 1;
3199         }
3200
3201         if (clear_free_space_tree) {
3202                 btrfs_info(fs_info, "clearing free space tree");
3203                 ret = btrfs_clear_free_space_tree(fs_info);
3204                 if (ret) {
3205                         btrfs_warn(fs_info,
3206                                    "failed to clear free space tree: %d", ret);
3207                         close_ctree(fs_info);
3208                         return ret;
3209                 }
3210         }
3211
3212         if (btrfs_test_opt(fs_info, FREE_SPACE_TREE) &&
3213             !btrfs_fs_compat_ro(fs_info, FREE_SPACE_TREE)) {
3214                 btrfs_info(fs_info, "creating free space tree");
3215                 ret = btrfs_create_free_space_tree(fs_info);
3216                 if (ret) {
3217                         btrfs_warn(fs_info,
3218                                 "failed to create free space tree: %d", ret);
3219                         close_ctree(fs_info);
3220                         return ret;
3221                 }
3222         }
3223
3224         down_read(&fs_info->cleanup_work_sem);
3225         if ((ret = btrfs_orphan_cleanup(fs_info->fs_root)) ||
3226             (ret = btrfs_orphan_cleanup(fs_info->tree_root))) {
3227                 up_read(&fs_info->cleanup_work_sem);
3228                 close_ctree(fs_info);
3229                 return ret;
3230         }
3231         up_read(&fs_info->cleanup_work_sem);
3232
3233         ret = btrfs_resume_balance_async(fs_info);
3234         if (ret) {
3235                 btrfs_warn(fs_info, "failed to resume balance: %d", ret);
3236                 close_ctree(fs_info);
3237                 return ret;
3238         }
3239
3240         ret = btrfs_resume_dev_replace_async(fs_info);
3241         if (ret) {
3242                 btrfs_warn(fs_info, "failed to resume device replace: %d", ret);
3243                 close_ctree(fs_info);
3244                 return ret;
3245         }
3246
3247         btrfs_qgroup_rescan_resume(fs_info);
3248         btrfs_discard_resume(fs_info);
3249
3250         if (!fs_info->uuid_root) {
3251                 btrfs_info(fs_info, "creating UUID tree");
3252                 ret = btrfs_create_uuid_tree(fs_info);
3253                 if (ret) {
3254                         btrfs_warn(fs_info,
3255                                 "failed to create the UUID tree: %d", ret);
3256                         close_ctree(fs_info);
3257                         return ret;
3258                 }
3259         } else if (btrfs_test_opt(fs_info, RESCAN_UUID_TREE) ||
3260                    fs_info->generation !=
3261                                 btrfs_super_uuid_tree_generation(disk_super)) {
3262                 btrfs_info(fs_info, "checking UUID tree");
3263                 ret = btrfs_check_uuid_tree(fs_info);
3264                 if (ret) {
3265                         btrfs_warn(fs_info,
3266                                 "failed to check the UUID tree: %d", ret);
3267                         close_ctree(fs_info);
3268                         return ret;
3269                 }
3270         } else {
3271                 set_bit(BTRFS_FS_UPDATE_UUID_TREE_GEN, &fs_info->flags);
3272         }
3273         set_bit(BTRFS_FS_OPEN, &fs_info->flags);
3274
3275         /*
3276          * backuproot only affect mount behavior, and if open_ctree succeeded,
3277          * no need to keep the flag
3278          */
3279         btrfs_clear_opt(fs_info->mount_opt, USEBACKUPROOT);
3280
3281         return 0;
3282
3283 fail_qgroup:
3284         btrfs_free_qgroup_config(fs_info);
3285 fail_trans_kthread:
3286         kthread_stop(fs_info->transaction_kthread);
3287         btrfs_cleanup_transaction(fs_info);
3288         btrfs_free_fs_roots(fs_info);
3289 fail_cleaner:
3290         kthread_stop(fs_info->cleaner_kthread);
3291
3292         /*
3293          * make sure we're done with the btree inode before we stop our
3294          * kthreads
3295          */
3296         filemap_write_and_wait(fs_info->btree_inode->i_mapping);
3297
3298 fail_sysfs:
3299         btrfs_sysfs_remove_mounted(fs_info);
3300
3301 fail_fsdev_sysfs:
3302         btrfs_sysfs_remove_fsid(fs_info->fs_devices);
3303
3304 fail_block_groups:
3305         btrfs_put_block_group_cache(fs_info);
3306
3307 fail_tree_roots:
3308         free_root_pointers(fs_info, true);
3309         invalidate_inode_pages2(fs_info->btree_inode->i_mapping);
3310
3311 fail_sb_buffer:
3312         btrfs_stop_all_workers(fs_info);
3313         btrfs_free_block_groups(fs_info);
3314 fail_csum:
3315         btrfs_free_csum_hash(fs_info);
3316 fail_alloc:
3317 fail_iput:
3318         btrfs_mapping_tree_free(&fs_info->mapping_tree);
3319
3320         iput(fs_info->btree_inode);
3321 fail_bio_counter:
3322         percpu_counter_destroy(&fs_info->dev_replace.bio_counter);
3323 fail_delalloc_bytes:
3324         percpu_counter_destroy(&fs_info->delalloc_bytes);
3325 fail_dirty_metadata_bytes:
3326         percpu_counter_destroy(&fs_info->dirty_metadata_bytes);
3327 fail_dio_bytes:
3328         percpu_counter_destroy(&fs_info->dio_bytes);
3329 fail_srcu:
3330         cleanup_srcu_struct(&fs_info->subvol_srcu);
3331 fail:
3332         btrfs_free_stripe_hash_table(fs_info);
3333         btrfs_close_devices(fs_info->fs_devices);
3334         return err;
3335 }
3336 ALLOW_ERROR_INJECTION(open_ctree, ERRNO);
3337
3338 static void btrfs_end_buffer_write_sync(struct buffer_head *bh, int uptodate)
3339 {
3340         if (uptodate) {
3341                 set_buffer_uptodate(bh);
3342         } else {
3343                 struct btrfs_device *device = (struct btrfs_device *)
3344                         bh->b_private;
3345
3346                 btrfs_warn_rl_in_rcu(device->fs_info,
3347                                 "lost page write due to IO error on %s",
3348                                           rcu_str_deref(device->name));
3349                 /* note, we don't set_buffer_write_io_error because we have
3350                  * our own ways of dealing with the IO errors
3351                  */
3352                 clear_buffer_uptodate(bh);
3353                 btrfs_dev_stat_inc_and_print(device, BTRFS_DEV_STAT_WRITE_ERRS);
3354         }
3355         unlock_buffer(bh);
3356         put_bh(bh);
3357 }
3358
3359 int btrfs_read_dev_one_super(struct block_device *bdev, int copy_num,
3360                         struct buffer_head **bh_ret)
3361 {
3362         struct buffer_head *bh;
3363         struct btrfs_super_block *super;
3364         u64 bytenr;
3365
3366         bytenr = btrfs_sb_offset(copy_num);
3367         if (bytenr + BTRFS_SUPER_INFO_SIZE >= i_size_read(bdev->bd_inode))
3368                 return -EINVAL;
3369
3370         bh = __bread(bdev, bytenr / BTRFS_BDEV_BLOCKSIZE, BTRFS_SUPER_INFO_SIZE);
3371         /*
3372          * If we fail to read from the underlying devices, as of now
3373          * the best option we have is to mark it EIO.
3374          */
3375         if (!bh)
3376                 return -EIO;
3377
3378         super = (struct btrfs_super_block *)bh->b_data;
3379         if (btrfs_super_bytenr(super) != bytenr ||
3380                     btrfs_super_magic(super) != BTRFS_MAGIC) {
3381                 brelse(bh);
3382                 return -EINVAL;
3383         }
3384
3385         *bh_ret = bh;
3386         return 0;
3387 }
3388
3389
3390 struct buffer_head *btrfs_read_dev_super(struct block_device *bdev)
3391 {
3392         struct buffer_head *bh;
3393         struct buffer_head *latest = NULL;
3394         struct btrfs_super_block *super;
3395         int i;
3396         u64 transid = 0;
3397         int ret = -EINVAL;
3398
3399         /* we would like to check all the supers, but that would make
3400          * a btrfs mount succeed after a mkfs from a different FS.
3401          * So, we need to add a special mount option to scan for
3402          * later supers, using BTRFS_SUPER_MIRROR_MAX instead
3403          */
3404         for (i = 0; i < 1; i++) {
3405                 ret = btrfs_read_dev_one_super(bdev, i, &bh);
3406                 if (ret)
3407                         continue;
3408
3409                 super = (struct btrfs_super_block *)bh->b_data;
3410
3411                 if (!latest || btrfs_super_generation(super) > transid) {
3412                         brelse(latest);
3413                         latest = bh;
3414                         transid = btrfs_super_generation(super);
3415                 } else {
3416                         brelse(bh);
3417                 }
3418         }
3419
3420         if (!latest)
3421                 return ERR_PTR(ret);
3422
3423         return latest;
3424 }
3425
3426 /*
3427  * Write superblock @sb to the @device. Do not wait for completion, all the
3428  * buffer heads we write are pinned.
3429  *
3430  * Write @max_mirrors copies of the superblock, where 0 means default that fit
3431  * the expected device size at commit time. Note that max_mirrors must be
3432  * same for write and wait phases.
3433  *
3434  * Return number of errors when buffer head is not found or submission fails.
3435  */
3436 static int write_dev_supers(struct btrfs_device *device,
3437                             struct btrfs_super_block *sb, int max_mirrors)
3438 {
3439         struct btrfs_fs_info *fs_info = device->fs_info;
3440         SHASH_DESC_ON_STACK(shash, fs_info->csum_shash);
3441         struct buffer_head *bh;
3442         int i;
3443         int ret;
3444         int errors = 0;
3445         u64 bytenr;
3446         int op_flags;
3447
3448         if (max_mirrors == 0)
3449                 max_mirrors = BTRFS_SUPER_MIRROR_MAX;
3450
3451         shash->tfm = fs_info->csum_shash;
3452
3453         for (i = 0; i < max_mirrors; i++) {
3454                 bytenr = btrfs_sb_offset(i);
3455                 if (bytenr + BTRFS_SUPER_INFO_SIZE >=
3456                     device->commit_total_bytes)
3457                         break;
3458
3459                 btrfs_set_super_bytenr(sb, bytenr);
3460
3461                 crypto_shash_init(shash);
3462                 crypto_shash_update(shash, (const char *)sb + BTRFS_CSUM_SIZE,
3463                                     BTRFS_SUPER_INFO_SIZE - BTRFS_CSUM_SIZE);
3464                 crypto_shash_final(shash, sb->csum);
3465
3466                 /* One reference for us, and we leave it for the caller */
3467                 bh = __getblk(device->bdev, bytenr / BTRFS_BDEV_BLOCKSIZE,
3468                               BTRFS_SUPER_INFO_SIZE);
3469                 if (!bh) {
3470                         btrfs_err(device->fs_info,
3471                             "couldn't get super buffer head for bytenr %llu",
3472                             bytenr);
3473                         errors++;
3474                         continue;
3475                 }
3476
3477                 memcpy(bh->b_data, sb, BTRFS_SUPER_INFO_SIZE);
3478
3479                 /* one reference for submit_bh */
3480                 get_bh(bh);
3481
3482                 set_buffer_uptodate(bh);
3483                 lock_buffer(bh);
3484                 bh->b_end_io = btrfs_end_buffer_write_sync;
3485                 bh->b_private = device;
3486
3487                 /*
3488                  * we fua the first super.  The others we allow
3489                  * to go down lazy.
3490                  */
3491                 op_flags = REQ_SYNC | REQ_META | REQ_PRIO;
3492                 if (i == 0 && !btrfs_test_opt(device->fs_info, NOBARRIER))
3493                         op_flags |= REQ_FUA;
3494                 ret = btrfsic_submit_bh(REQ_OP_WRITE, op_flags, bh);
3495                 if (ret)
3496                         errors++;
3497         }
3498         return errors < i ? 0 : -1;
3499 }
3500
3501 /*
3502  * Wait for write completion of superblocks done by write_dev_supers,
3503  * @max_mirrors same for write and wait phases.
3504  *
3505  * Return number of errors when buffer head is not found or not marked up to
3506  * date.
3507  */
3508 static int wait_dev_supers(struct btrfs_device *device, int max_mirrors)
3509 {
3510         struct buffer_head *bh;
3511         int i;
3512         int errors = 0;
3513         bool primary_failed = false;
3514         u64 bytenr;
3515
3516         if (max_mirrors == 0)
3517                 max_mirrors = BTRFS_SUPER_MIRROR_MAX;
3518
3519         for (i = 0; i < max_mirrors; i++) {
3520                 bytenr = btrfs_sb_offset(i);
3521                 if (bytenr + BTRFS_SUPER_INFO_SIZE >=
3522                     device->commit_total_bytes)
3523                         break;
3524
3525                 bh = __find_get_block(device->bdev,
3526                                       bytenr / BTRFS_BDEV_BLOCKSIZE,
3527                                       BTRFS_SUPER_INFO_SIZE);
3528                 if (!bh) {
3529                         errors++;
3530                         if (i == 0)
3531                                 primary_failed = true;
3532                         continue;
3533                 }
3534                 wait_on_buffer(bh);
3535                 if (!buffer_uptodate(bh)) {
3536                         errors++;
3537                         if (i == 0)
3538                                 primary_failed = true;
3539                 }
3540
3541                 /* drop our reference */
3542                 brelse(bh);
3543
3544                 /* drop the reference from the writing run */
3545                 brelse(bh);
3546         }
3547
3548         /* log error, force error return */
3549         if (primary_failed) {
3550                 btrfs_err(device->fs_info, "error writing primary super block to device %llu",
3551                           device->devid);
3552                 return -1;
3553         }
3554
3555         return errors < i ? 0 : -1;
3556 }
3557
3558 /*
3559  * endio for the write_dev_flush, this will wake anyone waiting
3560  * for the barrier when it is done
3561  */
3562 static void btrfs_end_empty_barrier(struct bio *bio)
3563 {
3564         complete(bio->bi_private);
3565 }
3566
3567 /*
3568  * Submit a flush request to the device if it supports it. Error handling is
3569  * done in the waiting counterpart.
3570  */
3571 static void write_dev_flush(struct btrfs_device *device)
3572 {
3573         struct request_queue *q = bdev_get_queue(device->bdev);
3574         struct bio *bio = device->flush_bio;
3575
3576         if (!test_bit(QUEUE_FLAG_WC, &q->queue_flags))
3577                 return;
3578
3579         bio_reset(bio);
3580         bio->bi_end_io = btrfs_end_empty_barrier;
3581         bio_set_dev(bio, device->bdev);
3582         bio->bi_opf = REQ_OP_WRITE | REQ_SYNC | REQ_PREFLUSH;
3583         init_completion(&device->flush_wait);
3584         bio->bi_private = &device->flush_wait;
3585
3586         btrfsic_submit_bio(bio);
3587         set_bit(BTRFS_DEV_STATE_FLUSH_SENT, &device->dev_state);
3588 }
3589
3590 /*
3591  * If the flush bio has been submitted by write_dev_flush, wait for it.
3592  */
3593 static blk_status_t wait_dev_flush(struct btrfs_device *device)
3594 {
3595         struct bio *bio = device->flush_bio;
3596
3597         if (!test_bit(BTRFS_DEV_STATE_FLUSH_SENT, &device->dev_state))
3598                 return BLK_STS_OK;
3599
3600         clear_bit(BTRFS_DEV_STATE_FLUSH_SENT, &device->dev_state);
3601         wait_for_completion_io(&device->flush_wait);
3602
3603         return bio->bi_status;
3604 }
3605
3606 static int check_barrier_error(struct btrfs_fs_info *fs_info)
3607 {
3608         if (!btrfs_check_rw_degradable(fs_info, NULL))
3609                 return -EIO;
3610         return 0;
3611 }
3612
3613 /*
3614  * send an empty flush down to each device in parallel,
3615  * then wait for them
3616  */
3617 static int barrier_all_devices(struct btrfs_fs_info *info)
3618 {
3619         struct list_head *head;
3620         struct btrfs_device *dev;
3621         int errors_wait = 0;
3622         blk_status_t ret;
3623
3624         lockdep_assert_held(&info->fs_devices->device_list_mutex);
3625         /* send down all the barriers */
3626         head = &info->fs_devices->devices;
3627         list_for_each_entry(dev, head, dev_list) {
3628                 if (test_bit(BTRFS_DEV_STATE_MISSING, &dev->dev_state))
3629                         continue;
3630                 if (!dev->bdev)
3631                         continue;
3632                 if (!test_bit(BTRFS_DEV_STATE_IN_FS_METADATA, &dev->dev_state) ||
3633                     !test_bit(BTRFS_DEV_STATE_WRITEABLE, &dev->dev_state))
3634                         continue;
3635
3636                 write_dev_flush(dev);
3637                 dev->last_flush_error = BLK_STS_OK;
3638         }
3639
3640         /* wait for all the barriers */
3641         list_for_each_entry(dev, head, dev_list) {
3642                 if (test_bit(BTRFS_DEV_STATE_MISSING, &dev->dev_state))
3643                         continue;
3644                 if (!dev->bdev) {
3645                         errors_wait++;
3646                         continue;
3647                 }
3648                 if (!test_bit(BTRFS_DEV_STATE_IN_FS_METADATA, &dev->dev_state) ||
3649                     !test_bit(BTRFS_DEV_STATE_WRITEABLE, &dev->dev_state))
3650                         continue;
3651
3652                 ret = wait_dev_flush(dev);
3653                 if (ret) {
3654                         dev->last_flush_error = ret;
3655                         btrfs_dev_stat_inc_and_print(dev,
3656                                         BTRFS_DEV_STAT_FLUSH_ERRS);
3657                         errors_wait++;
3658                 }
3659         }
3660
3661         if (errors_wait) {
3662                 /*
3663                  * At some point we need the status of all disks
3664                  * to arrive at the volume status. So error checking
3665                  * is being pushed to a separate loop.
3666                  */
3667                 return check_barrier_error(info);
3668         }
3669         return 0;
3670 }
3671
3672 int btrfs_get_num_tolerated_disk_barrier_failures(u64 flags)
3673 {
3674         int raid_type;
3675         int min_tolerated = INT_MAX;
3676
3677         if ((flags & BTRFS_BLOCK_GROUP_PROFILE_MASK) == 0 ||
3678             (flags & BTRFS_AVAIL_ALLOC_BIT_SINGLE))
3679                 min_tolerated = min_t(int, min_tolerated,
3680                                     btrfs_raid_array[BTRFS_RAID_SINGLE].
3681                                     tolerated_failures);
3682
3683         for (raid_type = 0; raid_type < BTRFS_NR_RAID_TYPES; raid_type++) {
3684                 if (raid_type == BTRFS_RAID_SINGLE)
3685                         continue;
3686                 if (!(flags & btrfs_raid_array[raid_type].bg_flag))
3687                         continue;
3688                 min_tolerated = min_t(int, min_tolerated,
3689                                     btrfs_raid_array[raid_type].
3690                                     tolerated_failures);
3691         }
3692
3693         if (min_tolerated == INT_MAX) {
3694                 pr_warn("BTRFS: unknown raid flag: %llu", flags);
3695                 min_tolerated = 0;
3696         }
3697
3698         return min_tolerated;
3699 }
3700
3701 int write_all_supers(struct btrfs_fs_info *fs_info, int max_mirrors)
3702 {
3703         struct list_head *head;
3704         struct btrfs_device *dev;
3705         struct btrfs_super_block *sb;
3706         struct btrfs_dev_item *dev_item;
3707         int ret;
3708         int do_barriers;
3709         int max_errors;
3710         int total_errors = 0;
3711         u64 flags;
3712
3713         do_barriers = !btrfs_test_opt(fs_info, NOBARRIER);
3714
3715         /*
3716          * max_mirrors == 0 indicates we're from commit_transaction,
3717          * not from fsync where the tree roots in fs_info have not
3718          * been consistent on disk.
3719          */
3720         if (max_mirrors == 0)
3721                 backup_super_roots(fs_info);
3722
3723         sb = fs_info->super_for_commit;
3724         dev_item = &sb->dev_item;
3725
3726         mutex_lock(&fs_info->fs_devices->device_list_mutex);
3727         head = &fs_info->fs_devices->devices;
3728         max_errors = btrfs_super_num_devices(fs_info->super_copy) - 1;
3729
3730         if (do_barriers) {
3731                 ret = barrier_all_devices(fs_info);
3732                 if (ret) {
3733                         mutex_unlock(
3734                                 &fs_info->fs_devices->device_list_mutex);
3735                         btrfs_handle_fs_error(fs_info, ret,
3736                                               "errors while submitting device barriers.");
3737                         return ret;
3738                 }
3739         }
3740
3741         list_for_each_entry(dev, head, dev_list) {
3742                 if (!dev->bdev) {
3743                         total_errors++;
3744                         continue;
3745                 }
3746                 if (!test_bit(BTRFS_DEV_STATE_IN_FS_METADATA, &dev->dev_state) ||
3747                     !test_bit(BTRFS_DEV_STATE_WRITEABLE, &dev->dev_state))
3748                         continue;
3749
3750                 btrfs_set_stack_device_generation(dev_item, 0);
3751                 btrfs_set_stack_device_type(dev_item, dev->type);
3752                 btrfs_set_stack_device_id(dev_item, dev->devid);
3753                 btrfs_set_stack_device_total_bytes(dev_item,
3754                                                    dev->commit_total_bytes);
3755                 btrfs_set_stack_device_bytes_used(dev_item,
3756                                                   dev->commit_bytes_used);
3757                 btrfs_set_stack_device_io_align(dev_item, dev->io_align);
3758                 btrfs_set_stack_device_io_width(dev_item, dev->io_width);
3759                 btrfs_set_stack_device_sector_size(dev_item, dev->sector_size);
3760                 memcpy(dev_item->uuid, dev->uuid, BTRFS_UUID_SIZE);
3761                 memcpy(dev_item->fsid, dev->fs_devices->metadata_uuid,
3762                        BTRFS_FSID_SIZE);
3763
3764                 flags = btrfs_super_flags(sb);
3765                 btrfs_set_super_flags(sb, flags | BTRFS_HEADER_FLAG_WRITTEN);
3766
3767                 ret = btrfs_validate_write_super(fs_info, sb);
3768                 if (ret < 0) {
3769                         mutex_unlock(&fs_info->fs_devices->device_list_mutex);
3770                         btrfs_handle_fs_error(fs_info, -EUCLEAN,
3771                                 "unexpected superblock corruption detected");
3772                         return -EUCLEAN;
3773                 }
3774
3775                 ret = write_dev_supers(dev, sb, max_mirrors);
3776                 if (ret)
3777                         total_errors++;
3778         }
3779         if (total_errors > max_errors) {
3780                 btrfs_err(fs_info, "%d errors while writing supers",
3781                           total_errors);
3782                 mutex_unlock(&fs_info->fs_devices->device_list_mutex);
3783
3784                 /* FUA is masked off if unsupported and can't be the reason */
3785                 btrfs_handle_fs_error(fs_info, -EIO,
3786                                       "%d errors while writing supers",
3787                                       total_errors);
3788                 return -EIO;
3789         }
3790
3791         total_errors = 0;
3792         list_for_each_entry(dev, head, dev_list) {
3793                 if (!dev->bdev)
3794                         continue;
3795                 if (!test_bit(BTRFS_DEV_STATE_IN_FS_METADATA, &dev->dev_state) ||
3796                     !test_bit(BTRFS_DEV_STATE_WRITEABLE, &dev->dev_state))
3797                         continue;
3798
3799                 ret = wait_dev_supers(dev, max_mirrors);
3800                 if (ret)
3801                         total_errors++;
3802         }
3803         mutex_unlock(&fs_info->fs_devices->device_list_mutex);
3804         if (total_errors > max_errors) {
3805                 btrfs_handle_fs_error(fs_info, -EIO,
3806                                       "%d errors while writing supers",
3807                                       total_errors);
3808                 return -EIO;
3809         }
3810         return 0;
3811 }
3812
3813 /* Drop a fs root from the radix tree and free it. */
3814 void btrfs_drop_and_free_fs_root(struct btrfs_fs_info *fs_info,
3815                                   struct btrfs_root *root)
3816 {
3817         spin_lock(&fs_info->fs_roots_radix_lock);
3818         radix_tree_delete(&fs_info->fs_roots_radix,
3819                           (unsigned long)root->root_key.objectid);
3820         spin_unlock(&fs_info->fs_roots_radix_lock);
3821
3822         if (btrfs_root_refs(&root->root_item) == 0)
3823                 synchronize_srcu(&fs_info->subvol_srcu);
3824
3825         if (test_bit(BTRFS_FS_STATE_ERROR, &fs_info->fs_state)) {
3826                 btrfs_free_log(NULL, root);
3827                 if (root->reloc_root) {
3828                         free_extent_buffer(root->reloc_root->node);
3829                         free_extent_buffer(root->reloc_root->commit_root);
3830                         btrfs_put_fs_root(root->reloc_root);
3831                         root->reloc_root = NULL;
3832                 }
3833         }
3834
3835         if (root->free_ino_pinned)
3836                 __btrfs_remove_free_space_cache(root->free_ino_pinned);
3837         if (root->free_ino_ctl)
3838                 __btrfs_remove_free_space_cache(root->free_ino_ctl);
3839         btrfs_free_fs_root(root);
3840 }
3841
3842 void btrfs_free_fs_root(struct btrfs_root *root)
3843 {
3844         iput(root->ino_cache_inode);
3845         WARN_ON(!RB_EMPTY_ROOT(&root->inode_tree));
3846         if (root->anon_dev)
3847                 free_anon_bdev(root->anon_dev);
3848         if (root->subv_writers)
3849                 btrfs_free_subvolume_writers(root->subv_writers);
3850         free_extent_buffer(root->node);
3851         free_extent_buffer(root->commit_root);
3852         kfree(root->free_ino_ctl);
3853         kfree(root->free_ino_pinned);
3854         btrfs_put_fs_root(root);
3855 }
3856
3857 int btrfs_cleanup_fs_roots(struct btrfs_fs_info *fs_info)
3858 {
3859         u64 root_objectid = 0;
3860         struct btrfs_root *gang[8];
3861         int i = 0;
3862         int err = 0;
3863         unsigned int ret = 0;
3864         int index;
3865
3866         while (1) {
3867                 index = srcu_read_lock(&fs_info->subvol_srcu);
3868                 ret = radix_tree_gang_lookup(&fs_info->fs_roots_radix,
3869                                              (void **)gang, root_objectid,
3870                                              ARRAY_SIZE(gang));
3871                 if (!ret) {
3872                         srcu_read_unlock(&fs_info->subvol_srcu, index);
3873                         break;
3874                 }
3875                 root_objectid = gang[ret - 1]->root_key.objectid + 1;
3876
3877                 for (i = 0; i < ret; i++) {
3878                         /* Avoid to grab roots in dead_roots */
3879                         if (btrfs_root_refs(&gang[i]->root_item) == 0) {
3880                                 gang[i] = NULL;
3881                                 continue;
3882                         }
3883                         /* grab all the search result for later use */
3884                         gang[i] = btrfs_grab_fs_root(gang[i]);
3885                 }
3886                 srcu_read_unlock(&fs_info->subvol_srcu, index);
3887
3888                 for (i = 0; i < ret; i++) {
3889                         if (!gang[i])
3890                                 continue;
3891                         root_objectid = gang[i]->root_key.objectid;
3892                         err = btrfs_orphan_cleanup(gang[i]);
3893                         if (err)
3894                                 break;
3895                         btrfs_put_fs_root(gang[i]);
3896                 }
3897                 root_objectid++;
3898         }
3899
3900         /* release the uncleaned roots due to error */
3901         for (; i < ret; i++) {
3902                 if (gang[i])
3903                         btrfs_put_fs_root(gang[i]);
3904         }
3905         return err;
3906 }
3907
3908 int btrfs_commit_super(struct btrfs_fs_info *fs_info)
3909 {
3910         struct btrfs_root *root = fs_info->tree_root;
3911         struct btrfs_trans_handle *trans;
3912
3913         mutex_lock(&fs_info->cleaner_mutex);
3914         btrfs_run_delayed_iputs(fs_info);
3915         mutex_unlock(&fs_info->cleaner_mutex);
3916         wake_up_process(fs_info->cleaner_kthread);
3917
3918         /* wait until ongoing cleanup work done */
3919         down_write(&fs_info->cleanup_work_sem);
3920         up_write(&fs_info->cleanup_work_sem);
3921
3922         trans = btrfs_join_transaction(root);
3923         if (IS_ERR(trans))
3924                 return PTR_ERR(trans);
3925         return btrfs_commit_transaction(trans);
3926 }
3927
3928 void __cold close_ctree(struct btrfs_fs_info *fs_info)
3929 {
3930         int ret;
3931
3932         set_bit(BTRFS_FS_CLOSING_START, &fs_info->flags);
3933         /*
3934          * We don't want the cleaner to start new transactions, add more delayed
3935          * iputs, etc. while we're closing. We can't use kthread_stop() yet
3936          * because that frees the task_struct, and the transaction kthread might
3937          * still try to wake up the cleaner.
3938          */
3939         kthread_park(fs_info->cleaner_kthread);
3940
3941         /* wait for the qgroup rescan worker to stop */
3942         btrfs_qgroup_wait_for_completion(fs_info, false);
3943
3944         /* wait for the uuid_scan task to finish */
3945         down(&fs_info->uuid_tree_rescan_sem);
3946         /* avoid complains from lockdep et al., set sem back to initial state */
3947         up(&fs_info->uuid_tree_rescan_sem);
3948
3949         /* pause restriper - we want to resume on mount */
3950         btrfs_pause_balance(fs_info);
3951
3952         btrfs_dev_replace_suspend_for_unmount(fs_info);
3953
3954         btrfs_scrub_cancel(fs_info);
3955
3956         /* wait for any defraggers to finish */
3957         wait_event(fs_info->transaction_wait,
3958                    (atomic_read(&fs_info->defrag_running) == 0));
3959
3960         /* clear out the rbtree of defraggable inodes */
3961         btrfs_cleanup_defrag_inodes(fs_info);
3962
3963         cancel_work_sync(&fs_info->async_reclaim_work);
3964
3965         /* Cancel or finish ongoing discard work */
3966         btrfs_discard_cleanup(fs_info);
3967
3968         if (!sb_rdonly(fs_info->sb)) {
3969                 /*
3970                  * The cleaner kthread is stopped, so do one final pass over
3971                  * unused block groups.
3972                  */
3973                 btrfs_delete_unused_bgs(fs_info);
3974
3975                 ret = btrfs_commit_super(fs_info);
3976                 if (ret)
3977                         btrfs_err(fs_info, "commit super ret %d", ret);
3978         }
3979
3980         if (test_bit(BTRFS_FS_STATE_ERROR, &fs_info->fs_state) ||
3981             test_bit(BTRFS_FS_STATE_TRANS_ABORTED, &fs_info->fs_state))
3982                 btrfs_error_commit_super(fs_info);
3983
3984         kthread_stop(fs_info->transaction_kthread);
3985         kthread_stop(fs_info->cleaner_kthread);
3986
3987         ASSERT(list_empty(&fs_info->delayed_iputs));
3988         set_bit(BTRFS_FS_CLOSING_DONE, &fs_info->flags);
3989
3990         btrfs_free_qgroup_config(fs_info);
3991         ASSERT(list_empty(&fs_info->delalloc_roots));
3992
3993         if (percpu_counter_sum(&fs_info->delalloc_bytes)) {
3994                 btrfs_info(fs_info, "at unmount delalloc count %lld",
3995                        percpu_counter_sum(&fs_info->delalloc_bytes));
3996         }
3997
3998         if (percpu_counter_sum(&fs_info->dio_bytes))
3999                 btrfs_info(fs_info, "at unmount dio bytes count %lld",
4000                            percpu_counter_sum(&fs_info->dio_bytes));
4001
4002         btrfs_sysfs_remove_mounted(fs_info);
4003         btrfs_sysfs_remove_fsid(fs_info->fs_devices);
4004
4005         btrfs_free_fs_roots(fs_info);
4006
4007         btrfs_put_block_group_cache(fs_info);
4008
4009         /*
4010          * we must make sure there is not any read request to
4011          * submit after we stopping all workers.
4012          */
4013         invalidate_inode_pages2(fs_info->btree_inode->i_mapping);
4014         btrfs_stop_all_workers(fs_info);
4015
4016         clear_bit(BTRFS_FS_OPEN, &fs_info->flags);
4017         free_root_pointers(fs_info, true);
4018
4019         /*
4020          * We must free the block groups after dropping the fs_roots as we could
4021          * have had an IO error and have left over tree log blocks that aren't
4022          * cleaned up until the fs roots are freed.  This makes the block group
4023          * accounting appear to be wrong because there's pending reserved bytes,
4024          * so make sure we do the block group cleanup afterwards.
4025          */
4026         btrfs_free_block_groups(fs_info);
4027
4028         iput(fs_info->btree_inode);
4029
4030 #ifdef CONFIG_BTRFS_FS_CHECK_INTEGRITY
4031         if (btrfs_test_opt(fs_info, CHECK_INTEGRITY))
4032                 btrfsic_unmount(fs_info->fs_devices);
4033 #endif
4034
4035         btrfs_mapping_tree_free(&fs_info->mapping_tree);
4036         btrfs_close_devices(fs_info->fs_devices);
4037
4038         percpu_counter_destroy(&fs_info->dirty_metadata_bytes);
4039         percpu_counter_destroy(&fs_info->delalloc_bytes);
4040         percpu_counter_destroy(&fs_info->dio_bytes);
4041         percpu_counter_destroy(&fs_info->dev_replace.bio_counter);
4042         cleanup_srcu_struct(&fs_info->subvol_srcu);
4043
4044         btrfs_free_csum_hash(fs_info);
4045         btrfs_free_stripe_hash_table(fs_info);
4046         btrfs_free_ref_cache(fs_info);
4047 }
4048
4049 int btrfs_buffer_uptodate(struct extent_buffer *buf, u64 parent_transid,
4050                           int atomic)
4051 {
4052         int ret;
4053         struct inode *btree_inode = buf->pages[0]->mapping->host;
4054
4055         ret = extent_buffer_uptodate(buf);
4056         if (!ret)
4057                 return ret;
4058
4059         ret = verify_parent_transid(&BTRFS_I(btree_inode)->io_tree, buf,
4060                                     parent_transid, atomic);
4061         if (ret == -EAGAIN)
4062                 return ret;
4063         return !ret;
4064 }
4065
4066 void btrfs_mark_buffer_dirty(struct extent_buffer *buf)
4067 {
4068         struct btrfs_fs_info *fs_info;
4069         struct btrfs_root *root;
4070         u64 transid = btrfs_header_generation(buf);
4071         int was_dirty;
4072
4073 #ifdef CONFIG_BTRFS_FS_RUN_SANITY_TESTS
4074         /*
4075          * This is a fast path so only do this check if we have sanity tests
4076          * enabled.  Normal people shouldn't be using unmapped buffers as dirty
4077          * outside of the sanity tests.
4078          */
4079         if (unlikely(test_bit(EXTENT_BUFFER_UNMAPPED, &buf->bflags)))
4080                 return;
4081 #endif
4082         root = BTRFS_I(buf->pages[0]->mapping->host)->root;
4083         fs_info = root->fs_info;
4084         btrfs_assert_tree_locked(buf);
4085         if (transid != fs_info->generation)
4086                 WARN(1, KERN_CRIT "btrfs transid mismatch buffer %llu, found %llu running %llu\n",
4087                         buf->start, transid, fs_info->generation);
4088         was_dirty = set_extent_buffer_dirty(buf);
4089         if (!was_dirty)
4090                 percpu_counter_add_batch(&fs_info->dirty_metadata_bytes,
4091                                          buf->len,
4092                                          fs_info->dirty_metadata_batch);
4093 #ifdef CONFIG_BTRFS_FS_CHECK_INTEGRITY
4094         /*
4095          * Since btrfs_mark_buffer_dirty() can be called with item pointer set
4096          * but item data not updated.
4097          * So here we should only check item pointers, not item data.
4098          */
4099         if (btrfs_header_level(buf) == 0 &&
4100             btrfs_check_leaf_relaxed(buf)) {
4101                 btrfs_print_leaf(buf);
4102                 ASSERT(0);
4103         }
4104 #endif
4105 }
4106
4107 static void __btrfs_btree_balance_dirty(struct btrfs_fs_info *fs_info,
4108                                         int flush_delayed)
4109 {
4110         /*
4111          * looks as though older kernels can get into trouble with
4112          * this code, they end up stuck in balance_dirty_pages forever
4113          */
4114         int ret;
4115
4116         if (current->flags & PF_MEMALLOC)
4117                 return;
4118
4119         if (flush_delayed)
4120                 btrfs_balance_delayed_items(fs_info);
4121
4122         ret = __percpu_counter_compare(&fs_info->dirty_metadata_bytes,
4123                                      BTRFS_DIRTY_METADATA_THRESH,
4124                                      fs_info->dirty_metadata_batch);
4125         if (ret > 0) {
4126                 balance_dirty_pages_ratelimited(fs_info->btree_inode->i_mapping);
4127         }
4128 }
4129
4130 void btrfs_btree_balance_dirty(struct btrfs_fs_info *fs_info)
4131 {
4132         __btrfs_btree_balance_dirty(fs_info, 1);
4133 }
4134
4135 void btrfs_btree_balance_dirty_nodelay(struct btrfs_fs_info *fs_info)
4136 {
4137         __btrfs_btree_balance_dirty(fs_info, 0);
4138 }
4139
4140 int btrfs_read_buffer(struct extent_buffer *buf, u64 parent_transid, int level,
4141                       struct btrfs_key *first_key)
4142 {
4143         return btree_read_extent_buffer_pages(buf, parent_transid,
4144                                               level, first_key);
4145 }
4146
4147 static void btrfs_error_commit_super(struct btrfs_fs_info *fs_info)
4148 {
4149         /* cleanup FS via transaction */
4150         btrfs_cleanup_transaction(fs_info);
4151
4152         mutex_lock(&fs_info->cleaner_mutex);
4153         btrfs_run_delayed_iputs(fs_info);
4154         mutex_unlock(&fs_info->cleaner_mutex);
4155
4156         down_write(&fs_info->cleanup_work_sem);
4157         up_write(&fs_info->cleanup_work_sem);
4158 }
4159
4160 static void btrfs_destroy_ordered_extents(struct btrfs_root *root)
4161 {
4162         struct btrfs_ordered_extent *ordered;
4163
4164         spin_lock(&root->ordered_extent_lock);
4165         /*
4166          * This will just short circuit the ordered completion stuff which will
4167          * make sure the ordered extent gets properly cleaned up.
4168          */
4169         list_for_each_entry(ordered, &root->ordered_extents,
4170                             root_extent_list)
4171                 set_bit(BTRFS_ORDERED_IOERR, &ordered->flags);
4172         spin_unlock(&root->ordered_extent_lock);
4173 }
4174
4175 static void btrfs_destroy_all_ordered_extents(struct btrfs_fs_info *fs_info)
4176 {
4177         struct btrfs_root *root;
4178         struct list_head splice;
4179
4180         INIT_LIST_HEAD(&splice);
4181
4182         spin_lock(&fs_info->ordered_root_lock);
4183         list_splice_init(&fs_info->ordered_roots, &splice);
4184         while (!list_empty(&splice)) {
4185                 root = list_first_entry(&splice, struct btrfs_root,
4186                                         ordered_root);
4187                 list_move_tail(&root->ordered_root,
4188                                &fs_info->ordered_roots);
4189
4190                 spin_unlock(&fs_info->ordered_root_lock);
4191                 btrfs_destroy_ordered_extents(root);
4192
4193                 cond_resched();
4194                 spin_lock(&fs_info->ordered_root_lock);
4195         }
4196         spin_unlock(&fs_info->ordered_root_lock);
4197
4198         /*
4199          * We need this here because if we've been flipped read-only we won't
4200          * get sync() from the umount, so we need to make sure any ordered
4201          * extents that haven't had their dirty pages IO start writeout yet
4202          * actually get run and error out properly.
4203          */
4204         btrfs_wait_ordered_roots(fs_info, U64_MAX, 0, (u64)-1);
4205 }
4206
4207 static int btrfs_destroy_delayed_refs(struct btrfs_transaction *trans,
4208                                       struct btrfs_fs_info *fs_info)
4209 {
4210         struct rb_node *node;
4211         struct btrfs_delayed_ref_root *delayed_refs;
4212         struct btrfs_delayed_ref_node *ref;
4213         int ret = 0;
4214
4215         delayed_refs = &trans->delayed_refs;
4216
4217         spin_lock(&delayed_refs->lock);
4218         if (atomic_read(&delayed_refs->num_entries) == 0) {
4219                 spin_unlock(&delayed_refs->lock);
4220                 btrfs_info(fs_info, "delayed_refs has NO entry");
4221                 return ret;
4222         }
4223
4224         while ((node = rb_first_cached(&delayed_refs->href_root)) != NULL) {
4225                 struct btrfs_delayed_ref_head *head;
4226                 struct rb_node *n;
4227                 bool pin_bytes = false;
4228
4229                 head = rb_entry(node, struct btrfs_delayed_ref_head,
4230                                 href_node);
4231                 if (btrfs_delayed_ref_lock(delayed_refs, head))
4232                         continue;
4233
4234                 spin_lock(&head->lock);
4235                 while ((n = rb_first_cached(&head->ref_tree)) != NULL) {
4236                         ref = rb_entry(n, struct btrfs_delayed_ref_node,
4237                                        ref_node);
4238                         ref->in_tree = 0;
4239                         rb_erase_cached(&ref->ref_node, &head->ref_tree);
4240                         RB_CLEAR_NODE(&ref->ref_node);
4241                         if (!list_empty(&ref->add_list))
4242                                 list_del(&ref->add_list);
4243                         atomic_dec(&delayed_refs->num_entries);
4244                         btrfs_put_delayed_ref(ref);
4245                 }
4246                 if (head->must_insert_reserved)
4247                         pin_bytes = true;
4248                 btrfs_free_delayed_extent_op(head->extent_op);
4249                 btrfs_delete_ref_head(delayed_refs, head);
4250                 spin_unlock(&head->lock);
4251                 spin_unlock(&delayed_refs->lock);
4252                 mutex_unlock(&head->mutex);
4253
4254                 if (pin_bytes)
4255                         btrfs_pin_extent(fs_info, head->bytenr,
4256                                          head->num_bytes, 1);
4257                 btrfs_cleanup_ref_head_accounting(fs_info, delayed_refs, head);
4258                 btrfs_put_delayed_ref_head(head);
4259                 cond_resched();
4260                 spin_lock(&delayed_refs->lock);
4261         }
4262         btrfs_qgroup_destroy_extent_records(trans);
4263
4264         spin_unlock(&delayed_refs->lock);
4265
4266         return ret;
4267 }
4268
4269 static void btrfs_destroy_delalloc_inodes(struct btrfs_root *root)
4270 {
4271         struct btrfs_inode *btrfs_inode;
4272         struct list_head splice;
4273
4274         INIT_LIST_HEAD(&splice);
4275
4276         spin_lock(&root->delalloc_lock);
4277         list_splice_init(&root->delalloc_inodes, &splice);
4278
4279         while (!list_empty(&splice)) {
4280                 struct inode *inode = NULL;
4281                 btrfs_inode = list_first_entry(&splice, struct btrfs_inode,
4282                                                delalloc_inodes);
4283                 __btrfs_del_delalloc_inode(root, btrfs_inode);
4284                 spin_unlock(&root->delalloc_lock);
4285
4286                 /*
4287                  * Make sure we get a live inode and that it'll not disappear
4288                  * meanwhile.
4289                  */
4290                 inode = igrab(&btrfs_inode->vfs_inode);
4291                 if (inode) {
4292                         invalidate_inode_pages2(inode->i_mapping);
4293                         iput(inode);
4294                 }
4295                 spin_lock(&root->delalloc_lock);
4296         }
4297         spin_unlock(&root->delalloc_lock);
4298 }
4299
4300 static void btrfs_destroy_all_delalloc_inodes(struct btrfs_fs_info *fs_info)
4301 {
4302         struct btrfs_root *root;
4303         struct list_head splice;
4304
4305         INIT_LIST_HEAD(&splice);
4306
4307         spin_lock(&fs_info->delalloc_root_lock);
4308         list_splice_init(&fs_info->delalloc_roots, &splice);
4309         while (!list_empty(&splice)) {
4310                 root = list_first_entry(&splice, struct btrfs_root,
4311                                          delalloc_root);
4312                 root = btrfs_grab_fs_root(root);
4313                 BUG_ON(!root);
4314                 spin_unlock(&fs_info->delalloc_root_lock);
4315
4316                 btrfs_destroy_delalloc_inodes(root);
4317                 btrfs_put_fs_root(root);
4318
4319                 spin_lock(&fs_info->delalloc_root_lock);
4320         }
4321         spin_unlock(&fs_info->delalloc_root_lock);
4322 }
4323
4324 static int btrfs_destroy_marked_extents(struct btrfs_fs_info *fs_info,
4325                                         struct extent_io_tree *dirty_pages,
4326                                         int mark)
4327 {
4328         int ret;
4329         struct extent_buffer *eb;
4330         u64 start = 0;
4331         u64 end;
4332
4333         while (1) {
4334                 ret = find_first_extent_bit(dirty_pages, start, &start, &end,
4335                                             mark, NULL);
4336                 if (ret)
4337                         break;
4338
4339                 clear_extent_bits(dirty_pages, start, end, mark);
4340                 while (start <= end) {
4341                         eb = find_extent_buffer(fs_info, start);
4342                         start += fs_info->nodesize;
4343                         if (!eb)
4344                                 continue;
4345                         wait_on_extent_buffer_writeback(eb);
4346
4347                         if (test_and_clear_bit(EXTENT_BUFFER_DIRTY,
4348                                                &eb->bflags))
4349                                 clear_extent_buffer_dirty(eb);
4350                         free_extent_buffer_stale(eb);
4351                 }
4352         }
4353
4354         return ret;
4355 }
4356
4357 static int btrfs_destroy_pinned_extent(struct btrfs_fs_info *fs_info,
4358                                        struct extent_io_tree *pinned_extents)
4359 {
4360         struct extent_io_tree *unpin;
4361         u64 start;
4362         u64 end;
4363         int ret;
4364         bool loop = true;
4365
4366         unpin = pinned_extents;
4367 again:
4368         while (1) {
4369                 struct extent_state *cached_state = NULL;
4370
4371                 /*
4372                  * The btrfs_finish_extent_commit() may get the same range as
4373                  * ours between find_first_extent_bit and clear_extent_dirty.
4374                  * Hence, hold the unused_bg_unpin_mutex to avoid double unpin
4375                  * the same extent range.
4376                  */
4377                 mutex_lock(&fs_info->unused_bg_unpin_mutex);
4378                 ret = find_first_extent_bit(unpin, 0, &start, &end,
4379                                             EXTENT_DIRTY, &cached_state);
4380                 if (ret) {
4381                         mutex_unlock(&fs_info->unused_bg_unpin_mutex);
4382                         break;
4383                 }
4384
4385                 clear_extent_dirty(unpin, start, end, &cached_state);
4386                 free_extent_state(cached_state);
4387                 btrfs_error_unpin_extent_range(fs_info, start, end);
4388                 mutex_unlock(&fs_info->unused_bg_unpin_mutex);
4389                 cond_resched();
4390         }
4391
4392         if (loop) {
4393                 if (unpin == &fs_info->freed_extents[0])
4394                         unpin = &fs_info->freed_extents[1];
4395                 else
4396                         unpin = &fs_info->freed_extents[0];
4397                 loop = false;
4398                 goto again;
4399         }
4400
4401         return 0;
4402 }
4403
4404 static void btrfs_cleanup_bg_io(struct btrfs_block_group *cache)
4405 {
4406         struct inode *inode;
4407
4408         inode = cache->io_ctl.inode;
4409         if (inode) {
4410                 invalidate_inode_pages2(inode->i_mapping);
4411                 BTRFS_I(inode)->generation = 0;
4412                 cache->io_ctl.inode = NULL;
4413                 iput(inode);
4414         }
4415         btrfs_put_block_group(cache);
4416 }
4417
4418 void btrfs_cleanup_dirty_bgs(struct btrfs_transaction *cur_trans,
4419                              struct btrfs_fs_info *fs_info)
4420 {
4421         struct btrfs_block_group *cache;
4422
4423         spin_lock(&cur_trans->dirty_bgs_lock);
4424         while (!list_empty(&cur_trans->dirty_bgs)) {
4425                 cache = list_first_entry(&cur_trans->dirty_bgs,
4426                                          struct btrfs_block_group,
4427                                          dirty_list);
4428
4429                 if (!list_empty(&cache->io_list)) {
4430                         spin_unlock(&cur_trans->dirty_bgs_lock);
4431                         list_del_init(&cache->io_list);
4432                         btrfs_cleanup_bg_io(cache);
4433                         spin_lock(&cur_trans->dirty_bgs_lock);
4434                 }
4435
4436                 list_del_init(&cache->dirty_list);
4437                 spin_lock(&cache->lock);
4438                 cache->disk_cache_state = BTRFS_DC_ERROR;
4439                 spin_unlock(&cache->lock);
4440
4441                 spin_unlock(&cur_trans->dirty_bgs_lock);
4442                 btrfs_put_block_group(cache);
4443                 btrfs_delayed_refs_rsv_release(fs_info, 1);
4444                 spin_lock(&cur_trans->dirty_bgs_lock);
4445         }
4446         spin_unlock(&cur_trans->dirty_bgs_lock);
4447
4448         /*
4449          * Refer to the definition of io_bgs member for details why it's safe
4450          * to use it without any locking
4451          */
4452         while (!list_empty(&cur_trans->io_bgs)) {
4453                 cache = list_first_entry(&cur_trans->io_bgs,
4454                                          struct btrfs_block_group,
4455                                          io_list);
4456
4457                 list_del_init(&cache->io_list);
4458                 spin_lock(&cache->lock);
4459                 cache->disk_cache_state = BTRFS_DC_ERROR;
4460                 spin_unlock(&cache->lock);
4461                 btrfs_cleanup_bg_io(cache);
4462         }
4463 }
4464
4465 void btrfs_cleanup_one_transaction(struct btrfs_transaction *cur_trans,
4466                                    struct btrfs_fs_info *fs_info)
4467 {
4468         struct btrfs_device *dev, *tmp;
4469
4470         btrfs_cleanup_dirty_bgs(cur_trans, fs_info);
4471         ASSERT(list_empty(&cur_trans->dirty_bgs));
4472         ASSERT(list_empty(&cur_trans->io_bgs));
4473
4474         list_for_each_entry_safe(dev, tmp, &cur_trans->dev_update_list,
4475                                  post_commit_list) {
4476                 list_del_init(&dev->post_commit_list);
4477         }
4478
4479         btrfs_destroy_delayed_refs(cur_trans, fs_info);
4480
4481         cur_trans->state = TRANS_STATE_COMMIT_START;
4482         wake_up(&fs_info->transaction_blocked_wait);
4483
4484         cur_trans->state = TRANS_STATE_UNBLOCKED;
4485         wake_up(&fs_info->transaction_wait);
4486
4487         btrfs_destroy_delayed_inodes(fs_info);
4488
4489         btrfs_destroy_marked_extents(fs_info, &cur_trans->dirty_pages,
4490                                      EXTENT_DIRTY);
4491         btrfs_destroy_pinned_extent(fs_info,
4492                                     fs_info->pinned_extents);
4493
4494         cur_trans->state =TRANS_STATE_COMPLETED;
4495         wake_up(&cur_trans->commit_wait);
4496 }
4497
4498 static int btrfs_cleanup_transaction(struct btrfs_fs_info *fs_info)
4499 {
4500         struct btrfs_transaction *t;
4501
4502         mutex_lock(&fs_info->transaction_kthread_mutex);
4503
4504         spin_lock(&fs_info->trans_lock);
4505         while (!list_empty(&fs_info->trans_list)) {
4506                 t = list_first_entry(&fs_info->trans_list,
4507                                      struct btrfs_transaction, list);
4508                 if (t->state >= TRANS_STATE_COMMIT_START) {
4509                         refcount_inc(&t->use_count);
4510                         spin_unlock(&fs_info->trans_lock);
4511                         btrfs_wait_for_commit(fs_info, t->transid);
4512                         btrfs_put_transaction(t);
4513                         spin_lock(&fs_info->trans_lock);
4514                         continue;
4515                 }
4516                 if (t == fs_info->running_transaction) {
4517                         t->state = TRANS_STATE_COMMIT_DOING;
4518                         spin_unlock(&fs_info->trans_lock);
4519                         /*
4520                          * We wait for 0 num_writers since we don't hold a trans
4521                          * handle open currently for this transaction.
4522                          */
4523                         wait_event(t->writer_wait,
4524                                    atomic_read(&t->num_writers) == 0);
4525                 } else {
4526                         spin_unlock(&fs_info->trans_lock);
4527                 }
4528                 btrfs_cleanup_one_transaction(t, fs_info);
4529
4530                 spin_lock(&fs_info->trans_lock);
4531                 if (t == fs_info->running_transaction)
4532                         fs_info->running_transaction = NULL;
4533                 list_del_init(&t->list);
4534                 spin_unlock(&fs_info->trans_lock);
4535
4536                 btrfs_put_transaction(t);
4537                 trace_btrfs_transaction_commit(fs_info->tree_root);
4538                 spin_lock(&fs_info->trans_lock);
4539         }
4540         spin_unlock(&fs_info->trans_lock);
4541         btrfs_destroy_all_ordered_extents(fs_info);
4542         btrfs_destroy_delayed_inodes(fs_info);
4543         btrfs_assert_delayed_root_empty(fs_info);
4544         btrfs_destroy_pinned_extent(fs_info, fs_info->pinned_extents);
4545         btrfs_destroy_all_delalloc_inodes(fs_info);
4546         mutex_unlock(&fs_info->transaction_kthread_mutex);
4547
4548         return 0;
4549 }
4550
4551 static const struct extent_io_ops btree_extent_io_ops = {
4552         /* mandatory callbacks */
4553         .submit_bio_hook = btree_submit_bio_hook,
4554         .readpage_end_io_hook = btree_readpage_end_io_hook,
4555 };