fs/btrfs/block-group.c

   1 // SPDX-License-Identifier: GPL-2.0
   2
   3 #include "misc.h"
   4 #include "ctree.h"
   5 #include "block-group.h"
   6 #include "space-info.h"
   7 #include "disk-io.h"
   8 #include "free-space-cache.h"
   9 #include "free-space-tree.h"
  10 #include "disk-io.h"
  11 #include "volumes.h"
  12 #include "transaction.h"
  13 #include "ref-verify.h"
  14 #include "sysfs.h"
  15 #include "tree-log.h"
  16 #include "delalloc-space.h"
  17 #include "discard.h"
  18 #include "raid56.h"
  19
  20 /*
  21  * Return target flags in extended format or 0 if restripe for this chunk_type
  22  * is not in progress
  23  *
  24  * Should be called with balance_lock held
  25  */
  26 static u64 get_restripe_target(struct btrfs_fs_info *fs_info, u64 flags)
  27 {
  28         struct btrfs_balance_control *bctl = fs_info->balance_ctl;
  29         u64 target = 0;
  30
  31         if (!bctl)
  32                 return 0;
  33
  34         if (flags & BTRFS_BLOCK_GROUP_DATA &&
  35             bctl->data.flags & BTRFS_BALANCE_ARGS_CONVERT) {
  36                 target = BTRFS_BLOCK_GROUP_DATA | bctl->data.target;
  37         } else if (flags & BTRFS_BLOCK_GROUP_SYSTEM &&
  38                    bctl->sys.flags & BTRFS_BALANCE_ARGS_CONVERT) {
  39                 target = BTRFS_BLOCK_GROUP_SYSTEM | bctl->sys.target;
  40         } else if (flags & BTRFS_BLOCK_GROUP_METADATA &&
  41                    bctl->meta.flags & BTRFS_BALANCE_ARGS_CONVERT) {
  42                 target = BTRFS_BLOCK_GROUP_METADATA | bctl->meta.target;
  43         }
  44
  45         return target;
  46 }
  47
  48 /*
  49  * @flags: available profiles in extended format (see ctree.h)
  50  *
  51  * Return reduced profile in chunk format.  If profile changing is in progress
  52  * (either running or paused) picks the target profile (if it's already
  53  * available), otherwise falls back to plain reducing.
  54  */
  55 static u64 btrfs_reduce_alloc_profile(struct btrfs_fs_info *fs_info, u64 flags)
  56 {
  57         u64 num_devices = fs_info->fs_devices->rw_devices;
  58         u64 target;
  59         u64 raid_type;
  60         u64 allowed = 0;
  61
  62         /*
  63          * See if restripe for this chunk_type is in progress, if so try to
  64          * reduce to the target profile
  65          */
  66         spin_lock(&fs_info->balance_lock);
  67         target = get_restripe_target(fs_info, flags);
  68         if (target) {
  69                 /* Pick target profile only if it's already available */
  70                 if ((flags & target) & BTRFS_EXTENDED_PROFILE_MASK) {
  71                         spin_unlock(&fs_info->balance_lock);
  72                         return extended_to_chunk(target);
  73                 }
  74         }
  75         spin_unlock(&fs_info->balance_lock);
  76
  77         /* First, mask out the RAID levels which aren't possible */
  78         for (raid_type = 0; raid_type < BTRFS_NR_RAID_TYPES; raid_type++) {
  79                 if (num_devices >= btrfs_raid_array[raid_type].devs_min)
  80                         allowed |= btrfs_raid_array[raid_type].bg_flag;
  81         }
  82         allowed &= flags;
  83
  84         if (allowed & BTRFS_BLOCK_GROUP_RAID6)
  85                 allowed = BTRFS_BLOCK_GROUP_RAID6;
  86         else if (allowed & BTRFS_BLOCK_GROUP_RAID5)
  87                 allowed = BTRFS_BLOCK_GROUP_RAID5;
  88         else if (allowed & BTRFS_BLOCK_GROUP_RAID10)
  89                 allowed = BTRFS_BLOCK_GROUP_RAID10;
  90         else if (allowed & BTRFS_BLOCK_GROUP_RAID1)
  91                 allowed = BTRFS_BLOCK_GROUP_RAID1;
  92         else if (allowed & BTRFS_BLOCK_GROUP_RAID0)
  93                 allowed = BTRFS_BLOCK_GROUP_RAID0;
  94
  95         flags &= ~BTRFS_BLOCK_GROUP_PROFILE_MASK;
  96
  97         return extended_to_chunk(flags | allowed);
  98 }
  99
 100 u64 btrfs_get_alloc_profile(struct btrfs_fs_info *fs_info, u64 orig_flags)
 101 {
 102         unsigned seq;
 103         u64 flags;
 104
 105         do {
 106                 flags = orig_flags;
 107                 seq = read_seqbegin(&fs_info->profiles_lock);
 108
 109                 if (flags & BTRFS_BLOCK_GROUP_DATA)
 110                         flags |= fs_info->avail_data_alloc_bits;
 111                 else if (flags & BTRFS_BLOCK_GROUP_SYSTEM)
 112                         flags |= fs_info->avail_system_alloc_bits;
 113                 else if (flags & BTRFS_BLOCK_GROUP_METADATA)
 114                         flags |= fs_info->avail_metadata_alloc_bits;
 115         } while (read_seqretry(&fs_info->profiles_lock, seq));
 116
 117         return btrfs_reduce_alloc_profile(fs_info, flags);
 118 }
 119
 120 void btrfs_get_block_group(struct btrfs_block_group *cache)
 121 {
 122         atomic_inc(&cache->count);
 123 }
 124
 125 void btrfs_put_block_group(struct btrfs_block_group *cache)
 126 {
 127         if (atomic_dec_and_test(&cache->count)) {
 128                 WARN_ON(cache->pinned > 0);
 129                 WARN_ON(cache->reserved > 0);
 130
 131                 /*
 132                  * A block_group shouldn't be on the discard_list anymore.
 133                  * Remove the block_group from the discard_list to prevent us
 134                  * from causing a panic due to NULL pointer dereference.
 135                  */
 136                 if (WARN_ON(!list_empty(&cache->discard_list)))
 137                         btrfs_discard_cancel_work(&cache->fs_info->discard_ctl,
 138                                                   cache);
 139
 140                 /*
 141                  * If not empty, someone is still holding mutex of
 142                  * full_stripe_lock, which can only be released by caller.
 143                  * And it will definitely cause use-after-free when caller
 144                  * tries to release full stripe lock.
 145                  *
 146                  * No better way to resolve, but only to warn.
 147                  */
 148                 WARN_ON(!RB_EMPTY_ROOT(&cache->full_stripe_locks_root.root));
 149                 kfree(cache->free_space_ctl);
 150                 kfree(cache);
 151         }
 152 }
 153
 154 /*
 155  * This adds the block group to the fs_info rb tree for the block group cache
 156  */
 157 static int btrfs_add_block_group_cache(struct btrfs_fs_info *info,
 158                                        struct btrfs_block_group *block_group)
 159 {
 160         struct rb_node **p;
 161         struct rb_node *parent = NULL;
 162         struct btrfs_block_group *cache;
 163
 164         spin_lock(&info->block_group_cache_lock);
 165         p = &info->block_group_cache_tree.rb_node;
 166
 167         while (*p) {
 168                 parent = *p;
 169                 cache = rb_entry(parent, struct btrfs_block_group, cache_node);
 170                 if (block_group->start < cache->start) {
 171                         p = &(*p)->rb_left;
 172                 } else if (block_group->start > cache->start) {
 173                         p = &(*p)->rb_right;
 174                 } else {
 175                         spin_unlock(&info->block_group_cache_lock);
 176                         return -EEXIST;
 177                 }
 178         }
 179
 180         rb_link_node(&block_group->cache_node, parent, p);
 181         rb_insert_color(&block_group->cache_node,
 182                         &info->block_group_cache_tree);
 183
 184         if (info->first_logical_byte > block_group->start)
 185                 info->first_logical_byte = block_group->start;
 186
 187         spin_unlock(&info->block_group_cache_lock);
 188
 189         return 0;
 190 }
 191
 192 /*
 193  * This will return the block group at or after bytenr if contains is 0, else
 194  * it will return the block group that contains the bytenr
 195  */
 196 static struct btrfs_block_group *block_group_cache_tree_search(
 197                 struct btrfs_fs_info *info, u64 bytenr, int contains)
 198 {
 199         struct btrfs_block_group *cache, *ret = NULL;
 200         struct rb_node *n;
 201         u64 end, start;
 202
 203         spin_lock(&info->block_group_cache_lock);
 204         n = info->block_group_cache_tree.rb_node;
 205
 206         while (n) {
 207                 cache = rb_entry(n, struct btrfs_block_group, cache_node);
 208                 end = cache->start + cache->length - 1;
 209                 start = cache->start;
 210
 211                 if (bytenr < start) {
 212                         if (!contains && (!ret || start < ret->start))
 213                                 ret = cache;
 214                         n = n->rb_left;
 215                 } else if (bytenr > start) {
 216                         if (contains && bytenr <= end) {
 217                                 ret = cache;
 218                                 break;
 219                         }
 220                         n = n->rb_right;
 221                 } else {
 222                         ret = cache;
 223                         break;
 224                 }
 225         }
 226         if (ret) {
 227                 btrfs_get_block_group(ret);
 228                 if (bytenr == 0 && info->first_logical_byte > ret->start)
 229                         info->first_logical_byte = ret->start;
 230         }
 231         spin_unlock(&info->block_group_cache_lock);
 232
 233         return ret;
 234 }
 235
 236 /*
 237  * Return the block group that starts at or after bytenr
 238  */
 239 struct btrfs_block_group *btrfs_lookup_first_block_group(
 240                 struct btrfs_fs_info *info, u64 bytenr)
 241 {
 242         return block_group_cache_tree_search(info, bytenr, 0);
 243 }
 244
 245 /*
 246  * Return the block group that contains the given bytenr
 247  */
 248 struct btrfs_block_group *btrfs_lookup_block_group(
 249                 struct btrfs_fs_info *info, u64 bytenr)
 250 {
 251         return block_group_cache_tree_search(info, bytenr, 1);
 252 }
 253
 254 struct btrfs_block_group *btrfs_next_block_group(
 255                 struct btrfs_block_group *cache)
 256 {
 257         struct btrfs_fs_info *fs_info = cache->fs_info;
 258         struct rb_node *node;
 259
 260         spin_lock(&fs_info->block_group_cache_lock);
 261
 262         /* If our block group was removed, we need a full search. */
 263         if (RB_EMPTY_NODE(&cache->cache_node)) {
 264                 const u64 next_bytenr = cache->start + cache->length;
 265
 266                 spin_unlock(&fs_info->block_group_cache_lock);
 267                 btrfs_put_block_group(cache);
 268                 cache = btrfs_lookup_first_block_group(fs_info, next_bytenr); return cache;
 269         }
 270         node = rb_next(&cache->cache_node);
 271         btrfs_put_block_group(cache);
 272         if (node) {
 273                 cache = rb_entry(node, struct btrfs_block_group, cache_node);
 274                 btrfs_get_block_group(cache);
 275         } else
 276                 cache = NULL;
 277         spin_unlock(&fs_info->block_group_cache_lock);
 278         return cache;
 279 }
 280
 281 bool btrfs_inc_nocow_writers(struct btrfs_fs_info *fs_info, u64 bytenr)
 282 {
 283         struct btrfs_block_group *bg;
 284         bool ret = true;
 285
 286         bg = btrfs_lookup_block_group(fs_info, bytenr);
 287         if (!bg)
 288                 return false;
 289
 290         spin_lock(&bg->lock);
 291         if (bg->ro)
 292                 ret = false;
 293         else
 294                 atomic_inc(&bg->nocow_writers);
 295         spin_unlock(&bg->lock);
 296
 297         /* No put on block group, done by btrfs_dec_nocow_writers */
 298         if (!ret)
 299                 btrfs_put_block_group(bg);
 300
 301         return ret;
 302 }
 303
 304 void btrfs_dec_nocow_writers(struct btrfs_fs_info *fs_info, u64 bytenr)
 305 {
 306         struct btrfs_block_group *bg;
 307
 308         bg = btrfs_lookup_block_group(fs_info, bytenr);
 309         ASSERT(bg);
 310         if (atomic_dec_and_test(&bg->nocow_writers))
 311                 wake_up_var(&bg->nocow_writers);
 312         /*
 313          * Once for our lookup and once for the lookup done by a previous call
 314          * to btrfs_inc_nocow_writers()
 315          */
 316         btrfs_put_block_group(bg);
 317         btrfs_put_block_group(bg);
 318 }
 319
 320 void btrfs_wait_nocow_writers(struct btrfs_block_group *bg)
 321 {
 322         wait_var_event(&bg->nocow_writers, !atomic_read(&bg->nocow_writers));
 323 }
 324
 325 void btrfs_dec_block_group_reservations(struct btrfs_fs_info *fs_info,
 326                                         const u64 start)
 327 {
 328         struct btrfs_block_group *bg;
 329
 330         bg = btrfs_lookup_block_group(fs_info, start);
 331         ASSERT(bg);
 332         if (atomic_dec_and_test(&bg->reservations))
 333                 wake_up_var(&bg->reservations);
 334         btrfs_put_block_group(bg);
 335 }
 336
 337 void btrfs_wait_block_group_reservations(struct btrfs_block_group *bg)
 338 {
 339         struct btrfs_space_info *space_info = bg->space_info;
 340
 341         ASSERT(bg->ro);
 342
 343         if (!(bg->flags & BTRFS_BLOCK_GROUP_DATA))
 344                 return;
 345
 346         /*
 347          * Our block group is read only but before we set it to read only,
 348          * some task might have had allocated an extent from it already, but it
 349          * has not yet created a respective ordered extent (and added it to a
 350          * root's list of ordered extents).
 351          * Therefore wait for any task currently allocating extents, since the
 352          * block group's reservations counter is incremented while a read lock
 353          * on the groups' semaphore is held and decremented after releasing
 354          * the read access on that semaphore and creating the ordered extent.
 355          */
 356         down_write(&space_info->groups_sem);
 357         up_write(&space_info->groups_sem);
 358
 359         wait_var_event(&bg->reservations, !atomic_read(&bg->reservations));
 360 }
 361
 362 struct btrfs_caching_control *btrfs_get_caching_control(
 363                 struct btrfs_block_group *cache)
 364 {
 365         struct btrfs_caching_control *ctl;
 366
 367         spin_lock(&cache->lock);
 368         if (!cache->caching_ctl) {
 369                 spin_unlock(&cache->lock);
 370                 return NULL;
 371         }
 372
 373         ctl = cache->caching_ctl;
 374         refcount_inc(&ctl->count);
 375         spin_unlock(&cache->lock);
 376         return ctl;
 377 }
 378
 379 void btrfs_put_caching_control(struct btrfs_caching_control *ctl)
 380 {
 381         if (refcount_dec_and_test(&ctl->count))
 382                 kfree(ctl);
 383 }
 384
 385 /*
 386  * When we wait for progress in the block group caching, its because our
 387  * allocation attempt failed at least once.  So, we must sleep and let some
 388  * progress happen before we try again.
 389  *
 390  * This function will sleep at least once waiting for new free space to show
 391  * up, and then it will check the block group free space numbers for our min
 392  * num_bytes.  Another option is to have it go ahead and look in the rbtree for
 393  * a free extent of a given size, but this is a good start.
 394  *
 395  * Callers of this must check if cache->cached == BTRFS_CACHE_ERROR before using
 396  * any of the information in this block group.
 397  */
 398 void btrfs_wait_block_group_cache_progress(struct btrfs_block_group *cache,
 399                                            u64 num_bytes)
 400 {
 401         struct btrfs_caching_control *caching_ctl;
 402
 403         caching_ctl = btrfs_get_caching_control(cache);
 404         if (!caching_ctl)
 405                 return;
 406
 407         wait_event(caching_ctl->wait, btrfs_block_group_done(cache) ||
 408                    (cache->free_space_ctl->free_space >= num_bytes));
 409
 410         btrfs_put_caching_control(caching_ctl);
 411 }
 412
 413 int btrfs_wait_block_group_cache_done(struct btrfs_block_group *cache)
 414 {
 415         struct btrfs_caching_control *caching_ctl;
 416         int ret = 0;
 417
 418         caching_ctl = btrfs_get_caching_control(cache);
 419         if (!caching_ctl)
 420                 return (cache->cached == BTRFS_CACHE_ERROR) ? -EIO : 0;
 421
 422         wait_event(caching_ctl->wait, btrfs_block_group_done(cache));
 423         if (cache->cached == BTRFS_CACHE_ERROR)
 424                 ret = -EIO;
 425         btrfs_put_caching_control(caching_ctl);
 426         return ret;
 427 }
 428
 429 #ifdef CONFIG_BTRFS_DEBUG
 430 static void fragment_free_space(struct btrfs_block_group *block_group)
 431 {
 432         struct btrfs_fs_info *fs_info = block_group->fs_info;
 433         u64 start = block_group->start;
 434         u64 len = block_group->length;
 435         u64 chunk = block_group->flags & BTRFS_BLOCK_GROUP_METADATA ?
 436                 fs_info->nodesize : fs_info->sectorsize;
 437         u64 step = chunk << 1;
 438
 439         while (len > chunk) {
 440                 btrfs_remove_free_space(block_group, start, chunk);
 441                 start += step;
 442                 if (len < step)
 443                         len = 0;
 444                 else
 445                         len -= step;
 446         }
 447 }
 448 #endif
 449
 450 /*
 451  * This is only called by btrfs_cache_block_group, since we could have freed
 452  * extents we need to check the pinned_extents for any extents that can't be
 453  * used yet since their free space will be released as soon as the transaction
 454  * commits.
 455  */
 456 u64 add_new_free_space(struct btrfs_block_group *block_group, u64 start, u64 end)
 457 {
 458         struct btrfs_fs_info *info = block_group->fs_info;
 459         u64 extent_start, extent_end, size, total_added = 0;
 460         int ret;
 461
 462         while (start < end) {
 463                 ret = find_first_extent_bit(&info->excluded_extents, start,
 464                                             &extent_start, &extent_end,
 465                                             EXTENT_DIRTY | EXTENT_UPTODATE,
 466                                             NULL);
 467                 if (ret)
 468                         break;
 469
 470                 if (extent_start <= start) {
 471                         start = extent_end + 1;
 472                 } else if (extent_start > start && extent_start < end) {
 473                         size = extent_start - start;
 474                         total_added += size;
 475                         ret = btrfs_add_free_space_async_trimmed(block_group,
 476                                                                  start, size);
 477                         BUG_ON(ret); /* -ENOMEM or logic error */
 478                         start = extent_end + 1;
 479                 } else {
 480                         break;
 481                 }
 482         }
 483
 484         if (start < end) {
 485                 size = end - start;
 486                 total_added += size;
 487                 ret = btrfs_add_free_space_async_trimmed(block_group, start,
 488                                                          size);
 489                 BUG_ON(ret); /* -ENOMEM or logic error */
 490         }
 491
 492         return total_added;
 493 }
 494
 495 static int load_extent_tree_free(struct btrfs_caching_control *caching_ctl)
 496 {
 497         struct btrfs_block_group *block_group = caching_ctl->block_group;
 498         struct btrfs_fs_info *fs_info = block_group->fs_info;
 499         struct btrfs_root *extent_root = fs_info->extent_root;
 500         struct btrfs_path *path;
 501         struct extent_buffer *leaf;
 502         struct btrfs_key key;
 503         u64 total_found = 0;
 504         u64 last = 0;
 505         u32 nritems;
 506         int ret;
 507         bool wakeup = true;
 508
 509         path = btrfs_alloc_path();
 510         if (!path)
 511                 return -ENOMEM;
 512
 513         last = max_t(u64, block_group->start, BTRFS_SUPER_INFO_OFFSET);
 514
 515 #ifdef CONFIG_BTRFS_DEBUG
 516         /*
 517          * If we're fragmenting we don't want to make anybody think we can
 518          * allocate from this block group until we've had a chance to fragment
 519          * the free space.
 520          */
 521         if (btrfs_should_fragment_free_space(block_group))
 522                 wakeup = false;
 523 #endif
 524         /*
 525          * We don't want to deadlock with somebody trying to allocate a new
 526          * extent for the extent root while also trying to search the extent
 527          * root to add free space.  So we skip locking and search the commit
 528          * root, since its read-only
 529          */
 530         path->skip_locking = 1;
 531         path->search_commit_root = 1;
 532         path->reada = READA_FORWARD;
 533
 534         key.objectid = last;
 535         key.offset = 0;
 536         key.type = BTRFS_EXTENT_ITEM_KEY;
 537
 538 next:
 539         ret = btrfs_search_slot(NULL, extent_root, &key, path, 0, 0);
 540         if (ret < 0)
 541                 goto out;
 542
 543         leaf = path->nodes[0];
 544         nritems = btrfs_header_nritems(leaf);
 545
 546         while (1) {
 547                 if (btrfs_fs_closing(fs_info) > 1) {
 548                         last = (u64)-1;
 549                         break;
 550                 }
 551
 552                 if (path->slots[0] < nritems) {
 553                         btrfs_item_key_to_cpu(leaf, &key, path->slots[0]);
 554                 } else {
 555                         ret = btrfs_find_next_key(extent_root, path, &key, 0, 0);
 556                         if (ret)
 557                                 break;
 558
 559                         if (need_resched() ||
 560                             rwsem_is_contended(&fs_info->commit_root_sem)) {
 561                                 if (wakeup)
 562                                         caching_ctl->progress = last;
 563                                 btrfs_release_path(path);
 564                                 up_read(&fs_info->commit_root_sem);
 565                                 mutex_unlock(&caching_ctl->mutex);
 566                                 cond_resched();
 567                                 mutex_lock(&caching_ctl->mutex);
 568                                 down_read(&fs_info->commit_root_sem);
 569                                 goto next;
 570                         }
 571
 572                         ret = btrfs_next_leaf(extent_root, path);
 573                         if (ret < 0)
 574                                 goto out;
 575                         if (ret)
 576                                 break;
 577                         leaf = path->nodes[0];
 578                         nritems = btrfs_header_nritems(leaf);
 579                         continue;
 580                 }
 581
 582                 if (key.objectid < last) {
 583                         key.objectid = last;
 584                         key.offset = 0;
 585                         key.type = BTRFS_EXTENT_ITEM_KEY;
 586
 587                         if (wakeup)
 588                                 caching_ctl->progress = last;
 589                         btrfs_release_path(path);
 590                         goto next;
 591                 }
 592
 593                 if (key.objectid < block_group->start) {
 594                         path->slots[0]++;
 595                         continue;
 596                 }
 597
 598                 if (key.objectid >= block_group->start + block_group->length)
 599                         break;
 600
 601                 if (key.type == BTRFS_EXTENT_ITEM_KEY ||
 602                     key.type == BTRFS_METADATA_ITEM_KEY) {
 603                         total_found += add_new_free_space(block_group, last,
 604                                                           key.objectid);
 605                         if (key.type == BTRFS_METADATA_ITEM_KEY)
 606                                 last = key.objectid +
 607                                         fs_info->nodesize;
 608                         else
 609                                 last = key.objectid + key.offset;
 610
 611                         if (total_found > CACHING_CTL_WAKE_UP) {
 612                                 total_found = 0;
 613                                 if (wakeup)
 614                                         wake_up(&caching_ctl->wait);
 615                         }
 616                 }
 617                 path->slots[0]++;
 618         }
 619         ret = 0;
 620
 621         total_found += add_new_free_space(block_group, last,
 622                                 block_group->start + block_group->length);
 623         caching_ctl->progress = (u64)-1;
 624
 625 out:
 626         btrfs_free_path(path);
 627         return ret;
 628 }
 629
 630 static noinline void caching_thread(struct btrfs_work *work)
 631 {
 632         struct btrfs_block_group *block_group;
 633         struct btrfs_fs_info *fs_info;
 634         struct btrfs_caching_control *caching_ctl;
 635         int ret;
 636
 637         caching_ctl = container_of(work, struct btrfs_caching_control, work);
 638         block_group = caching_ctl->block_group;
 639         fs_info = block_group->fs_info;
 640
 641         mutex_lock(&caching_ctl->mutex);
 642         down_read(&fs_info->commit_root_sem);
 643
 644         if (btrfs_fs_compat_ro(fs_info, FREE_SPACE_TREE))
 645                 ret = load_free_space_tree(caching_ctl);
 646         else
 647                 ret = load_extent_tree_free(caching_ctl);
 648
 649         spin_lock(&block_group->lock);
 650         block_group->caching_ctl = NULL;
 651         block_group->cached = ret ? BTRFS_CACHE_ERROR : BTRFS_CACHE_FINISHED;
 652         spin_unlock(&block_group->lock);
 653
 654 #ifdef CONFIG_BTRFS_DEBUG
 655         if (btrfs_should_fragment_free_space(block_group)) {
 656                 u64 bytes_used;
 657
 658                 spin_lock(&block_group->space_info->lock);
 659                 spin_lock(&block_group->lock);
 660                 bytes_used = block_group->length - block_group->used;
 661                 block_group->space_info->bytes_used += bytes_used >> 1;
 662                 spin_unlock(&block_group->lock);
 663                 spin_unlock(&block_group->space_info->lock);
 664                 fragment_free_space(block_group);
 665         }
 666 #endif
 667
 668         caching_ctl->progress = (u64)-1;
 669
 670         up_read(&fs_info->commit_root_sem);
 671         btrfs_free_excluded_extents(block_group);
 672         mutex_unlock(&caching_ctl->mutex);
 673
 674         wake_up(&caching_ctl->wait);
 675
 676         btrfs_put_caching_control(caching_ctl);
 677         btrfs_put_block_group(block_group);
 678 }
 679
 680 int btrfs_cache_block_group(struct btrfs_block_group *cache, int load_cache_only)
 681 {
 682         DEFINE_WAIT(wait);
 683         struct btrfs_fs_info *fs_info = cache->fs_info;
 684         struct btrfs_caching_control *caching_ctl;
 685         int ret = 0;
 686
 687         caching_ctl = kzalloc(sizeof(*caching_ctl), GFP_NOFS);
 688         if (!caching_ctl)
 689                 return -ENOMEM;
 690
 691         INIT_LIST_HEAD(&caching_ctl->list);
 692         mutex_init(&caching_ctl->mutex);
 693         init_waitqueue_head(&caching_ctl->wait);
 694         caching_ctl->block_group = cache;
 695         caching_ctl->progress = cache->start;
 696         refcount_set(&caching_ctl->count, 1);
 697         btrfs_init_work(&caching_ctl->work, caching_thread, NULL, NULL);
 698
 699         spin_lock(&cache->lock);
 700         /*
 701          * This should be a rare occasion, but this could happen I think in the
 702          * case where one thread starts to load the space cache info, and then
 703          * some other thread starts a transaction commit which tries to do an
 704          * allocation while the other thread is still loading the space cache
 705          * info.  The previous loop should have kept us from choosing this block
 706          * group, but if we've moved to the state where we will wait on caching
 707          * block groups we need to first check if we're doing a fast load here,
 708          * so we can wait for it to finish, otherwise we could end up allocating
 709          * from a block group who's cache gets evicted for one reason or
 710          * another.
 711          */
 712         while (cache->cached == BTRFS_CACHE_FAST) {
 713                 struct btrfs_caching_control *ctl;
 714
 715                 ctl = cache->caching_ctl;
 716                 refcount_inc(&ctl->count);
 717                 prepare_to_wait(&ctl->wait, &wait, TASK_UNINTERRUPTIBLE);
 718                 spin_unlock(&cache->lock);
 719
 720                 schedule();
 721
 722                 finish_wait(&ctl->wait, &wait);
 723                 btrfs_put_caching_control(ctl);
 724                 spin_lock(&cache->lock);
 725         }
 726
 727         if (cache->cached != BTRFS_CACHE_NO) {
 728                 spin_unlock(&cache->lock);
 729                 kfree(caching_ctl);
 730                 return 0;
 731         }
 732         WARN_ON(cache->caching_ctl);
 733         cache->caching_ctl = caching_ctl;
 734         cache->cached = BTRFS_CACHE_FAST;
 735         spin_unlock(&cache->lock);
 736
 737         if (btrfs_test_opt(fs_info, SPACE_CACHE)) {
 738                 mutex_lock(&caching_ctl->mutex);
 739                 ret = load_free_space_cache(cache);
 740
 741                 spin_lock(&cache->lock);
 742                 if (ret == 1) {
 743                         cache->caching_ctl = NULL;
 744                         cache->cached = BTRFS_CACHE_FINISHED;
 745                         cache->last_byte_to_unpin = (u64)-1;
 746                         caching_ctl->progress = (u64)-1;
 747                 } else {
 748                         if (load_cache_only) {
 749                                 cache->caching_ctl = NULL;
 750                                 cache->cached = BTRFS_CACHE_NO;
 751                         } else {
 752                                 cache->cached = BTRFS_CACHE_STARTED;
 753                                 cache->has_caching_ctl = 1;
 754                         }
 755                 }
 756                 spin_unlock(&cache->lock);
 757 #ifdef CONFIG_BTRFS_DEBUG
 758                 if (ret == 1 &&
 759                     btrfs_should_fragment_free_space(cache)) {
 760                         u64 bytes_used;
 761
 762                         spin_lock(&cache->space_info->lock);
 763                         spin_lock(&cache->lock);
 764                         bytes_used = cache->length - cache->used;
 765                         cache->space_info->bytes_used += bytes_used >> 1;
 766                         spin_unlock(&cache->lock);
 767                         spin_unlock(&cache->space_info->lock);
 768                         fragment_free_space(cache);
 769                 }
 770 #endif
 771                 mutex_unlock(&caching_ctl->mutex);
 772
 773                 wake_up(&caching_ctl->wait);
 774                 if (ret == 1) {
 775                         btrfs_put_caching_control(caching_ctl);
 776                         btrfs_free_excluded_extents(cache);
 777                         return 0;
 778                 }
 779         } else {
 780                 /*
 781                  * We're either using the free space tree or no caching at all.
 782                  * Set cached to the appropriate value and wakeup any waiters.
 783                  */
 784                 spin_lock(&cache->lock);
 785                 if (load_cache_only) {
 786                         cache->caching_ctl = NULL;
 787                         cache->cached = BTRFS_CACHE_NO;
 788                 } else {
 789                         cache->cached = BTRFS_CACHE_STARTED;
 790                         cache->has_caching_ctl = 1;
 791                 }
 792                 spin_unlock(&cache->lock);
 793                 wake_up(&caching_ctl->wait);
 794         }
 795
 796         if (load_cache_only) {
 797                 btrfs_put_caching_control(caching_ctl);
 798                 return 0;
 799         }
 800
 801         down_write(&fs_info->commit_root_sem);
 802         refcount_inc(&caching_ctl->count);
 803         list_add_tail(&caching_ctl->list, &fs_info->caching_block_groups);
 804         up_write(&fs_info->commit_root_sem);
 805
 806         btrfs_get_block_group(cache);
 807
 808         btrfs_queue_work(fs_info->caching_workers, &caching_ctl->work);
 809
 810         return ret;
 811 }
 812
 813 static void clear_avail_alloc_bits(struct btrfs_fs_info *fs_info, u64 flags)
 814 {
 815         u64 extra_flags = chunk_to_extended(flags) &
 816                                 BTRFS_EXTENDED_PROFILE_MASK;
 817
 818         write_seqlock(&fs_info->profiles_lock);
 819         if (flags & BTRFS_BLOCK_GROUP_DATA)
 820                 fs_info->avail_data_alloc_bits &= ~extra_flags;
 821         if (flags & BTRFS_BLOCK_GROUP_METADATA)
 822                 fs_info->avail_metadata_alloc_bits &= ~extra_flags;
 823         if (flags & BTRFS_BLOCK_GROUP_SYSTEM)
 824                 fs_info->avail_system_alloc_bits &= ~extra_flags;
 825         write_sequnlock(&fs_info->profiles_lock);
 826 }
 827
 828 /*
 829  * Clear incompat bits for the following feature(s):
 830  *
 831  * - RAID56 - in case there's neither RAID5 nor RAID6 profile block group
 832  *            in the whole filesystem
 833  *
 834  * - RAID1C34 - same as above for RAID1C3 and RAID1C4 block groups
 835  */
 836 static void clear_incompat_bg_bits(struct btrfs_fs_info *fs_info, u64 flags)
 837 {
 838         bool found_raid56 = false;
 839         bool found_raid1c34 = false;
 840
 841         if ((flags & BTRFS_BLOCK_GROUP_RAID56_MASK) ||
 842             (flags & BTRFS_BLOCK_GROUP_RAID1C3) ||
 843             (flags & BTRFS_BLOCK_GROUP_RAID1C4)) {
 844                 struct list_head *head = &fs_info->space_info;
 845                 struct btrfs_space_info *sinfo;
 846
 847                 list_for_each_entry_rcu(sinfo, head, list) {
 848                         down_read(&sinfo->groups_sem);
 849                         if (!list_empty(&sinfo->block_groups[BTRFS_RAID_RAID5]))
 850                                 found_raid56 = true;
 851                         if (!list_empty(&sinfo->block_groups[BTRFS_RAID_RAID6]))
 852                                 found_raid56 = true;
 853                         if (!list_empty(&sinfo->block_groups[BTRFS_RAID_RAID1C3]))
 854                                 found_raid1c34 = true;
 855                         if (!list_empty(&sinfo->block_groups[BTRFS_RAID_RAID1C4]))
 856                                 found_raid1c34 = true;
 857                         up_read(&sinfo->groups_sem);
 858                 }
 859                 if (!found_raid56)
 860                         btrfs_clear_fs_incompat(fs_info, RAID56);
 861                 if (!found_raid1c34)
 862                         btrfs_clear_fs_incompat(fs_info, RAID1C34);
 863         }
 864 }
 865
 866 int btrfs_remove_block_group(struct btrfs_trans_handle *trans,
 867                              u64 group_start, struct extent_map *em)
 868 {
 869         struct btrfs_fs_info *fs_info = trans->fs_info;
 870         struct btrfs_root *root = fs_info->extent_root;
 871         struct btrfs_path *path;
 872         struct btrfs_block_group *block_group;
 873         struct btrfs_free_cluster *cluster;
 874         struct btrfs_root *tree_root = fs_info->tree_root;
 875         struct btrfs_key key;
 876         struct inode *inode;
 877         struct kobject *kobj = NULL;
 878         int ret;
 879         int index;
 880         int factor;
 881         struct btrfs_caching_control *caching_ctl = NULL;
 882         bool remove_em;
 883         bool remove_rsv = false;
 884
 885         block_group = btrfs_lookup_block_group(fs_info, group_start);
 886         BUG_ON(!block_group);
 887         BUG_ON(!block_group->ro);
 888
 889         trace_btrfs_remove_block_group(block_group);
 890         /*
 891          * Free the reserved super bytes from this block group before
 892          * remove it.
 893          */
 894         btrfs_free_excluded_extents(block_group);
 895         btrfs_free_ref_tree_range(fs_info, block_group->start,
 896                                   block_group->length);
 897
 898         index = btrfs_bg_flags_to_raid_index(block_group->flags);
 899         factor = btrfs_bg_type_to_factor(block_group->flags);
 900
 901         /* make sure this block group isn't part of an allocation cluster */
 902         cluster = &fs_info->data_alloc_cluster;
 903         spin_lock(&cluster->refill_lock);
 904         btrfs_return_cluster_to_free_space(block_group, cluster);
 905         spin_unlock(&cluster->refill_lock);
 906
 907         /*
 908          * make sure this block group isn't part of a metadata
 909          * allocation cluster
 910          */
 911         cluster = &fs_info->meta_alloc_cluster;
 912         spin_lock(&cluster->refill_lock);
 913         btrfs_return_cluster_to_free_space(block_group, cluster);
 914         spin_unlock(&cluster->refill_lock);
 915
 916         path = btrfs_alloc_path();
 917         if (!path) {
 918                 ret = -ENOMEM;
 919                 goto out_put_group;
 920         }
 921
 922         /*
 923          * get the inode first so any iput calls done for the io_list
 924          * aren't the final iput (no unlinks allowed now)
 925          */
 926         inode = lookup_free_space_inode(block_group, path);
 927
 928         mutex_lock(&trans->transaction->cache_write_mutex);
 929         /*
 930          * Make sure our free space cache IO is done before removing the
 931          * free space inode
 932          */
 933         spin_lock(&trans->transaction->dirty_bgs_lock);
 934         if (!list_empty(&block_group->io_list)) {
 935                 list_del_init(&block_group->io_list);
 936
 937                 WARN_ON(!IS_ERR(inode) && inode != block_group->io_ctl.inode);
 938
 939                 spin_unlock(&trans->transaction->dirty_bgs_lock);
 940                 btrfs_wait_cache_io(trans, block_group, path);
 941                 btrfs_put_block_group(block_group);
 942                 spin_lock(&trans->transaction->dirty_bgs_lock);
 943         }
 944
 945         if (!list_empty(&block_group->dirty_list)) {
 946                 list_del_init(&block_group->dirty_list);
 947                 remove_rsv = true;
 948                 btrfs_put_block_group(block_group);
 949         }
 950         spin_unlock(&trans->transaction->dirty_bgs_lock);
 951         mutex_unlock(&trans->transaction->cache_write_mutex);
 952
 953         if (!IS_ERR(inode)) {
 954                 ret = btrfs_orphan_add(trans, BTRFS_I(inode));
 955                 if (ret) {
 956                         btrfs_add_delayed_iput(inode);
 957                         goto out_put_group;
 958                 }
 959                 clear_nlink(inode);
 960                 /* One for the block groups ref */
 961                 spin_lock(&block_group->lock);
 962                 if (block_group->iref) {
 963                         block_group->iref = 0;
 964                         block_group->inode = NULL;
 965                         spin_unlock(&block_group->lock);
 966                         iput(inode);
 967                 } else {
 968                         spin_unlock(&block_group->lock);
 969                 }
 970                 /* One for our lookup ref */
 971                 btrfs_add_delayed_iput(inode);
 972         }
 973
 974         key.objectid = BTRFS_FREE_SPACE_OBJECTID;
 975         key.type = 0;
 976         key.offset = block_group->start;
 977
 978         ret = btrfs_search_slot(trans, tree_root, &key, path, -1, 1);
 979         if (ret < 0)
 980                 goto out_put_group;
 981         if (ret > 0)
 982                 btrfs_release_path(path);
 983         if (ret == 0) {
 984                 ret = btrfs_del_item(trans, tree_root, path);
 985                 if (ret)
 986                         goto out_put_group;
 987                 btrfs_release_path(path);
 988         }
 989
 990         spin_lock(&fs_info->block_group_cache_lock);
 991         rb_erase(&block_group->cache_node,
 992                  &fs_info->block_group_cache_tree);
 993         RB_CLEAR_NODE(&block_group->cache_node);
 994
 995         if (fs_info->first_logical_byte == block_group->start)
 996                 fs_info->first_logical_byte = (u64)-1;
 997         spin_unlock(&fs_info->block_group_cache_lock);
 998
 999         down_write(&block_group->space_info->groups_sem);
1000         /*
1001          * we must use list_del_init so people can check to see if they
1002          * are still on the list after taking the semaphore
1003          */
1004         list_del_init(&block_group->list);
1005         if (list_empty(&block_group->space_info->block_groups[index])) {
1006                 kobj = block_group->space_info->block_group_kobjs[index];
1007                 block_group->space_info->block_group_kobjs[index] = NULL;
1008                 clear_avail_alloc_bits(fs_info, block_group->flags);
1009         }
1010         up_write(&block_group->space_info->groups_sem);
1011         clear_incompat_bg_bits(fs_info, block_group->flags);
1012         if (kobj) {
1013                 kobject_del(kobj);
1014                 kobject_put(kobj);
1015         }
1016
1017         if (block_group->has_caching_ctl)
1018                 caching_ctl = btrfs_get_caching_control(block_group);
1019         if (block_group->cached == BTRFS_CACHE_STARTED)
1020                 btrfs_wait_block_group_cache_done(block_group);
1021         if (block_group->has_caching_ctl) {
1022                 down_write(&fs_info->commit_root_sem);
1023                 if (!caching_ctl) {
1024                         struct btrfs_caching_control *ctl;
1025
1026                         list_for_each_entry(ctl,
1027                                     &fs_info->caching_block_groups, list)
1028                                 if (ctl->block_group == block_group) {
1029                                         caching_ctl = ctl;
1030                                         refcount_inc(&caching_ctl->count);
1031                                         break;
1032                                 }
1033                 }
1034                 if (caching_ctl)
1035                         list_del_init(&caching_ctl->list);
1036                 up_write(&fs_info->commit_root_sem);
1037                 if (caching_ctl) {
1038                         /* Once for the caching bgs list and once for us. */
1039                         btrfs_put_caching_control(caching_ctl);
1040                         btrfs_put_caching_control(caching_ctl);
1041                 }
1042         }
1043
1044         spin_lock(&trans->transaction->dirty_bgs_lock);
1045         WARN_ON(!list_empty(&block_group->dirty_list));
1046         WARN_ON(!list_empty(&block_group->io_list));
1047         spin_unlock(&trans->transaction->dirty_bgs_lock);
1048
1049         btrfs_remove_free_space_cache(block_group);
1050
1051         spin_lock(&block_group->space_info->lock);
1052         list_del_init(&block_group->ro_list);
1053
1054         if (btrfs_test_opt(fs_info, ENOSPC_DEBUG)) {
1055                 WARN_ON(block_group->space_info->total_bytes
1056                         < block_group->length);
1057                 WARN_ON(block_group->space_info->bytes_readonly
1058                         < block_group->length);
1059                 WARN_ON(block_group->space_info->disk_total
1060                         < block_group->length * factor);
1061         }
1062         block_group->space_info->total_bytes -= block_group->length;
1063         block_group->space_info->bytes_readonly -= block_group->length;
1064         block_group->space_info->disk_total -= block_group->length * factor;
1065
1066         spin_unlock(&block_group->space_info->lock);
1067
1068         key.objectid = block_group->start;
1069         key.type = BTRFS_BLOCK_GROUP_ITEM_KEY;
1070         key.offset = block_group->length;
1071
1072         mutex_lock(&fs_info->chunk_mutex);
1073         spin_lock(&block_group->lock);
1074         block_group->removed = 1;
1075         /*
1076          * At this point trimming can't start on this block group, because we
1077          * removed the block group from the tree fs_info->block_group_cache_tree
1078          * so no one can't find it anymore and even if someone already got this
1079          * block group before we removed it from the rbtree, they have already
1080          * incremented block_group->trimming - if they didn't, they won't find
1081          * any free space entries because we already removed them all when we
1082          * called btrfs_remove_free_space_cache().
1083          *
1084          * And we must not remove the extent map from the fs_info->mapping_tree
1085          * to prevent the same logical address range and physical device space
1086          * ranges from being reused for a new block group. This is because our
1087          * fs trim operation (btrfs_trim_fs() / btrfs_ioctl_fitrim()) is
1088          * completely transactionless, so while it is trimming a range the
1089          * currently running transaction might finish and a new one start,
1090          * allowing for new block groups to be created that can reuse the same
1091          * physical device locations unless we take this special care.
1092          *
1093          * There may also be an implicit trim operation if the file system
1094          * is mounted with -odiscard. The same protections must remain
1095          * in place until the extents have been discarded completely when
1096          * the transaction commit has completed.
1097          */
1098         remove_em = (atomic_read(&block_group->trimming) == 0);
1099         spin_unlock(&block_group->lock);
1100
1101         mutex_unlock(&fs_info->chunk_mutex);
1102
1103         ret = remove_block_group_free_space(trans, block_group);
1104         if (ret)
1105                 goto out_put_group;
1106
1107         /* Once for the block groups rbtree */
1108         btrfs_put_block_group(block_group);
1109
1110         ret = btrfs_search_slot(trans, root, &key, path, -1, 1);
1111         if (ret > 0)
1112                 ret = -EIO;
1113         if (ret < 0)
1114                 goto out;
1115
1116         ret = btrfs_del_item(trans, root, path);
1117         if (ret)
1118                 goto out;
1119
1120         if (remove_em) {
1121                 struct extent_map_tree *em_tree;
1122
1123                 em_tree = &fs_info->mapping_tree;
1124                 write_lock(&em_tree->lock);
1125                 remove_extent_mapping(em_tree, em);
1126                 write_unlock(&em_tree->lock);
1127                 /* once for the tree */
1128                 free_extent_map(em);
1129         }
1130
1131 out_put_group:
1132         /* Once for the lookup reference */
1133         btrfs_put_block_group(block_group);
1134 out:
1135         if (remove_rsv)
1136                 btrfs_delayed_refs_rsv_release(fs_info, 1);
1137         btrfs_free_path(path);
1138         return ret;
1139 }
1140
1141 struct btrfs_trans_handle *btrfs_start_trans_remove_block_group(
1142                 struct btrfs_fs_info *fs_info, const u64 chunk_offset)
1143 {
1144         struct extent_map_tree *em_tree = &fs_info->mapping_tree;
1145         struct extent_map *em;
1146         struct map_lookup *map;
1147         unsigned int num_items;
1148
1149         read_lock(&em_tree->lock);
1150         em = lookup_extent_mapping(em_tree, chunk_offset, 1);
1151         read_unlock(&em_tree->lock);
1152         ASSERT(em && em->start == chunk_offset);
1153
1154         /*
1155          * We need to reserve 3 + N units from the metadata space info in order
1156          * to remove a block group (done at btrfs_remove_chunk() and at
1157          * btrfs_remove_block_group()), which are used for:
1158          *
1159          * 1 unit for adding the free space inode's orphan (located in the tree
1160          * of tree roots).
1161          * 1 unit for deleting the block group item (located in the extent
1162          * tree).
1163          * 1 unit for deleting the free space item (located in tree of tree
1164          * roots).
1165          * N units for deleting N device extent items corresponding to each
1166          * stripe (located in the device tree).
1167          *
1168          * In order to remove a block group we also need to reserve units in the
1169          * system space info in order to update the chunk tree (update one or
1170          * more device items and remove one chunk item), but this is done at
1171          * btrfs_remove_chunk() through a call to check_system_chunk().
1172          */
1173         map = em->map_lookup;
1174         num_items = 3 + map->num_stripes;
1175         free_extent_map(em);
1176
1177         return btrfs_start_transaction_fallback_global_rsv(fs_info->extent_root,
1178                                                            num_items, 1);
1179 }
1180
1181 /*
1182  * Mark block group @cache read-only, so later write won't happen to block
1183  * group @cache.
1184  *
1185  * If @force is not set, this function will only mark the block group readonly
1186  * if we have enough free space (1M) in other metadata/system block groups.
1187  * If @force is not set, this function will mark the block group readonly
1188  * without checking free space.
1189  *
1190  * NOTE: This function doesn't care if other block groups can contain all the
1191  * data in this block group. That check should be done by relocation routine,
1192  * not this function.
1193  */
1194 static int inc_block_group_ro(struct btrfs_block_group *cache, int force)
1195 {
1196         struct btrfs_space_info *sinfo = cache->space_info;
1197         u64 num_bytes;
1198         int ret = -ENOSPC;
1199
1200         spin_lock(&sinfo->lock);
1201         spin_lock(&cache->lock);
1202
1203         if (cache->ro) {
1204                 cache->ro++;
1205                 ret = 0;
1206                 goto out;
1207         }
1208
1209         num_bytes = cache->length - cache->reserved - cache->pinned -
1210                     cache->bytes_super - cache->used;
1211
1212         /*
1213          * Data never overcommits, even in mixed mode, so do just the straight
1214          * check of left over space in how much we have allocated.
1215          */
1216         if (force) {
1217                 ret = 0;
1218         } else if (sinfo->flags & BTRFS_BLOCK_GROUP_DATA) {
1219                 u64 sinfo_used = btrfs_space_info_used(sinfo, true);
1220
1221                 /*
1222                  * Here we make sure if we mark this bg RO, we still have enough
1223                  * free space as buffer.
1224                  */
1225                 if (sinfo_used + num_bytes <= sinfo->total_bytes)
1226                         ret = 0;
1227         } else {
1228                 /*
1229                  * We overcommit metadata, so we need to do the
1230                  * btrfs_can_overcommit check here, and we need to pass in
1231                  * BTRFS_RESERVE_NO_FLUSH to give ourselves the most amount of
1232                  * leeway to allow us to mark this block group as read only.
1233                  */
1234                 if (btrfs_can_overcommit(cache->fs_info, sinfo, num_bytes,
1235                                          BTRFS_RESERVE_NO_FLUSH))
1236                         ret = 0;
1237         }
1238
1239         if (!ret) {
1240                 sinfo->bytes_readonly += num_bytes;
1241                 cache->ro++;
1242                 list_add_tail(&cache->ro_list, &sinfo->ro_bgs);
1243         }
1244 out:
1245         spin_unlock(&cache->lock);
1246         spin_unlock(&sinfo->lock);
1247         if (ret == -ENOSPC && btrfs_test_opt(cache->fs_info, ENOSPC_DEBUG)) {
1248                 btrfs_info(cache->fs_info,
1249                         "unable to make block group %llu ro", cache->start);
1250                 btrfs_dump_space_info(cache->fs_info, cache->space_info, 0, 0);
1251         }
1252         return ret;
1253 }
1254
1255 static bool clean_pinned_extents(struct btrfs_trans_handle *trans,
1256                                  struct btrfs_block_group *bg)
1257 {
1258         struct btrfs_fs_info *fs_info = bg->fs_info;
1259         struct btrfs_transaction *prev_trans = NULL;
1260         const u64 start = bg->start;
1261         const u64 end = start + bg->length - 1;
1262         int ret;
1263
1264         spin_lock(&fs_info->trans_lock);
1265         if (trans->transaction->list.prev != &fs_info->trans_list) {
1266                 prev_trans = list_last_entry(&trans->transaction->list,
1267                                              struct btrfs_transaction, list);
1268                 refcount_inc(&prev_trans->use_count);
1269         }
1270         spin_unlock(&fs_info->trans_lock);
1271
1272         /*
1273          * Hold the unused_bg_unpin_mutex lock to avoid racing with
1274          * btrfs_finish_extent_commit(). If we are at transaction N, another
1275          * task might be running finish_extent_commit() for the previous
1276          * transaction N - 1, and have seen a range belonging to the block
1277          * group in pinned_extents before we were able to clear the whole block
1278          * group range from pinned_extents. This means that task can lookup for
1279          * the block group after we unpinned it from pinned_extents and removed
1280          * it, leading to a BUG_ON() at unpin_extent_range().
1281          */
1282         mutex_lock(&fs_info->unused_bg_unpin_mutex);
1283         if (prev_trans) {
1284                 ret = clear_extent_bits(&prev_trans->pinned_extents, start, end,
1285                                         EXTENT_DIRTY);
1286                 if (ret)
1287                         goto err;
1288         }
1289
1290         ret = clear_extent_bits(&trans->transaction->pinned_extents, start, end,
1291                                 EXTENT_DIRTY);
1292         if (ret)
1293                 goto err;
1294         mutex_unlock(&fs_info->unused_bg_unpin_mutex);
1295         if (prev_trans)
1296                 btrfs_put_transaction(prev_trans);
1297
1298         return true;
1299
1300 err:
1301         mutex_unlock(&fs_info->unused_bg_unpin_mutex);
1302         if (prev_trans)
1303                 btrfs_put_transaction(prev_trans);
1304         btrfs_dec_block_group_ro(bg);
1305         return false;
1306 }
1307
1308 /*
1309  * Process the unused_bgs list and remove any that don't have any allocated
1310  * space inside of them.
1311  */
1312 void btrfs_delete_unused_bgs(struct btrfs_fs_info *fs_info)
1313 {
1314         struct btrfs_block_group *block_group;
1315         struct btrfs_space_info *space_info;
1316         struct btrfs_trans_handle *trans;
1317         const bool async_trim_enabled = btrfs_test_opt(fs_info, DISCARD_ASYNC);
1318         int ret = 0;
1319
1320         if (!test_bit(BTRFS_FS_OPEN, &fs_info->flags))
1321                 return;
1322
1323         spin_lock(&fs_info->unused_bgs_lock);
1324         while (!list_empty(&fs_info->unused_bgs)) {
1325                 int trimming;
1326
1327                 block_group = list_first_entry(&fs_info->unused_bgs,
1328                                                struct btrfs_block_group,
1329                                                bg_list);
1330                 list_del_init(&block_group->bg_list);
1331
1332                 space_info = block_group->space_info;
1333
1334                 if (ret || btrfs_mixed_space_info(space_info)) {
1335                         btrfs_put_block_group(block_group);
1336                         continue;
1337                 }
1338                 spin_unlock(&fs_info->unused_bgs_lock);
1339
1340                 btrfs_discard_cancel_work(&fs_info->discard_ctl, block_group);
1341
1342                 mutex_lock(&fs_info->delete_unused_bgs_mutex);
1343
1344                 /* Don't want to race with allocators so take the groups_sem */
1345                 down_write(&space_info->groups_sem);
1346
1347                 /*
1348                  * Async discard moves the final block group discard to be prior
1349                  * to the unused_bgs code path.  Therefore, if it's not fully
1350                  * trimmed, punt it back to the async discard lists.
1351                  */
1352                 if (btrfs_test_opt(fs_info, DISCARD_ASYNC) &&
1353                     !btrfs_is_free_space_trimmed(block_group)) {
1354                         trace_btrfs_skip_unused_block_group(block_group);
1355                         up_write(&space_info->groups_sem);
1356                         /* Requeue if we failed because of async discard */
1357                         btrfs_discard_queue_work(&fs_info->discard_ctl,
1358                                                  block_group);
1359                         goto next;
1360                 }
1361
1362                 spin_lock(&block_group->lock);
1363                 if (block_group->reserved || block_group->pinned ||
1364                     block_group->used || block_group->ro ||
1365                     list_is_singular(&block_group->list)) {
1366                         /*
1367                          * We want to bail if we made new allocations or have
1368                          * outstanding allocations in this block group.  We do
1369                          * the ro check in case balance is currently acting on
1370                          * this block group.
1371                          */
1372                         trace_btrfs_skip_unused_block_group(block_group);
1373                         spin_unlock(&block_group->lock);
1374                         up_write(&space_info->groups_sem);
1375                         goto next;
1376                 }
1377                 spin_unlock(&block_group->lock);
1378
1379                 /* We don't want to force the issue, only flip if it's ok. */
1380                 ret = inc_block_group_ro(block_group, 0);
1381                 up_write(&space_info->groups_sem);
1382                 if (ret < 0) {
1383                         ret = 0;
1384                         goto next;
1385                 }
1386
1387                 /*
1388                  * Want to do this before we do anything else so we can recover
1389                  * properly if we fail to join the transaction.
1390                  */
1391                 trans = btrfs_start_trans_remove_block_group(fs_info,
1392                                                      block_group->start);
1393                 if (IS_ERR(trans)) {
1394                         btrfs_dec_block_group_ro(block_group);
1395                         ret = PTR_ERR(trans);
1396                         goto next;
1397                 }
1398
1399                 /*
1400                  * We could have pending pinned extents for this block group,
1401                  * just delete them, we don't care about them anymore.
1402                  */
1403                 if (!clean_pinned_extents(trans, block_group))
1404                         goto end_trans;
1405
1406                 /*
1407                  * At this point, the block_group is read only and should fail
1408                  * new allocations.  However, btrfs_finish_extent_commit() can
1409                  * cause this block_group to be placed back on the discard
1410                  * lists because now the block_group isn't fully discarded.
1411                  * Bail here and try again later after discarding everything.
1412                  */
1413                 spin_lock(&fs_info->discard_ctl.lock);
1414                 if (!list_empty(&block_group->discard_list)) {
1415                         spin_unlock(&fs_info->discard_ctl.lock);
1416                         btrfs_dec_block_group_ro(block_group);
1417                         btrfs_discard_queue_work(&fs_info->discard_ctl,
1418                                                  block_group);
1419                         goto end_trans;
1420                 }
1421                 spin_unlock(&fs_info->discard_ctl.lock);
1422
1423                 /* Reset pinned so btrfs_put_block_group doesn't complain */
1424                 spin_lock(&space_info->lock);
1425                 spin_lock(&block_group->lock);
1426
1427                 btrfs_space_info_update_bytes_pinned(fs_info, space_info,
1428                                                      -block_group->pinned);
1429                 space_info->bytes_readonly += block_group->pinned;
1430                 percpu_counter_add_batch(&space_info->total_bytes_pinned,
1431                                    -block_group->pinned,
1432                                    BTRFS_TOTAL_BYTES_PINNED_BATCH);
1433                 block_group->pinned = 0;
1434
1435                 spin_unlock(&block_group->lock);
1436                 spin_unlock(&space_info->lock);
1437
1438                 /*
1439                  * The normal path here is an unused block group is passed here,
1440                  * then trimming is handled in the transaction commit path.
1441                  * Async discard interposes before this to do the trimming
1442                  * before coming down the unused block group path as trimming
1443                  * will no longer be done later in the transaction commit path.
1444                  */
1445                 if (!async_trim_enabled && btrfs_test_opt(fs_info, DISCARD_ASYNC))
1446                         goto flip_async;
1447
1448                 /* DISCARD can flip during remount */
1449                 trimming = btrfs_test_opt(fs_info, DISCARD_SYNC);
1450
1451                 /* Implicit trim during transaction commit. */
1452                 if (trimming)
1453                         btrfs_get_block_group_trimming(block_group);
1454
1455                 /*
1456                  * Btrfs_remove_chunk will abort the transaction if things go
1457                  * horribly wrong.
1458                  */
1459                 ret = btrfs_remove_chunk(trans, block_group->start);
1460
1461                 if (ret) {
1462                         if (trimming)
1463                                 btrfs_put_block_group_trimming(block_group);
1464                         goto end_trans;
1465                 }
1466
1467                 /*
1468                  * If we're not mounted with -odiscard, we can just forget
1469                  * about this block group. Otherwise we'll need to wait
1470                  * until transaction commit to do the actual discard.
1471                  */
1472                 if (trimming) {
1473                         spin_lock(&fs_info->unused_bgs_lock);
1474                         /*
1475                          * A concurrent scrub might have added us to the list
1476                          * fs_info->unused_bgs, so use a list_move operation
1477                          * to add the block group to the deleted_bgs list.
1478                          */
1479                         list_move(&block_group->bg_list,
1480                                   &trans->transaction->deleted_bgs);
1481                         spin_unlock(&fs_info->unused_bgs_lock);
1482                         btrfs_get_block_group(block_group);
1483                 }
1484 end_trans:
1485                 btrfs_end_transaction(trans);
1486 next:
1487                 mutex_unlock(&fs_info->delete_unused_bgs_mutex);
1488                 btrfs_put_block_group(block_group);
1489                 spin_lock(&fs_info->unused_bgs_lock);
1490         }
1491         spin_unlock(&fs_info->unused_bgs_lock);
1492         return;
1493
1494 flip_async:
1495         btrfs_end_transaction(trans);
1496         mutex_unlock(&fs_info->delete_unused_bgs_mutex);
1497         btrfs_put_block_group(block_group);
1498         btrfs_discard_punt_unused_bgs_list(fs_info);
1499 }
1500
1501 void btrfs_mark_bg_unused(struct btrfs_block_group *bg)
1502 {
1503         struct btrfs_fs_info *fs_info = bg->fs_info;
1504
1505         spin_lock(&fs_info->unused_bgs_lock);
1506         if (list_empty(&bg->bg_list)) {
1507                 btrfs_get_block_group(bg);
1508                 trace_btrfs_add_unused_block_group(bg);
1509                 list_add_tail(&bg->bg_list, &fs_info->unused_bgs);
1510         }
1511         spin_unlock(&fs_info->unused_bgs_lock);
1512 }
1513
1514 static int find_first_block_group(struct btrfs_fs_info *fs_info,
1515                                   struct btrfs_path *path,
1516                                   struct btrfs_key *key)
1517 {
1518         struct btrfs_root *root = fs_info->extent_root;
1519         int ret = 0;
1520         struct btrfs_key found_key;
1521         struct extent_buffer *leaf;
1522         struct btrfs_block_group_item bg;
1523         u64 flags;
1524         int slot;
1525
1526         ret = btrfs_search_slot(NULL, root, key, path, 0, 0);
1527         if (ret < 0)
1528                 goto out;
1529
1530         while (1) {
1531                 slot = path->slots[0];
1532                 leaf = path->nodes[0];
1533                 if (slot >= btrfs_header_nritems(leaf)) {
1534                         ret = btrfs_next_leaf(root, path);
1535                         if (ret == 0)
1536                                 continue;
1537                         if (ret < 0)
1538                                 goto out;
1539                         break;
1540                 }
1541                 btrfs_item_key_to_cpu(leaf, &found_key, slot);
1542
1543                 if (found_key.objectid >= key->objectid &&
1544                     found_key.type == BTRFS_BLOCK_GROUP_ITEM_KEY) {
1545                         struct extent_map_tree *em_tree;
1546                         struct extent_map *em;
1547
1548                         em_tree = &root->fs_info->mapping_tree;
1549                         read_lock(&em_tree->lock);
1550                         em = lookup_extent_mapping(em_tree, found_key.objectid,
1551                                                    found_key.offset);
1552                         read_unlock(&em_tree->lock);
1553                         if (!em) {
1554                                 btrfs_err(fs_info,
1555                         "logical %llu len %llu found bg but no related chunk",
1556                                           found_key.objectid, found_key.offset);
1557                                 ret = -ENOENT;
1558                         } else if (em->start != found_key.objectid ||
1559                                    em->len != found_key.offset) {
1560                                 btrfs_err(fs_info,
1561                 "block group %llu len %llu mismatch with chunk %llu len %llu",
1562                                           found_key.objectid, found_key.offset,
1563                                           em->start, em->len);
1564                                 ret = -EUCLEAN;
1565                         } else {
1566                                 read_extent_buffer(leaf, &bg,
1567                                         btrfs_item_ptr_offset(leaf, slot),
1568                                         sizeof(bg));
1569                                 flags = btrfs_stack_block_group_flags(&bg) &
1570                                         BTRFS_BLOCK_GROUP_TYPE_MASK;
1571
1572                                 if (flags != (em->map_lookup->type &
1573                                               BTRFS_BLOCK_GROUP_TYPE_MASK)) {
1574                                         btrfs_err(fs_info,
1575 "block group %llu len %llu type flags 0x%llx mismatch with chunk type flags 0x%llx",
1576                                                 found_key.objectid,
1577                                                 found_key.offset, flags,
1578                                                 (BTRFS_BLOCK_GROUP_TYPE_MASK &
1579                                                  em->map_lookup->type));
1580                                         ret = -EUCLEAN;
1581                                 } else {
1582                                         ret = 0;
1583                                 }
1584                         }
1585                         free_extent_map(em);
1586                         goto out;
1587                 }
1588                 path->slots[0]++;
1589         }
1590 out:
1591         return ret;
1592 }
1593
1594 static void set_avail_alloc_bits(struct btrfs_fs_info *fs_info, u64 flags)
1595 {
1596         u64 extra_flags = chunk_to_extended(flags) &
1597                                 BTRFS_EXTENDED_PROFILE_MASK;
1598
1599         write_seqlock(&fs_info->profiles_lock);
1600         if (flags & BTRFS_BLOCK_GROUP_DATA)
1601                 fs_info->avail_data_alloc_bits |= extra_flags;
1602         if (flags & BTRFS_BLOCK_GROUP_METADATA)
1603                 fs_info->avail_metadata_alloc_bits |= extra_flags;
1604         if (flags & BTRFS_BLOCK_GROUP_SYSTEM)
1605                 fs_info->avail_system_alloc_bits |= extra_flags;
1606         write_sequnlock(&fs_info->profiles_lock);
1607 }
1608
1609 /**
1610  * btrfs_rmap_block - Map a physical disk address to a list of logical addresses
1611  * @chunk_start:   logical address of block group
1612  * @physical:      physical address to map to logical addresses
1613  * @logical:       return array of logical addresses which map to @physical
1614  * @naddrs:        length of @logical
1615  * @stripe_len:    size of IO stripe for the given block group
1616  *
1617  * Maps a particular @physical disk address to a list of @logical addresses.
1618  * Used primarily to exclude those portions of a block group that contain super
1619  * block copies.
1620  */
1621 EXPORT_FOR_TESTS
1622 int btrfs_rmap_block(struct btrfs_fs_info *fs_info, u64 chunk_start,
1623                      u64 physical, u64 **logical, int *naddrs, int *stripe_len)
1624 {
1625         struct extent_map *em;
1626         struct map_lookup *map;
1627         u64 *buf;
1628         u64 bytenr;
1629         u64 data_stripe_length;
1630         u64 io_stripe_size;
1631         int i, nr = 0;
1632         int ret = 0;
1633
1634         em = btrfs_get_chunk_map(fs_info, chunk_start, 1);
1635         if (IS_ERR(em))
1636                 return -EIO;
1637
1638         map = em->map_lookup;
1639         data_stripe_length = em->len;
1640         io_stripe_size = map->stripe_len;
1641
1642         if (map->type & BTRFS_BLOCK_GROUP_RAID10)
1643                 data_stripe_length = div_u64(data_stripe_length,
1644                                              map->num_stripes / map->sub_stripes);
1645         else if (map->type & BTRFS_BLOCK_GROUP_RAID0)
1646                 data_stripe_length = div_u64(data_stripe_length, map->num_stripes);
1647         else if (map->type & BTRFS_BLOCK_GROUP_RAID56_MASK) {
1648                 data_stripe_length = div_u64(data_stripe_length,
1649                                              nr_data_stripes(map));
1650                 io_stripe_size = map->stripe_len * nr_data_stripes(map);
1651         }
1652
1653         buf = kcalloc(map->num_stripes, sizeof(u64), GFP_NOFS);
1654         if (!buf) {
1655                 ret = -ENOMEM;
1656                 goto out;
1657         }
1658
1659         for (i = 0; i < map->num_stripes; i++) {
1660                 bool already_inserted = false;
1661                 u64 stripe_nr;
1662                 int j;
1663
1664                 if (!in_range(physical, map->stripes[i].physical,
1665                               data_stripe_length))
1666                         continue;
1667
1668                 stripe_nr = physical - map->stripes[i].physical;
1669                 stripe_nr = div64_u64(stripe_nr, map->stripe_len);
1670
1671                 if (map->type & BTRFS_BLOCK_GROUP_RAID10) {
1672                         stripe_nr = stripe_nr * map->num_stripes + i;
1673                         stripe_nr = div_u64(stripe_nr, map->sub_stripes);
1674                 } else if (map->type & BTRFS_BLOCK_GROUP_RAID0) {
1675                         stripe_nr = stripe_nr * map->num_stripes + i;
1676                 }
1677                 /*
1678                  * The remaining case would be for RAID56, multiply by
1679                  * nr_data_stripes().  Alternatively, just use rmap_len below
1680                  * instead of map->stripe_len
1681                  */
1682
1683                 bytenr = chunk_start + stripe_nr * io_stripe_size;
1684
1685                 /* Ensure we don't add duplicate addresses */
1686                 for (j = 0; j < nr; j++) {
1687                         if (buf[j] == bytenr) {
1688                                 already_inserted = true;
1689                                 break;
1690                         }
1691                 }
1692
1693                 if (!already_inserted)
1694                         buf[nr++] = bytenr;
1695         }
1696
1697         *logical = buf;
1698         *naddrs = nr;
1699         *stripe_len = io_stripe_size;
1700 out:
1701         free_extent_map(em);
1702         return ret;
1703 }
1704
1705 static int exclude_super_stripes(struct btrfs_block_group *cache)
1706 {
1707         struct btrfs_fs_info *fs_info = cache->fs_info;
1708         u64 bytenr;
1709         u64 *logical;
1710         int stripe_len;
1711         int i, nr, ret;
1712
1713         if (cache->start < BTRFS_SUPER_INFO_OFFSET) {
1714                 stripe_len = BTRFS_SUPER_INFO_OFFSET - cache->start;
1715                 cache->bytes_super += stripe_len;
1716                 ret = btrfs_add_excluded_extent(fs_info, cache->start,
1717                                                 stripe_len);
1718                 if (ret)
1719                         return ret;
1720         }
1721
1722         for (i = 0; i < BTRFS_SUPER_MIRROR_MAX; i++) {
1723                 bytenr = btrfs_sb_offset(i);
1724                 ret = btrfs_rmap_block(fs_info, cache->start,
1725                                        bytenr, &logical, &nr, &stripe_len);
1726                 if (ret)
1727                         return ret;
1728
1729                 while (nr--) {
1730                         u64 start, len;
1731
1732                         if (logical[nr] > cache->start + cache->length)
1733                                 continue;
1734
1735                         if (logical[nr] + stripe_len <= cache->start)
1736                                 continue;
1737
1738                         start = logical[nr];
1739                         if (start < cache->start) {
1740                                 start = cache->start;
1741                                 len = (logical[nr] + stripe_len) - start;
1742                         } else {
1743                                 len = min_t(u64, stripe_len,
1744                                             cache->start + cache->length - start);
1745                         }
1746
1747                         cache->bytes_super += len;
1748                         ret = btrfs_add_excluded_extent(fs_info, start, len);
1749                         if (ret) {
1750                                 kfree(logical);
1751                                 return ret;
1752                         }
1753                 }
1754
1755                 kfree(logical);
1756         }
1757         return 0;
1758 }
1759
1760 static void link_block_group(struct btrfs_block_group *cache)
1761 {
1762         struct btrfs_space_info *space_info = cache->space_info;
1763         int index = btrfs_bg_flags_to_raid_index(cache->flags);
1764         bool first = false;
1765
1766         down_write(&space_info->groups_sem);
1767         if (list_empty(&space_info->block_groups[index]))
1768                 first = true;
1769         list_add_tail(&cache->list, &space_info->block_groups[index]);
1770         up_write(&space_info->groups_sem);
1771
1772         if (first)
1773                 btrfs_sysfs_add_block_group_type(cache);
1774 }
1775
1776 static struct btrfs_block_group *btrfs_create_block_group_cache(
1777                 struct btrfs_fs_info *fs_info, u64 start, u64 size)
1778 {
1779         struct btrfs_block_group *cache;
1780
1781         cache = kzalloc(sizeof(*cache), GFP_NOFS);
1782         if (!cache)
1783                 return NULL;
1784
1785         cache->free_space_ctl = kzalloc(sizeof(*cache->free_space_ctl),
1786                                         GFP_NOFS);
1787         if (!cache->free_space_ctl) {
1788                 kfree(cache);
1789                 return NULL;
1790         }
1791
1792         cache->start = start;
1793         cache->length = size;
1794
1795         cache->fs_info = fs_info;
1796         cache->full_stripe_len = btrfs_full_stripe_len(fs_info, start);
1797         set_free_space_tree_thresholds(cache);
1798
1799         cache->discard_index = BTRFS_DISCARD_INDEX_UNUSED;
1800
1801         atomic_set(&cache->count, 1);
1802         spin_lock_init(&cache->lock);
1803         init_rwsem(&cache->data_rwsem);
1804         INIT_LIST_HEAD(&cache->list);
1805         INIT_LIST_HEAD(&cache->cluster_list);
1806         INIT_LIST_HEAD(&cache->bg_list);
1807         INIT_LIST_HEAD(&cache->ro_list);
1808         INIT_LIST_HEAD(&cache->discard_list);
1809         INIT_LIST_HEAD(&cache->dirty_list);
1810         INIT_LIST_HEAD(&cache->io_list);
1811         btrfs_init_free_space_ctl(cache);
1812         atomic_set(&cache->trimming, 0);
1813         mutex_init(&cache->free_space_lock);
1814         btrfs_init_full_stripe_locks_tree(&cache->full_stripe_locks_root);
1815
1816         return cache;
1817 }
1818
1819 /*
1820  * Iterate all chunks and verify that each of them has the corresponding block
1821  * group
1822  */
1823 static int check_chunk_block_group_mappings(struct btrfs_fs_info *fs_info)
1824 {
1825         struct extent_map_tree *map_tree = &fs_info->mapping_tree;
1826         struct extent_map *em;
1827         struct btrfs_block_group *bg;
1828         u64 start = 0;
1829         int ret = 0;
1830
1831         while (1) {
1832                 read_lock(&map_tree->lock);
1833                 /*
1834                  * lookup_extent_mapping will return the first extent map
1835                  * intersecting the range, so setting @len to 1 is enough to
1836                  * get the first chunk.
1837                  */
1838                 em = lookup_extent_mapping(map_tree, start, 1);
1839                 read_unlock(&map_tree->lock);
1840                 if (!em)
1841                         break;
1842
1843                 bg = btrfs_lookup_block_group(fs_info, em->start);
1844                 if (!bg) {
1845                         btrfs_err(fs_info,
1846         "chunk start=%llu len=%llu doesn't have corresponding block group",
1847                                      em->start, em->len);
1848                         ret = -EUCLEAN;
1849                         free_extent_map(em);
1850                         break;
1851                 }
1852                 if (bg->start != em->start || bg->length != em->len ||
1853                     (bg->flags & BTRFS_BLOCK_GROUP_TYPE_MASK) !=
1854                     (em->map_lookup->type & BTRFS_BLOCK_GROUP_TYPE_MASK)) {
1855                         btrfs_err(fs_info,
1856 "chunk start=%llu len=%llu flags=0x%llx doesn't match block group start=%llu len=%llu flags=0x%llx",
1857                                 em->start, em->len,
1858                                 em->map_lookup->type & BTRFS_BLOCK_GROUP_TYPE_MASK,
1859                                 bg->start, bg->length,
1860                                 bg->flags & BTRFS_BLOCK_GROUP_TYPE_MASK);
1861                         ret = -EUCLEAN;
1862                         free_extent_map(em);
1863                         btrfs_put_block_group(bg);
1864                         break;
1865                 }
1866                 start = em->start + em->len;
1867                 free_extent_map(em);
1868                 btrfs_put_block_group(bg);
1869         }
1870         return ret;
1871 }
1872
1873 static int read_one_block_group(struct btrfs_fs_info *info,
1874                                 struct btrfs_path *path,
1875                                 const struct btrfs_key *key,
1876                                 int need_clear)
1877 {
1878         struct extent_buffer *leaf = path->nodes[0];
1879         struct btrfs_block_group *cache;
1880         struct btrfs_space_info *space_info;
1881         struct btrfs_block_group_item bgi;
1882         const bool mixed = btrfs_fs_incompat(info, MIXED_GROUPS);
1883         int slot = path->slots[0];
1884         int ret;
1885
1886         ASSERT(key->type == BTRFS_BLOCK_GROUP_ITEM_KEY);
1887
1888         cache = btrfs_create_block_group_cache(info, key->objectid, key->offset);
1889         if (!cache)
1890                 return -ENOMEM;
1891
1892         if (need_clear) {
1893                 /*
1894                  * When we mount with old space cache, we need to
1895                  * set BTRFS_DC_CLEAR and set dirty flag.
1896                  *
1897                  * a) Setting 'BTRFS_DC_CLEAR' makes sure that we
1898                  *    truncate the old free space cache inode and
1899                  *    setup a new one.
1900                  * b) Setting 'dirty flag' makes sure that we flush
1901                  *    the new space cache info onto disk.
1902                  */
1903                 if (btrfs_test_opt(info, SPACE_CACHE))
1904                         cache->disk_cache_state = BTRFS_DC_CLEAR;
1905         }
1906         read_extent_buffer(leaf, &bgi, btrfs_item_ptr_offset(leaf, slot),
1907                            sizeof(bgi));
1908         cache->used = btrfs_stack_block_group_used(&bgi);
1909         cache->flags = btrfs_stack_block_group_flags(&bgi);
1910         if (!mixed && ((cache->flags & BTRFS_BLOCK_GROUP_METADATA) &&
1911             (cache->flags & BTRFS_BLOCK_GROUP_DATA))) {
1912                         btrfs_err(info,
1913 "bg %llu is a mixed block group but filesystem hasn't enabled mixed block groups",
1914                                   cache->start);
1915                         ret = -EINVAL;
1916                         goto error;
1917         }
1918
1919         /*
1920          * We need to exclude the super stripes now so that the space info has
1921          * super bytes accounted for, otherwise we'll think we have more space
1922          * than we actually do.
1923          */
1924         ret = exclude_super_stripes(cache);
1925         if (ret) {
1926                 /* We may have excluded something, so call this just in case. */
1927                 btrfs_free_excluded_extents(cache);
1928                 goto error;
1929         }
1930
1931         /*
1932          * Check for two cases, either we are full, and therefore don't need
1933          * to bother with the caching work since we won't find any space, or we
1934          * are empty, and we can just add all the space in and be done with it.
1935          * This saves us _a_lot_ of time, particularly in the full case.
1936          */
1937         if (key->offset == cache->used) {
1938                 cache->last_byte_to_unpin = (u64)-1;
1939                 cache->cached = BTRFS_CACHE_FINISHED;
1940                 btrfs_free_excluded_extents(cache);
1941         } else if (cache->used == 0) {
1942                 cache->last_byte_to_unpin = (u64)-1;
1943                 cache->cached = BTRFS_CACHE_FINISHED;
1944                 add_new_free_space(cache, key->objectid,
1945                                    key->objectid + key->offset);
1946                 btrfs_free_excluded_extents(cache);
1947         }
1948
1949         ret = btrfs_add_block_group_cache(info, cache);
1950         if (ret) {
1951                 btrfs_remove_free_space_cache(cache);
1952                 goto error;
1953         }
1954         trace_btrfs_add_block_group(info, cache, 0);
1955         btrfs_update_space_info(info, cache->flags, key->offset,
1956                                 cache->used, cache->bytes_super, &space_info);
1957
1958         cache->space_info = space_info;
1959
1960         link_block_group(cache);
1961
1962         set_avail_alloc_bits(info, cache->flags);
1963         if (btrfs_chunk_readonly(info, cache->start)) {
1964                 inc_block_group_ro(cache, 1);
1965         } else if (cache->used == 0) {
1966                 ASSERT(list_empty(&cache->bg_list));
1967                 if (btrfs_test_opt(info, DISCARD_ASYNC))
1968                         btrfs_discard_queue_work(&info->discard_ctl, cache);
1969                 else
1970                         btrfs_mark_bg_unused(cache);
1971         }
1972         return 0;
1973 error:
1974         btrfs_put_block_group(cache);
1975         return ret;
1976 }
1977
1978 int btrfs_read_block_groups(struct btrfs_fs_info *info)
1979 {
1980         struct btrfs_path *path;
1981         int ret;
1982         struct btrfs_block_group *cache;
1983         struct btrfs_space_info *space_info;
1984         struct btrfs_key key;
1985         int need_clear = 0;
1986         u64 cache_gen;
1987
1988         key.objectid = 0;
1989         key.offset = 0;
1990         key.type = BTRFS_BLOCK_GROUP_ITEM_KEY;
1991         path = btrfs_alloc_path();
1992         if (!path)
1993                 return -ENOMEM;
1994         path->reada = READA_FORWARD;
1995
1996         cache_gen = btrfs_super_cache_generation(info->super_copy);
1997         if (btrfs_test_opt(info, SPACE_CACHE) &&
1998             btrfs_super_generation(info->super_copy) != cache_gen)
1999                 need_clear = 1;
2000         if (btrfs_test_opt(info, CLEAR_CACHE))
2001                 need_clear = 1;
2002
2003         while (1) {
2004                 ret = find_first_block_group(info, path, &key);
2005                 if (ret > 0)
2006                         break;
2007                 if (ret != 0)
2008                         goto error;
2009
2010                 btrfs_item_key_to_cpu(path->nodes[0], &key, path->slots[0]);
2011                 ret = read_one_block_group(info, path, &key, need_clear);
2012                 if (ret < 0)
2013                         goto error;
2014                 key.objectid += key.offset;
2015                 key.offset = 0;
2016                 btrfs_release_path(path);
2017         }
2018
2019         rcu_read_lock();
2020         list_for_each_entry_rcu(space_info, &info->space_info, list) {
2021                 if (!(btrfs_get_alloc_profile(info, space_info->flags) &
2022                       (BTRFS_BLOCK_GROUP_RAID10 |
2023                        BTRFS_BLOCK_GROUP_RAID1_MASK |
2024                        BTRFS_BLOCK_GROUP_RAID56_MASK |
2025                        BTRFS_BLOCK_GROUP_DUP)))
2026                         continue;
2027                 /*
2028                  * Avoid allocating from un-mirrored block group if there are
2029                  * mirrored block groups.
2030                  */
2031                 list_for_each_entry(cache,
2032                                 &space_info->block_groups[BTRFS_RAID_RAID0],
2033                                 list)
2034                         inc_block_group_ro(cache, 1);
2035                 list_for_each_entry(cache,
2036                                 &space_info->block_groups[BTRFS_RAID_SINGLE],
2037                                 list)
2038                         inc_block_group_ro(cache, 1);
2039         }
2040         rcu_read_unlock();
2041
2042         btrfs_init_global_block_rsv(info);
2043         ret = check_chunk_block_group_mappings(info);
2044 error:
2045         btrfs_free_path(path);
2046         return ret;
2047 }
2048
2049 void btrfs_create_pending_block_groups(struct btrfs_trans_handle *trans)
2050 {
2051         struct btrfs_fs_info *fs_info = trans->fs_info;
2052         struct btrfs_block_group *block_group;
2053         struct btrfs_root *extent_root = fs_info->extent_root;
2054         struct btrfs_block_group_item item;
2055         struct btrfs_key key;
2056         int ret = 0;
2057
2058         if (!trans->can_flush_pending_bgs)
2059                 return;
2060
2061         while (!list_empty(&trans->new_bgs)) {
2062                 block_group = list_first_entry(&trans->new_bgs,
2063                                                struct btrfs_block_group,
2064                                                bg_list);
2065                 if (ret)
2066                         goto next;
2067
2068                 spin_lock(&block_group->lock);
2069                 btrfs_set_stack_block_group_used(&item, block_group->used);
2070                 btrfs_set_stack_block_group_chunk_objectid(&item,
2071                                 BTRFS_FIRST_CHUNK_TREE_OBJECTID);
2072                 btrfs_set_stack_block_group_flags(&item, block_group->flags);
2073                 key.objectid = block_group->start;
2074                 key.type = BTRFS_BLOCK_GROUP_ITEM_KEY;
2075                 key.offset = block_group->length;
2076                 spin_unlock(&block_group->lock);
2077
2078                 ret = btrfs_insert_item(trans, extent_root, &key, &item,
2079                                         sizeof(item));
2080                 if (ret)
2081                         btrfs_abort_transaction(trans, ret);
2082                 ret = btrfs_finish_chunk_alloc(trans, key.objectid, key.offset);
2083                 if (ret)
2084                         btrfs_abort_transaction(trans, ret);
2085                 add_block_group_free_space(trans, block_group);
2086                 /* Already aborted the transaction if it failed. */
2087 next:
2088                 btrfs_delayed_refs_rsv_release(fs_info, 1);
2089                 list_del_init(&block_group->bg_list);
2090         }
2091         btrfs_trans_release_chunk_metadata(trans);
2092 }
2093
2094 int btrfs_make_block_group(struct btrfs_trans_handle *trans, u64 bytes_used,
2095                            u64 type, u64 chunk_offset, u64 size)
2096 {
2097         struct btrfs_fs_info *fs_info = trans->fs_info;
2098         struct btrfs_block_group *cache;
2099         int ret;
2100
2101         btrfs_set_log_full_commit(trans);
2102
2103         cache = btrfs_create_block_group_cache(fs_info, chunk_offset, size);
2104         if (!cache)
2105                 return -ENOMEM;
2106
2107         cache->used = bytes_used;
2108         cache->flags = type;
2109         cache->last_byte_to_unpin = (u64)-1;
2110         cache->cached = BTRFS_CACHE_FINISHED;
2111         cache->needs_free_space = 1;
2112         ret = exclude_super_stripes(cache);
2113         if (ret) {
2114                 /* We may have excluded something, so call this just in case */
2115                 btrfs_free_excluded_extents(cache);
2116                 btrfs_put_block_group(cache);
2117                 return ret;
2118         }
2119
2120         add_new_free_space(cache, chunk_offset, chunk_offset + size);
2121
2122         btrfs_free_excluded_extents(cache);
2123
2124 #ifdef CONFIG_BTRFS_DEBUG
2125         if (btrfs_should_fragment_free_space(cache)) {
2126                 u64 new_bytes_used = size - bytes_used;
2127
2128                 bytes_used += new_bytes_used >> 1;
2129                 fragment_free_space(cache);
2130         }
2131 #endif
2132         /*
2133          * Ensure the corresponding space_info object is created and
2134          * assigned to our block group. We want our bg to be added to the rbtree
2135          * with its ->space_info set.
2136          */
2137         cache->space_info = btrfs_find_space_info(fs_info, cache->flags);
2138         ASSERT(cache->space_info);
2139
2140         ret = btrfs_add_block_group_cache(fs_info, cache);
2141         if (ret) {
2142                 btrfs_remove_free_space_cache(cache);
2143                 btrfs_put_block_group(cache);
2144                 return ret;
2145         }
2146
2147         /*
2148          * Now that our block group has its ->space_info set and is inserted in
2149          * the rbtree, update the space info's counters.
2150          */
2151         trace_btrfs_add_block_group(fs_info, cache, 1);
2152         btrfs_update_space_info(fs_info, cache->flags, size, bytes_used,
2153                                 cache->bytes_super, &cache->space_info);
2154         btrfs_update_global_block_rsv(fs_info);
2155
2156         link_block_group(cache);
2157
2158         list_add_tail(&cache->bg_list, &trans->new_bgs);
2159         trans->delayed_ref_updates++;
2160         btrfs_update_delayed_refs_rsv(trans);
2161
2162         set_avail_alloc_bits(fs_info, type);
2163         return 0;
2164 }
2165
2166 static u64 update_block_group_flags(struct btrfs_fs_info *fs_info, u64 flags)
2167 {
2168         u64 num_devices;
2169         u64 stripped;
2170
2171         /*
2172          * if restripe for this chunk_type is on pick target profile and
2173          * return, otherwise do the usual balance
2174          */
2175         stripped = get_restripe_target(fs_info, flags);
2176         if (stripped)
2177                 return extended_to_chunk(stripped);
2178
2179         num_devices = fs_info->fs_devices->rw_devices;
2180
2181         stripped = BTRFS_BLOCK_GROUP_RAID0 | BTRFS_BLOCK_GROUP_RAID56_MASK |
2182                 BTRFS_BLOCK_GROUP_RAID1_MASK | BTRFS_BLOCK_GROUP_RAID10;
2183
2184         if (num_devices == 1) {
2185                 stripped |= BTRFS_BLOCK_GROUP_DUP;
2186                 stripped = flags & ~stripped;
2187
2188                 /* turn raid0 into single device chunks */
2189                 if (flags & BTRFS_BLOCK_GROUP_RAID0)
2190                         return stripped;
2191
2192                 /* turn mirroring into duplication */
2193                 if (flags & (BTRFS_BLOCK_GROUP_RAID1_MASK |
2194                              BTRFS_BLOCK_GROUP_RAID10))
2195                         return stripped | BTRFS_BLOCK_GROUP_DUP;
2196         } else {
2197                 /* they already had raid on here, just return */
2198                 if (flags & stripped)
2199                         return flags;
2200
2201                 stripped |= BTRFS_BLOCK_GROUP_DUP;
2202                 stripped = flags & ~stripped;
2203
2204                 /* switch duplicated blocks with raid1 */
2205                 if (flags & BTRFS_BLOCK_GROUP_DUP)
2206                         return stripped | BTRFS_BLOCK_GROUP_RAID1;
2207
2208                 /* this is drive concat, leave it alone */
2209         }
2210
2211         return flags;
2212 }
2213
2214 /*
2215  * Mark one block group RO, can be called several times for the same block
2216  * group.
2217  *
2218  * @cache:              the destination block group
2219  * @do_chunk_alloc:     whether need to do chunk pre-allocation, this is to
2220  *                      ensure we still have some free space after marking this
2221  *                      block group RO.
2222  */
2223 int btrfs_inc_block_group_ro(struct btrfs_block_group *cache,
2224                              bool do_chunk_alloc)
2225 {
2226         struct btrfs_fs_info *fs_info = cache->fs_info;
2227         struct btrfs_trans_handle *trans;
2228         u64 alloc_flags;
2229         int ret;
2230
2231 again:
2232         trans = btrfs_join_transaction(fs_info->extent_root);
2233         if (IS_ERR(trans))
2234                 return PTR_ERR(trans);
2235
2236         /*
2237          * we're not allowed to set block groups readonly after the dirty
2238          * block groups cache has started writing.  If it already started,
2239          * back off and let this transaction commit
2240          */
2241         mutex_lock(&fs_info->ro_block_group_mutex);
2242         if (test_bit(BTRFS_TRANS_DIRTY_BG_RUN, &trans->transaction->flags)) {
2243                 u64 transid = trans->transid;
2244
2245                 mutex_unlock(&fs_info->ro_block_group_mutex);
2246                 btrfs_end_transaction(trans);
2247
2248                 ret = btrfs_wait_for_commit(fs_info, transid);
2249                 if (ret)
2250                         return ret;
2251                 goto again;
2252         }
2253
2254         if (do_chunk_alloc) {
2255                 /*
2256                  * If we are changing raid levels, try to allocate a
2257                  * corresponding block group with the new raid level.
2258                  */
2259                 alloc_flags = update_block_group_flags(fs_info, cache->flags);
2260                 if (alloc_flags != cache->flags) {
2261                         ret = btrfs_chunk_alloc(trans, alloc_flags,
2262                                                 CHUNK_ALLOC_FORCE);
2263                         /*
2264                          * ENOSPC is allowed here, we may have enough space
2265                          * already allocated at the new raid level to carry on
2266                          */
2267                         if (ret == -ENOSPC)
2268                                 ret = 0;
2269                         if (ret < 0)
2270                                 goto out;
2271                 }
2272         }
2273
2274         ret = inc_block_group_ro(cache, 0);
2275         if (!do_chunk_alloc)
2276                 goto unlock_out;
2277         if (!ret)
2278                 goto out;
2279         alloc_flags = btrfs_get_alloc_profile(fs_info, cache->space_info->flags);
2280         ret = btrfs_chunk_alloc(trans, alloc_flags, CHUNK_ALLOC_FORCE);
2281         if (ret < 0)
2282                 goto out;
2283         ret = inc_block_group_ro(cache, 0);
2284 out:
2285         if (cache->flags & BTRFS_BLOCK_GROUP_SYSTEM) {
2286                 alloc_flags = update_block_group_flags(fs_info, cache->flags);
2287                 mutex_lock(&fs_info->chunk_mutex);
2288                 check_system_chunk(trans, alloc_flags);
2289                 mutex_unlock(&fs_info->chunk_mutex);
2290         }
2291 unlock_out:
2292         mutex_unlock(&fs_info->ro_block_group_mutex);
2293
2294         btrfs_end_transaction(trans);
2295         return ret;
2296 }
2297
2298 void btrfs_dec_block_group_ro(struct btrfs_block_group *cache)
2299 {
2300         struct btrfs_space_info *sinfo = cache->space_info;
2301         u64 num_bytes;
2302
2303         BUG_ON(!cache->ro);
2304
2305         spin_lock(&sinfo->lock);
2306         spin_lock(&cache->lock);
2307         if (!--cache->ro) {
2308                 num_bytes = cache->length - cache->reserved -
2309                             cache->pinned - cache->bytes_super - cache->used;
2310                 sinfo->bytes_readonly -= num_bytes;
2311                 list_del_init(&cache->ro_list);
2312         }
2313         spin_unlock(&cache->lock);
2314         spin_unlock(&sinfo->lock);
2315 }
2316
2317 static int write_one_cache_group(struct btrfs_trans_handle *trans,
2318                                  struct btrfs_path *path,
2319                                  struct btrfs_block_group *cache)
2320 {
2321         struct btrfs_fs_info *fs_info = trans->fs_info;
2322         int ret;
2323         struct btrfs_root *extent_root = fs_info->extent_root;
2324         unsigned long bi;
2325         struct extent_buffer *leaf;
2326         struct btrfs_block_group_item bgi;
2327         struct btrfs_key key;
2328
2329         key.objectid = cache->start;
2330         key.type = BTRFS_BLOCK_GROUP_ITEM_KEY;
2331         key.offset = cache->length;
2332
2333         ret = btrfs_search_slot(trans, extent_root, &key, path, 0, 1);
2334         if (ret) {
2335                 if (ret > 0)
2336                         ret = -ENOENT;
2337                 goto fail;
2338         }
2339
2340         leaf = path->nodes[0];
2341         bi = btrfs_item_ptr_offset(leaf, path->slots[0]);
2342         btrfs_set_stack_block_group_used(&bgi, cache->used);
2343         btrfs_set_stack_block_group_chunk_objectid(&bgi,
2344                         BTRFS_FIRST_CHUNK_TREE_OBJECTID);
2345         btrfs_set_stack_block_group_flags(&bgi, cache->flags);
2346         write_extent_buffer(leaf, &bgi, bi, sizeof(bgi));
2347         btrfs_mark_buffer_dirty(leaf);
2348 fail:
2349         btrfs_release_path(path);
2350         return ret;
2351
2352 }
2353
2354 static int cache_save_setup(struct btrfs_block_group *block_group,
2355                             struct btrfs_trans_handle *trans,
2356                             struct btrfs_path *path)
2357 {
2358         struct btrfs_fs_info *fs_info = block_group->fs_info;
2359         struct btrfs_root *root = fs_info->tree_root;
2360         struct inode *inode = NULL;
2361         struct extent_changeset *data_reserved = NULL;
2362         u64 alloc_hint = 0;
2363         int dcs = BTRFS_DC_ERROR;
2364         u64 num_pages = 0;
2365         int retries = 0;
2366         int ret = 0;
2367
2368         /*
2369          * If this block group is smaller than 100 megs don't bother caching the
2370          * block group.
2371          */
2372         if (block_group->length < (100 * SZ_1M)) {
2373                 spin_lock(&block_group->lock);
2374                 block_group->disk_cache_state = BTRFS_DC_WRITTEN;
2375                 spin_unlock(&block_group->lock);
2376                 return 0;
2377         }
2378
2379         if (TRANS_ABORTED(trans))
2380                 return 0;
2381 again:
2382         inode = lookup_free_space_inode(block_group, path);
2383         if (IS_ERR(inode) && PTR_ERR(inode) != -ENOENT) {
2384                 ret = PTR_ERR(inode);
2385                 btrfs_release_path(path);
2386                 goto out;
2387         }
2388
2389         if (IS_ERR(inode)) {
2390                 BUG_ON(retries);
2391                 retries++;
2392
2393                 if (block_group->ro)
2394                         goto out_free;
2395
2396                 ret = create_free_space_inode(trans, block_group, path);
2397                 if (ret)
2398                         goto out_free;
2399                 goto again;
2400         }
2401
2402         /*
2403          * We want to set the generation to 0, that way if anything goes wrong
2404          * from here on out we know not to trust this cache when we load up next
2405          * time.
2406          */
2407         BTRFS_I(inode)->generation = 0;
2408         ret = btrfs_update_inode(trans, root, inode);
2409         if (ret) {
2410                 /*
2411                  * So theoretically we could recover from this, simply set the
2412                  * super cache generation to 0 so we know to invalidate the
2413                  * cache, but then we'd have to keep track of the block groups
2414                  * that fail this way so we know we _have_ to reset this cache
2415                  * before the next commit or risk reading stale cache.  So to
2416                  * limit our exposure to horrible edge cases lets just abort the
2417                  * transaction, this only happens in really bad situations
2418                  * anyway.
2419                  */
2420                 btrfs_abort_transaction(trans, ret);
2421                 goto out_put;
2422         }
2423         WARN_ON(ret);
2424
2425         /* We've already setup this transaction, go ahead and exit */
2426         if (block_group->cache_generation == trans->transid &&
2427             i_size_read(inode)) {
2428                 dcs = BTRFS_DC_SETUP;
2429                 goto out_put;
2430         }
2431
2432         if (i_size_read(inode) > 0) {
2433                 ret = btrfs_check_trunc_cache_free_space(fs_info,
2434                                         &fs_info->global_block_rsv);
2435                 if (ret)
2436                         goto out_put;
2437
2438                 ret = btrfs_truncate_free_space_cache(trans, NULL, inode);
2439                 if (ret)
2440                         goto out_put;
2441         }
2442
2443         spin_lock(&block_group->lock);
2444         if (block_group->cached != BTRFS_CACHE_FINISHED ||
2445             !btrfs_test_opt(fs_info, SPACE_CACHE)) {
2446                 /*
2447                  * don't bother trying to write stuff out _if_
2448                  * a) we're not cached,
2449                  * b) we're with nospace_cache mount option,
2450                  * c) we're with v2 space_cache (FREE_SPACE_TREE).
2451                  */
2452                 dcs = BTRFS_DC_WRITTEN;
2453                 spin_unlock(&block_group->lock);
2454                 goto out_put;
2455         }
2456         spin_unlock(&block_group->lock);
2457
2458         /*
2459          * We hit an ENOSPC when setting up the cache in this transaction, just
2460          * skip doing the setup, we've already cleared the cache so we're safe.
2461          */
2462         if (test_bit(BTRFS_TRANS_CACHE_ENOSPC, &trans->transaction->flags)) {
2463                 ret = -ENOSPC;
2464                 goto out_put;
2465         }
2466
2467         /*
2468          * Try to preallocate enough space based on how big the block group is.
2469          * Keep in mind this has to include any pinned space which could end up
2470          * taking up quite a bit since it's not folded into the other space
2471          * cache.
2472          */
2473         num_pages = div_u64(block_group->length, SZ_256M);
2474         if (!num_pages)
2475                 num_pages = 1;
2476
2477         num_pages *= 16;
2478         num_pages *= PAGE_SIZE;
2479
2480         ret = btrfs_check_data_free_space(inode, &data_reserved, 0, num_pages);
2481         if (ret)
2482                 goto out_put;
2483
2484         ret = btrfs_prealloc_file_range_trans(inode, trans, 0, 0, num_pages,
2485                                               num_pages, num_pages,
2486                                               &alloc_hint);
2487         /*
2488          * Our cache requires contiguous chunks so that we don't modify a bunch
2489          * of metadata or split extents when writing the cache out, which means
2490          * we can enospc if we are heavily fragmented in addition to just normal
2491          * out of space conditions.  So if we hit this just skip setting up any
2492          * other block groups for this transaction, maybe we'll unpin enough
2493          * space the next time around.
2494          */
2495         if (!ret)
2496                 dcs = BTRFS_DC_SETUP;
2497         else if (ret == -ENOSPC)
2498                 set_bit(BTRFS_TRANS_CACHE_ENOSPC, &trans->transaction->flags);
2499
2500 out_put:
2501         iput(inode);
2502 out_free:
2503         btrfs_release_path(path);
2504 out:
2505         spin_lock(&block_group->lock);
2506         if (!ret && dcs == BTRFS_DC_SETUP)
2507                 block_group->cache_generation = trans->transid;
2508         block_group->disk_cache_state = dcs;
2509         spin_unlock(&block_group->lock);
2510
2511         extent_changeset_free(data_reserved);
2512         return ret;
2513 }
2514
2515 int btrfs_setup_space_cache(struct btrfs_trans_handle *trans)
2516 {
2517         struct btrfs_fs_info *fs_info = trans->fs_info;
2518         struct btrfs_block_group *cache, *tmp;
2519         struct btrfs_transaction *cur_trans = trans->transaction;
2520         struct btrfs_path *path;
2521
2522         if (list_empty(&cur_trans->dirty_bgs) ||
2523             !btrfs_test_opt(fs_info, SPACE_CACHE))
2524                 return 0;
2525
2526         path = btrfs_alloc_path();
2527         if (!path)
2528                 return -ENOMEM;
2529
2530         /* Could add new block groups, use _safe just in case */
2531         list_for_each_entry_safe(cache, tmp, &cur_trans->dirty_bgs,
2532                                  dirty_list) {
2533                 if (cache->disk_cache_state == BTRFS_DC_CLEAR)
2534                         cache_save_setup(cache, trans, path);
2535         }
2536
2537         btrfs_free_path(path);
2538         return 0;
2539 }
2540
2541 /*
2542  * Transaction commit does final block group cache writeback during a critical
2543  * section where nothing is allowed to change the FS.  This is required in
2544  * order for the cache to actually match the block group, but can introduce a
2545  * lot of latency into the commit.
2546  *
2547  * So, btrfs_start_dirty_block_groups is here to kick off block group cache IO.
2548  * There's a chance we'll have to redo some of it if the block group changes
2549  * again during the commit, but it greatly reduces the commit latency by
2550  * getting rid of the easy block groups while we're still allowing others to
2551  * join the commit.
2552  */
2553 int btrfs_start_dirty_block_groups(struct btrfs_trans_handle *trans)
2554 {
2555         struct btrfs_fs_info *fs_info = trans->fs_info;
2556         struct btrfs_block_group *cache;
2557         struct btrfs_transaction *cur_trans = trans->transaction;
2558         int ret = 0;
2559         int should_put;
2560         struct btrfs_path *path = NULL;
2561         LIST_HEAD(dirty);
2562         struct list_head *io = &cur_trans->io_bgs;
2563         int num_started = 0;
2564         int loops = 0;
2565
2566         spin_lock(&cur_trans->dirty_bgs_lock);
2567         if (list_empty(&cur_trans->dirty_bgs)) {
2568                 spin_unlock(&cur_trans->dirty_bgs_lock);
2569                 return 0;
2570         }
2571         list_splice_init(&cur_trans->dirty_bgs, &dirty);
2572         spin_unlock(&cur_trans->dirty_bgs_lock);
2573
2574 again:
2575         /* Make sure all the block groups on our dirty list actually exist */
2576         btrfs_create_pending_block_groups(trans);
2577
2578         if (!path) {
2579                 path = btrfs_alloc_path();
2580                 if (!path)
2581                         return -ENOMEM;
2582         }
2583
2584         /*
2585          * cache_write_mutex is here only to save us from balance or automatic
2586          * removal of empty block groups deleting this block group while we are
2587          * writing out the cache
2588          */
2589         mutex_lock(&trans->transaction->cache_write_mutex);
2590         while (!list_empty(&dirty)) {
2591                 bool drop_reserve = true;
2592
2593                 cache = list_first_entry(&dirty, struct btrfs_block_group,
2594                                          dirty_list);
2595                 /*
2596                  * This can happen if something re-dirties a block group that
2597                  * is already under IO.  Just wait for it to finish and then do
2598                  * it all again
2599                  */
2600                 if (!list_empty(&cache->io_list)) {
2601                         list_del_init(&cache->io_list);
2602                         btrfs_wait_cache_io(trans, cache, path);
2603                         btrfs_put_block_group(cache);
2604                 }
2605
2606
2607                 /*
2608                  * btrfs_wait_cache_io uses the cache->dirty_list to decide if
2609                  * it should update the cache_state.  Don't delete until after
2610                  * we wait.
2611                  *
2612                  * Since we're not running in the commit critical section
2613                  * we need the dirty_bgs_lock to protect from update_block_group
2614                  */
2615                 spin_lock(&cur_trans->dirty_bgs_lock);
2616                 list_del_init(&cache->dirty_list);
2617                 spin_unlock(&cur_trans->dirty_bgs_lock);
2618
2619                 should_put = 1;
2620
2621                 cache_save_setup(cache, trans, path);
2622
2623                 if (cache->disk_cache_state == BTRFS_DC_SETUP) {
2624                         cache->io_ctl.inode = NULL;
2625                         ret = btrfs_write_out_cache(trans, cache, path);
2626                         if (ret == 0 && cache->io_ctl.inode) {
2627                                 num_started++;
2628                                 should_put = 0;
2629
2630                                 /*
2631                                  * The cache_write_mutex is protecting the
2632                                  * io_list, also refer to the definition of
2633                                  * btrfs_transaction::io_bgs for more details
2634                                  */
2635                                 list_add_tail(&cache->io_list, io);
2636                         } else {
2637                                 /*
2638                                  * If we failed to write the cache, the
2639                                  * generation will be bad and life goes on
2640                                  */
2641                                 ret = 0;
2642                         }
2643                 }
2644                 if (!ret) {
2645                         ret = write_one_cache_group(trans, path, cache);
2646                         /*
2647                          * Our block group might still be attached to the list
2648                          * of new block groups in the transaction handle of some
2649                          * other task (struct btrfs_trans_handle->new_bgs). This
2650                          * means its block group item isn't yet in the extent
2651                          * tree. If this happens ignore the error, as we will
2652                          * try again later in the critical section of the
2653                          * transaction commit.
2654                          */
2655                         if (ret == -ENOENT) {
2656                                 ret = 0;
2657                                 spin_lock(&cur_trans->dirty_bgs_lock);
2658                                 if (list_empty(&cache->dirty_list)) {
2659                                         list_add_tail(&cache->dirty_list,
2660                                                       &cur_trans->dirty_bgs);
2661                                         btrfs_get_block_group(cache);
2662                                         drop_reserve = false;
2663                                 }
2664                                 spin_unlock(&cur_trans->dirty_bgs_lock);
2665                         } else if (ret) {
2666                                 btrfs_abort_transaction(trans, ret);
2667                         }
2668                 }
2669
2670                 /* If it's not on the io list, we need to put the block group */
2671                 if (should_put)
2672                         btrfs_put_block_group(cache);
2673                 if (drop_reserve)
2674                         btrfs_delayed_refs_rsv_release(fs_info, 1);
2675
2676                 if (ret)
2677                         break;
2678
2679                 /*
2680                  * Avoid blocking other tasks for too long. It might even save
2681                  * us from writing caches for block groups that are going to be
2682                  * removed.
2683                  */
2684                 mutex_unlock(&trans->transaction->cache_write_mutex);
2685                 mutex_lock(&trans->transaction->cache_write_mutex);
2686         }
2687         mutex_unlock(&trans->transaction->cache_write_mutex);
2688
2689         /*
2690          * Go through delayed refs for all the stuff we've just kicked off
2691          * and then loop back (just once)
2692          */
2693         ret = btrfs_run_delayed_refs(trans, 0);
2694         if (!ret && loops == 0) {
2695                 loops++;
2696                 spin_lock(&cur_trans->dirty_bgs_lock);
2697                 list_splice_init(&cur_trans->dirty_bgs, &dirty);
2698                 /*
2699                  * dirty_bgs_lock protects us from concurrent block group
2700                  * deletes too (not just cache_write_mutex).
2701                  */
2702                 if (!list_empty(&dirty)) {
2703                         spin_unlock(&cur_trans->dirty_bgs_lock);
2704                         goto again;
2705                 }
2706                 spin_unlock(&cur_trans->dirty_bgs_lock);
2707         } else if (ret < 0) {
2708                 btrfs_cleanup_dirty_bgs(cur_trans, fs_info);
2709         }
2710
2711         btrfs_free_path(path);
2712         return ret;
2713 }
2714
2715 int btrfs_write_dirty_block_groups(struct btrfs_trans_handle *trans)
2716 {
2717         struct btrfs_fs_info *fs_info = trans->fs_info;
2718         struct btrfs_block_group *cache;
2719         struct btrfs_transaction *cur_trans = trans->transaction;
2720         int ret = 0;
2721         int should_put;
2722         struct btrfs_path *path;
2723         struct list_head *io = &cur_trans->io_bgs;
2724         int num_started = 0;
2725
2726         path = btrfs_alloc_path();
2727         if (!path)
2728                 return -ENOMEM;
2729
2730         /*
2731          * Even though we are in the critical section of the transaction commit,
2732          * we can still have concurrent tasks adding elements to this
2733          * transaction's list of dirty block groups. These tasks correspond to
2734          * endio free space workers started when writeback finishes for a
2735          * space cache, which run inode.c:btrfs_finish_ordered_io(), and can
2736          * allocate new block groups as a result of COWing nodes of the root
2737          * tree when updating the free space inode. The writeback for the space
2738          * caches is triggered by an earlier call to
2739          * btrfs_start_dirty_block_groups() and iterations of the following
2740          * loop.
2741          * Also we want to do the cache_save_setup first and then run the
2742          * delayed refs to make sure we have the best chance at doing this all
2743          * in one shot.
2744          */
2745         spin_lock(&cur_trans->dirty_bgs_lock);
2746         while (!list_empty(&cur_trans->dirty_bgs)) {
2747                 cache = list_first_entry(&cur_trans->dirty_bgs,
2748                                          struct btrfs_block_group,
2749                                          dirty_list);
2750
2751                 /*
2752                  * This can happen if cache_save_setup re-dirties a block group
2753                  * that is already under IO.  Just wait for it to finish and
2754                  * then do it all again
2755                  */
2756                 if (!list_empty(&cache->io_list)) {
2757                         spin_unlock(&cur_trans->dirty_bgs_lock);
2758                         list_del_init(&cache->io_list);
2759                         btrfs_wait_cache_io(trans, cache, path);
2760                         btrfs_put_block_group(cache);
2761                         spin_lock(&cur_trans->dirty_bgs_lock);
2762                 }
2763
2764                 /*
2765                  * Don't remove from the dirty list until after we've waited on
2766                  * any pending IO
2767                  */
2768                 list_del_init(&cache->dirty_list);
2769                 spin_unlock(&cur_trans->dirty_bgs_lock);
2770                 should_put = 1;
2771
2772                 cache_save_setup(cache, trans, path);
2773
2774                 if (!ret)
2775                         ret = btrfs_run_delayed_refs(trans,
2776                                                      (unsigned long) -1);
2777
2778                 if (!ret && cache->disk_cache_state == BTRFS_DC_SETUP) {
2779                         cache->io_ctl.inode = NULL;
2780                         ret = btrfs_write_out_cache(trans, cache, path);
2781                         if (ret == 0 && cache->io_ctl.inode) {
2782                                 num_started++;
2783                                 should_put = 0;
2784                                 list_add_tail(&cache->io_list, io);
2785                         } else {
2786                                 /*
2787                                  * If we failed to write the cache, the
2788                                  * generation will be bad and life goes on
2789                                  */
2790                                 ret = 0;
2791                         }
2792                 }
2793                 if (!ret) {
2794                         ret = write_one_cache_group(trans, path, cache);
2795                         /*
2796                          * One of the free space endio workers might have
2797                          * created a new block group while updating a free space
2798                          * cache's inode (at inode.c:btrfs_finish_ordered_io())
2799                          * and hasn't released its transaction handle yet, in
2800                          * which case the new block group is still attached to
2801                          * its transaction handle and its creation has not
2802                          * finished yet (no block group item in the extent tree
2803                          * yet, etc). If this is the case, wait for all free
2804                          * space endio workers to finish and retry. This is a
2805                          * a very rare case so no need for a more efficient and
2806                          * complex approach.
2807                          */
2808                         if (ret == -ENOENT) {
2809                                 wait_event(cur_trans->writer_wait,
2810                                    atomic_read(&cur_trans->num_writers) == 1);
2811                                 ret = write_one_cache_group(trans, path, cache);
2812                         }
2813                         if (ret)
2814                                 btrfs_abort_transaction(trans, ret);
2815                 }
2816
2817                 /* If its not on the io list, we need to put the block group */
2818                 if (should_put)
2819                         btrfs_put_block_group(cache);
2820                 btrfs_delayed_refs_rsv_release(fs_info, 1);
2821                 spin_lock(&cur_trans->dirty_bgs_lock);
2822         }
2823         spin_unlock(&cur_trans->dirty_bgs_lock);
2824
2825         /*
2826          * Refer to the definition of io_bgs member for details why it's safe
2827          * to use it without any locking
2828          */
2829         while (!list_empty(io)) {
2830                 cache = list_first_entry(io, struct btrfs_block_group,
2831                                          io_list);
2832                 list_del_init(&cache->io_list);
2833                 btrfs_wait_cache_io(trans, cache, path);
2834                 btrfs_put_block_group(cache);
2835         }
2836
2837         btrfs_free_path(path);
2838         return ret;
2839 }
2840
2841 int btrfs_update_block_group(struct btrfs_trans_handle *trans,
2842                              u64 bytenr, u64 num_bytes, int alloc)
2843 {
2844         struct btrfs_fs_info *info = trans->fs_info;
2845         struct btrfs_block_group *cache = NULL;
2846         u64 total = num_bytes;
2847         u64 old_val;
2848         u64 byte_in_group;
2849         int factor;
2850         int ret = 0;
2851
2852         /* Block accounting for super block */
2853         spin_lock(&info->delalloc_root_lock);
2854         old_val = btrfs_super_bytes_used(info->super_copy);
2855         if (alloc)
2856                 old_val += num_bytes;
2857         else
2858                 old_val -= num_bytes;
2859         btrfs_set_super_bytes_used(info->super_copy, old_val);
2860         spin_unlock(&info->delalloc_root_lock);
2861
2862         while (total) {
2863                 cache = btrfs_lookup_block_group(info, bytenr);
2864                 if (!cache) {
2865                         ret = -ENOENT;
2866                         break;
2867                 }
2868                 factor = btrfs_bg_type_to_factor(cache->flags);
2869
2870                 /*
2871                  * If this block group has free space cache written out, we
2872                  * need to make sure to load it if we are removing space.  This
2873                  * is because we need the unpinning stage to actually add the
2874                  * space back to the block group, otherwise we will leak space.
2875                  */
2876                 if (!alloc && !btrfs_block_group_done(cache))
2877                         btrfs_cache_block_group(cache, 1);
2878
2879                 byte_in_group = bytenr - cache->start;
2880                 WARN_ON(byte_in_group > cache->length);
2881
2882                 spin_lock(&cache->space_info->lock);
2883                 spin_lock(&cache->lock);
2884
2885                 if (btrfs_test_opt(info, SPACE_CACHE) &&
2886                     cache->disk_cache_state < BTRFS_DC_CLEAR)
2887                         cache->disk_cache_state = BTRFS_DC_CLEAR;
2888
2889                 old_val = cache->used;
2890                 num_bytes = min(total, cache->length - byte_in_group);
2891                 if (alloc) {
2892                         old_val += num_bytes;
2893                         cache->used = old_val;
2894                         cache->reserved -= num_bytes;
2895                         cache->space_info->bytes_reserved -= num_bytes;
2896                         cache->space_info->bytes_used += num_bytes;
2897                         cache->space_info->disk_used += num_bytes * factor;
2898                         spin_unlock(&cache->lock);
2899                         spin_unlock(&cache->space_info->lock);
2900                 } else {
2901                         old_val -= num_bytes;
2902                         cache->used = old_val;
2903                         cache->pinned += num_bytes;
2904                         btrfs_space_info_update_bytes_pinned(info,
2905                                         cache->space_info, num_bytes);
2906                         cache->space_info->bytes_used -= num_bytes;
2907                         cache->space_info->disk_used -= num_bytes * factor;
2908                         spin_unlock(&cache->lock);
2909                         spin_unlock(&cache->space_info->lock);
2910
2911                         percpu_counter_add_batch(
2912                                         &cache->space_info->total_bytes_pinned,
2913                                         num_bytes,
2914                                         BTRFS_TOTAL_BYTES_PINNED_BATCH);
2915                         set_extent_dirty(&trans->transaction->pinned_extents,
2916                                          bytenr, bytenr + num_bytes - 1,
2917                                          GFP_NOFS | __GFP_NOFAIL);
2918                 }
2919
2920                 spin_lock(&trans->transaction->dirty_bgs_lock);
2921                 if (list_empty(&cache->dirty_list)) {
2922                         list_add_tail(&cache->dirty_list,
2923                                       &trans->transaction->dirty_bgs);
2924                         trans->delayed_ref_updates++;
2925                         btrfs_get_block_group(cache);
2926                 }
2927                 spin_unlock(&trans->transaction->dirty_bgs_lock);
2928
2929                 /*
2930                  * No longer have used bytes in this block group, queue it for
2931                  * deletion. We do this after adding the block group to the
2932                  * dirty list to avoid races between cleaner kthread and space
2933                  * cache writeout.
2934                  */
2935                 if (!alloc && old_val == 0) {
2936                         if (!btrfs_test_opt(info, DISCARD_ASYNC))
2937                                 btrfs_mark_bg_unused(cache);
2938                 }
2939
2940                 btrfs_put_block_group(cache);
2941                 total -= num_bytes;
2942                 bytenr += num_bytes;
2943         }
2944
2945         /* Modified block groups are accounted for in the delayed_refs_rsv. */
2946         btrfs_update_delayed_refs_rsv(trans);
2947         return ret;
2948 }
2949
2950 /**
2951  * btrfs_add_reserved_bytes - update the block_group and space info counters
2952  * @cache:      The cache we are manipulating
2953  * @ram_bytes:  The number of bytes of file content, and will be same to
2954  *              @num_bytes except for the compress path.
2955  * @num_bytes:  The number of bytes in question
2956  * @delalloc:   The blocks are allocated for the delalloc write
2957  *
2958  * This is called by the allocator when it reserves space. If this is a
2959  * reservation and the block group has become read only we cannot make the
2960  * reservation and return -EAGAIN, otherwise this function always succeeds.
2961  */
2962 int btrfs_add_reserved_bytes(struct btrfs_block_group *cache,
2963                              u64 ram_bytes, u64 num_bytes, int delalloc)
2964 {
2965         struct btrfs_space_info *space_info = cache->space_info;
2966         int ret = 0;
2967
2968         spin_lock(&space_info->lock);
2969         spin_lock(&cache->lock);
2970         if (cache->ro) {
2971                 ret = -EAGAIN;
2972         } else {
2973                 cache->reserved += num_bytes;
2974                 space_info->bytes_reserved += num_bytes;
2975                 trace_btrfs_space_reservation(cache->fs_info, "space_info",
2976                                               space_info->flags, num_bytes, 1);
2977                 btrfs_space_info_update_bytes_may_use(cache->fs_info,
2978                                                       space_info, -ram_bytes);
2979                 if (delalloc)
2980                         cache->delalloc_bytes += num_bytes;
2981         }
2982         spin_unlock(&cache->lock);
2983         spin_unlock(&space_info->lock);
2984         return ret;
2985 }
2986
2987 /**
2988  * btrfs_free_reserved_bytes - update the block_group and space info counters
2989  * @cache:      The cache we are manipulating
2990  * @num_bytes:  The number of bytes in question
2991  * @delalloc:   The blocks are allocated for the delalloc write
2992  *
2993  * This is called by somebody who is freeing space that was never actually used
2994  * on disk.  For example if you reserve some space for a new leaf in transaction
2995  * A and before transaction A commits you free that leaf, you call this with
2996  * reserve set to 0 in order to clear the reservation.
2997  */
2998 void btrfs_free_reserved_bytes(struct btrfs_block_group *cache,
2999                                u64 num_bytes, int delalloc)
3000 {
3001         struct btrfs_space_info *space_info = cache->space_info;
3002
3003         spin_lock(&space_info->lock);
3004         spin_lock(&cache->lock);
3005         if (cache->ro)
3006                 space_info->bytes_readonly += num_bytes;
3007         cache->reserved -= num_bytes;
3008         space_info->bytes_reserved -= num_bytes;
3009         space_info->max_extent_size = 0;
3010
3011         if (delalloc)
3012                 cache->delalloc_bytes -= num_bytes;
3013         spin_unlock(&cache->lock);
3014         spin_unlock(&space_info->lock);
3015 }
3016
3017 static void force_metadata_allocation(struct btrfs_fs_info *info)
3018 {
3019         struct list_head *head = &info->space_info;
3020         struct btrfs_space_info *found;
3021
3022         rcu_read_lock();
3023         list_for_each_entry_rcu(found, head, list) {
3024                 if (found->flags & BTRFS_BLOCK_GROUP_METADATA)
3025                         found->force_alloc = CHUNK_ALLOC_FORCE;
3026         }
3027         rcu_read_unlock();
3028 }
3029
3030 static int should_alloc_chunk(struct btrfs_fs_info *fs_info,
3031                               struct btrfs_space_info *sinfo, int force)
3032 {
3033         u64 bytes_used = btrfs_space_info_used(sinfo, false);
3034         u64 thresh;
3035
3036         if (force == CHUNK_ALLOC_FORCE)
3037                 return 1;
3038
3039         /*
3040          * in limited mode, we want to have some free space up to
3041          * about 1% of the FS size.
3042          */
3043         if (force == CHUNK_ALLOC_LIMITED) {
3044                 thresh = btrfs_super_total_bytes(fs_info->super_copy);
3045                 thresh = max_t(u64, SZ_64M, div_factor_fine(thresh, 1));
3046
3047                 if (sinfo->total_bytes - bytes_used < thresh)
3048                         return 1;
3049         }
3050
3051         if (bytes_used + SZ_2M < div_factor(sinfo->total_bytes, 8))
3052                 return 0;
3053         return 1;
3054 }
3055
3056 int btrfs_force_chunk_alloc(struct btrfs_trans_handle *trans, u64 type)
3057 {
3058         u64 alloc_flags = btrfs_get_alloc_profile(trans->fs_info, type);
3059
3060         return btrfs_chunk_alloc(trans, alloc_flags, CHUNK_ALLOC_FORCE);
3061 }
3062
3063 /*
3064  * If force is CHUNK_ALLOC_FORCE:
3065  *    - return 1 if it successfully allocates a chunk,
3066  *    - return errors including -ENOSPC otherwise.
3067  * If force is NOT CHUNK_ALLOC_FORCE:
3068  *    - return 0 if it doesn't need to allocate a new chunk,
3069  *    - return 1 if it successfully allocates a chunk,
3070  *    - return errors including -ENOSPC otherwise.
3071  */
3072 int btrfs_chunk_alloc(struct btrfs_trans_handle *trans, u64 flags,
3073                       enum btrfs_chunk_alloc_enum force)
3074 {
3075         struct btrfs_fs_info *fs_info = trans->fs_info;
3076         struct btrfs_space_info *space_info;
3077         bool wait_for_alloc = false;
3078         bool should_alloc = false;
3079         int ret = 0;
3080
3081         /* Don't re-enter if we're already allocating a chunk */
3082         if (trans->allocating_chunk)
3083                 return -ENOSPC;
3084
3085         space_info = btrfs_find_space_info(fs_info, flags);
3086         ASSERT(space_info);
3087
3088         do {
3089                 spin_lock(&space_info->lock);
3090                 if (force < space_info->force_alloc)
3091                         force = space_info->force_alloc;
3092                 should_alloc = should_alloc_chunk(fs_info, space_info, force);
3093                 if (space_info->full) {
3094                         /* No more free physical space */
3095                         if (should_alloc)
3096                                 ret = -ENOSPC;
3097                         else
3098                                 ret = 0;
3099                         spin_unlock(&space_info->lock);
3100                         return ret;
3101                 } else if (!should_alloc) {
3102                         spin_unlock(&space_info->lock);
3103                         return 0;
3104                 } else if (space_info->chunk_alloc) {
3105                         /*
3106                          * Someone is already allocating, so we need to block
3107                          * until this someone is finished and then loop to
3108                          * recheck if we should continue with our allocation
3109                          * attempt.
3110                          */
3111                         wait_for_alloc = true;
3112                         spin_unlock(&space_info->lock);
3113                         mutex_lock(&fs_info->chunk_mutex);
3114                         mutex_unlock(&fs_info->chunk_mutex);
3115                 } else {
3116                         /* Proceed with allocation */
3117                         space_info->chunk_alloc = 1;
3118                         wait_for_alloc = false;
3119                         spin_unlock(&space_info->lock);
3120                 }
3121
3122                 cond_resched();
3123         } while (wait_for_alloc);
3124
3125         mutex_lock(&fs_info->chunk_mutex);
3126         trans->allocating_chunk = true;
3127
3128         /*
3129          * If we have mixed data/metadata chunks we want to make sure we keep
3130          * allocating mixed chunks instead of individual chunks.
3131          */
3132         if (btrfs_mixed_space_info(space_info))
3133                 flags |= (BTRFS_BLOCK_GROUP_DATA | BTRFS_BLOCK_GROUP_METADATA);
3134
3135         /*
3136          * if we're doing a data chunk, go ahead and make sure that
3137          * we keep a reasonable number of metadata chunks allocated in the
3138          * FS as well.
3139          */
3140         if (flags & BTRFS_BLOCK_GROUP_DATA && fs_info->metadata_ratio) {
3141                 fs_info->data_chunk_allocations++;
3142                 if (!(fs_info->data_chunk_allocations %
3143                       fs_info->metadata_ratio))
3144                         force_metadata_allocation(fs_info);
3145         }
3146
3147         /*
3148          * Check if we have enough space in SYSTEM chunk because we may need
3149          * to update devices.
3150          */
3151         check_system_chunk(trans, flags);
3152
3153         ret = btrfs_alloc_chunk(trans, flags);
3154         trans->allocating_chunk = false;
3155
3156         spin_lock(&space_info->lock);
3157         if (ret < 0) {
3158                 if (ret == -ENOSPC)
3159                         space_info->full = 1;
3160                 else
3161                         goto out;
3162         } else {
3163                 ret = 1;
3164                 space_info->max_extent_size = 0;
3165         }
3166
3167         space_info->force_alloc = CHUNK_ALLOC_NO_FORCE;
3168 out:
3169         space_info->chunk_alloc = 0;
3170         spin_unlock(&space_info->lock);
3171         mutex_unlock(&fs_info->chunk_mutex);
3172         /*
3173          * When we allocate a new chunk we reserve space in the chunk block
3174          * reserve to make sure we can COW nodes/leafs in the chunk tree or
3175          * add new nodes/leafs to it if we end up needing to do it when
3176          * inserting the chunk item and updating device items as part of the
3177          * second phase of chunk allocation, performed by
3178          * btrfs_finish_chunk_alloc(). So make sure we don't accumulate a
3179          * large number of new block groups to create in our transaction
3180          * handle's new_bgs list to avoid exhausting the chunk block reserve
3181          * in extreme cases - like having a single transaction create many new
3182          * block groups when starting to write out the free space caches of all
3183          * the block groups that were made dirty during the lifetime of the
3184          * transaction.
3185          */
3186         if (trans->chunk_bytes_reserved >= (u64)SZ_2M)
3187                 btrfs_create_pending_block_groups(trans);
3188
3189         return ret;
3190 }
3191
3192 static u64 get_profile_num_devs(struct btrfs_fs_info *fs_info, u64 type)
3193 {
3194         u64 num_dev;
3195
3196         num_dev = btrfs_raid_array[btrfs_bg_flags_to_raid_index(type)].devs_max;
3197         if (!num_dev)
3198                 num_dev = fs_info->fs_devices->rw_devices;
3199
3200         return num_dev;
3201 }
3202
3203 /*
3204  * Reserve space in the system space for allocating or removing a chunk
3205  */
3206 void check_system_chunk(struct btrfs_trans_handle *trans, u64 type)
3207 {
3208         struct btrfs_fs_info *fs_info = trans->fs_info;
3209         struct btrfs_space_info *info;
3210         u64 left;
3211         u64 thresh;
3212         int ret = 0;
3213         u64 num_devs;
3214
3215         /*
3216          * Needed because we can end up allocating a system chunk and for an
3217          * atomic and race free space reservation in the chunk block reserve.
3218          */
3219         lockdep_assert_held(&fs_info->chunk_mutex);
3220
3221         info = btrfs_find_space_info(fs_info, BTRFS_BLOCK_GROUP_SYSTEM);
3222         spin_lock(&info->lock);
3223         left = info->total_bytes - btrfs_space_info_used(info, true);
3224         spin_unlock(&info->lock);
3225
3226         num_devs = get_profile_num_devs(fs_info, type);
3227
3228         /* num_devs device items to update and 1 chunk item to add or remove */
3229         thresh = btrfs_calc_metadata_size(fs_info, num_devs) +
3230                 btrfs_calc_insert_metadata_size(fs_info, 1);
3231
3232         if (left < thresh && btrfs_test_opt(fs_info, ENOSPC_DEBUG)) {
3233                 btrfs_info(fs_info, "left=%llu, need=%llu, flags=%llu",
3234                            left, thresh, type);
3235                 btrfs_dump_space_info(fs_info, info, 0, 0);
3236         }
3237
3238         if (left < thresh) {
3239                 u64 flags = btrfs_system_alloc_profile(fs_info);
3240
3241                 /*
3242                  * Ignore failure to create system chunk. We might end up not
3243                  * needing it, as we might not need to COW all nodes/leafs from
3244                  * the paths we visit in the chunk tree (they were already COWed
3245                  * or created in the current transaction for example).
3246                  */
3247                 ret = btrfs_alloc_chunk(trans, flags);
3248         }
3249
3250         if (!ret) {
3251                 ret = btrfs_block_rsv_add(fs_info->chunk_root,
3252                                           &fs_info->chunk_block_rsv,
3253                                           thresh, BTRFS_RESERVE_NO_FLUSH);
3254                 if (!ret)
3255                         trans->chunk_bytes_reserved += thresh;
3256         }
3257 }
3258
3259 void btrfs_put_block_group_cache(struct btrfs_fs_info *info)
3260 {
3261         struct btrfs_block_group *block_group;
3262         u64 last = 0;
3263
3264         while (1) {
3265                 struct inode *inode;
3266
3267                 block_group = btrfs_lookup_first_block_group(info, last);
3268                 while (block_group) {
3269                         btrfs_wait_block_group_cache_done(block_group);
3270                         spin_lock(&block_group->lock);
3271                         if (block_group->iref)
3272                                 break;
3273                         spin_unlock(&block_group->lock);
3274                         block_group = btrfs_next_block_group(block_group);
3275                 }
3276                 if (!block_group) {
3277                         if (last == 0)
3278                                 break;
3279                         last = 0;
3280                         continue;
3281                 }
3282
3283                 inode = block_group->inode;
3284                 block_group->iref = 0;
3285                 block_group->inode = NULL;
3286                 spin_unlock(&block_group->lock);
3287                 ASSERT(block_group->io_ctl.inode == NULL);
3288                 iput(inode);
3289                 last = block_group->start + block_group->length;
3290                 btrfs_put_block_group(block_group);
3291         }
3292 }
3293
3294 /*
3295  * Must be called only after stopping all workers, since we could have block
3296  * group caching kthreads running, and therefore they could race with us if we
3297  * freed the block groups before stopping them.
3298  */
3299 int btrfs_free_block_groups(struct btrfs_fs_info *info)
3300 {
3301         struct btrfs_block_group *block_group;
3302         struct btrfs_space_info *space_info;
3303         struct btrfs_caching_control *caching_ctl;
3304         struct rb_node *n;
3305
3306         down_write(&info->commit_root_sem);
3307         while (!list_empty(&info->caching_block_groups)) {
3308                 caching_ctl = list_entry(info->caching_block_groups.next,
3309                                          struct btrfs_caching_control, list);
3310                 list_del(&caching_ctl->list);
3311                 btrfs_put_caching_control(caching_ctl);
3312         }
3313         up_write(&info->commit_root_sem);
3314
3315         spin_lock(&info->unused_bgs_lock);
3316         while (!list_empty(&info->unused_bgs)) {
3317                 block_group = list_first_entry(&info->unused_bgs,
3318                                                struct btrfs_block_group,
3319                                                bg_list);
3320                 list_del_init(&block_group->bg_list);
3321                 btrfs_put_block_group(block_group);
3322         }
3323         spin_unlock(&info->unused_bgs_lock);
3324
3325         spin_lock(&info->block_group_cache_lock);
3326         while ((n = rb_last(&info->block_group_cache_tree)) != NULL) {
3327                 block_group = rb_entry(n, struct btrfs_block_group,
3328                                        cache_node);
3329                 rb_erase(&block_group->cache_node,
3330                          &info->block_group_cache_tree);
3331                 RB_CLEAR_NODE(&block_group->cache_node);
3332                 spin_unlock(&info->block_group_cache_lock);
3333
3334                 down_write(&block_group->space_info->groups_sem);
3335                 list_del(&block_group->list);
3336                 up_write(&block_group->space_info->groups_sem);
3337
3338                 /*
3339                  * We haven't cached this block group, which means we could
3340                  * possibly have excluded extents on this block group.
3341                  */
3342                 if (block_group->cached == BTRFS_CACHE_NO ||
3343                     block_group->cached == BTRFS_CACHE_ERROR)
3344                         btrfs_free_excluded_extents(block_group);
3345
3346                 btrfs_remove_free_space_cache(block_group);
3347                 ASSERT(block_group->cached != BTRFS_CACHE_STARTED);
3348                 ASSERT(list_empty(&block_group->dirty_list));
3349                 ASSERT(list_empty(&block_group->io_list));
3350                 ASSERT(list_empty(&block_group->bg_list));
3351                 ASSERT(atomic_read(&block_group->count) == 1);
3352                 btrfs_put_block_group(block_group);
3353
3354                 spin_lock(&info->block_group_cache_lock);
3355         }
3356         spin_unlock(&info->block_group_cache_lock);
3357
3358         /*
3359          * Now that all the block groups are freed, go through and free all the
3360          * space_info structs.  This is only called during the final stages of
3361          * unmount, and so we know nobody is using them.  We call
3362          * synchronize_rcu() once before we start, just to be on the safe side.
3363          */
3364         synchronize_rcu();
3365
3366         btrfs_release_global_block_rsv(info);
3367
3368         while (!list_empty(&info->space_info)) {
3369                 space_info = list_entry(info->space_info.next,
3370                                         struct btrfs_space_info,
3371                                         list);
3372
3373                 /*
3374                  * Do not hide this behind enospc_debug, this is actually
3375                  * important and indicates a real bug if this happens.
3376                  */
3377                 if (WARN_ON(space_info->bytes_pinned > 0 ||
3378                             space_info->bytes_reserved > 0 ||
3379                             space_info->bytes_may_use > 0))
3380                         btrfs_dump_space_info(info, space_info, 0, 0);
3381                 WARN_ON(space_info->reclaim_size > 0);
3382                 list_del(&space_info->list);
3383                 btrfs_sysfs_remove_space_info(space_info);
3384         }
3385         return 0;
3386 }