fs/btrfs/block-group.c

   1 // SPDX-License-Identifier: GPL-2.0
   2
   3 #include "misc.h"
   4 #include "ctree.h"
   5 #include "block-group.h"
   6 #include "space-info.h"
   7 #include "disk-io.h"
   8 #include "free-space-cache.h"
   9 #include "free-space-tree.h"
  10 #include "disk-io.h"
  11 #include "volumes.h"
  12 #include "transaction.h"
  13 #include "ref-verify.h"
  14 #include "sysfs.h"
  15 #include "tree-log.h"
  16 #include "delalloc-space.h"
  17 #include "discard.h"
  18 #include "raid56.h"
  19
  20 /*
  21  * Return target flags in extended format or 0 if restripe for this chunk_type
  22  * is not in progress
  23  *
  24  * Should be called with balance_lock held
  25  */
  26 static u64 get_restripe_target(struct btrfs_fs_info *fs_info, u64 flags)
  27 {
  28         struct btrfs_balance_control *bctl = fs_info->balance_ctl;
  29         u64 target = 0;
  30
  31         if (!bctl)
  32                 return 0;
  33
  34         if (flags & BTRFS_BLOCK_GROUP_DATA &&
  35             bctl->data.flags & BTRFS_BALANCE_ARGS_CONVERT) {
  36                 target = BTRFS_BLOCK_GROUP_DATA | bctl->data.target;
  37         } else if (flags & BTRFS_BLOCK_GROUP_SYSTEM &&
  38                    bctl->sys.flags & BTRFS_BALANCE_ARGS_CONVERT) {
  39                 target = BTRFS_BLOCK_GROUP_SYSTEM | bctl->sys.target;
  40         } else if (flags & BTRFS_BLOCK_GROUP_METADATA &&
  41                    bctl->meta.flags & BTRFS_BALANCE_ARGS_CONVERT) {
  42                 target = BTRFS_BLOCK_GROUP_METADATA | bctl->meta.target;
  43         }
  44
  45         return target;
  46 }
  47
  48 /*
  49  * @flags: available profiles in extended format (see ctree.h)
  50  *
  51  * Return reduced profile in chunk format.  If profile changing is in progress
  52  * (either running or paused) picks the target profile (if it's already
  53  * available), otherwise falls back to plain reducing.
  54  */
  55 static u64 btrfs_reduce_alloc_profile(struct btrfs_fs_info *fs_info, u64 flags)
  56 {
  57         u64 num_devices = fs_info->fs_devices->rw_devices;
  58         u64 target;
  59         u64 raid_type;
  60         u64 allowed = 0;
  61
  62         /*
  63          * See if restripe for this chunk_type is in progress, if so try to
  64          * reduce to the target profile
  65          */
  66         spin_lock(&fs_info->balance_lock);
  67         target = get_restripe_target(fs_info, flags);
  68         if (target) {
  69                 /* Pick target profile only if it's already available */
  70                 if ((flags & target) & BTRFS_EXTENDED_PROFILE_MASK) {
  71                         spin_unlock(&fs_info->balance_lock);
  72                         return extended_to_chunk(target);
  73                 }
  74         }
  75         spin_unlock(&fs_info->balance_lock);
  76
  77         /* First, mask out the RAID levels which aren't possible */
  78         for (raid_type = 0; raid_type < BTRFS_NR_RAID_TYPES; raid_type++) {
  79                 if (num_devices >= btrfs_raid_array[raid_type].devs_min)
  80                         allowed |= btrfs_raid_array[raid_type].bg_flag;
  81         }
  82         allowed &= flags;
  83
  84         if (allowed & BTRFS_BLOCK_GROUP_RAID6)
  85                 allowed = BTRFS_BLOCK_GROUP_RAID6;
  86         else if (allowed & BTRFS_BLOCK_GROUP_RAID5)
  87                 allowed = BTRFS_BLOCK_GROUP_RAID5;
  88         else if (allowed & BTRFS_BLOCK_GROUP_RAID10)
  89                 allowed = BTRFS_BLOCK_GROUP_RAID10;
  90         else if (allowed & BTRFS_BLOCK_GROUP_RAID1)
  91                 allowed = BTRFS_BLOCK_GROUP_RAID1;
  92         else if (allowed & BTRFS_BLOCK_GROUP_RAID0)
  93                 allowed = BTRFS_BLOCK_GROUP_RAID0;
  94
  95         flags &= ~BTRFS_BLOCK_GROUP_PROFILE_MASK;
  96
  97         return extended_to_chunk(flags | allowed);
  98 }
  99
 100 u64 btrfs_get_alloc_profile(struct btrfs_fs_info *fs_info, u64 orig_flags)
 101 {
 102         unsigned seq;
 103         u64 flags;
 104
 105         do {
 106                 flags = orig_flags;
 107                 seq = read_seqbegin(&fs_info->profiles_lock);
 108
 109                 if (flags & BTRFS_BLOCK_GROUP_DATA)
 110                         flags |= fs_info->avail_data_alloc_bits;
 111                 else if (flags & BTRFS_BLOCK_GROUP_SYSTEM)
 112                         flags |= fs_info->avail_system_alloc_bits;
 113                 else if (flags & BTRFS_BLOCK_GROUP_METADATA)
 114                         flags |= fs_info->avail_metadata_alloc_bits;
 115         } while (read_seqretry(&fs_info->profiles_lock, seq));
 116
 117         return btrfs_reduce_alloc_profile(fs_info, flags);
 118 }
 119
 120 void btrfs_get_block_group(struct btrfs_block_group *cache)
 121 {
 122         atomic_inc(&cache->count);
 123 }
 124
 125 void btrfs_put_block_group(struct btrfs_block_group *cache)
 126 {
 127         if (atomic_dec_and_test(&cache->count)) {
 128                 WARN_ON(cache->pinned > 0);
 129                 WARN_ON(cache->reserved > 0);
 130
 131                 /*
 132                  * A block_group shouldn't be on the discard_list anymore.
 133                  * Remove the block_group from the discard_list to prevent us
 134                  * from causing a panic due to NULL pointer dereference.
 135                  */
 136                 if (WARN_ON(!list_empty(&cache->discard_list)))
 137                         btrfs_discard_cancel_work(&cache->fs_info->discard_ctl,
 138                                                   cache);
 139
 140                 /*
 141                  * If not empty, someone is still holding mutex of
 142                  * full_stripe_lock, which can only be released by caller.
 143                  * And it will definitely cause use-after-free when caller
 144                  * tries to release full stripe lock.
 145                  *
 146                  * No better way to resolve, but only to warn.
 147                  */
 148                 WARN_ON(!RB_EMPTY_ROOT(&cache->full_stripe_locks_root.root));
 149                 kfree(cache->free_space_ctl);
 150                 kfree(cache);
 151         }
 152 }
 153
 154 /*
 155  * This adds the block group to the fs_info rb tree for the block group cache
 156  */
 157 static int btrfs_add_block_group_cache(struct btrfs_fs_info *info,
 158                                        struct btrfs_block_group *block_group)
 159 {
 160         struct rb_node **p;
 161         struct rb_node *parent = NULL;
 162         struct btrfs_block_group *cache;
 163
 164         spin_lock(&info->block_group_cache_lock);
 165         p = &info->block_group_cache_tree.rb_node;
 166
 167         while (*p) {
 168                 parent = *p;
 169                 cache = rb_entry(parent, struct btrfs_block_group, cache_node);
 170                 if (block_group->start < cache->start) {
 171                         p = &(*p)->rb_left;
 172                 } else if (block_group->start > cache->start) {
 173                         p = &(*p)->rb_right;
 174                 } else {
 175                         spin_unlock(&info->block_group_cache_lock);
 176                         return -EEXIST;
 177                 }
 178         }
 179
 180         rb_link_node(&block_group->cache_node, parent, p);
 181         rb_insert_color(&block_group->cache_node,
 182                         &info->block_group_cache_tree);
 183
 184         if (info->first_logical_byte > block_group->start)
 185                 info->first_logical_byte = block_group->start;
 186
 187         spin_unlock(&info->block_group_cache_lock);
 188
 189         return 0;
 190 }
 191
 192 /*
 193  * This will return the block group at or after bytenr if contains is 0, else
 194  * it will return the block group that contains the bytenr
 195  */
 196 static struct btrfs_block_group *block_group_cache_tree_search(
 197                 struct btrfs_fs_info *info, u64 bytenr, int contains)
 198 {
 199         struct btrfs_block_group *cache, *ret = NULL;
 200         struct rb_node *n;
 201         u64 end, start;
 202
 203         spin_lock(&info->block_group_cache_lock);
 204         n = info->block_group_cache_tree.rb_node;
 205
 206         while (n) {
 207                 cache = rb_entry(n, struct btrfs_block_group, cache_node);
 208                 end = cache->start + cache->length - 1;
 209                 start = cache->start;
 210
 211                 if (bytenr < start) {
 212                         if (!contains && (!ret || start < ret->start))
 213                                 ret = cache;
 214                         n = n->rb_left;
 215                 } else if (bytenr > start) {
 216                         if (contains && bytenr <= end) {
 217                                 ret = cache;
 218                                 break;
 219                         }
 220                         n = n->rb_right;
 221                 } else {
 222                         ret = cache;
 223                         break;
 224                 }
 225         }
 226         if (ret) {
 227                 btrfs_get_block_group(ret);
 228                 if (bytenr == 0 && info->first_logical_byte > ret->start)
 229                         info->first_logical_byte = ret->start;
 230         }
 231         spin_unlock(&info->block_group_cache_lock);
 232
 233         return ret;
 234 }
 235
 236 /*
 237  * Return the block group that starts at or after bytenr
 238  */
 239 struct btrfs_block_group *btrfs_lookup_first_block_group(
 240                 struct btrfs_fs_info *info, u64 bytenr)
 241 {
 242         return block_group_cache_tree_search(info, bytenr, 0);
 243 }
 244
 245 /*
 246  * Return the block group that contains the given bytenr
 247  */
 248 struct btrfs_block_group *btrfs_lookup_block_group(
 249                 struct btrfs_fs_info *info, u64 bytenr)
 250 {
 251         return block_group_cache_tree_search(info, bytenr, 1);
 252 }
 253
 254 struct btrfs_block_group *btrfs_next_block_group(
 255                 struct btrfs_block_group *cache)
 256 {
 257         struct btrfs_fs_info *fs_info = cache->fs_info;
 258         struct rb_node *node;
 259
 260         spin_lock(&fs_info->block_group_cache_lock);
 261
 262         /* If our block group was removed, we need a full search. */
 263         if (RB_EMPTY_NODE(&cache->cache_node)) {
 264                 const u64 next_bytenr = cache->start + cache->length;
 265
 266                 spin_unlock(&fs_info->block_group_cache_lock);
 267                 btrfs_put_block_group(cache);
 268                 cache = btrfs_lookup_first_block_group(fs_info, next_bytenr); return cache;
 269         }
 270         node = rb_next(&cache->cache_node);
 271         btrfs_put_block_group(cache);
 272         if (node) {
 273                 cache = rb_entry(node, struct btrfs_block_group, cache_node);
 274                 btrfs_get_block_group(cache);
 275         } else
 276                 cache = NULL;
 277         spin_unlock(&fs_info->block_group_cache_lock);
 278         return cache;
 279 }
 280
 281 bool btrfs_inc_nocow_writers(struct btrfs_fs_info *fs_info, u64 bytenr)
 282 {
 283         struct btrfs_block_group *bg;
 284         bool ret = true;
 285
 286         bg = btrfs_lookup_block_group(fs_info, bytenr);
 287         if (!bg)
 288                 return false;
 289
 290         spin_lock(&bg->lock);
 291         if (bg->ro)
 292                 ret = false;
 293         else
 294                 atomic_inc(&bg->nocow_writers);
 295         spin_unlock(&bg->lock);
 296
 297         /* No put on block group, done by btrfs_dec_nocow_writers */
 298         if (!ret)
 299                 btrfs_put_block_group(bg);
 300
 301         return ret;
 302 }
 303
 304 void btrfs_dec_nocow_writers(struct btrfs_fs_info *fs_info, u64 bytenr)
 305 {
 306         struct btrfs_block_group *bg;
 307
 308         bg = btrfs_lookup_block_group(fs_info, bytenr);
 309         ASSERT(bg);
 310         if (atomic_dec_and_test(&bg->nocow_writers))
 311                 wake_up_var(&bg->nocow_writers);
 312         /*
 313          * Once for our lookup and once for the lookup done by a previous call
 314          * to btrfs_inc_nocow_writers()
 315          */
 316         btrfs_put_block_group(bg);
 317         btrfs_put_block_group(bg);
 318 }
 319
 320 void btrfs_wait_nocow_writers(struct btrfs_block_group *bg)
 321 {
 322         wait_var_event(&bg->nocow_writers, !atomic_read(&bg->nocow_writers));
 323 }
 324
 325 void btrfs_dec_block_group_reservations(struct btrfs_fs_info *fs_info,
 326                                         const u64 start)
 327 {
 328         struct btrfs_block_group *bg;
 329
 330         bg = btrfs_lookup_block_group(fs_info, start);
 331         ASSERT(bg);
 332         if (atomic_dec_and_test(&bg->reservations))
 333                 wake_up_var(&bg->reservations);
 334         btrfs_put_block_group(bg);
 335 }
 336
 337 void btrfs_wait_block_group_reservations(struct btrfs_block_group *bg)
 338 {
 339         struct btrfs_space_info *space_info = bg->space_info;
 340
 341         ASSERT(bg->ro);
 342
 343         if (!(bg->flags & BTRFS_BLOCK_GROUP_DATA))
 344                 return;
 345
 346         /*
 347          * Our block group is read only but before we set it to read only,
 348          * some task might have had allocated an extent from it already, but it
 349          * has not yet created a respective ordered extent (and added it to a
 350          * root's list of ordered extents).
 351          * Therefore wait for any task currently allocating extents, since the
 352          * block group's reservations counter is incremented while a read lock
 353          * on the groups' semaphore is held and decremented after releasing
 354          * the read access on that semaphore and creating the ordered extent.
 355          */
 356         down_write(&space_info->groups_sem);
 357         up_write(&space_info->groups_sem);
 358
 359         wait_var_event(&bg->reservations, !atomic_read(&bg->reservations));
 360 }
 361
 362 struct btrfs_caching_control *btrfs_get_caching_control(
 363                 struct btrfs_block_group *cache)
 364 {
 365         struct btrfs_caching_control *ctl;
 366
 367         spin_lock(&cache->lock);
 368         if (!cache->caching_ctl) {
 369                 spin_unlock(&cache->lock);
 370                 return NULL;
 371         }
 372
 373         ctl = cache->caching_ctl;
 374         refcount_inc(&ctl->count);
 375         spin_unlock(&cache->lock);
 376         return ctl;
 377 }
 378
 379 void btrfs_put_caching_control(struct btrfs_caching_control *ctl)
 380 {
 381         if (refcount_dec_and_test(&ctl->count))
 382                 kfree(ctl);
 383 }
 384
 385 /*
 386  * When we wait for progress in the block group caching, its because our
 387  * allocation attempt failed at least once.  So, we must sleep and let some
 388  * progress happen before we try again.
 389  *
 390  * This function will sleep at least once waiting for new free space to show
 391  * up, and then it will check the block group free space numbers for our min
 392  * num_bytes.  Another option is to have it go ahead and look in the rbtree for
 393  * a free extent of a given size, but this is a good start.
 394  *
 395  * Callers of this must check if cache->cached == BTRFS_CACHE_ERROR before using
 396  * any of the information in this block group.
 397  */
 398 void btrfs_wait_block_group_cache_progress(struct btrfs_block_group *cache,
 399                                            u64 num_bytes)
 400 {
 401         struct btrfs_caching_control *caching_ctl;
 402
 403         caching_ctl = btrfs_get_caching_control(cache);
 404         if (!caching_ctl)
 405                 return;
 406
 407         wait_event(caching_ctl->wait, btrfs_block_group_done(cache) ||
 408                    (cache->free_space_ctl->free_space >= num_bytes));
 409
 410         btrfs_put_caching_control(caching_ctl);
 411 }
 412
 413 int btrfs_wait_block_group_cache_done(struct btrfs_block_group *cache)
 414 {
 415         struct btrfs_caching_control *caching_ctl;
 416         int ret = 0;
 417
 418         caching_ctl = btrfs_get_caching_control(cache);
 419         if (!caching_ctl)
 420                 return (cache->cached == BTRFS_CACHE_ERROR) ? -EIO : 0;
 421
 422         wait_event(caching_ctl->wait, btrfs_block_group_done(cache));
 423         if (cache->cached == BTRFS_CACHE_ERROR)
 424                 ret = -EIO;
 425         btrfs_put_caching_control(caching_ctl);
 426         return ret;
 427 }
 428
 429 #ifdef CONFIG_BTRFS_DEBUG
 430 static void fragment_free_space(struct btrfs_block_group *block_group)
 431 {
 432         struct btrfs_fs_info *fs_info = block_group->fs_info;
 433         u64 start = block_group->start;
 434         u64 len = block_group->length;
 435         u64 chunk = block_group->flags & BTRFS_BLOCK_GROUP_METADATA ?
 436                 fs_info->nodesize : fs_info->sectorsize;
 437         u64 step = chunk << 1;
 438
 439         while (len > chunk) {
 440                 btrfs_remove_free_space(block_group, start, chunk);
 441                 start += step;
 442                 if (len < step)
 443                         len = 0;
 444                 else
 445                         len -= step;
 446         }
 447 }
 448 #endif
 449
 450 /*
 451  * This is only called by btrfs_cache_block_group, since we could have freed
 452  * extents we need to check the pinned_extents for any extents that can't be
 453  * used yet since their free space will be released as soon as the transaction
 454  * commits.
 455  */
 456 u64 add_new_free_space(struct btrfs_block_group *block_group, u64 start, u64 end)
 457 {
 458         struct btrfs_fs_info *info = block_group->fs_info;
 459         u64 extent_start, extent_end, size, total_added = 0;
 460         int ret;
 461
 462         while (start < end) {
 463                 ret = find_first_extent_bit(&info->excluded_extents, start,
 464                                             &extent_start, &extent_end,
 465                                             EXTENT_DIRTY | EXTENT_UPTODATE,
 466                                             NULL);
 467                 if (ret)
 468                         break;
 469
 470                 if (extent_start <= start) {
 471                         start = extent_end + 1;
 472                 } else if (extent_start > start && extent_start < end) {
 473                         size = extent_start - start;
 474                         total_added += size;
 475                         ret = btrfs_add_free_space_async_trimmed(block_group,
 476                                                                  start, size);
 477                         BUG_ON(ret); /* -ENOMEM or logic error */
 478                         start = extent_end + 1;
 479                 } else {
 480                         break;
 481                 }
 482         }
 483
 484         if (start < end) {
 485                 size = end - start;
 486                 total_added += size;
 487                 ret = btrfs_add_free_space_async_trimmed(block_group, start,
 488                                                          size);
 489                 BUG_ON(ret); /* -ENOMEM or logic error */
 490         }
 491
 492         return total_added;
 493 }
 494
 495 static int load_extent_tree_free(struct btrfs_caching_control *caching_ctl)
 496 {
 497         struct btrfs_block_group *block_group = caching_ctl->block_group;
 498         struct btrfs_fs_info *fs_info = block_group->fs_info;
 499         struct btrfs_root *extent_root = fs_info->extent_root;
 500         struct btrfs_path *path;
 501         struct extent_buffer *leaf;
 502         struct btrfs_key key;
 503         u64 total_found = 0;
 504         u64 last = 0;
 505         u32 nritems;
 506         int ret;
 507         bool wakeup = true;
 508
 509         path = btrfs_alloc_path();
 510         if (!path)
 511                 return -ENOMEM;
 512
 513         last = max_t(u64, block_group->start, BTRFS_SUPER_INFO_OFFSET);
 514
 515 #ifdef CONFIG_BTRFS_DEBUG
 516         /*
 517          * If we're fragmenting we don't want to make anybody think we can
 518          * allocate from this block group until we've had a chance to fragment
 519          * the free space.
 520          */
 521         if (btrfs_should_fragment_free_space(block_group))
 522                 wakeup = false;
 523 #endif
 524         /*
 525          * We don't want to deadlock with somebody trying to allocate a new
 526          * extent for the extent root while also trying to search the extent
 527          * root to add free space.  So we skip locking and search the commit
 528          * root, since its read-only
 529          */
 530         path->skip_locking = 1;
 531         path->search_commit_root = 1;
 532         path->reada = READA_FORWARD;
 533
 534         key.objectid = last;
 535         key.offset = 0;
 536         key.type = BTRFS_EXTENT_ITEM_KEY;
 537
 538 next:
 539         ret = btrfs_search_slot(NULL, extent_root, &key, path, 0, 0);
 540         if (ret < 0)
 541                 goto out;
 542
 543         leaf = path->nodes[0];
 544         nritems = btrfs_header_nritems(leaf);
 545
 546         while (1) {
 547                 if (btrfs_fs_closing(fs_info) > 1) {
 548                         last = (u64)-1;
 549                         break;
 550                 }
 551
 552                 if (path->slots[0] < nritems) {
 553                         btrfs_item_key_to_cpu(leaf, &key, path->slots[0]);
 554                 } else {
 555                         ret = btrfs_find_next_key(extent_root, path, &key, 0, 0);
 556                         if (ret)
 557                                 break;
 558
 559                         if (need_resched() ||
 560                             rwsem_is_contended(&fs_info->commit_root_sem)) {
 561                                 if (wakeup)
 562                                         caching_ctl->progress = last;
 563                                 btrfs_release_path(path);
 564                                 up_read(&fs_info->commit_root_sem);
 565                                 mutex_unlock(&caching_ctl->mutex);
 566                                 cond_resched();
 567                                 mutex_lock(&caching_ctl->mutex);
 568                                 down_read(&fs_info->commit_root_sem);
 569                                 goto next;
 570                         }
 571
 572                         ret = btrfs_next_leaf(extent_root, path);
 573                         if (ret < 0)
 574                                 goto out;
 575                         if (ret)
 576                                 break;
 577                         leaf = path->nodes[0];
 578                         nritems = btrfs_header_nritems(leaf);
 579                         continue;
 580                 }
 581
 582                 if (key.objectid < last) {
 583                         key.objectid = last;
 584                         key.offset = 0;
 585                         key.type = BTRFS_EXTENT_ITEM_KEY;
 586
 587                         if (wakeup)
 588                                 caching_ctl->progress = last;
 589                         btrfs_release_path(path);
 590                         goto next;
 591                 }
 592
 593                 if (key.objectid < block_group->start) {
 594                         path->slots[0]++;
 595                         continue;
 596                 }
 597
 598                 if (key.objectid >= block_group->start + block_group->length)
 599                         break;
 600
 601                 if (key.type == BTRFS_EXTENT_ITEM_KEY ||
 602                     key.type == BTRFS_METADATA_ITEM_KEY) {
 603                         total_found += add_new_free_space(block_group, last,
 604                                                           key.objectid);
 605                         if (key.type == BTRFS_METADATA_ITEM_KEY)
 606                                 last = key.objectid +
 607                                         fs_info->nodesize;
 608                         else
 609                                 last = key.objectid + key.offset;
 610
 611                         if (total_found > CACHING_CTL_WAKE_UP) {
 612                                 total_found = 0;
 613                                 if (wakeup)
 614                                         wake_up(&caching_ctl->wait);
 615                         }
 616                 }
 617                 path->slots[0]++;
 618         }
 619         ret = 0;
 620
 621         total_found += add_new_free_space(block_group, last,
 622                                 block_group->start + block_group->length);
 623         caching_ctl->progress = (u64)-1;
 624
 625 out:
 626         btrfs_free_path(path);
 627         return ret;
 628 }
 629
 630 static noinline void caching_thread(struct btrfs_work *work)
 631 {
 632         struct btrfs_block_group *block_group;
 633         struct btrfs_fs_info *fs_info;
 634         struct btrfs_caching_control *caching_ctl;
 635         int ret;
 636
 637         caching_ctl = container_of(work, struct btrfs_caching_control, work);
 638         block_group = caching_ctl->block_group;
 639         fs_info = block_group->fs_info;
 640
 641         mutex_lock(&caching_ctl->mutex);
 642         down_read(&fs_info->commit_root_sem);
 643
 644         if (btrfs_fs_compat_ro(fs_info, FREE_SPACE_TREE))
 645                 ret = load_free_space_tree(caching_ctl);
 646         else
 647                 ret = load_extent_tree_free(caching_ctl);
 648
 649         spin_lock(&block_group->lock);
 650         block_group->caching_ctl = NULL;
 651         block_group->cached = ret ? BTRFS_CACHE_ERROR : BTRFS_CACHE_FINISHED;
 652         spin_unlock(&block_group->lock);
 653
 654 #ifdef CONFIG_BTRFS_DEBUG
 655         if (btrfs_should_fragment_free_space(block_group)) {
 656                 u64 bytes_used;
 657
 658                 spin_lock(&block_group->space_info->lock);
 659                 spin_lock(&block_group->lock);
 660                 bytes_used = block_group->length - block_group->used;
 661                 block_group->space_info->bytes_used += bytes_used >> 1;
 662                 spin_unlock(&block_group->lock);
 663                 spin_unlock(&block_group->space_info->lock);
 664                 fragment_free_space(block_group);
 665         }
 666 #endif
 667
 668         caching_ctl->progress = (u64)-1;
 669
 670         up_read(&fs_info->commit_root_sem);
 671         btrfs_free_excluded_extents(block_group);
 672         mutex_unlock(&caching_ctl->mutex);
 673
 674         wake_up(&caching_ctl->wait);
 675
 676         btrfs_put_caching_control(caching_ctl);
 677         btrfs_put_block_group(block_group);
 678 }
 679
 680 int btrfs_cache_block_group(struct btrfs_block_group *cache, int load_cache_only)
 681 {
 682         DEFINE_WAIT(wait);
 683         struct btrfs_fs_info *fs_info = cache->fs_info;
 684         struct btrfs_caching_control *caching_ctl;
 685         int ret = 0;
 686
 687         caching_ctl = kzalloc(sizeof(*caching_ctl), GFP_NOFS);
 688         if (!caching_ctl)
 689                 return -ENOMEM;
 690
 691         INIT_LIST_HEAD(&caching_ctl->list);
 692         mutex_init(&caching_ctl->mutex);
 693         init_waitqueue_head(&caching_ctl->wait);
 694         caching_ctl->block_group = cache;
 695         caching_ctl->progress = cache->start;
 696         refcount_set(&caching_ctl->count, 1);
 697         btrfs_init_work(&caching_ctl->work, caching_thread, NULL, NULL);
 698
 699         spin_lock(&cache->lock);
 700         /*
 701          * This should be a rare occasion, but this could happen I think in the
 702          * case where one thread starts to load the space cache info, and then
 703          * some other thread starts a transaction commit which tries to do an
 704          * allocation while the other thread is still loading the space cache
 705          * info.  The previous loop should have kept us from choosing this block
 706          * group, but if we've moved to the state where we will wait on caching
 707          * block groups we need to first check if we're doing a fast load here,
 708          * so we can wait for it to finish, otherwise we could end up allocating
 709          * from a block group who's cache gets evicted for one reason or
 710          * another.
 711          */
 712         while (cache->cached == BTRFS_CACHE_FAST) {
 713                 struct btrfs_caching_control *ctl;
 714
 715                 ctl = cache->caching_ctl;
 716                 refcount_inc(&ctl->count);
 717                 prepare_to_wait(&ctl->wait, &wait, TASK_UNINTERRUPTIBLE);
 718                 spin_unlock(&cache->lock);
 719
 720                 schedule();
 721
 722                 finish_wait(&ctl->wait, &wait);
 723                 btrfs_put_caching_control(ctl);
 724                 spin_lock(&cache->lock);
 725         }
 726
 727         if (cache->cached != BTRFS_CACHE_NO) {
 728                 spin_unlock(&cache->lock);
 729                 kfree(caching_ctl);
 730                 return 0;
 731         }
 732         WARN_ON(cache->caching_ctl);
 733         cache->caching_ctl = caching_ctl;
 734         cache->cached = BTRFS_CACHE_FAST;
 735         spin_unlock(&cache->lock);
 736
 737         if (btrfs_test_opt(fs_info, SPACE_CACHE)) {
 738                 mutex_lock(&caching_ctl->mutex);
 739                 ret = load_free_space_cache(cache);
 740
 741                 spin_lock(&cache->lock);
 742                 if (ret == 1) {
 743                         cache->caching_ctl = NULL;
 744                         cache->cached = BTRFS_CACHE_FINISHED;
 745                         cache->last_byte_to_unpin = (u64)-1;
 746                         caching_ctl->progress = (u64)-1;
 747                 } else {
 748                         if (load_cache_only) {
 749                                 cache->caching_ctl = NULL;
 750                                 cache->cached = BTRFS_CACHE_NO;
 751                         } else {
 752                                 cache->cached = BTRFS_CACHE_STARTED;
 753                                 cache->has_caching_ctl = 1;
 754                         }
 755                 }
 756                 spin_unlock(&cache->lock);
 757 #ifdef CONFIG_BTRFS_DEBUG
 758                 if (ret == 1 &&
 759                     btrfs_should_fragment_free_space(cache)) {
 760                         u64 bytes_used;
 761
 762                         spin_lock(&cache->space_info->lock);
 763                         spin_lock(&cache->lock);
 764                         bytes_used = cache->length - cache->used;
 765                         cache->space_info->bytes_used += bytes_used >> 1;
 766                         spin_unlock(&cache->lock);
 767                         spin_unlock(&cache->space_info->lock);
 768                         fragment_free_space(cache);
 769                 }
 770 #endif
 771                 mutex_unlock(&caching_ctl->mutex);
 772
 773                 wake_up(&caching_ctl->wait);
 774                 if (ret == 1) {
 775                         btrfs_put_caching_control(caching_ctl);
 776                         btrfs_free_excluded_extents(cache);
 777                         return 0;
 778                 }
 779         } else {
 780                 /*
 781                  * We're either using the free space tree or no caching at all.
 782                  * Set cached to the appropriate value and wakeup any waiters.
 783                  */
 784                 spin_lock(&cache->lock);
 785                 if (load_cache_only) {
 786                         cache->caching_ctl = NULL;
 787                         cache->cached = BTRFS_CACHE_NO;
 788                 } else {
 789                         cache->cached = BTRFS_CACHE_STARTED;
 790                         cache->has_caching_ctl = 1;
 791                 }
 792                 spin_unlock(&cache->lock);
 793                 wake_up(&caching_ctl->wait);
 794         }
 795
 796         if (load_cache_only) {
 797                 btrfs_put_caching_control(caching_ctl);
 798                 return 0;
 799         }
 800
 801         down_write(&fs_info->commit_root_sem);
 802         refcount_inc(&caching_ctl->count);
 803         list_add_tail(&caching_ctl->list, &fs_info->caching_block_groups);
 804         up_write(&fs_info->commit_root_sem);
 805
 806         btrfs_get_block_group(cache);
 807
 808         btrfs_queue_work(fs_info->caching_workers, &caching_ctl->work);
 809
 810         return ret;
 811 }
 812
 813 static void clear_avail_alloc_bits(struct btrfs_fs_info *fs_info, u64 flags)
 814 {
 815         u64 extra_flags = chunk_to_extended(flags) &
 816                                 BTRFS_EXTENDED_PROFILE_MASK;
 817
 818         write_seqlock(&fs_info->profiles_lock);
 819         if (flags & BTRFS_BLOCK_GROUP_DATA)
 820                 fs_info->avail_data_alloc_bits &= ~extra_flags;
 821         if (flags & BTRFS_BLOCK_GROUP_METADATA)
 822                 fs_info->avail_metadata_alloc_bits &= ~extra_flags;
 823         if (flags & BTRFS_BLOCK_GROUP_SYSTEM)
 824                 fs_info->avail_system_alloc_bits &= ~extra_flags;
 825         write_sequnlock(&fs_info->profiles_lock);
 826 }
 827
 828 /*
 829  * Clear incompat bits for the following feature(s):
 830  *
 831  * - RAID56 - in case there's neither RAID5 nor RAID6 profile block group
 832  *            in the whole filesystem
 833  *
 834  * - RAID1C34 - same as above for RAID1C3 and RAID1C4 block groups
 835  */
 836 static void clear_incompat_bg_bits(struct btrfs_fs_info *fs_info, u64 flags)
 837 {
 838         bool found_raid56 = false;
 839         bool found_raid1c34 = false;
 840
 841         if ((flags & BTRFS_BLOCK_GROUP_RAID56_MASK) ||
 842             (flags & BTRFS_BLOCK_GROUP_RAID1C3) ||
 843             (flags & BTRFS_BLOCK_GROUP_RAID1C4)) {
 844                 struct list_head *head = &fs_info->space_info;
 845                 struct btrfs_space_info *sinfo;
 846
 847                 list_for_each_entry_rcu(sinfo, head, list) {
 848                         down_read(&sinfo->groups_sem);
 849                         if (!list_empty(&sinfo->block_groups[BTRFS_RAID_RAID5]))
 850                                 found_raid56 = true;
 851                         if (!list_empty(&sinfo->block_groups[BTRFS_RAID_RAID6]))
 852                                 found_raid56 = true;
 853                         if (!list_empty(&sinfo->block_groups[BTRFS_RAID_RAID1C3]))
 854                                 found_raid1c34 = true;
 855                         if (!list_empty(&sinfo->block_groups[BTRFS_RAID_RAID1C4]))
 856                                 found_raid1c34 = true;
 857                         up_read(&sinfo->groups_sem);
 858                 }
 859                 if (!found_raid56)
 860                         btrfs_clear_fs_incompat(fs_info, RAID56);
 861                 if (!found_raid1c34)
 862                         btrfs_clear_fs_incompat(fs_info, RAID1C34);
 863         }
 864 }
 865
 866 int btrfs_remove_block_group(struct btrfs_trans_handle *trans,
 867                              u64 group_start, struct extent_map *em)
 868 {
 869         struct btrfs_fs_info *fs_info = trans->fs_info;
 870         struct btrfs_root *root = fs_info->extent_root;
 871         struct btrfs_path *path;
 872         struct btrfs_block_group *block_group;
 873         struct btrfs_free_cluster *cluster;
 874         struct btrfs_root *tree_root = fs_info->tree_root;
 875         struct btrfs_key key;
 876         struct inode *inode;
 877         struct kobject *kobj = NULL;
 878         int ret;
 879         int index;
 880         int factor;
 881         struct btrfs_caching_control *caching_ctl = NULL;
 882         bool remove_em;
 883         bool remove_rsv = false;
 884
 885         block_group = btrfs_lookup_block_group(fs_info, group_start);
 886         BUG_ON(!block_group);
 887         BUG_ON(!block_group->ro);
 888
 889         trace_btrfs_remove_block_group(block_group);
 890         /*
 891          * Free the reserved super bytes from this block group before
 892          * remove it.
 893          */
 894         btrfs_free_excluded_extents(block_group);
 895         btrfs_free_ref_tree_range(fs_info, block_group->start,
 896                                   block_group->length);
 897
 898         index = btrfs_bg_flags_to_raid_index(block_group->flags);
 899         factor = btrfs_bg_type_to_factor(block_group->flags);
 900
 901         /* make sure this block group isn't part of an allocation cluster */
 902         cluster = &fs_info->data_alloc_cluster;
 903         spin_lock(&cluster->refill_lock);
 904         btrfs_return_cluster_to_free_space(block_group, cluster);
 905         spin_unlock(&cluster->refill_lock);
 906
 907         /*
 908          * make sure this block group isn't part of a metadata
 909          * allocation cluster
 910          */
 911         cluster = &fs_info->meta_alloc_cluster;
 912         spin_lock(&cluster->refill_lock);
 913         btrfs_return_cluster_to_free_space(block_group, cluster);
 914         spin_unlock(&cluster->refill_lock);
 915
 916         path = btrfs_alloc_path();
 917         if (!path) {
 918                 ret = -ENOMEM;
 919                 goto out;
 920         }
 921
 922         /*
 923          * get the inode first so any iput calls done for the io_list
 924          * aren't the final iput (no unlinks allowed now)
 925          */
 926         inode = lookup_free_space_inode(block_group, path);
 927
 928         mutex_lock(&trans->transaction->cache_write_mutex);
 929         /*
 930          * Make sure our free space cache IO is done before removing the
 931          * free space inode
 932          */
 933         spin_lock(&trans->transaction->dirty_bgs_lock);
 934         if (!list_empty(&block_group->io_list)) {
 935                 list_del_init(&block_group->io_list);
 936
 937                 WARN_ON(!IS_ERR(inode) && inode != block_group->io_ctl.inode);
 938
 939                 spin_unlock(&trans->transaction->dirty_bgs_lock);
 940                 btrfs_wait_cache_io(trans, block_group, path);
 941                 btrfs_put_block_group(block_group);
 942                 spin_lock(&trans->transaction->dirty_bgs_lock);
 943         }
 944
 945         if (!list_empty(&block_group->dirty_list)) {
 946                 list_del_init(&block_group->dirty_list);
 947                 remove_rsv = true;
 948                 btrfs_put_block_group(block_group);
 949         }
 950         spin_unlock(&trans->transaction->dirty_bgs_lock);
 951         mutex_unlock(&trans->transaction->cache_write_mutex);
 952
 953         if (!IS_ERR(inode)) {
 954                 ret = btrfs_orphan_add(trans, BTRFS_I(inode));
 955                 if (ret) {
 956                         btrfs_add_delayed_iput(inode);
 957                         goto out;
 958                 }
 959                 clear_nlink(inode);
 960                 /* One for the block groups ref */
 961                 spin_lock(&block_group->lock);
 962                 if (block_group->iref) {
 963                         block_group->iref = 0;
 964                         block_group->inode = NULL;
 965                         spin_unlock(&block_group->lock);
 966                         iput(inode);
 967                 } else {
 968                         spin_unlock(&block_group->lock);
 969                 }
 970                 /* One for our lookup ref */
 971                 btrfs_add_delayed_iput(inode);
 972         }
 973
 974         key.objectid = BTRFS_FREE_SPACE_OBJECTID;
 975         key.type = 0;
 976         key.offset = block_group->start;
 977
 978         ret = btrfs_search_slot(trans, tree_root, &key, path, -1, 1);
 979         if (ret < 0)
 980                 goto out;
 981         if (ret > 0)
 982                 btrfs_release_path(path);
 983         if (ret == 0) {
 984                 ret = btrfs_del_item(trans, tree_root, path);
 985                 if (ret)
 986                         goto out;
 987                 btrfs_release_path(path);
 988         }
 989
 990         spin_lock(&fs_info->block_group_cache_lock);
 991         rb_erase(&block_group->cache_node,
 992                  &fs_info->block_group_cache_tree);
 993         RB_CLEAR_NODE(&block_group->cache_node);
 994
 995         if (fs_info->first_logical_byte == block_group->start)
 996                 fs_info->first_logical_byte = (u64)-1;
 997         spin_unlock(&fs_info->block_group_cache_lock);
 998
 999         down_write(&block_group->space_info->groups_sem);
1000         /*
1001          * we must use list_del_init so people can check to see if they
1002          * are still on the list after taking the semaphore
1003          */
1004         list_del_init(&block_group->list);
1005         if (list_empty(&block_group->space_info->block_groups[index])) {
1006                 kobj = block_group->space_info->block_group_kobjs[index];
1007                 block_group->space_info->block_group_kobjs[index] = NULL;
1008                 clear_avail_alloc_bits(fs_info, block_group->flags);
1009         }
1010         up_write(&block_group->space_info->groups_sem);
1011         clear_incompat_bg_bits(fs_info, block_group->flags);
1012         if (kobj) {
1013                 kobject_del(kobj);
1014                 kobject_put(kobj);
1015         }
1016
1017         if (block_group->has_caching_ctl)
1018                 caching_ctl = btrfs_get_caching_control(block_group);
1019         if (block_group->cached == BTRFS_CACHE_STARTED)
1020                 btrfs_wait_block_group_cache_done(block_group);
1021         if (block_group->has_caching_ctl) {
1022                 down_write(&fs_info->commit_root_sem);
1023                 if (!caching_ctl) {
1024                         struct btrfs_caching_control *ctl;
1025
1026                         list_for_each_entry(ctl,
1027                                     &fs_info->caching_block_groups, list)
1028                                 if (ctl->block_group == block_group) {
1029                                         caching_ctl = ctl;
1030                                         refcount_inc(&caching_ctl->count);
1031                                         break;
1032                                 }
1033                 }
1034                 if (caching_ctl)
1035                         list_del_init(&caching_ctl->list);
1036                 up_write(&fs_info->commit_root_sem);
1037                 if (caching_ctl) {
1038                         /* Once for the caching bgs list and once for us. */
1039                         btrfs_put_caching_control(caching_ctl);
1040                         btrfs_put_caching_control(caching_ctl);
1041                 }
1042         }
1043
1044         spin_lock(&trans->transaction->dirty_bgs_lock);
1045         WARN_ON(!list_empty(&block_group->dirty_list));
1046         WARN_ON(!list_empty(&block_group->io_list));
1047         spin_unlock(&trans->transaction->dirty_bgs_lock);
1048
1049         btrfs_remove_free_space_cache(block_group);
1050
1051         spin_lock(&block_group->space_info->lock);
1052         list_del_init(&block_group->ro_list);
1053
1054         if (btrfs_test_opt(fs_info, ENOSPC_DEBUG)) {
1055                 WARN_ON(block_group->space_info->total_bytes
1056                         < block_group->length);
1057                 WARN_ON(block_group->space_info->bytes_readonly
1058                         < block_group->length);
1059                 WARN_ON(block_group->space_info->disk_total
1060                         < block_group->length * factor);
1061         }
1062         block_group->space_info->total_bytes -= block_group->length;
1063         block_group->space_info->bytes_readonly -= block_group->length;
1064         block_group->space_info->disk_total -= block_group->length * factor;
1065
1066         spin_unlock(&block_group->space_info->lock);
1067
1068         key.objectid = block_group->start;
1069         key.type = BTRFS_BLOCK_GROUP_ITEM_KEY;
1070         key.offset = block_group->length;
1071
1072         mutex_lock(&fs_info->chunk_mutex);
1073         spin_lock(&block_group->lock);
1074         block_group->removed = 1;
1075         /*
1076          * At this point trimming can't start on this block group, because we
1077          * removed the block group from the tree fs_info->block_group_cache_tree
1078          * so no one can't find it anymore and even if someone already got this
1079          * block group before we removed it from the rbtree, they have already
1080          * incremented block_group->trimming - if they didn't, they won't find
1081          * any free space entries because we already removed them all when we
1082          * called btrfs_remove_free_space_cache().
1083          *
1084          * And we must not remove the extent map from the fs_info->mapping_tree
1085          * to prevent the same logical address range and physical device space
1086          * ranges from being reused for a new block group. This is because our
1087          * fs trim operation (btrfs_trim_fs() / btrfs_ioctl_fitrim()) is
1088          * completely transactionless, so while it is trimming a range the
1089          * currently running transaction might finish and a new one start,
1090          * allowing for new block groups to be created that can reuse the same
1091          * physical device locations unless we take this special care.
1092          *
1093          * There may also be an implicit trim operation if the file system
1094          * is mounted with -odiscard. The same protections must remain
1095          * in place until the extents have been discarded completely when
1096          * the transaction commit has completed.
1097          */
1098         remove_em = (atomic_read(&block_group->trimming) == 0);
1099         spin_unlock(&block_group->lock);
1100
1101         mutex_unlock(&fs_info->chunk_mutex);
1102
1103         ret = remove_block_group_free_space(trans, block_group);
1104         if (ret)
1105                 goto out;
1106
1107         btrfs_put_block_group(block_group);
1108         btrfs_put_block_group(block_group);
1109
1110         ret = btrfs_search_slot(trans, root, &key, path, -1, 1);
1111         if (ret > 0)
1112                 ret = -EIO;
1113         if (ret < 0)
1114                 goto out;
1115
1116         ret = btrfs_del_item(trans, root, path);
1117         if (ret)
1118                 goto out;
1119
1120         if (remove_em) {
1121                 struct extent_map_tree *em_tree;
1122
1123                 em_tree = &fs_info->mapping_tree;
1124                 write_lock(&em_tree->lock);
1125                 remove_extent_mapping(em_tree, em);
1126                 write_unlock(&em_tree->lock);
1127                 /* once for the tree */
1128                 free_extent_map(em);
1129         }
1130 out:
1131         if (remove_rsv)
1132                 btrfs_delayed_refs_rsv_release(fs_info, 1);
1133         btrfs_free_path(path);
1134         return ret;
1135 }
1136
1137 struct btrfs_trans_handle *btrfs_start_trans_remove_block_group(
1138                 struct btrfs_fs_info *fs_info, const u64 chunk_offset)
1139 {
1140         struct extent_map_tree *em_tree = &fs_info->mapping_tree;
1141         struct extent_map *em;
1142         struct map_lookup *map;
1143         unsigned int num_items;
1144
1145         read_lock(&em_tree->lock);
1146         em = lookup_extent_mapping(em_tree, chunk_offset, 1);
1147         read_unlock(&em_tree->lock);
1148         ASSERT(em && em->start == chunk_offset);
1149
1150         /*
1151          * We need to reserve 3 + N units from the metadata space info in order
1152          * to remove a block group (done at btrfs_remove_chunk() and at
1153          * btrfs_remove_block_group()), which are used for:
1154          *
1155          * 1 unit for adding the free space inode's orphan (located in the tree
1156          * of tree roots).
1157          * 1 unit for deleting the block group item (located in the extent
1158          * tree).
1159          * 1 unit for deleting the free space item (located in tree of tree
1160          * roots).
1161          * N units for deleting N device extent items corresponding to each
1162          * stripe (located in the device tree).
1163          *
1164          * In order to remove a block group we also need to reserve units in the
1165          * system space info in order to update the chunk tree (update one or
1166          * more device items and remove one chunk item), but this is done at
1167          * btrfs_remove_chunk() through a call to check_system_chunk().
1168          */
1169         map = em->map_lookup;
1170         num_items = 3 + map->num_stripes;
1171         free_extent_map(em);
1172
1173         return btrfs_start_transaction_fallback_global_rsv(fs_info->extent_root,
1174                                                            num_items, 1);
1175 }
1176
1177 /*
1178  * Mark block group @cache read-only, so later write won't happen to block
1179  * group @cache.
1180  *
1181  * If @force is not set, this function will only mark the block group readonly
1182  * if we have enough free space (1M) in other metadata/system block groups.
1183  * If @force is not set, this function will mark the block group readonly
1184  * without checking free space.
1185  *
1186  * NOTE: This function doesn't care if other block groups can contain all the
1187  * data in this block group. That check should be done by relocation routine,
1188  * not this function.
1189  */
1190 static int inc_block_group_ro(struct btrfs_block_group *cache, int force)
1191 {
1192         struct btrfs_space_info *sinfo = cache->space_info;
1193         u64 num_bytes;
1194         int ret = -ENOSPC;
1195
1196         spin_lock(&sinfo->lock);
1197         spin_lock(&cache->lock);
1198
1199         if (cache->ro) {
1200                 cache->ro++;
1201                 ret = 0;
1202                 goto out;
1203         }
1204
1205         num_bytes = cache->length - cache->reserved - cache->pinned -
1206                     cache->bytes_super - cache->used;
1207
1208         /*
1209          * Data never overcommits, even in mixed mode, so do just the straight
1210          * check of left over space in how much we have allocated.
1211          */
1212         if (force) {
1213                 ret = 0;
1214         } else if (sinfo->flags & BTRFS_BLOCK_GROUP_DATA) {
1215                 u64 sinfo_used = btrfs_space_info_used(sinfo, true);
1216
1217                 /*
1218                  * Here we make sure if we mark this bg RO, we still have enough
1219                  * free space as buffer.
1220                  */
1221                 if (sinfo_used + num_bytes <= sinfo->total_bytes)
1222                         ret = 0;
1223         } else {
1224                 /*
1225                  * We overcommit metadata, so we need to do the
1226                  * btrfs_can_overcommit check here, and we need to pass in
1227                  * BTRFS_RESERVE_NO_FLUSH to give ourselves the most amount of
1228                  * leeway to allow us to mark this block group as read only.
1229                  */
1230                 if (btrfs_can_overcommit(cache->fs_info, sinfo, num_bytes,
1231                                          BTRFS_RESERVE_NO_FLUSH))
1232                         ret = 0;
1233         }
1234
1235         if (!ret) {
1236                 sinfo->bytes_readonly += num_bytes;
1237                 cache->ro++;
1238                 list_add_tail(&cache->ro_list, &sinfo->ro_bgs);
1239         }
1240 out:
1241         spin_unlock(&cache->lock);
1242         spin_unlock(&sinfo->lock);
1243         if (ret == -ENOSPC && btrfs_test_opt(cache->fs_info, ENOSPC_DEBUG)) {
1244                 btrfs_info(cache->fs_info,
1245                         "unable to make block group %llu ro", cache->start);
1246                 btrfs_dump_space_info(cache->fs_info, cache->space_info, 0, 0);
1247         }
1248         return ret;
1249 }
1250
1251 static bool clean_pinned_extents(struct btrfs_trans_handle *trans,
1252                                  struct btrfs_block_group *bg)
1253 {
1254         struct btrfs_fs_info *fs_info = bg->fs_info;
1255         struct btrfs_transaction *prev_trans = NULL;
1256         const u64 start = bg->start;
1257         const u64 end = start + bg->length - 1;
1258         int ret;
1259
1260         spin_lock(&fs_info->trans_lock);
1261         if (trans->transaction->list.prev != &fs_info->trans_list) {
1262                 prev_trans = list_last_entry(&trans->transaction->list,
1263                                              struct btrfs_transaction, list);
1264                 refcount_inc(&prev_trans->use_count);
1265         }
1266         spin_unlock(&fs_info->trans_lock);
1267
1268         /*
1269          * Hold the unused_bg_unpin_mutex lock to avoid racing with
1270          * btrfs_finish_extent_commit(). If we are at transaction N, another
1271          * task might be running finish_extent_commit() for the previous
1272          * transaction N - 1, and have seen a range belonging to the block
1273          * group in pinned_extents before we were able to clear the whole block
1274          * group range from pinned_extents. This means that task can lookup for
1275          * the block group after we unpinned it from pinned_extents and removed
1276          * it, leading to a BUG_ON() at unpin_extent_range().
1277          */
1278         mutex_lock(&fs_info->unused_bg_unpin_mutex);
1279         if (prev_trans) {
1280                 ret = clear_extent_bits(&prev_trans->pinned_extents, start, end,
1281                                         EXTENT_DIRTY);
1282                 if (ret)
1283                         goto err;
1284         }
1285
1286         ret = clear_extent_bits(&trans->transaction->pinned_extents, start, end,
1287                                 EXTENT_DIRTY);
1288         if (ret)
1289                 goto err;
1290         mutex_unlock(&fs_info->unused_bg_unpin_mutex);
1291
1292         return true;
1293
1294 err:
1295         mutex_unlock(&fs_info->unused_bg_unpin_mutex);
1296         btrfs_dec_block_group_ro(bg);
1297         return false;
1298 }
1299
1300 /*
1301  * Process the unused_bgs list and remove any that don't have any allocated
1302  * space inside of them.
1303  */
1304 void btrfs_delete_unused_bgs(struct btrfs_fs_info *fs_info)
1305 {
1306         struct btrfs_block_group *block_group;
1307         struct btrfs_space_info *space_info;
1308         struct btrfs_trans_handle *trans;
1309         const bool async_trim_enabled = btrfs_test_opt(fs_info, DISCARD_ASYNC);
1310         int ret = 0;
1311
1312         if (!test_bit(BTRFS_FS_OPEN, &fs_info->flags))
1313                 return;
1314
1315         spin_lock(&fs_info->unused_bgs_lock);
1316         while (!list_empty(&fs_info->unused_bgs)) {
1317                 int trimming;
1318
1319                 block_group = list_first_entry(&fs_info->unused_bgs,
1320                                                struct btrfs_block_group,
1321                                                bg_list);
1322                 list_del_init(&block_group->bg_list);
1323
1324                 space_info = block_group->space_info;
1325
1326                 if (ret || btrfs_mixed_space_info(space_info)) {
1327                         btrfs_put_block_group(block_group);
1328                         continue;
1329                 }
1330                 spin_unlock(&fs_info->unused_bgs_lock);
1331
1332                 btrfs_discard_cancel_work(&fs_info->discard_ctl, block_group);
1333
1334                 mutex_lock(&fs_info->delete_unused_bgs_mutex);
1335
1336                 /* Don't want to race with allocators so take the groups_sem */
1337                 down_write(&space_info->groups_sem);
1338
1339                 /*
1340                  * Async discard moves the final block group discard to be prior
1341                  * to the unused_bgs code path.  Therefore, if it's not fully
1342                  * trimmed, punt it back to the async discard lists.
1343                  */
1344                 if (btrfs_test_opt(fs_info, DISCARD_ASYNC) &&
1345                     !btrfs_is_free_space_trimmed(block_group)) {
1346                         trace_btrfs_skip_unused_block_group(block_group);
1347                         up_write(&space_info->groups_sem);
1348                         /* Requeue if we failed because of async discard */
1349                         btrfs_discard_queue_work(&fs_info->discard_ctl,
1350                                                  block_group);
1351                         goto next;
1352                 }
1353
1354                 spin_lock(&block_group->lock);
1355                 if (block_group->reserved || block_group->pinned ||
1356                     block_group->used || block_group->ro ||
1357                     list_is_singular(&block_group->list)) {
1358                         /*
1359                          * We want to bail if we made new allocations or have
1360                          * outstanding allocations in this block group.  We do
1361                          * the ro check in case balance is currently acting on
1362                          * this block group.
1363                          */
1364                         trace_btrfs_skip_unused_block_group(block_group);
1365                         spin_unlock(&block_group->lock);
1366                         up_write(&space_info->groups_sem);
1367                         goto next;
1368                 }
1369                 spin_unlock(&block_group->lock);
1370
1371                 /* We don't want to force the issue, only flip if it's ok. */
1372                 ret = inc_block_group_ro(block_group, 0);
1373                 up_write(&space_info->groups_sem);
1374                 if (ret < 0) {
1375                         ret = 0;
1376                         goto next;
1377                 }
1378
1379                 /*
1380                  * Want to do this before we do anything else so we can recover
1381                  * properly if we fail to join the transaction.
1382                  */
1383                 trans = btrfs_start_trans_remove_block_group(fs_info,
1384                                                      block_group->start);
1385                 if (IS_ERR(trans)) {
1386                         btrfs_dec_block_group_ro(block_group);
1387                         ret = PTR_ERR(trans);
1388                         goto next;
1389                 }
1390
1391                 /*
1392                  * We could have pending pinned extents for this block group,
1393                  * just delete them, we don't care about them anymore.
1394                  */
1395                 if (!clean_pinned_extents(trans, block_group))
1396                         goto end_trans;
1397
1398                 /*
1399                  * At this point, the block_group is read only and should fail
1400                  * new allocations.  However, btrfs_finish_extent_commit() can
1401                  * cause this block_group to be placed back on the discard
1402                  * lists because now the block_group isn't fully discarded.
1403                  * Bail here and try again later after discarding everything.
1404                  */
1405                 spin_lock(&fs_info->discard_ctl.lock);
1406                 if (!list_empty(&block_group->discard_list)) {
1407                         spin_unlock(&fs_info->discard_ctl.lock);
1408                         btrfs_dec_block_group_ro(block_group);
1409                         btrfs_discard_queue_work(&fs_info->discard_ctl,
1410                                                  block_group);
1411                         goto end_trans;
1412                 }
1413                 spin_unlock(&fs_info->discard_ctl.lock);
1414
1415                 /* Reset pinned so btrfs_put_block_group doesn't complain */
1416                 spin_lock(&space_info->lock);
1417                 spin_lock(&block_group->lock);
1418
1419                 btrfs_space_info_update_bytes_pinned(fs_info, space_info,
1420                                                      -block_group->pinned);
1421                 space_info->bytes_readonly += block_group->pinned;
1422                 percpu_counter_add_batch(&space_info->total_bytes_pinned,
1423                                    -block_group->pinned,
1424                                    BTRFS_TOTAL_BYTES_PINNED_BATCH);
1425                 block_group->pinned = 0;
1426
1427                 spin_unlock(&block_group->lock);
1428                 spin_unlock(&space_info->lock);
1429
1430                 /*
1431                  * The normal path here is an unused block group is passed here,
1432                  * then trimming is handled in the transaction commit path.
1433                  * Async discard interposes before this to do the trimming
1434                  * before coming down the unused block group path as trimming
1435                  * will no longer be done later in the transaction commit path.
1436                  */
1437                 if (!async_trim_enabled && btrfs_test_opt(fs_info, DISCARD_ASYNC))
1438                         goto flip_async;
1439
1440                 /* DISCARD can flip during remount */
1441                 trimming = btrfs_test_opt(fs_info, DISCARD_SYNC);
1442
1443                 /* Implicit trim during transaction commit. */
1444                 if (trimming)
1445                         btrfs_get_block_group_trimming(block_group);
1446
1447                 /*
1448                  * Btrfs_remove_chunk will abort the transaction if things go
1449                  * horribly wrong.
1450                  */
1451                 ret = btrfs_remove_chunk(trans, block_group->start);
1452
1453                 if (ret) {
1454                         if (trimming)
1455                                 btrfs_put_block_group_trimming(block_group);
1456                         goto end_trans;
1457                 }
1458
1459                 /*
1460                  * If we're not mounted with -odiscard, we can just forget
1461                  * about this block group. Otherwise we'll need to wait
1462                  * until transaction commit to do the actual discard.
1463                  */
1464                 if (trimming) {
1465                         spin_lock(&fs_info->unused_bgs_lock);
1466                         /*
1467                          * A concurrent scrub might have added us to the list
1468                          * fs_info->unused_bgs, so use a list_move operation
1469                          * to add the block group to the deleted_bgs list.
1470                          */
1471                         list_move(&block_group->bg_list,
1472                                   &trans->transaction->deleted_bgs);
1473                         spin_unlock(&fs_info->unused_bgs_lock);
1474                         btrfs_get_block_group(block_group);
1475                 }
1476 end_trans:
1477                 btrfs_end_transaction(trans);
1478 next:
1479                 mutex_unlock(&fs_info->delete_unused_bgs_mutex);
1480                 btrfs_put_block_group(block_group);
1481                 spin_lock(&fs_info->unused_bgs_lock);
1482         }
1483         spin_unlock(&fs_info->unused_bgs_lock);
1484         return;
1485
1486 flip_async:
1487         btrfs_end_transaction(trans);
1488         mutex_unlock(&fs_info->delete_unused_bgs_mutex);
1489         btrfs_put_block_group(block_group);
1490         btrfs_discard_punt_unused_bgs_list(fs_info);
1491 }
1492
1493 void btrfs_mark_bg_unused(struct btrfs_block_group *bg)
1494 {
1495         struct btrfs_fs_info *fs_info = bg->fs_info;
1496
1497         spin_lock(&fs_info->unused_bgs_lock);
1498         if (list_empty(&bg->bg_list)) {
1499                 btrfs_get_block_group(bg);
1500                 trace_btrfs_add_unused_block_group(bg);
1501                 list_add_tail(&bg->bg_list, &fs_info->unused_bgs);
1502         }
1503         spin_unlock(&fs_info->unused_bgs_lock);
1504 }
1505
1506 static int find_first_block_group(struct btrfs_fs_info *fs_info,
1507                                   struct btrfs_path *path,
1508                                   struct btrfs_key *key)
1509 {
1510         struct btrfs_root *root = fs_info->extent_root;
1511         int ret = 0;
1512         struct btrfs_key found_key;
1513         struct extent_buffer *leaf;
1514         struct btrfs_block_group_item bg;
1515         u64 flags;
1516         int slot;
1517
1518         ret = btrfs_search_slot(NULL, root, key, path, 0, 0);
1519         if (ret < 0)
1520                 goto out;
1521
1522         while (1) {
1523                 slot = path->slots[0];
1524                 leaf = path->nodes[0];
1525                 if (slot >= btrfs_header_nritems(leaf)) {
1526                         ret = btrfs_next_leaf(root, path);
1527                         if (ret == 0)
1528                                 continue;
1529                         if (ret < 0)
1530                                 goto out;
1531                         break;
1532                 }
1533                 btrfs_item_key_to_cpu(leaf, &found_key, slot);
1534
1535                 if (found_key.objectid >= key->objectid &&
1536                     found_key.type == BTRFS_BLOCK_GROUP_ITEM_KEY) {
1537                         struct extent_map_tree *em_tree;
1538                         struct extent_map *em;
1539
1540                         em_tree = &root->fs_info->mapping_tree;
1541                         read_lock(&em_tree->lock);
1542                         em = lookup_extent_mapping(em_tree, found_key.objectid,
1543                                                    found_key.offset);
1544                         read_unlock(&em_tree->lock);
1545                         if (!em) {
1546                                 btrfs_err(fs_info,
1547                         "logical %llu len %llu found bg but no related chunk",
1548                                           found_key.objectid, found_key.offset);
1549                                 ret = -ENOENT;
1550                         } else if (em->start != found_key.objectid ||
1551                                    em->len != found_key.offset) {
1552                                 btrfs_err(fs_info,
1553                 "block group %llu len %llu mismatch with chunk %llu len %llu",
1554                                           found_key.objectid, found_key.offset,
1555                                           em->start, em->len);
1556                                 ret = -EUCLEAN;
1557                         } else {
1558                                 read_extent_buffer(leaf, &bg,
1559                                         btrfs_item_ptr_offset(leaf, slot),
1560                                         sizeof(bg));
1561                                 flags = btrfs_stack_block_group_flags(&bg) &
1562                                         BTRFS_BLOCK_GROUP_TYPE_MASK;
1563
1564                                 if (flags != (em->map_lookup->type &
1565                                               BTRFS_BLOCK_GROUP_TYPE_MASK)) {
1566                                         btrfs_err(fs_info,
1567 "block group %llu len %llu type flags 0x%llx mismatch with chunk type flags 0x%llx",
1568                                                 found_key.objectid,
1569                                                 found_key.offset, flags,
1570                                                 (BTRFS_BLOCK_GROUP_TYPE_MASK &
1571                                                  em->map_lookup->type));
1572                                         ret = -EUCLEAN;
1573                                 } else {
1574                                         ret = 0;
1575                                 }
1576                         }
1577                         free_extent_map(em);
1578                         goto out;
1579                 }
1580                 path->slots[0]++;
1581         }
1582 out:
1583         return ret;
1584 }
1585
1586 static void set_avail_alloc_bits(struct btrfs_fs_info *fs_info, u64 flags)
1587 {
1588         u64 extra_flags = chunk_to_extended(flags) &
1589                                 BTRFS_EXTENDED_PROFILE_MASK;
1590
1591         write_seqlock(&fs_info->profiles_lock);
1592         if (flags & BTRFS_BLOCK_GROUP_DATA)
1593                 fs_info->avail_data_alloc_bits |= extra_flags;
1594         if (flags & BTRFS_BLOCK_GROUP_METADATA)
1595                 fs_info->avail_metadata_alloc_bits |= extra_flags;
1596         if (flags & BTRFS_BLOCK_GROUP_SYSTEM)
1597                 fs_info->avail_system_alloc_bits |= extra_flags;
1598         write_sequnlock(&fs_info->profiles_lock);
1599 }
1600
1601 /**
1602  * btrfs_rmap_block - Map a physical disk address to a list of logical addresses
1603  * @chunk_start:   logical address of block group
1604  * @physical:      physical address to map to logical addresses
1605  * @logical:       return array of logical addresses which map to @physical
1606  * @naddrs:        length of @logical
1607  * @stripe_len:    size of IO stripe for the given block group
1608  *
1609  * Maps a particular @physical disk address to a list of @logical addresses.
1610  * Used primarily to exclude those portions of a block group that contain super
1611  * block copies.
1612  */
1613 EXPORT_FOR_TESTS
1614 int btrfs_rmap_block(struct btrfs_fs_info *fs_info, u64 chunk_start,
1615                      u64 physical, u64 **logical, int *naddrs, int *stripe_len)
1616 {
1617         struct extent_map *em;
1618         struct map_lookup *map;
1619         u64 *buf;
1620         u64 bytenr;
1621         u64 data_stripe_length;
1622         u64 io_stripe_size;
1623         int i, nr = 0;
1624         int ret = 0;
1625
1626         em = btrfs_get_chunk_map(fs_info, chunk_start, 1);
1627         if (IS_ERR(em))
1628                 return -EIO;
1629
1630         map = em->map_lookup;
1631         data_stripe_length = em->len;
1632         io_stripe_size = map->stripe_len;
1633
1634         if (map->type & BTRFS_BLOCK_GROUP_RAID10)
1635                 data_stripe_length = div_u64(data_stripe_length,
1636                                              map->num_stripes / map->sub_stripes);
1637         else if (map->type & BTRFS_BLOCK_GROUP_RAID0)
1638                 data_stripe_length = div_u64(data_stripe_length, map->num_stripes);
1639         else if (map->type & BTRFS_BLOCK_GROUP_RAID56_MASK) {
1640                 data_stripe_length = div_u64(data_stripe_length,
1641                                              nr_data_stripes(map));
1642                 io_stripe_size = map->stripe_len * nr_data_stripes(map);
1643         }
1644
1645         buf = kcalloc(map->num_stripes, sizeof(u64), GFP_NOFS);
1646         if (!buf) {
1647                 ret = -ENOMEM;
1648                 goto out;
1649         }
1650
1651         for (i = 0; i < map->num_stripes; i++) {
1652                 bool already_inserted = false;
1653                 u64 stripe_nr;
1654                 int j;
1655
1656                 if (!in_range(physical, map->stripes[i].physical,
1657                               data_stripe_length))
1658                         continue;
1659
1660                 stripe_nr = physical - map->stripes[i].physical;
1661                 stripe_nr = div64_u64(stripe_nr, map->stripe_len);
1662
1663                 if (map->type & BTRFS_BLOCK_GROUP_RAID10) {
1664                         stripe_nr = stripe_nr * map->num_stripes + i;
1665                         stripe_nr = div_u64(stripe_nr, map->sub_stripes);
1666                 } else if (map->type & BTRFS_BLOCK_GROUP_RAID0) {
1667                         stripe_nr = stripe_nr * map->num_stripes + i;
1668                 }
1669                 /*
1670                  * The remaining case would be for RAID56, multiply by
1671                  * nr_data_stripes().  Alternatively, just use rmap_len below
1672                  * instead of map->stripe_len
1673                  */
1674
1675                 bytenr = chunk_start + stripe_nr * io_stripe_size;
1676
1677                 /* Ensure we don't add duplicate addresses */
1678                 for (j = 0; j < nr; j++) {
1679                         if (buf[j] == bytenr) {
1680                                 already_inserted = true;
1681                                 break;
1682                         }
1683                 }
1684
1685                 if (!already_inserted)
1686                         buf[nr++] = bytenr;
1687         }
1688
1689         *logical = buf;
1690         *naddrs = nr;
1691         *stripe_len = io_stripe_size;
1692 out:
1693         free_extent_map(em);
1694         return ret;
1695 }
1696
1697 static int exclude_super_stripes(struct btrfs_block_group *cache)
1698 {
1699         struct btrfs_fs_info *fs_info = cache->fs_info;
1700         u64 bytenr;
1701         u64 *logical;
1702         int stripe_len;
1703         int i, nr, ret;
1704
1705         if (cache->start < BTRFS_SUPER_INFO_OFFSET) {
1706                 stripe_len = BTRFS_SUPER_INFO_OFFSET - cache->start;
1707                 cache->bytes_super += stripe_len;
1708                 ret = btrfs_add_excluded_extent(fs_info, cache->start,
1709                                                 stripe_len);
1710                 if (ret)
1711                         return ret;
1712         }
1713
1714         for (i = 0; i < BTRFS_SUPER_MIRROR_MAX; i++) {
1715                 bytenr = btrfs_sb_offset(i);
1716                 ret = btrfs_rmap_block(fs_info, cache->start,
1717                                        bytenr, &logical, &nr, &stripe_len);
1718                 if (ret)
1719                         return ret;
1720
1721                 while (nr--) {
1722                         u64 start, len;
1723
1724                         if (logical[nr] > cache->start + cache->length)
1725                                 continue;
1726
1727                         if (logical[nr] + stripe_len <= cache->start)
1728                                 continue;
1729
1730                         start = logical[nr];
1731                         if (start < cache->start) {
1732                                 start = cache->start;
1733                                 len = (logical[nr] + stripe_len) - start;
1734                         } else {
1735                                 len = min_t(u64, stripe_len,
1736                                             cache->start + cache->length - start);
1737                         }
1738
1739                         cache->bytes_super += len;
1740                         ret = btrfs_add_excluded_extent(fs_info, start, len);
1741                         if (ret) {
1742                                 kfree(logical);
1743                                 return ret;
1744                         }
1745                 }
1746
1747                 kfree(logical);
1748         }
1749         return 0;
1750 }
1751
1752 static void link_block_group(struct btrfs_block_group *cache)
1753 {
1754         struct btrfs_space_info *space_info = cache->space_info;
1755         int index = btrfs_bg_flags_to_raid_index(cache->flags);
1756         bool first = false;
1757
1758         down_write(&space_info->groups_sem);
1759         if (list_empty(&space_info->block_groups[index]))
1760                 first = true;
1761         list_add_tail(&cache->list, &space_info->block_groups[index]);
1762         up_write(&space_info->groups_sem);
1763
1764         if (first)
1765                 btrfs_sysfs_add_block_group_type(cache);
1766 }
1767
1768 static struct btrfs_block_group *btrfs_create_block_group_cache(
1769                 struct btrfs_fs_info *fs_info, u64 start, u64 size)
1770 {
1771         struct btrfs_block_group *cache;
1772
1773         cache = kzalloc(sizeof(*cache), GFP_NOFS);
1774         if (!cache)
1775                 return NULL;
1776
1777         cache->free_space_ctl = kzalloc(sizeof(*cache->free_space_ctl),
1778                                         GFP_NOFS);
1779         if (!cache->free_space_ctl) {
1780                 kfree(cache);
1781                 return NULL;
1782         }
1783
1784         cache->start = start;
1785         cache->length = size;
1786
1787         cache->fs_info = fs_info;
1788         cache->full_stripe_len = btrfs_full_stripe_len(fs_info, start);
1789         set_free_space_tree_thresholds(cache);
1790
1791         cache->discard_index = BTRFS_DISCARD_INDEX_UNUSED;
1792
1793         atomic_set(&cache->count, 1);
1794         spin_lock_init(&cache->lock);
1795         init_rwsem(&cache->data_rwsem);
1796         INIT_LIST_HEAD(&cache->list);
1797         INIT_LIST_HEAD(&cache->cluster_list);
1798         INIT_LIST_HEAD(&cache->bg_list);
1799         INIT_LIST_HEAD(&cache->ro_list);
1800         INIT_LIST_HEAD(&cache->discard_list);
1801         INIT_LIST_HEAD(&cache->dirty_list);
1802         INIT_LIST_HEAD(&cache->io_list);
1803         btrfs_init_free_space_ctl(cache);
1804         atomic_set(&cache->trimming, 0);
1805         mutex_init(&cache->free_space_lock);
1806         btrfs_init_full_stripe_locks_tree(&cache->full_stripe_locks_root);
1807
1808         return cache;
1809 }
1810
1811 /*
1812  * Iterate all chunks and verify that each of them has the corresponding block
1813  * group
1814  */
1815 static int check_chunk_block_group_mappings(struct btrfs_fs_info *fs_info)
1816 {
1817         struct extent_map_tree *map_tree = &fs_info->mapping_tree;
1818         struct extent_map *em;
1819         struct btrfs_block_group *bg;
1820         u64 start = 0;
1821         int ret = 0;
1822
1823         while (1) {
1824                 read_lock(&map_tree->lock);
1825                 /*
1826                  * lookup_extent_mapping will return the first extent map
1827                  * intersecting the range, so setting @len to 1 is enough to
1828                  * get the first chunk.
1829                  */
1830                 em = lookup_extent_mapping(map_tree, start, 1);
1831                 read_unlock(&map_tree->lock);
1832                 if (!em)
1833                         break;
1834
1835                 bg = btrfs_lookup_block_group(fs_info, em->start);
1836                 if (!bg) {
1837                         btrfs_err(fs_info,
1838         "chunk start=%llu len=%llu doesn't have corresponding block group",
1839                                      em->start, em->len);
1840                         ret = -EUCLEAN;
1841                         free_extent_map(em);
1842                         break;
1843                 }
1844                 if (bg->start != em->start || bg->length != em->len ||
1845                     (bg->flags & BTRFS_BLOCK_GROUP_TYPE_MASK) !=
1846                     (em->map_lookup->type & BTRFS_BLOCK_GROUP_TYPE_MASK)) {
1847                         btrfs_err(fs_info,
1848 "chunk start=%llu len=%llu flags=0x%llx doesn't match block group start=%llu len=%llu flags=0x%llx",
1849                                 em->start, em->len,
1850                                 em->map_lookup->type & BTRFS_BLOCK_GROUP_TYPE_MASK,
1851                                 bg->start, bg->length,
1852                                 bg->flags & BTRFS_BLOCK_GROUP_TYPE_MASK);
1853                         ret = -EUCLEAN;
1854                         free_extent_map(em);
1855                         btrfs_put_block_group(bg);
1856                         break;
1857                 }
1858                 start = em->start + em->len;
1859                 free_extent_map(em);
1860                 btrfs_put_block_group(bg);
1861         }
1862         return ret;
1863 }
1864
1865 static int read_one_block_group(struct btrfs_fs_info *info,
1866                                 struct btrfs_path *path,
1867                                 const struct btrfs_key *key,
1868                                 int need_clear)
1869 {
1870         struct extent_buffer *leaf = path->nodes[0];
1871         struct btrfs_block_group *cache;
1872         struct btrfs_space_info *space_info;
1873         struct btrfs_block_group_item bgi;
1874         const bool mixed = btrfs_fs_incompat(info, MIXED_GROUPS);
1875         int slot = path->slots[0];
1876         int ret;
1877
1878         ASSERT(key->type == BTRFS_BLOCK_GROUP_ITEM_KEY);
1879
1880         cache = btrfs_create_block_group_cache(info, key->objectid, key->offset);
1881         if (!cache)
1882                 return -ENOMEM;
1883
1884         if (need_clear) {
1885                 /*
1886                  * When we mount with old space cache, we need to
1887                  * set BTRFS_DC_CLEAR and set dirty flag.
1888                  *
1889                  * a) Setting 'BTRFS_DC_CLEAR' makes sure that we
1890                  *    truncate the old free space cache inode and
1891                  *    setup a new one.
1892                  * b) Setting 'dirty flag' makes sure that we flush
1893                  *    the new space cache info onto disk.
1894                  */
1895                 if (btrfs_test_opt(info, SPACE_CACHE))
1896                         cache->disk_cache_state = BTRFS_DC_CLEAR;
1897         }
1898         read_extent_buffer(leaf, &bgi, btrfs_item_ptr_offset(leaf, slot),
1899                            sizeof(bgi));
1900         cache->used = btrfs_stack_block_group_used(&bgi);
1901         cache->flags = btrfs_stack_block_group_flags(&bgi);
1902         if (!mixed && ((cache->flags & BTRFS_BLOCK_GROUP_METADATA) &&
1903             (cache->flags & BTRFS_BLOCK_GROUP_DATA))) {
1904                         btrfs_err(info,
1905 "bg %llu is a mixed block group but filesystem hasn't enabled mixed block groups",
1906                                   cache->start);
1907                         ret = -EINVAL;
1908                         goto error;
1909         }
1910
1911         /*
1912          * We need to exclude the super stripes now so that the space info has
1913          * super bytes accounted for, otherwise we'll think we have more space
1914          * than we actually do.
1915          */
1916         ret = exclude_super_stripes(cache);
1917         if (ret) {
1918                 /* We may have excluded something, so call this just in case. */
1919                 btrfs_free_excluded_extents(cache);
1920                 goto error;
1921         }
1922
1923         /*
1924          * Check for two cases, either we are full, and therefore don't need
1925          * to bother with the caching work since we won't find any space, or we
1926          * are empty, and we can just add all the space in and be done with it.
1927          * This saves us _a_lot_ of time, particularly in the full case.
1928          */
1929         if (key->offset == cache->used) {
1930                 cache->last_byte_to_unpin = (u64)-1;
1931                 cache->cached = BTRFS_CACHE_FINISHED;
1932                 btrfs_free_excluded_extents(cache);
1933         } else if (cache->used == 0) {
1934                 cache->last_byte_to_unpin = (u64)-1;
1935                 cache->cached = BTRFS_CACHE_FINISHED;
1936                 add_new_free_space(cache, key->objectid,
1937                                    key->objectid + key->offset);
1938                 btrfs_free_excluded_extents(cache);
1939         }
1940
1941         ret = btrfs_add_block_group_cache(info, cache);
1942         if (ret) {
1943                 btrfs_remove_free_space_cache(cache);
1944                 goto error;
1945         }
1946         trace_btrfs_add_block_group(info, cache, 0);
1947         btrfs_update_space_info(info, cache->flags, key->offset,
1948                                 cache->used, cache->bytes_super, &space_info);
1949
1950         cache->space_info = space_info;
1951
1952         link_block_group(cache);
1953
1954         set_avail_alloc_bits(info, cache->flags);
1955         if (btrfs_chunk_readonly(info, cache->start)) {
1956                 inc_block_group_ro(cache, 1);
1957         } else if (cache->used == 0) {
1958                 ASSERT(list_empty(&cache->bg_list));
1959                 if (btrfs_test_opt(info, DISCARD_ASYNC))
1960                         btrfs_discard_queue_work(&info->discard_ctl, cache);
1961                 else
1962                         btrfs_mark_bg_unused(cache);
1963         }
1964         return 0;
1965 error:
1966         btrfs_put_block_group(cache);
1967         return ret;
1968 }
1969
1970 int btrfs_read_block_groups(struct btrfs_fs_info *info)
1971 {
1972         struct btrfs_path *path;
1973         int ret;
1974         struct btrfs_block_group *cache;
1975         struct btrfs_space_info *space_info;
1976         struct btrfs_key key;
1977         int need_clear = 0;
1978         u64 cache_gen;
1979
1980         key.objectid = 0;
1981         key.offset = 0;
1982         key.type = BTRFS_BLOCK_GROUP_ITEM_KEY;
1983         path = btrfs_alloc_path();
1984         if (!path)
1985                 return -ENOMEM;
1986         path->reada = READA_FORWARD;
1987
1988         cache_gen = btrfs_super_cache_generation(info->super_copy);
1989         if (btrfs_test_opt(info, SPACE_CACHE) &&
1990             btrfs_super_generation(info->super_copy) != cache_gen)
1991                 need_clear = 1;
1992         if (btrfs_test_opt(info, CLEAR_CACHE))
1993                 need_clear = 1;
1994
1995         while (1) {
1996                 ret = find_first_block_group(info, path, &key);
1997                 if (ret > 0)
1998                         break;
1999                 if (ret != 0)
2000                         goto error;
2001
2002                 btrfs_item_key_to_cpu(path->nodes[0], &key, path->slots[0]);
2003                 ret = read_one_block_group(info, path, &key, need_clear);
2004                 if (ret < 0)
2005                         goto error;
2006                 key.objectid += key.offset;
2007                 key.offset = 0;
2008                 btrfs_release_path(path);
2009         }
2010
2011         rcu_read_lock();
2012         list_for_each_entry_rcu(space_info, &info->space_info, list) {
2013                 if (!(btrfs_get_alloc_profile(info, space_info->flags) &
2014                       (BTRFS_BLOCK_GROUP_RAID10 |
2015                        BTRFS_BLOCK_GROUP_RAID1_MASK |
2016                        BTRFS_BLOCK_GROUP_RAID56_MASK |
2017                        BTRFS_BLOCK_GROUP_DUP)))
2018                         continue;
2019                 /*
2020                  * Avoid allocating from un-mirrored block group if there are
2021                  * mirrored block groups.
2022                  */
2023                 list_for_each_entry(cache,
2024                                 &space_info->block_groups[BTRFS_RAID_RAID0],
2025                                 list)
2026                         inc_block_group_ro(cache, 1);
2027                 list_for_each_entry(cache,
2028                                 &space_info->block_groups[BTRFS_RAID_SINGLE],
2029                                 list)
2030                         inc_block_group_ro(cache, 1);
2031         }
2032         rcu_read_unlock();
2033
2034         btrfs_init_global_block_rsv(info);
2035         ret = check_chunk_block_group_mappings(info);
2036 error:
2037         btrfs_free_path(path);
2038         return ret;
2039 }
2040
2041 void btrfs_create_pending_block_groups(struct btrfs_trans_handle *trans)
2042 {
2043         struct btrfs_fs_info *fs_info = trans->fs_info;
2044         struct btrfs_block_group *block_group;
2045         struct btrfs_root *extent_root = fs_info->extent_root;
2046         struct btrfs_block_group_item item;
2047         struct btrfs_key key;
2048         int ret = 0;
2049
2050         if (!trans->can_flush_pending_bgs)
2051                 return;
2052
2053         while (!list_empty(&trans->new_bgs)) {
2054                 block_group = list_first_entry(&trans->new_bgs,
2055                                                struct btrfs_block_group,
2056                                                bg_list);
2057                 if (ret)
2058                         goto next;
2059
2060                 spin_lock(&block_group->lock);
2061                 btrfs_set_stack_block_group_used(&item, block_group->used);
2062                 btrfs_set_stack_block_group_chunk_objectid(&item,
2063                                 BTRFS_FIRST_CHUNK_TREE_OBJECTID);
2064                 btrfs_set_stack_block_group_flags(&item, block_group->flags);
2065                 key.objectid = block_group->start;
2066                 key.type = BTRFS_BLOCK_GROUP_ITEM_KEY;
2067                 key.offset = block_group->length;
2068                 spin_unlock(&block_group->lock);
2069
2070                 ret = btrfs_insert_item(trans, extent_root, &key, &item,
2071                                         sizeof(item));
2072                 if (ret)
2073                         btrfs_abort_transaction(trans, ret);
2074                 ret = btrfs_finish_chunk_alloc(trans, key.objectid, key.offset);
2075                 if (ret)
2076                         btrfs_abort_transaction(trans, ret);
2077                 add_block_group_free_space(trans, block_group);
2078                 /* Already aborted the transaction if it failed. */
2079 next:
2080                 btrfs_delayed_refs_rsv_release(fs_info, 1);
2081                 list_del_init(&block_group->bg_list);
2082         }
2083         btrfs_trans_release_chunk_metadata(trans);
2084 }
2085
2086 int btrfs_make_block_group(struct btrfs_trans_handle *trans, u64 bytes_used,
2087                            u64 type, u64 chunk_offset, u64 size)
2088 {
2089         struct btrfs_fs_info *fs_info = trans->fs_info;
2090         struct btrfs_block_group *cache;
2091         int ret;
2092
2093         btrfs_set_log_full_commit(trans);
2094
2095         cache = btrfs_create_block_group_cache(fs_info, chunk_offset, size);
2096         if (!cache)
2097                 return -ENOMEM;
2098
2099         cache->used = bytes_used;
2100         cache->flags = type;
2101         cache->last_byte_to_unpin = (u64)-1;
2102         cache->cached = BTRFS_CACHE_FINISHED;
2103         cache->needs_free_space = 1;
2104         ret = exclude_super_stripes(cache);
2105         if (ret) {
2106                 /* We may have excluded something, so call this just in case */
2107                 btrfs_free_excluded_extents(cache);
2108                 btrfs_put_block_group(cache);
2109                 return ret;
2110         }
2111
2112         add_new_free_space(cache, chunk_offset, chunk_offset + size);
2113
2114         btrfs_free_excluded_extents(cache);
2115
2116 #ifdef CONFIG_BTRFS_DEBUG
2117         if (btrfs_should_fragment_free_space(cache)) {
2118                 u64 new_bytes_used = size - bytes_used;
2119
2120                 bytes_used += new_bytes_used >> 1;
2121                 fragment_free_space(cache);
2122         }
2123 #endif
2124         /*
2125          * Ensure the corresponding space_info object is created and
2126          * assigned to our block group. We want our bg to be added to the rbtree
2127          * with its ->space_info set.
2128          */
2129         cache->space_info = btrfs_find_space_info(fs_info, cache->flags);
2130         ASSERT(cache->space_info);
2131
2132         ret = btrfs_add_block_group_cache(fs_info, cache);
2133         if (ret) {
2134                 btrfs_remove_free_space_cache(cache);
2135                 btrfs_put_block_group(cache);
2136                 return ret;
2137         }
2138
2139         /*
2140          * Now that our block group has its ->space_info set and is inserted in
2141          * the rbtree, update the space info's counters.
2142          */
2143         trace_btrfs_add_block_group(fs_info, cache, 1);
2144         btrfs_update_space_info(fs_info, cache->flags, size, bytes_used,
2145                                 cache->bytes_super, &cache->space_info);
2146         btrfs_update_global_block_rsv(fs_info);
2147
2148         link_block_group(cache);
2149
2150         list_add_tail(&cache->bg_list, &trans->new_bgs);
2151         trans->delayed_ref_updates++;
2152         btrfs_update_delayed_refs_rsv(trans);
2153
2154         set_avail_alloc_bits(fs_info, type);
2155         return 0;
2156 }
2157
2158 static u64 update_block_group_flags(struct btrfs_fs_info *fs_info, u64 flags)
2159 {
2160         u64 num_devices;
2161         u64 stripped;
2162
2163         /*
2164          * if restripe for this chunk_type is on pick target profile and
2165          * return, otherwise do the usual balance
2166          */
2167         stripped = get_restripe_target(fs_info, flags);
2168         if (stripped)
2169                 return extended_to_chunk(stripped);
2170
2171         num_devices = fs_info->fs_devices->rw_devices;
2172
2173         stripped = BTRFS_BLOCK_GROUP_RAID0 | BTRFS_BLOCK_GROUP_RAID56_MASK |
2174                 BTRFS_BLOCK_GROUP_RAID1_MASK | BTRFS_BLOCK_GROUP_RAID10;
2175
2176         if (num_devices == 1) {
2177                 stripped |= BTRFS_BLOCK_GROUP_DUP;
2178                 stripped = flags & ~stripped;
2179
2180                 /* turn raid0 into single device chunks */
2181                 if (flags & BTRFS_BLOCK_GROUP_RAID0)
2182                         return stripped;
2183
2184                 /* turn mirroring into duplication */
2185                 if (flags & (BTRFS_BLOCK_GROUP_RAID1_MASK |
2186                              BTRFS_BLOCK_GROUP_RAID10))
2187                         return stripped | BTRFS_BLOCK_GROUP_DUP;
2188         } else {
2189                 /* they already had raid on here, just return */
2190                 if (flags & stripped)
2191                         return flags;
2192
2193                 stripped |= BTRFS_BLOCK_GROUP_DUP;
2194                 stripped = flags & ~stripped;
2195
2196                 /* switch duplicated blocks with raid1 */
2197                 if (flags & BTRFS_BLOCK_GROUP_DUP)
2198                         return stripped | BTRFS_BLOCK_GROUP_RAID1;
2199
2200                 /* this is drive concat, leave it alone */
2201         }
2202
2203         return flags;
2204 }
2205
2206 /*
2207  * Mark one block group RO, can be called several times for the same block
2208  * group.
2209  *
2210  * @cache:              the destination block group
2211  * @do_chunk_alloc:     whether need to do chunk pre-allocation, this is to
2212  *                      ensure we still have some free space after marking this
2213  *                      block group RO.
2214  */
2215 int btrfs_inc_block_group_ro(struct btrfs_block_group *cache,
2216                              bool do_chunk_alloc)
2217 {
2218         struct btrfs_fs_info *fs_info = cache->fs_info;
2219         struct btrfs_trans_handle *trans;
2220         u64 alloc_flags;
2221         int ret;
2222
2223 again:
2224         trans = btrfs_join_transaction(fs_info->extent_root);
2225         if (IS_ERR(trans))
2226                 return PTR_ERR(trans);
2227
2228         /*
2229          * we're not allowed to set block groups readonly after the dirty
2230          * block groups cache has started writing.  If it already started,
2231          * back off and let this transaction commit
2232          */
2233         mutex_lock(&fs_info->ro_block_group_mutex);
2234         if (test_bit(BTRFS_TRANS_DIRTY_BG_RUN, &trans->transaction->flags)) {
2235                 u64 transid = trans->transid;
2236
2237                 mutex_unlock(&fs_info->ro_block_group_mutex);
2238                 btrfs_end_transaction(trans);
2239
2240                 ret = btrfs_wait_for_commit(fs_info, transid);
2241                 if (ret)
2242                         return ret;
2243                 goto again;
2244         }
2245
2246         if (do_chunk_alloc) {
2247                 /*
2248                  * If we are changing raid levels, try to allocate a
2249                  * corresponding block group with the new raid level.
2250                  */
2251                 alloc_flags = update_block_group_flags(fs_info, cache->flags);
2252                 if (alloc_flags != cache->flags) {
2253                         ret = btrfs_chunk_alloc(trans, alloc_flags,
2254                                                 CHUNK_ALLOC_FORCE);
2255                         /*
2256                          * ENOSPC is allowed here, we may have enough space
2257                          * already allocated at the new raid level to carry on
2258                          */
2259                         if (ret == -ENOSPC)
2260                                 ret = 0;
2261                         if (ret < 0)
2262                                 goto out;
2263                 }
2264         }
2265
2266         ret = inc_block_group_ro(cache, 0);
2267         if (!do_chunk_alloc)
2268                 goto unlock_out;
2269         if (!ret)
2270                 goto out;
2271         alloc_flags = btrfs_get_alloc_profile(fs_info, cache->space_info->flags);
2272         ret = btrfs_chunk_alloc(trans, alloc_flags, CHUNK_ALLOC_FORCE);
2273         if (ret < 0)
2274                 goto out;
2275         ret = inc_block_group_ro(cache, 0);
2276 out:
2277         if (cache->flags & BTRFS_BLOCK_GROUP_SYSTEM) {
2278                 alloc_flags = update_block_group_flags(fs_info, cache->flags);
2279                 mutex_lock(&fs_info->chunk_mutex);
2280                 check_system_chunk(trans, alloc_flags);
2281                 mutex_unlock(&fs_info->chunk_mutex);
2282         }
2283 unlock_out:
2284         mutex_unlock(&fs_info->ro_block_group_mutex);
2285
2286         btrfs_end_transaction(trans);
2287         return ret;
2288 }
2289
2290 void btrfs_dec_block_group_ro(struct btrfs_block_group *cache)
2291 {
2292         struct btrfs_space_info *sinfo = cache->space_info;
2293         u64 num_bytes;
2294
2295         BUG_ON(!cache->ro);
2296
2297         spin_lock(&sinfo->lock);
2298         spin_lock(&cache->lock);
2299         if (!--cache->ro) {
2300                 num_bytes = cache->length - cache->reserved -
2301                             cache->pinned - cache->bytes_super - cache->used;
2302                 sinfo->bytes_readonly -= num_bytes;
2303                 list_del_init(&cache->ro_list);
2304         }
2305         spin_unlock(&cache->lock);
2306         spin_unlock(&sinfo->lock);
2307 }
2308
2309 static int write_one_cache_group(struct btrfs_trans_handle *trans,
2310                                  struct btrfs_path *path,
2311                                  struct btrfs_block_group *cache)
2312 {
2313         struct btrfs_fs_info *fs_info = trans->fs_info;
2314         int ret;
2315         struct btrfs_root *extent_root = fs_info->extent_root;
2316         unsigned long bi;
2317         struct extent_buffer *leaf;
2318         struct btrfs_block_group_item bgi;
2319         struct btrfs_key key;
2320
2321         key.objectid = cache->start;
2322         key.type = BTRFS_BLOCK_GROUP_ITEM_KEY;
2323         key.offset = cache->length;
2324
2325         ret = btrfs_search_slot(trans, extent_root, &key, path, 0, 1);
2326         if (ret) {
2327                 if (ret > 0)
2328                         ret = -ENOENT;
2329                 goto fail;
2330         }
2331
2332         leaf = path->nodes[0];
2333         bi = btrfs_item_ptr_offset(leaf, path->slots[0]);
2334         btrfs_set_stack_block_group_used(&bgi, cache->used);
2335         btrfs_set_stack_block_group_chunk_objectid(&bgi,
2336                         BTRFS_FIRST_CHUNK_TREE_OBJECTID);
2337         btrfs_set_stack_block_group_flags(&bgi, cache->flags);
2338         write_extent_buffer(leaf, &bgi, bi, sizeof(bgi));
2339         btrfs_mark_buffer_dirty(leaf);
2340 fail:
2341         btrfs_release_path(path);
2342         return ret;
2343
2344 }
2345
2346 static int cache_save_setup(struct btrfs_block_group *block_group,
2347                             struct btrfs_trans_handle *trans,
2348                             struct btrfs_path *path)
2349 {
2350         struct btrfs_fs_info *fs_info = block_group->fs_info;
2351         struct btrfs_root *root = fs_info->tree_root;
2352         struct inode *inode = NULL;
2353         struct extent_changeset *data_reserved = NULL;
2354         u64 alloc_hint = 0;
2355         int dcs = BTRFS_DC_ERROR;
2356         u64 num_pages = 0;
2357         int retries = 0;
2358         int ret = 0;
2359
2360         /*
2361          * If this block group is smaller than 100 megs don't bother caching the
2362          * block group.
2363          */
2364         if (block_group->length < (100 * SZ_1M)) {
2365                 spin_lock(&block_group->lock);
2366                 block_group->disk_cache_state = BTRFS_DC_WRITTEN;
2367                 spin_unlock(&block_group->lock);
2368                 return 0;
2369         }
2370
2371         if (TRANS_ABORTED(trans))
2372                 return 0;
2373 again:
2374         inode = lookup_free_space_inode(block_group, path);
2375         if (IS_ERR(inode) && PTR_ERR(inode) != -ENOENT) {
2376                 ret = PTR_ERR(inode);
2377                 btrfs_release_path(path);
2378                 goto out;
2379         }
2380
2381         if (IS_ERR(inode)) {
2382                 BUG_ON(retries);
2383                 retries++;
2384
2385                 if (block_group->ro)
2386                         goto out_free;
2387
2388                 ret = create_free_space_inode(trans, block_group, path);
2389                 if (ret)
2390                         goto out_free;
2391                 goto again;
2392         }
2393
2394         /*
2395          * We want to set the generation to 0, that way if anything goes wrong
2396          * from here on out we know not to trust this cache when we load up next
2397          * time.
2398          */
2399         BTRFS_I(inode)->generation = 0;
2400         ret = btrfs_update_inode(trans, root, inode);
2401         if (ret) {
2402                 /*
2403                  * So theoretically we could recover from this, simply set the
2404                  * super cache generation to 0 so we know to invalidate the
2405                  * cache, but then we'd have to keep track of the block groups
2406                  * that fail this way so we know we _have_ to reset this cache
2407                  * before the next commit or risk reading stale cache.  So to
2408                  * limit our exposure to horrible edge cases lets just abort the
2409                  * transaction, this only happens in really bad situations
2410                  * anyway.
2411                  */
2412                 btrfs_abort_transaction(trans, ret);
2413                 goto out_put;
2414         }
2415         WARN_ON(ret);
2416
2417         /* We've already setup this transaction, go ahead and exit */
2418         if (block_group->cache_generation == trans->transid &&
2419             i_size_read(inode)) {
2420                 dcs = BTRFS_DC_SETUP;
2421                 goto out_put;
2422         }
2423
2424         if (i_size_read(inode) > 0) {
2425                 ret = btrfs_check_trunc_cache_free_space(fs_info,
2426                                         &fs_info->global_block_rsv);
2427                 if (ret)
2428                         goto out_put;
2429
2430                 ret = btrfs_truncate_free_space_cache(trans, NULL, inode);
2431                 if (ret)
2432                         goto out_put;
2433         }
2434
2435         spin_lock(&block_group->lock);
2436         if (block_group->cached != BTRFS_CACHE_FINISHED ||
2437             !btrfs_test_opt(fs_info, SPACE_CACHE)) {
2438                 /*
2439                  * don't bother trying to write stuff out _if_
2440                  * a) we're not cached,
2441                  * b) we're with nospace_cache mount option,
2442                  * c) we're with v2 space_cache (FREE_SPACE_TREE).
2443                  */
2444                 dcs = BTRFS_DC_WRITTEN;
2445                 spin_unlock(&block_group->lock);
2446                 goto out_put;
2447         }
2448         spin_unlock(&block_group->lock);
2449
2450         /*
2451          * We hit an ENOSPC when setting up the cache in this transaction, just
2452          * skip doing the setup, we've already cleared the cache so we're safe.
2453          */
2454         if (test_bit(BTRFS_TRANS_CACHE_ENOSPC, &trans->transaction->flags)) {
2455                 ret = -ENOSPC;
2456                 goto out_put;
2457         }
2458
2459         /*
2460          * Try to preallocate enough space based on how big the block group is.
2461          * Keep in mind this has to include any pinned space which could end up
2462          * taking up quite a bit since it's not folded into the other space
2463          * cache.
2464          */
2465         num_pages = div_u64(block_group->length, SZ_256M);
2466         if (!num_pages)
2467                 num_pages = 1;
2468
2469         num_pages *= 16;
2470         num_pages *= PAGE_SIZE;
2471
2472         ret = btrfs_check_data_free_space(inode, &data_reserved, 0, num_pages);
2473         if (ret)
2474                 goto out_put;
2475
2476         ret = btrfs_prealloc_file_range_trans(inode, trans, 0, 0, num_pages,
2477                                               num_pages, num_pages,
2478                                               &alloc_hint);
2479         /*
2480          * Our cache requires contiguous chunks so that we don't modify a bunch
2481          * of metadata or split extents when writing the cache out, which means
2482          * we can enospc if we are heavily fragmented in addition to just normal
2483          * out of space conditions.  So if we hit this just skip setting up any
2484          * other block groups for this transaction, maybe we'll unpin enough
2485          * space the next time around.
2486          */
2487         if (!ret)
2488                 dcs = BTRFS_DC_SETUP;
2489         else if (ret == -ENOSPC)
2490                 set_bit(BTRFS_TRANS_CACHE_ENOSPC, &trans->transaction->flags);
2491
2492 out_put:
2493         iput(inode);
2494 out_free:
2495         btrfs_release_path(path);
2496 out:
2497         spin_lock(&block_group->lock);
2498         if (!ret && dcs == BTRFS_DC_SETUP)
2499                 block_group->cache_generation = trans->transid;
2500         block_group->disk_cache_state = dcs;
2501         spin_unlock(&block_group->lock);
2502
2503         extent_changeset_free(data_reserved);
2504         return ret;
2505 }
2506
2507 int btrfs_setup_space_cache(struct btrfs_trans_handle *trans)
2508 {
2509         struct btrfs_fs_info *fs_info = trans->fs_info;
2510         struct btrfs_block_group *cache, *tmp;
2511         struct btrfs_transaction *cur_trans = trans->transaction;
2512         struct btrfs_path *path;
2513
2514         if (list_empty(&cur_trans->dirty_bgs) ||
2515             !btrfs_test_opt(fs_info, SPACE_CACHE))
2516                 return 0;
2517
2518         path = btrfs_alloc_path();
2519         if (!path)
2520                 return -ENOMEM;
2521
2522         /* Could add new block groups, use _safe just in case */
2523         list_for_each_entry_safe(cache, tmp, &cur_trans->dirty_bgs,
2524                                  dirty_list) {
2525                 if (cache->disk_cache_state == BTRFS_DC_CLEAR)
2526                         cache_save_setup(cache, trans, path);
2527         }
2528
2529         btrfs_free_path(path);
2530         return 0;
2531 }
2532
2533 /*
2534  * Transaction commit does final block group cache writeback during a critical
2535  * section where nothing is allowed to change the FS.  This is required in
2536  * order for the cache to actually match the block group, but can introduce a
2537  * lot of latency into the commit.
2538  *
2539  * So, btrfs_start_dirty_block_groups is here to kick off block group cache IO.
2540  * There's a chance we'll have to redo some of it if the block group changes
2541  * again during the commit, but it greatly reduces the commit latency by
2542  * getting rid of the easy block groups while we're still allowing others to
2543  * join the commit.
2544  */
2545 int btrfs_start_dirty_block_groups(struct btrfs_trans_handle *trans)
2546 {
2547         struct btrfs_fs_info *fs_info = trans->fs_info;
2548         struct btrfs_block_group *cache;
2549         struct btrfs_transaction *cur_trans = trans->transaction;
2550         int ret = 0;
2551         int should_put;
2552         struct btrfs_path *path = NULL;
2553         LIST_HEAD(dirty);
2554         struct list_head *io = &cur_trans->io_bgs;
2555         int num_started = 0;
2556         int loops = 0;
2557
2558         spin_lock(&cur_trans->dirty_bgs_lock);
2559         if (list_empty(&cur_trans->dirty_bgs)) {
2560                 spin_unlock(&cur_trans->dirty_bgs_lock);
2561                 return 0;
2562         }
2563         list_splice_init(&cur_trans->dirty_bgs, &dirty);
2564         spin_unlock(&cur_trans->dirty_bgs_lock);
2565
2566 again:
2567         /* Make sure all the block groups on our dirty list actually exist */
2568         btrfs_create_pending_block_groups(trans);
2569
2570         if (!path) {
2571                 path = btrfs_alloc_path();
2572                 if (!path)
2573                         return -ENOMEM;
2574         }
2575
2576         /*
2577          * cache_write_mutex is here only to save us from balance or automatic
2578          * removal of empty block groups deleting this block group while we are
2579          * writing out the cache
2580          */
2581         mutex_lock(&trans->transaction->cache_write_mutex);
2582         while (!list_empty(&dirty)) {
2583                 bool drop_reserve = true;
2584
2585                 cache = list_first_entry(&dirty, struct btrfs_block_group,
2586                                          dirty_list);
2587                 /*
2588                  * This can happen if something re-dirties a block group that
2589                  * is already under IO.  Just wait for it to finish and then do
2590                  * it all again
2591                  */
2592                 if (!list_empty(&cache->io_list)) {
2593                         list_del_init(&cache->io_list);
2594                         btrfs_wait_cache_io(trans, cache, path);
2595                         btrfs_put_block_group(cache);
2596                 }
2597
2598
2599                 /*
2600                  * btrfs_wait_cache_io uses the cache->dirty_list to decide if
2601                  * it should update the cache_state.  Don't delete until after
2602                  * we wait.
2603                  *
2604                  * Since we're not running in the commit critical section
2605                  * we need the dirty_bgs_lock to protect from update_block_group
2606                  */
2607                 spin_lock(&cur_trans->dirty_bgs_lock);
2608                 list_del_init(&cache->dirty_list);
2609                 spin_unlock(&cur_trans->dirty_bgs_lock);
2610
2611                 should_put = 1;
2612
2613                 cache_save_setup(cache, trans, path);
2614
2615                 if (cache->disk_cache_state == BTRFS_DC_SETUP) {
2616                         cache->io_ctl.inode = NULL;
2617                         ret = btrfs_write_out_cache(trans, cache, path);
2618                         if (ret == 0 && cache->io_ctl.inode) {
2619                                 num_started++;
2620                                 should_put = 0;
2621
2622                                 /*
2623                                  * The cache_write_mutex is protecting the
2624                                  * io_list, also refer to the definition of
2625                                  * btrfs_transaction::io_bgs for more details
2626                                  */
2627                                 list_add_tail(&cache->io_list, io);
2628                         } else {
2629                                 /*
2630                                  * If we failed to write the cache, the
2631                                  * generation will be bad and life goes on
2632                                  */
2633                                 ret = 0;
2634                         }
2635                 }
2636                 if (!ret) {
2637                         ret = write_one_cache_group(trans, path, cache);
2638                         /*
2639                          * Our block group might still be attached to the list
2640                          * of new block groups in the transaction handle of some
2641                          * other task (struct btrfs_trans_handle->new_bgs). This
2642                          * means its block group item isn't yet in the extent
2643                          * tree. If this happens ignore the error, as we will
2644                          * try again later in the critical section of the
2645                          * transaction commit.
2646                          */
2647                         if (ret == -ENOENT) {
2648                                 ret = 0;
2649                                 spin_lock(&cur_trans->dirty_bgs_lock);
2650                                 if (list_empty(&cache->dirty_list)) {
2651                                         list_add_tail(&cache->dirty_list,
2652                                                       &cur_trans->dirty_bgs);
2653                                         btrfs_get_block_group(cache);
2654                                         drop_reserve = false;
2655                                 }
2656                                 spin_unlock(&cur_trans->dirty_bgs_lock);
2657                         } else if (ret) {
2658                                 btrfs_abort_transaction(trans, ret);
2659                         }
2660                 }
2661
2662                 /* If it's not on the io list, we need to put the block group */
2663                 if (should_put)
2664                         btrfs_put_block_group(cache);
2665                 if (drop_reserve)
2666                         btrfs_delayed_refs_rsv_release(fs_info, 1);
2667
2668                 if (ret)
2669                         break;
2670
2671                 /*
2672                  * Avoid blocking other tasks for too long. It might even save
2673                  * us from writing caches for block groups that are going to be
2674                  * removed.
2675                  */
2676                 mutex_unlock(&trans->transaction->cache_write_mutex);
2677                 mutex_lock(&trans->transaction->cache_write_mutex);
2678         }
2679         mutex_unlock(&trans->transaction->cache_write_mutex);
2680
2681         /*
2682          * Go through delayed refs for all the stuff we've just kicked off
2683          * and then loop back (just once)
2684          */
2685         ret = btrfs_run_delayed_refs(trans, 0);
2686         if (!ret && loops == 0) {
2687                 loops++;
2688                 spin_lock(&cur_trans->dirty_bgs_lock);
2689                 list_splice_init(&cur_trans->dirty_bgs, &dirty);
2690                 /*
2691                  * dirty_bgs_lock protects us from concurrent block group
2692                  * deletes too (not just cache_write_mutex).
2693                  */
2694                 if (!list_empty(&dirty)) {
2695                         spin_unlock(&cur_trans->dirty_bgs_lock);
2696                         goto again;
2697                 }
2698                 spin_unlock(&cur_trans->dirty_bgs_lock);
2699         } else if (ret < 0) {
2700                 btrfs_cleanup_dirty_bgs(cur_trans, fs_info);
2701         }
2702
2703         btrfs_free_path(path);
2704         return ret;
2705 }
2706
2707 int btrfs_write_dirty_block_groups(struct btrfs_trans_handle *trans)
2708 {
2709         struct btrfs_fs_info *fs_info = trans->fs_info;
2710         struct btrfs_block_group *cache;
2711         struct btrfs_transaction *cur_trans = trans->transaction;
2712         int ret = 0;
2713         int should_put;
2714         struct btrfs_path *path;
2715         struct list_head *io = &cur_trans->io_bgs;
2716         int num_started = 0;
2717
2718         path = btrfs_alloc_path();
2719         if (!path)
2720                 return -ENOMEM;
2721
2722         /*
2723          * Even though we are in the critical section of the transaction commit,
2724          * we can still have concurrent tasks adding elements to this
2725          * transaction's list of dirty block groups. These tasks correspond to
2726          * endio free space workers started when writeback finishes for a
2727          * space cache, which run inode.c:btrfs_finish_ordered_io(), and can
2728          * allocate new block groups as a result of COWing nodes of the root
2729          * tree when updating the free space inode. The writeback for the space
2730          * caches is triggered by an earlier call to
2731          * btrfs_start_dirty_block_groups() and iterations of the following
2732          * loop.
2733          * Also we want to do the cache_save_setup first and then run the
2734          * delayed refs to make sure we have the best chance at doing this all
2735          * in one shot.
2736          */
2737         spin_lock(&cur_trans->dirty_bgs_lock);
2738         while (!list_empty(&cur_trans->dirty_bgs)) {
2739                 cache = list_first_entry(&cur_trans->dirty_bgs,
2740                                          struct btrfs_block_group,
2741                                          dirty_list);
2742
2743                 /*
2744                  * This can happen if cache_save_setup re-dirties a block group
2745                  * that is already under IO.  Just wait for it to finish and
2746                  * then do it all again
2747                  */
2748                 if (!list_empty(&cache->io_list)) {
2749                         spin_unlock(&cur_trans->dirty_bgs_lock);
2750                         list_del_init(&cache->io_list);
2751                         btrfs_wait_cache_io(trans, cache, path);
2752                         btrfs_put_block_group(cache);
2753                         spin_lock(&cur_trans->dirty_bgs_lock);
2754                 }
2755
2756                 /*
2757                  * Don't remove from the dirty list until after we've waited on
2758                  * any pending IO
2759                  */
2760                 list_del_init(&cache->dirty_list);
2761                 spin_unlock(&cur_trans->dirty_bgs_lock);
2762                 should_put = 1;
2763
2764                 cache_save_setup(cache, trans, path);
2765
2766                 if (!ret)
2767                         ret = btrfs_run_delayed_refs(trans,
2768                                                      (unsigned long) -1);
2769
2770                 if (!ret && cache->disk_cache_state == BTRFS_DC_SETUP) {
2771                         cache->io_ctl.inode = NULL;
2772                         ret = btrfs_write_out_cache(trans, cache, path);
2773                         if (ret == 0 && cache->io_ctl.inode) {
2774                                 num_started++;
2775                                 should_put = 0;
2776                                 list_add_tail(&cache->io_list, io);
2777                         } else {
2778                                 /*
2779                                  * If we failed to write the cache, the
2780                                  * generation will be bad and life goes on
2781                                  */
2782                                 ret = 0;
2783                         }
2784                 }
2785                 if (!ret) {
2786                         ret = write_one_cache_group(trans, path, cache);
2787                         /*
2788                          * One of the free space endio workers might have
2789                          * created a new block group while updating a free space
2790                          * cache's inode (at inode.c:btrfs_finish_ordered_io())
2791                          * and hasn't released its transaction handle yet, in
2792                          * which case the new block group is still attached to
2793                          * its transaction handle and its creation has not
2794                          * finished yet (no block group item in the extent tree
2795                          * yet, etc). If this is the case, wait for all free
2796                          * space endio workers to finish and retry. This is a
2797                          * a very rare case so no need for a more efficient and
2798                          * complex approach.
2799                          */
2800                         if (ret == -ENOENT) {
2801                                 wait_event(cur_trans->writer_wait,
2802                                    atomic_read(&cur_trans->num_writers) == 1);
2803                                 ret = write_one_cache_group(trans, path, cache);
2804                         }
2805                         if (ret)
2806                                 btrfs_abort_transaction(trans, ret);
2807                 }
2808
2809                 /* If its not on the io list, we need to put the block group */
2810                 if (should_put)
2811                         btrfs_put_block_group(cache);
2812                 btrfs_delayed_refs_rsv_release(fs_info, 1);
2813                 spin_lock(&cur_trans->dirty_bgs_lock);
2814         }
2815         spin_unlock(&cur_trans->dirty_bgs_lock);
2816
2817         /*
2818          * Refer to the definition of io_bgs member for details why it's safe
2819          * to use it without any locking
2820          */
2821         while (!list_empty(io)) {
2822                 cache = list_first_entry(io, struct btrfs_block_group,
2823                                          io_list);
2824                 list_del_init(&cache->io_list);
2825                 btrfs_wait_cache_io(trans, cache, path);
2826                 btrfs_put_block_group(cache);
2827         }
2828
2829         btrfs_free_path(path);
2830         return ret;
2831 }
2832
2833 int btrfs_update_block_group(struct btrfs_trans_handle *trans,
2834                              u64 bytenr, u64 num_bytes, int alloc)
2835 {
2836         struct btrfs_fs_info *info = trans->fs_info;
2837         struct btrfs_block_group *cache = NULL;
2838         u64 total = num_bytes;
2839         u64 old_val;
2840         u64 byte_in_group;
2841         int factor;
2842         int ret = 0;
2843
2844         /* Block accounting for super block */
2845         spin_lock(&info->delalloc_root_lock);
2846         old_val = btrfs_super_bytes_used(info->super_copy);
2847         if (alloc)
2848                 old_val += num_bytes;
2849         else
2850                 old_val -= num_bytes;
2851         btrfs_set_super_bytes_used(info->super_copy, old_val);
2852         spin_unlock(&info->delalloc_root_lock);
2853
2854         while (total) {
2855                 cache = btrfs_lookup_block_group(info, bytenr);
2856                 if (!cache) {
2857                         ret = -ENOENT;
2858                         break;
2859                 }
2860                 factor = btrfs_bg_type_to_factor(cache->flags);
2861
2862                 /*
2863                  * If this block group has free space cache written out, we
2864                  * need to make sure to load it if we are removing space.  This
2865                  * is because we need the unpinning stage to actually add the
2866                  * space back to the block group, otherwise we will leak space.
2867                  */
2868                 if (!alloc && !btrfs_block_group_done(cache))
2869                         btrfs_cache_block_group(cache, 1);
2870
2871                 byte_in_group = bytenr - cache->start;
2872                 WARN_ON(byte_in_group > cache->length);
2873
2874                 spin_lock(&cache->space_info->lock);
2875                 spin_lock(&cache->lock);
2876
2877                 if (btrfs_test_opt(info, SPACE_CACHE) &&
2878                     cache->disk_cache_state < BTRFS_DC_CLEAR)
2879                         cache->disk_cache_state = BTRFS_DC_CLEAR;
2880
2881                 old_val = cache->used;
2882                 num_bytes = min(total, cache->length - byte_in_group);
2883                 if (alloc) {
2884                         old_val += num_bytes;
2885                         cache->used = old_val;
2886                         cache->reserved -= num_bytes;
2887                         cache->space_info->bytes_reserved -= num_bytes;
2888                         cache->space_info->bytes_used += num_bytes;
2889                         cache->space_info->disk_used += num_bytes * factor;
2890                         spin_unlock(&cache->lock);
2891                         spin_unlock(&cache->space_info->lock);
2892                 } else {
2893                         old_val -= num_bytes;
2894                         cache->used = old_val;
2895                         cache->pinned += num_bytes;
2896                         btrfs_space_info_update_bytes_pinned(info,
2897                                         cache->space_info, num_bytes);
2898                         cache->space_info->bytes_used -= num_bytes;
2899                         cache->space_info->disk_used -= num_bytes * factor;
2900                         spin_unlock(&cache->lock);
2901                         spin_unlock(&cache->space_info->lock);
2902
2903                         percpu_counter_add_batch(
2904                                         &cache->space_info->total_bytes_pinned,
2905                                         num_bytes,
2906                                         BTRFS_TOTAL_BYTES_PINNED_BATCH);
2907                         set_extent_dirty(&trans->transaction->pinned_extents,
2908                                          bytenr, bytenr + num_bytes - 1,
2909                                          GFP_NOFS | __GFP_NOFAIL);
2910                 }
2911
2912                 spin_lock(&trans->transaction->dirty_bgs_lock);
2913                 if (list_empty(&cache->dirty_list)) {
2914                         list_add_tail(&cache->dirty_list,
2915                                       &trans->transaction->dirty_bgs);
2916                         trans->delayed_ref_updates++;
2917                         btrfs_get_block_group(cache);
2918                 }
2919                 spin_unlock(&trans->transaction->dirty_bgs_lock);
2920
2921                 /*
2922                  * No longer have used bytes in this block group, queue it for
2923                  * deletion. We do this after adding the block group to the
2924                  * dirty list to avoid races between cleaner kthread and space
2925                  * cache writeout.
2926                  */
2927                 if (!alloc && old_val == 0) {
2928                         if (!btrfs_test_opt(info, DISCARD_ASYNC))
2929                                 btrfs_mark_bg_unused(cache);
2930                 }
2931
2932                 btrfs_put_block_group(cache);
2933                 total -= num_bytes;
2934                 bytenr += num_bytes;
2935         }
2936
2937         /* Modified block groups are accounted for in the delayed_refs_rsv. */
2938         btrfs_update_delayed_refs_rsv(trans);
2939         return ret;
2940 }
2941
2942 /**
2943  * btrfs_add_reserved_bytes - update the block_group and space info counters
2944  * @cache:      The cache we are manipulating
2945  * @ram_bytes:  The number of bytes of file content, and will be same to
2946  *              @num_bytes except for the compress path.
2947  * @num_bytes:  The number of bytes in question
2948  * @delalloc:   The blocks are allocated for the delalloc write
2949  *
2950  * This is called by the allocator when it reserves space. If this is a
2951  * reservation and the block group has become read only we cannot make the
2952  * reservation and return -EAGAIN, otherwise this function always succeeds.
2953  */
2954 int btrfs_add_reserved_bytes(struct btrfs_block_group *cache,
2955                              u64 ram_bytes, u64 num_bytes, int delalloc)
2956 {
2957         struct btrfs_space_info *space_info = cache->space_info;
2958         int ret = 0;
2959
2960         spin_lock(&space_info->lock);
2961         spin_lock(&cache->lock);
2962         if (cache->ro) {
2963                 ret = -EAGAIN;
2964         } else {
2965                 cache->reserved += num_bytes;
2966                 space_info->bytes_reserved += num_bytes;
2967                 trace_btrfs_space_reservation(cache->fs_info, "space_info",
2968                                               space_info->flags, num_bytes, 1);
2969                 btrfs_space_info_update_bytes_may_use(cache->fs_info,
2970                                                       space_info, -ram_bytes);
2971                 if (delalloc)
2972                         cache->delalloc_bytes += num_bytes;
2973         }
2974         spin_unlock(&cache->lock);
2975         spin_unlock(&space_info->lock);
2976         return ret;
2977 }
2978
2979 /**
2980  * btrfs_free_reserved_bytes - update the block_group and space info counters
2981  * @cache:      The cache we are manipulating
2982  * @num_bytes:  The number of bytes in question
2983  * @delalloc:   The blocks are allocated for the delalloc write
2984  *
2985  * This is called by somebody who is freeing space that was never actually used
2986  * on disk.  For example if you reserve some space for a new leaf in transaction
2987  * A and before transaction A commits you free that leaf, you call this with
2988  * reserve set to 0 in order to clear the reservation.
2989  */
2990 void btrfs_free_reserved_bytes(struct btrfs_block_group *cache,
2991                                u64 num_bytes, int delalloc)
2992 {
2993         struct btrfs_space_info *space_info = cache->space_info;
2994
2995         spin_lock(&space_info->lock);
2996         spin_lock(&cache->lock);
2997         if (cache->ro)
2998                 space_info->bytes_readonly += num_bytes;
2999         cache->reserved -= num_bytes;
3000         space_info->bytes_reserved -= num_bytes;
3001         space_info->max_extent_size = 0;
3002
3003         if (delalloc)
3004                 cache->delalloc_bytes -= num_bytes;
3005         spin_unlock(&cache->lock);
3006         spin_unlock(&space_info->lock);
3007 }
3008
3009 static void force_metadata_allocation(struct btrfs_fs_info *info)
3010 {
3011         struct list_head *head = &info->space_info;
3012         struct btrfs_space_info *found;
3013
3014         rcu_read_lock();
3015         list_for_each_entry_rcu(found, head, list) {
3016                 if (found->flags & BTRFS_BLOCK_GROUP_METADATA)
3017                         found->force_alloc = CHUNK_ALLOC_FORCE;
3018         }
3019         rcu_read_unlock();
3020 }
3021
3022 static int should_alloc_chunk(struct btrfs_fs_info *fs_info,
3023                               struct btrfs_space_info *sinfo, int force)
3024 {
3025         u64 bytes_used = btrfs_space_info_used(sinfo, false);
3026         u64 thresh;
3027
3028         if (force == CHUNK_ALLOC_FORCE)
3029                 return 1;
3030
3031         /*
3032          * in limited mode, we want to have some free space up to
3033          * about 1% of the FS size.
3034          */
3035         if (force == CHUNK_ALLOC_LIMITED) {
3036                 thresh = btrfs_super_total_bytes(fs_info->super_copy);
3037                 thresh = max_t(u64, SZ_64M, div_factor_fine(thresh, 1));
3038
3039                 if (sinfo->total_bytes - bytes_used < thresh)
3040                         return 1;
3041         }
3042
3043         if (bytes_used + SZ_2M < div_factor(sinfo->total_bytes, 8))
3044                 return 0;
3045         return 1;
3046 }
3047
3048 int btrfs_force_chunk_alloc(struct btrfs_trans_handle *trans, u64 type)
3049 {
3050         u64 alloc_flags = btrfs_get_alloc_profile(trans->fs_info, type);
3051
3052         return btrfs_chunk_alloc(trans, alloc_flags, CHUNK_ALLOC_FORCE);
3053 }
3054
3055 /*
3056  * If force is CHUNK_ALLOC_FORCE:
3057  *    - return 1 if it successfully allocates a chunk,
3058  *    - return errors including -ENOSPC otherwise.
3059  * If force is NOT CHUNK_ALLOC_FORCE:
3060  *    - return 0 if it doesn't need to allocate a new chunk,
3061  *    - return 1 if it successfully allocates a chunk,
3062  *    - return errors including -ENOSPC otherwise.
3063  */
3064 int btrfs_chunk_alloc(struct btrfs_trans_handle *trans, u64 flags,
3065                       enum btrfs_chunk_alloc_enum force)
3066 {
3067         struct btrfs_fs_info *fs_info = trans->fs_info;
3068         struct btrfs_space_info *space_info;
3069         bool wait_for_alloc = false;
3070         bool should_alloc = false;
3071         int ret = 0;
3072
3073         /* Don't re-enter if we're already allocating a chunk */
3074         if (trans->allocating_chunk)
3075                 return -ENOSPC;
3076
3077         space_info = btrfs_find_space_info(fs_info, flags);
3078         ASSERT(space_info);
3079
3080         do {
3081                 spin_lock(&space_info->lock);
3082                 if (force < space_info->force_alloc)
3083                         force = space_info->force_alloc;
3084                 should_alloc = should_alloc_chunk(fs_info, space_info, force);
3085                 if (space_info->full) {
3086                         /* No more free physical space */
3087                         if (should_alloc)
3088                                 ret = -ENOSPC;
3089                         else
3090                                 ret = 0;
3091                         spin_unlock(&space_info->lock);
3092                         return ret;
3093                 } else if (!should_alloc) {
3094                         spin_unlock(&space_info->lock);
3095                         return 0;
3096                 } else if (space_info->chunk_alloc) {
3097                         /*
3098                          * Someone is already allocating, so we need to block
3099                          * until this someone is finished and then loop to
3100                          * recheck if we should continue with our allocation
3101                          * attempt.
3102                          */
3103                         wait_for_alloc = true;
3104                         spin_unlock(&space_info->lock);
3105                         mutex_lock(&fs_info->chunk_mutex);
3106                         mutex_unlock(&fs_info->chunk_mutex);
3107                 } else {
3108                         /* Proceed with allocation */
3109                         space_info->chunk_alloc = 1;
3110                         wait_for_alloc = false;
3111                         spin_unlock(&space_info->lock);
3112                 }
3113
3114                 cond_resched();
3115         } while (wait_for_alloc);
3116
3117         mutex_lock(&fs_info->chunk_mutex);
3118         trans->allocating_chunk = true;
3119
3120         /*
3121          * If we have mixed data/metadata chunks we want to make sure we keep
3122          * allocating mixed chunks instead of individual chunks.
3123          */
3124         if (btrfs_mixed_space_info(space_info))
3125                 flags |= (BTRFS_BLOCK_GROUP_DATA | BTRFS_BLOCK_GROUP_METADATA);
3126
3127         /*
3128          * if we're doing a data chunk, go ahead and make sure that
3129          * we keep a reasonable number of metadata chunks allocated in the
3130          * FS as well.
3131          */
3132         if (flags & BTRFS_BLOCK_GROUP_DATA && fs_info->metadata_ratio) {
3133                 fs_info->data_chunk_allocations++;
3134                 if (!(fs_info->data_chunk_allocations %
3135                       fs_info->metadata_ratio))
3136                         force_metadata_allocation(fs_info);
3137         }
3138
3139         /*
3140          * Check if we have enough space in SYSTEM chunk because we may need
3141          * to update devices.
3142          */
3143         check_system_chunk(trans, flags);
3144
3145         ret = btrfs_alloc_chunk(trans, flags);
3146         trans->allocating_chunk = false;
3147
3148         spin_lock(&space_info->lock);
3149         if (ret < 0) {
3150                 if (ret == -ENOSPC)
3151                         space_info->full = 1;
3152                 else
3153                         goto out;
3154         } else {
3155                 ret = 1;
3156                 space_info->max_extent_size = 0;
3157         }
3158
3159         space_info->force_alloc = CHUNK_ALLOC_NO_FORCE;
3160 out:
3161         space_info->chunk_alloc = 0;
3162         spin_unlock(&space_info->lock);
3163         mutex_unlock(&fs_info->chunk_mutex);
3164         /*
3165          * When we allocate a new chunk we reserve space in the chunk block
3166          * reserve to make sure we can COW nodes/leafs in the chunk tree or
3167          * add new nodes/leafs to it if we end up needing to do it when
3168          * inserting the chunk item and updating device items as part of the
3169          * second phase of chunk allocation, performed by
3170          * btrfs_finish_chunk_alloc(). So make sure we don't accumulate a
3171          * large number of new block groups to create in our transaction
3172          * handle's new_bgs list to avoid exhausting the chunk block reserve
3173          * in extreme cases - like having a single transaction create many new
3174          * block groups when starting to write out the free space caches of all
3175          * the block groups that were made dirty during the lifetime of the
3176          * transaction.
3177          */
3178         if (trans->chunk_bytes_reserved >= (u64)SZ_2M)
3179                 btrfs_create_pending_block_groups(trans);
3180
3181         return ret;
3182 }
3183
3184 static u64 get_profile_num_devs(struct btrfs_fs_info *fs_info, u64 type)
3185 {
3186         u64 num_dev;
3187
3188         num_dev = btrfs_raid_array[btrfs_bg_flags_to_raid_index(type)].devs_max;
3189         if (!num_dev)
3190                 num_dev = fs_info->fs_devices->rw_devices;
3191
3192         return num_dev;
3193 }
3194
3195 /*
3196  * Reserve space in the system space for allocating or removing a chunk
3197  */
3198 void check_system_chunk(struct btrfs_trans_handle *trans, u64 type)
3199 {
3200         struct btrfs_fs_info *fs_info = trans->fs_info;
3201         struct btrfs_space_info *info;
3202         u64 left;
3203         u64 thresh;
3204         int ret = 0;
3205         u64 num_devs;
3206
3207         /*
3208          * Needed because we can end up allocating a system chunk and for an
3209          * atomic and race free space reservation in the chunk block reserve.
3210          */
3211         lockdep_assert_held(&fs_info->chunk_mutex);
3212
3213         info = btrfs_find_space_info(fs_info, BTRFS_BLOCK_GROUP_SYSTEM);
3214         spin_lock(&info->lock);
3215         left = info->total_bytes - btrfs_space_info_used(info, true);
3216         spin_unlock(&info->lock);
3217
3218         num_devs = get_profile_num_devs(fs_info, type);
3219
3220         /* num_devs device items to update and 1 chunk item to add or remove */
3221         thresh = btrfs_calc_metadata_size(fs_info, num_devs) +
3222                 btrfs_calc_insert_metadata_size(fs_info, 1);
3223
3224         if (left < thresh && btrfs_test_opt(fs_info, ENOSPC_DEBUG)) {
3225                 btrfs_info(fs_info, "left=%llu, need=%llu, flags=%llu",
3226                            left, thresh, type);
3227                 btrfs_dump_space_info(fs_info, info, 0, 0);
3228         }
3229
3230         if (left < thresh) {
3231                 u64 flags = btrfs_system_alloc_profile(fs_info);
3232
3233                 /*
3234                  * Ignore failure to create system chunk. We might end up not
3235                  * needing it, as we might not need to COW all nodes/leafs from
3236                  * the paths we visit in the chunk tree (they were already COWed
3237                  * or created in the current transaction for example).
3238                  */
3239                 ret = btrfs_alloc_chunk(trans, flags);
3240         }
3241
3242         if (!ret) {
3243                 ret = btrfs_block_rsv_add(fs_info->chunk_root,
3244                                           &fs_info->chunk_block_rsv,
3245                                           thresh, BTRFS_RESERVE_NO_FLUSH);
3246                 if (!ret)
3247                         trans->chunk_bytes_reserved += thresh;
3248         }
3249 }
3250
3251 void btrfs_put_block_group_cache(struct btrfs_fs_info *info)
3252 {
3253         struct btrfs_block_group *block_group;
3254         u64 last = 0;
3255
3256         while (1) {
3257                 struct inode *inode;
3258
3259                 block_group = btrfs_lookup_first_block_group(info, last);
3260                 while (block_group) {
3261                         btrfs_wait_block_group_cache_done(block_group);
3262                         spin_lock(&block_group->lock);
3263                         if (block_group->iref)
3264                                 break;
3265                         spin_unlock(&block_group->lock);
3266                         block_group = btrfs_next_block_group(block_group);
3267                 }
3268                 if (!block_group) {
3269                         if (last == 0)
3270                                 break;
3271                         last = 0;
3272                         continue;
3273                 }
3274
3275                 inode = block_group->inode;
3276                 block_group->iref = 0;
3277                 block_group->inode = NULL;
3278                 spin_unlock(&block_group->lock);
3279                 ASSERT(block_group->io_ctl.inode == NULL);
3280                 iput(inode);
3281                 last = block_group->start + block_group->length;
3282                 btrfs_put_block_group(block_group);
3283         }
3284 }
3285
3286 /*
3287  * Must be called only after stopping all workers, since we could have block
3288  * group caching kthreads running, and therefore they could race with us if we
3289  * freed the block groups before stopping them.
3290  */
3291 int btrfs_free_block_groups(struct btrfs_fs_info *info)
3292 {
3293         struct btrfs_block_group *block_group;
3294         struct btrfs_space_info *space_info;
3295         struct btrfs_caching_control *caching_ctl;
3296         struct rb_node *n;
3297
3298         down_write(&info->commit_root_sem);
3299         while (!list_empty(&info->caching_block_groups)) {
3300                 caching_ctl = list_entry(info->caching_block_groups.next,
3301                                          struct btrfs_caching_control, list);
3302                 list_del(&caching_ctl->list);
3303                 btrfs_put_caching_control(caching_ctl);
3304         }
3305         up_write(&info->commit_root_sem);
3306
3307         spin_lock(&info->unused_bgs_lock);
3308         while (!list_empty(&info->unused_bgs)) {
3309                 block_group = list_first_entry(&info->unused_bgs,
3310                                                struct btrfs_block_group,
3311                                                bg_list);
3312                 list_del_init(&block_group->bg_list);
3313                 btrfs_put_block_group(block_group);
3314         }
3315         spin_unlock(&info->unused_bgs_lock);
3316
3317         spin_lock(&info->block_group_cache_lock);
3318         while ((n = rb_last(&info->block_group_cache_tree)) != NULL) {
3319                 block_group = rb_entry(n, struct btrfs_block_group,
3320                                        cache_node);
3321                 rb_erase(&block_group->cache_node,
3322                          &info->block_group_cache_tree);
3323                 RB_CLEAR_NODE(&block_group->cache_node);
3324                 spin_unlock(&info->block_group_cache_lock);
3325
3326                 down_write(&block_group->space_info->groups_sem);
3327                 list_del(&block_group->list);
3328                 up_write(&block_group->space_info->groups_sem);
3329
3330                 /*
3331                  * We haven't cached this block group, which means we could
3332                  * possibly have excluded extents on this block group.
3333                  */
3334                 if (block_group->cached == BTRFS_CACHE_NO ||
3335                     block_group->cached == BTRFS_CACHE_ERROR)
3336                         btrfs_free_excluded_extents(block_group);
3337
3338                 btrfs_remove_free_space_cache(block_group);
3339                 ASSERT(block_group->cached != BTRFS_CACHE_STARTED);
3340                 ASSERT(list_empty(&block_group->dirty_list));
3341                 ASSERT(list_empty(&block_group->io_list));
3342                 ASSERT(list_empty(&block_group->bg_list));
3343                 ASSERT(atomic_read(&block_group->count) == 1);
3344                 btrfs_put_block_group(block_group);
3345
3346                 spin_lock(&info->block_group_cache_lock);
3347         }
3348         spin_unlock(&info->block_group_cache_lock);
3349
3350         /*
3351          * Now that all the block groups are freed, go through and free all the
3352          * space_info structs.  This is only called during the final stages of
3353          * unmount, and so we know nobody is using them.  We call
3354          * synchronize_rcu() once before we start, just to be on the safe side.
3355          */
3356         synchronize_rcu();
3357
3358         btrfs_release_global_block_rsv(info);
3359
3360         while (!list_empty(&info->space_info)) {
3361                 space_info = list_entry(info->space_info.next,
3362                                         struct btrfs_space_info,
3363                                         list);
3364
3365                 /*
3366                  * Do not hide this behind enospc_debug, this is actually
3367                  * important and indicates a real bug if this happens.
3368                  */
3369                 if (WARN_ON(space_info->bytes_pinned > 0 ||
3370                             space_info->bytes_reserved > 0 ||
3371                             space_info->bytes_may_use > 0))
3372                         btrfs_dump_space_info(info, space_info, 0, 0);
3373                 list_del(&space_info->list);
3374                 btrfs_sysfs_remove_space_info(space_info);
3375         }
3376         return 0;
3377 }