fs/f2fs/data.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * fs/f2fs/data.c
   4  *
   5  * Copyright (c) 2012 Samsung Electronics Co., Ltd.
   6  *             http://www.samsung.com/
   7  */
   8 #include <linux/fs.h>
   9 #include <linux/f2fs_fs.h>
  10 #include <linux/buffer_head.h>
  11 #include <linux/mpage.h>
  12 #include <linux/writeback.h>
  13 #include <linux/backing-dev.h>
  14 #include <linux/pagevec.h>
  15 #include <linux/blkdev.h>
  16 #include <linux/bio.h>
  17 #include <linux/swap.h>
  18 #include <linux/prefetch.h>
  19 #include <linux/uio.h>
  20 #include <linux/cleancache.h>
  21
  22 #include "f2fs.h"
  23 #include "node.h"
  24 #include "segment.h"
  25 #include "trace.h"
  26 #include <trace/events/f2fs.h>
  27 #include <trace/events/android_fs.h>
  28
  29 #define NUM_PREALLOC_POST_READ_CTXS     128
  30
  31 static struct kmem_cache *bio_post_read_ctx_cache;
  32 static mempool_t *bio_post_read_ctx_pool;
  33
  34 static bool __is_cp_guaranteed(struct page *page)
  35 {
  36         struct address_space *mapping = page->mapping;
  37         struct inode *inode;
  38         struct f2fs_sb_info *sbi;
  39
  40         if (!mapping)
  41                 return false;
  42
  43         inode = mapping->host;
  44         sbi = F2FS_I_SB(inode);
  45
  46         if (inode->i_ino == F2FS_META_INO(sbi) ||
  47                         inode->i_ino ==  F2FS_NODE_INO(sbi) ||
  48                         S_ISDIR(inode->i_mode) ||
  49                         (S_ISREG(inode->i_mode) &&
  50                         (f2fs_is_atomic_file(inode) || IS_NOQUOTA(inode))) ||
  51                         is_cold_data(page))
  52                 return true;
  53         return false;
  54 }
  55
  56 static enum count_type __read_io_type(struct page *page)
  57 {
  58         struct address_space *mapping = page_file_mapping(page);
  59
  60         if (mapping) {
  61                 struct inode *inode = mapping->host;
  62                 struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
  63
  64                 if (inode->i_ino == F2FS_META_INO(sbi))
  65                         return F2FS_RD_META;
  66
  67                 if (inode->i_ino == F2FS_NODE_INO(sbi))
  68                         return F2FS_RD_NODE;
  69         }
  70         return F2FS_RD_DATA;
  71 }
  72
  73 /* postprocessing steps for read bios */
  74 enum bio_post_read_step {
  75         STEP_INITIAL = 0,
  76         STEP_DECRYPT,
  77 };
  78
  79 struct bio_post_read_ctx {
  80         struct bio *bio;
  81         struct work_struct work;
  82         unsigned int cur_step;
  83         unsigned int enabled_steps;
  84 };
  85
  86 static void __read_end_io(struct bio *bio)
  87 {
  88         struct page *page;
  89         struct bio_vec *bv;
  90         int i;
  91
  92         bio_for_each_segment_all(bv, bio, i) {
  93                 page = bv->bv_page;
  94
  95                 /* PG_error was set if any post_read step failed */
  96                 if (bio->bi_error || PageError(page)) {
  97                         ClearPageUptodate(page);
  98                         /* will re-read again later */
  99                         ClearPageError(page);
 100                 } else {
 101                         SetPageUptodate(page);
 102                 }
 103                 dec_page_count(F2FS_P_SB(page), __read_io_type(page));
 104                 unlock_page(page);
 105         }
 106         if (bio->bi_private)
 107                 mempool_free(bio->bi_private, bio_post_read_ctx_pool);
 108         bio_put(bio);
 109 }
 110
 111 static void bio_post_read_processing(struct bio_post_read_ctx *ctx);
 112
 113 static void decrypt_work(struct work_struct *work)
 114 {
 115         struct bio_post_read_ctx *ctx =
 116                 container_of(work, struct bio_post_read_ctx, work);
 117
 118         fscrypt_decrypt_bio(ctx->bio);
 119
 120         bio_post_read_processing(ctx);
 121 }
 122
 123 static void bio_post_read_processing(struct bio_post_read_ctx *ctx)
 124 {
 125         switch (++ctx->cur_step) {
 126         case STEP_DECRYPT:
 127                 if (ctx->enabled_steps & (1 << STEP_DECRYPT)) {
 128                         INIT_WORK(&ctx->work, decrypt_work);
 129                         fscrypt_enqueue_decrypt_work(&ctx->work);
 130                         return;
 131                 }
 132                 ctx->cur_step++;
 133                 /* fall-through */
 134         default:
 135                 __read_end_io(ctx->bio);
 136         }
 137 }
 138
 139 static bool f2fs_bio_post_read_required(struct bio *bio)
 140 {
 141         return bio->bi_private && !bio->bi_error;
 142 }
 143
 144 static void f2fs_read_end_io(struct bio *bio)
 145 {
 146         struct page *first_page = bio->bi_io_vec[0].bv_page;
 147
 148         if (time_to_inject(F2FS_P_SB(bio->bi_io_vec->bv_page), FAULT_READ_IO)) {
 149                 f2fs_show_injection_info(FAULT_READ_IO);
 150                 bio->bi_error = -EIO;
 151         }
 152
 153         if (f2fs_bio_post_read_required(bio)) {
 154                 struct bio_post_read_ctx *ctx = bio->bi_private;
 155
 156                 ctx->cur_step = STEP_INITIAL;
 157                 bio_post_read_processing(ctx);
 158                 return;
 159         }
 160
 161         if (first_page != NULL &&
 162                 __read_io_type(first_page) == F2FS_RD_DATA) {
 163                 trace_android_fs_dataread_end(first_page->mapping->host,
 164                                                 page_offset(first_page),
 165                                                 bio->bi_iter.bi_size);
 166         }
 167
 168         __read_end_io(bio);
 169 }
 170
 171 static void f2fs_write_end_io(struct bio *bio)
 172 {
 173         struct f2fs_sb_info *sbi = bio->bi_private;
 174         struct bio_vec *bvec;
 175         int i;
 176
 177         if (time_to_inject(sbi, FAULT_WRITE_IO)) {
 178                 f2fs_show_injection_info(FAULT_WRITE_IO);
 179                 bio->bi_error = -EIO;
 180         }
 181
 182         bio_for_each_segment_all(bvec, bio, i) {
 183                 struct page *page = bvec->bv_page;
 184                 enum count_type type = WB_DATA_TYPE(page);
 185
 186                 if (IS_DUMMY_WRITTEN_PAGE(page)) {
 187                         set_page_private(page, (unsigned long)NULL);
 188                         ClearPagePrivate(page);
 189                         unlock_page(page);
 190                         mempool_free(page, sbi->write_io_dummy);
 191
 192                         if (unlikely(bio->bi_error))
 193                                 f2fs_stop_checkpoint(sbi, true);
 194                         continue;
 195                 }
 196
 197                 fscrypt_pullback_bio_page(&page, true);
 198
 199                 if (unlikely(bio->bi_error)) {
 200                         set_bit(AS_EIO, &page->mapping->flags);
 201                         if (type == F2FS_WB_CP_DATA)
 202                                 f2fs_stop_checkpoint(sbi, true);
 203                 }
 204
 205                 f2fs_bug_on(sbi, page->mapping == NODE_MAPPING(sbi) &&
 206                                         page->index != nid_of_node(page));
 207
 208                 dec_page_count(sbi, type);
 209                 if (f2fs_in_warm_node_list(sbi, page))
 210                         f2fs_del_fsync_node_entry(sbi, page);
 211                 clear_cold_data(page);
 212                 end_page_writeback(page);
 213         }
 214         if (!get_pages(sbi, F2FS_WB_CP_DATA) &&
 215                                 wq_has_sleeper(&sbi->cp_wait))
 216                 wake_up(&sbi->cp_wait);
 217
 218         bio_put(bio);
 219 }
 220
 221 /*
 222  * Return true, if pre_bio's bdev is same as its target device.
 223  */
 224 struct block_device *f2fs_target_device(struct f2fs_sb_info *sbi,
 225                                 block_t blk_addr, struct bio *bio)
 226 {
 227         struct block_device *bdev = sbi->sb->s_bdev;
 228         int i;
 229
 230         if (f2fs_is_multi_device(sbi)) {
 231                 for (i = 0; i < sbi->s_ndevs; i++) {
 232                         if (FDEV(i).start_blk <= blk_addr &&
 233                             FDEV(i).end_blk >= blk_addr) {
 234                                 blk_addr -= FDEV(i).start_blk;
 235                                 bdev = FDEV(i).bdev;
 236                                 break;
 237                         }
 238                 }
 239         }
 240         if (bio) {
 241                 bio->bi_bdev = bdev;
 242                 bio->bi_iter.bi_sector = SECTOR_FROM_BLOCK(blk_addr);
 243         }
 244         return bdev;
 245 }
 246
 247 int f2fs_target_device_index(struct f2fs_sb_info *sbi, block_t blkaddr)
 248 {
 249         int i;
 250
 251         if (!f2fs_is_multi_device(sbi))
 252                 return 0;
 253
 254         for (i = 0; i < sbi->s_ndevs; i++)
 255                 if (FDEV(i).start_blk <= blkaddr && FDEV(i).end_blk >= blkaddr)
 256                         return i;
 257         return 0;
 258 }
 259
 260 static bool __same_bdev(struct f2fs_sb_info *sbi,
 261                                 block_t blk_addr, struct bio *bio)
 262 {
 263         return f2fs_target_device(sbi, blk_addr, NULL) == bio->bi_bdev;
 264 }
 265
 266 /*
 267  * Low-level block read/write IO operations.
 268  */
 269 static struct bio *__bio_alloc(struct f2fs_sb_info *sbi, block_t blk_addr,
 270                                 struct writeback_control *wbc,
 271                                 int npages, bool is_read,
 272                                 enum page_type type, enum temp_type temp)
 273 {
 274         struct bio *bio;
 275
 276         bio = f2fs_bio_alloc(sbi, npages, true);
 277
 278         f2fs_target_device(sbi, blk_addr, bio);
 279         if (is_read) {
 280                 bio->bi_end_io = f2fs_read_end_io;
 281                 bio->bi_private = NULL;
 282         } else {
 283                 bio->bi_end_io = f2fs_write_end_io;
 284                 bio->bi_private = sbi;
 285                 bio->bi_write_hint = f2fs_io_type_to_rw_hint(sbi, type, temp);
 286         }
 287         if (wbc)
 288                 wbc_init_bio(wbc, bio);
 289
 290         return bio;
 291 }
 292
 293 static inline void __submit_bio(struct f2fs_sb_info *sbi,
 294                                 struct bio *bio, enum page_type type)
 295 {
 296         if (!is_read_io(bio_op(bio))) {
 297                 unsigned int start;
 298
 299                 if (type != DATA && type != NODE)
 300                         goto submit_io;
 301
 302                 if (test_opt(sbi, LFS) && current->plug)
 303                         blk_finish_plug(current->plug);
 304
 305                 start = bio->bi_iter.bi_size >> F2FS_BLKSIZE_BITS;
 306                 start %= F2FS_IO_SIZE(sbi);
 307
 308                 if (start == 0)
 309                         goto submit_io;
 310
 311                 /* fill dummy pages */
 312                 for (; start < F2FS_IO_SIZE(sbi); start++) {
 313                         struct page *page =
 314                                 mempool_alloc(sbi->write_io_dummy,
 315                                               GFP_NOIO | __GFP_NOFAIL);
 316                         f2fs_bug_on(sbi, !page);
 317
 318                         zero_user_segment(page, 0, PAGE_SIZE);
 319                         SetPagePrivate(page);
 320                         set_page_private(page, (unsigned long)DUMMY_WRITTEN_PAGE);
 321                         lock_page(page);
 322                         if (bio_add_page(bio, page, PAGE_SIZE, 0) < PAGE_SIZE)
 323                                 f2fs_bug_on(sbi, 1);
 324                 }
 325                 /*
 326                  * In the NODE case, we lose next block address chain. So, we
 327                  * need to do checkpoint in f2fs_sync_file.
 328                  */
 329                 if (type == NODE)
 330                         set_sbi_flag(sbi, SBI_NEED_CP);
 331         }
 332 submit_io:
 333         if (is_read_io(bio_op(bio)))
 334                 trace_f2fs_submit_read_bio(sbi->sb, type, bio);
 335         else
 336                 trace_f2fs_submit_write_bio(sbi->sb, type, bio);
 337         submit_bio(bio_op(bio), bio);
 338 }
 339
 340 static void __f2fs_submit_read_bio(struct f2fs_sb_info *sbi,
 341                                 struct bio *bio, enum page_type type)
 342 {
 343         if (trace_android_fs_dataread_start_enabled() && (type == DATA)) {
 344                 struct page *first_page = bio->bi_io_vec[0].bv_page;
 345
 346                 if (first_page != NULL &&
 347                         __read_io_type(first_page) == F2FS_RD_DATA) {
 348                         char *path, pathbuf[MAX_TRACE_PATHBUF_LEN];
 349
 350                         path = android_fstrace_get_pathname(pathbuf,
 351                                                 MAX_TRACE_PATHBUF_LEN,
 352                                                 first_page->mapping->host);
 353
 354                         trace_android_fs_dataread_start(
 355                                 first_page->mapping->host,
 356                                 page_offset(first_page),
 357                                 bio->bi_iter.bi_size,
 358                                 current->pid,
 359                                 path,
 360                                 current->comm);
 361                 }
 362         }
 363         __submit_bio(sbi, bio, type);
 364 }
 365
 366 static void __submit_merged_bio(struct f2fs_bio_info *io)
 367 {
 368         struct f2fs_io_info *fio = &io->fio;
 369
 370         if (!io->bio)
 371                 return;
 372
 373         bio_set_op_attrs(io->bio, fio->op, fio->op_flags);
 374
 375         if (is_read_io(fio->op))
 376                 trace_f2fs_prepare_read_bio(io->sbi->sb, fio->type, io->bio);
 377         else
 378                 trace_f2fs_prepare_write_bio(io->sbi->sb, fio->type, io->bio);
 379
 380         __submit_bio(io->sbi, io->bio, fio->type);
 381         io->bio = NULL;
 382 }
 383
 384 static bool __has_merged_page(struct bio *bio, struct inode *inode,
 385                                                 struct page *page, nid_t ino)
 386 {
 387         struct bio_vec *bvec;
 388         struct page *target;
 389         int i;
 390
 391         if (!bio)
 392                 return false;
 393
 394         if (!inode && !page && !ino)
 395                 return true;
 396
 397         bio_for_each_segment_all(bvec, bio, i) {
 398
 399                 if (bvec->bv_page->mapping)
 400                         target = bvec->bv_page;
 401                 else
 402                         target = fscrypt_control_page(bvec->bv_page);
 403
 404                 if (inode && inode == target->mapping->host)
 405                         return true;
 406                 if (page && page == target)
 407                         return true;
 408                 if (ino && ino == ino_of_node(target))
 409                         return true;
 410         }
 411
 412         return false;
 413 }
 414
 415 static void __f2fs_submit_merged_write(struct f2fs_sb_info *sbi,
 416                                 enum page_type type, enum temp_type temp)
 417 {
 418         enum page_type btype = PAGE_TYPE_OF_BIO(type);
 419         struct f2fs_bio_info *io = sbi->write_io[btype] + temp;
 420
 421         down_write(&io->io_rwsem);
 422
 423         /* change META to META_FLUSH in the checkpoint procedure */
 424         if (type >= META_FLUSH) {
 425                 io->fio.type = META_FLUSH;
 426                 io->fio.op = REQ_OP_WRITE;
 427                 io->fio.op_flags = REQ_META | REQ_PRIO;
 428                 if (!test_opt(sbi, NOBARRIER))
 429                         io->fio.op_flags |= WRITE_FLUSH | REQ_FUA;
 430         }
 431         __submit_merged_bio(io);
 432         up_write(&io->io_rwsem);
 433 }
 434
 435 static void __submit_merged_write_cond(struct f2fs_sb_info *sbi,
 436                                 struct inode *inode, struct page *page,
 437                                 nid_t ino, enum page_type type, bool force)
 438 {
 439         enum temp_type temp;
 440         bool ret = true;
 441
 442         for (temp = HOT; temp < NR_TEMP_TYPE; temp++) {
 443                 if (!force)     {
 444                         enum page_type btype = PAGE_TYPE_OF_BIO(type);
 445                         struct f2fs_bio_info *io = sbi->write_io[btype] + temp;
 446
 447                         down_read(&io->io_rwsem);
 448                         ret = __has_merged_page(io->bio, inode, page, ino);
 449                         up_read(&io->io_rwsem);
 450                 }
 451                 if (ret)
 452                         __f2fs_submit_merged_write(sbi, type, temp);
 453
 454                 /* TODO: use HOT temp only for meta pages now. */
 455                 if (type >= META)
 456                         break;
 457         }
 458 }
 459
 460 void f2fs_submit_merged_write(struct f2fs_sb_info *sbi, enum page_type type)
 461 {
 462         __submit_merged_write_cond(sbi, NULL, NULL, 0, type, true);
 463 }
 464
 465 void f2fs_submit_merged_write_cond(struct f2fs_sb_info *sbi,
 466                                 struct inode *inode, struct page *page,
 467                                 nid_t ino, enum page_type type)
 468 {
 469         __submit_merged_write_cond(sbi, inode, page, ino, type, false);
 470 }
 471
 472 void f2fs_flush_merged_writes(struct f2fs_sb_info *sbi)
 473 {
 474         f2fs_submit_merged_write(sbi, DATA);
 475         f2fs_submit_merged_write(sbi, NODE);
 476         f2fs_submit_merged_write(sbi, META);
 477 }
 478
 479 /*
 480  * Fill the locked page with data located in the block address.
 481  * A caller needs to unlock the page on failure.
 482  */
 483 int f2fs_submit_page_bio(struct f2fs_io_info *fio)
 484 {
 485         struct bio *bio;
 486         struct page *page = fio->encrypted_page ?
 487                         fio->encrypted_page : fio->page;
 488
 489         if (!f2fs_is_valid_blkaddr(fio->sbi, fio->new_blkaddr,
 490                         fio->is_por ? META_POR : (__is_meta_io(fio) ?
 491                         META_GENERIC : DATA_GENERIC_ENHANCE)))
 492                 return -EFSCORRUPTED;
 493
 494         trace_f2fs_submit_page_bio(page, fio);
 495         f2fs_trace_ios(fio, 0);
 496
 497         /* Allocate a new bio */
 498         bio = __bio_alloc(fio->sbi, fio->new_blkaddr, fio->io_wbc,
 499                                 1, is_read_io(fio->op), fio->type, fio->temp);
 500
 501         if (bio_add_page(bio, page, PAGE_SIZE, 0) < PAGE_SIZE) {
 502                 bio_put(bio);
 503                 return -EFAULT;
 504         }
 505
 506         if (fio->io_wbc && !is_read_io(fio->op))
 507                 wbc_account_io(fio->io_wbc, page, PAGE_SIZE);
 508
 509         bio_set_op_attrs(bio, fio->op, fio->op_flags);
 510
 511         inc_page_count(fio->sbi, is_read_io(fio->op) ?
 512                         __read_io_type(page): WB_DATA_TYPE(fio->page));
 513
 514         __f2fs_submit_read_bio(fio->sbi, bio, fio->type);
 515         return 0;
 516 }
 517
 518 int f2fs_merge_page_bio(struct f2fs_io_info *fio)
 519 {
 520         struct bio *bio = *fio->bio;
 521         struct page *page = fio->encrypted_page ?
 522                         fio->encrypted_page : fio->page;
 523
 524         if (!f2fs_is_valid_blkaddr(fio->sbi, fio->new_blkaddr,
 525                         __is_meta_io(fio) ? META_GENERIC : DATA_GENERIC))
 526                 return -EFSCORRUPTED;
 527
 528         trace_f2fs_submit_page_bio(page, fio);
 529         f2fs_trace_ios(fio, 0);
 530
 531         if (bio && (*fio->last_block + 1 != fio->new_blkaddr ||
 532                         !__same_bdev(fio->sbi, fio->new_blkaddr, bio))) {
 533                 __submit_bio(fio->sbi, bio, fio->type);
 534                 bio = NULL;
 535         }
 536 alloc_new:
 537         if (!bio) {
 538                 bio = __bio_alloc(fio->sbi, fio->new_blkaddr, fio->io_wbc,
 539                                 BIO_MAX_PAGES, false, fio->type, fio->temp);
 540                 bio_set_op_attrs(bio, fio->op, fio->op_flags);
 541         }
 542
 543         if (bio_add_page(bio, page, PAGE_SIZE, 0) < PAGE_SIZE) {
 544                 __submit_bio(fio->sbi, bio, fio->type);
 545                 bio = NULL;
 546                 goto alloc_new;
 547         }
 548
 549         if (fio->io_wbc)
 550                 wbc_account_io(fio->io_wbc, page, PAGE_SIZE);
 551
 552         inc_page_count(fio->sbi, WB_DATA_TYPE(page));
 553
 554         *fio->last_block = fio->new_blkaddr;
 555         *fio->bio = bio;
 556
 557         return 0;
 558 }
 559
 560 static void f2fs_submit_ipu_bio(struct f2fs_sb_info *sbi, struct bio **bio,
 561                                                         struct page *page)
 562 {
 563         if (!bio)
 564                 return;
 565
 566         if (!__has_merged_page(*bio, NULL, page, 0))
 567                 return;
 568
 569         __submit_bio(sbi, *bio, DATA);
 570         *bio = NULL;
 571 }
 572
 573 void f2fs_submit_page_write(struct f2fs_io_info *fio)
 574 {
 575         struct f2fs_sb_info *sbi = fio->sbi;
 576         enum page_type btype = PAGE_TYPE_OF_BIO(fio->type);
 577         struct f2fs_bio_info *io = sbi->write_io[btype] + fio->temp;
 578         struct page *bio_page;
 579
 580         f2fs_bug_on(sbi, is_read_io(fio->op));
 581
 582         down_write(&io->io_rwsem);
 583 next:
 584         if (fio->in_list) {
 585                 spin_lock(&io->io_lock);
 586                 if (list_empty(&io->io_list)) {
 587                         spin_unlock(&io->io_lock);
 588                         goto out;
 589                 }
 590                 fio = list_first_entry(&io->io_list,
 591                                                 struct f2fs_io_info, list);
 592                 list_del(&fio->list);
 593                 spin_unlock(&io->io_lock);
 594         }
 595
 596         verify_fio_blkaddr(fio);
 597
 598         bio_page = fio->encrypted_page ? fio->encrypted_page : fio->page;
 599
 600         /* set submitted = true as a return value */
 601         fio->submitted = true;
 602
 603         inc_page_count(sbi, WB_DATA_TYPE(bio_page));
 604
 605         if (io->bio && (io->last_block_in_bio != fio->new_blkaddr - 1 ||
 606             (io->fio.op != fio->op || io->fio.op_flags != fio->op_flags) ||
 607                         !__same_bdev(sbi, fio->new_blkaddr, io->bio)))
 608                 __submit_merged_bio(io);
 609 alloc_new:
 610         if (io->bio == NULL) {
 611                 if ((fio->type == DATA || fio->type == NODE) &&
 612                                 fio->new_blkaddr & F2FS_IO_SIZE_MASK(sbi)) {
 613                         dec_page_count(sbi, WB_DATA_TYPE(bio_page));
 614                         fio->retry = true;
 615                         goto skip;
 616                 }
 617                 io->bio = __bio_alloc(sbi, fio->new_blkaddr, fio->io_wbc,
 618                                                 BIO_MAX_PAGES, false,
 619                                                 fio->type, fio->temp);
 620                 io->fio = *fio;
 621         }
 622
 623         if (bio_add_page(io->bio, bio_page, PAGE_SIZE, 0) < PAGE_SIZE) {
 624                 __submit_merged_bio(io);
 625                 goto alloc_new;
 626         }
 627
 628         if (fio->io_wbc)
 629                 wbc_account_io(fio->io_wbc, bio_page, PAGE_SIZE);
 630
 631         io->last_block_in_bio = fio->new_blkaddr;
 632         f2fs_trace_ios(fio, 0);
 633
 634         trace_f2fs_submit_page_write(fio->page, fio);
 635 skip:
 636         if (fio->in_list)
 637                 goto next;
 638 out:
 639         if (is_sbi_flag_set(sbi, SBI_IS_SHUTDOWN) ||
 640                                 f2fs_is_checkpoint_ready(sbi))
 641                 __submit_merged_bio(io);
 642         up_write(&io->io_rwsem);
 643 }
 644
 645 static struct bio *f2fs_grab_read_bio(struct inode *inode, block_t blkaddr,
 646                                         unsigned nr_pages, unsigned op_flag)
 647 {
 648         struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
 649         struct bio *bio;
 650         struct bio_post_read_ctx *ctx;
 651         unsigned int post_read_steps = 0;
 652
 653         bio = f2fs_bio_alloc(sbi, min_t(int, nr_pages, BIO_MAX_PAGES), false);
 654         if (!bio)
 655                 return ERR_PTR(-ENOMEM);
 656         f2fs_target_device(sbi, blkaddr, bio);
 657         bio->bi_end_io = f2fs_read_end_io;
 658         bio_set_op_attrs(bio, REQ_OP_READ, op_flag);
 659
 660         if (f2fs_encrypted_file(inode))
 661                 post_read_steps |= 1 << STEP_DECRYPT;
 662         if (post_read_steps) {
 663                 ctx = mempool_alloc(bio_post_read_ctx_pool, GFP_NOFS);
 664                 if (!ctx) {
 665                         bio_put(bio);
 666                         return ERR_PTR(-ENOMEM);
 667                 }
 668                 ctx->bio = bio;
 669                 ctx->enabled_steps = post_read_steps;
 670                 bio->bi_private = ctx;
 671         }
 672
 673         return bio;
 674 }
 675
 676 /* This can handle encryption stuffs */
 677 static int f2fs_submit_page_read(struct inode *inode, struct page *page,
 678                                                         block_t blkaddr)
 679 {
 680         struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
 681         struct bio *bio;
 682
 683         bio = f2fs_grab_read_bio(inode, blkaddr, 1, 0);
 684         if (IS_ERR(bio))
 685                 return PTR_ERR(bio);
 686
 687         /* wait for GCed page writeback via META_MAPPING */
 688         f2fs_wait_on_block_writeback(inode, blkaddr);
 689
 690         if (bio_add_page(bio, page, PAGE_SIZE, 0) < PAGE_SIZE) {
 691                 bio_put(bio);
 692                 return -EFAULT;
 693         }
 694         ClearPageError(page);
 695         inc_page_count(sbi, F2FS_RD_DATA);
 696         __f2fs_submit_read_bio(sbi, bio, DATA);
 697         return 0;
 698 }
 699
 700 static void __set_data_blkaddr(struct dnode_of_data *dn)
 701 {
 702         struct f2fs_node *rn = F2FS_NODE(dn->node_page);
 703         __le32 *addr_array;
 704         int base = 0;
 705
 706         if (IS_INODE(dn->node_page) && f2fs_has_extra_attr(dn->inode))
 707                 base = get_extra_isize(dn->inode);
 708
 709         /* Get physical address of data block */
 710         addr_array = blkaddr_in_node(rn);
 711         addr_array[base + dn->ofs_in_node] = cpu_to_le32(dn->data_blkaddr);
 712 }
 713
 714 /*
 715  * Lock ordering for the change of data block address:
 716  * ->data_page
 717  *  ->node_page
 718  *    update block addresses in the node page
 719  */
 720 void f2fs_set_data_blkaddr(struct dnode_of_data *dn)
 721 {
 722         f2fs_wait_on_page_writeback(dn->node_page, NODE, true, true);
 723         __set_data_blkaddr(dn);
 724         if (set_page_dirty(dn->node_page))
 725                 dn->node_changed = true;
 726 }
 727
 728 void f2fs_update_data_blkaddr(struct dnode_of_data *dn, block_t blkaddr)
 729 {
 730         dn->data_blkaddr = blkaddr;
 731         f2fs_set_data_blkaddr(dn);
 732         f2fs_update_extent_cache(dn);
 733 }
 734
 735 /* dn->ofs_in_node will be returned with up-to-date last block pointer */
 736 int f2fs_reserve_new_blocks(struct dnode_of_data *dn, blkcnt_t count)
 737 {
 738         struct f2fs_sb_info *sbi = F2FS_I_SB(dn->inode);
 739         int err;
 740
 741         if (!count)
 742                 return 0;
 743
 744         if (unlikely(is_inode_flag_set(dn->inode, FI_NO_ALLOC)))
 745                 return -EPERM;
 746         if (unlikely((err = inc_valid_block_count(sbi, dn->inode, &count))))
 747                 return err;
 748
 749         trace_f2fs_reserve_new_blocks(dn->inode, dn->nid,
 750                                                 dn->ofs_in_node, count);
 751
 752         f2fs_wait_on_page_writeback(dn->node_page, NODE, true, true);
 753
 754         for (; count > 0; dn->ofs_in_node++) {
 755                 block_t blkaddr = datablock_addr(dn->inode,
 756                                         dn->node_page, dn->ofs_in_node);
 757                 if (blkaddr == NULL_ADDR) {
 758                         dn->data_blkaddr = NEW_ADDR;
 759                         __set_data_blkaddr(dn);
 760                         count--;
 761                 }
 762         }
 763
 764         if (set_page_dirty(dn->node_page))
 765                 dn->node_changed = true;
 766         return 0;
 767 }
 768
 769 /* Should keep dn->ofs_in_node unchanged */
 770 int f2fs_reserve_new_block(struct dnode_of_data *dn)
 771 {
 772         unsigned int ofs_in_node = dn->ofs_in_node;
 773         int ret;
 774
 775         ret = f2fs_reserve_new_blocks(dn, 1);
 776         dn->ofs_in_node = ofs_in_node;
 777         return ret;
 778 }
 779
 780 int f2fs_reserve_block(struct dnode_of_data *dn, pgoff_t index)
 781 {
 782         bool need_put = dn->inode_page ? false : true;
 783         int err;
 784
 785         err = f2fs_get_dnode_of_data(dn, index, ALLOC_NODE);
 786         if (err)
 787                 return err;
 788
 789         if (dn->data_blkaddr == NULL_ADDR)
 790                 err = f2fs_reserve_new_block(dn);
 791         if (err || need_put)
 792                 f2fs_put_dnode(dn);
 793         return err;
 794 }
 795
 796 int f2fs_get_block(struct dnode_of_data *dn, pgoff_t index)
 797 {
 798         struct extent_info ei  = {0,0,0};
 799         struct inode *inode = dn->inode;
 800
 801         if (f2fs_lookup_extent_cache(inode, index, &ei)) {
 802                 dn->data_blkaddr = ei.blk + index - ei.fofs;
 803                 return 0;
 804         }
 805
 806         return f2fs_reserve_block(dn, index);
 807 }
 808
 809 struct page *f2fs_get_read_data_page(struct inode *inode, pgoff_t index,
 810                                                 int op_flags, bool for_write)
 811 {
 812         struct address_space *mapping = inode->i_mapping;
 813         struct dnode_of_data dn;
 814         struct page *page;
 815         struct extent_info ei = {0,0,0};
 816         int err;
 817
 818         page = f2fs_grab_cache_page(mapping, index, for_write);
 819         if (!page)
 820                 return ERR_PTR(-ENOMEM);
 821
 822         if (f2fs_lookup_extent_cache(inode, index, &ei)) {
 823                 dn.data_blkaddr = ei.blk + index - ei.fofs;
 824                 if (!f2fs_is_valid_blkaddr(F2FS_I_SB(inode), dn.data_blkaddr,
 825                                                 DATA_GENERIC_ENHANCE_READ)) {
 826                         err = -EFSCORRUPTED;
 827                         goto put_err;
 828                 }
 829                 goto got_it;
 830         }
 831
 832         set_new_dnode(&dn, inode, NULL, NULL, 0);
 833         err = f2fs_get_dnode_of_data(&dn, index, LOOKUP_NODE);
 834         if (err)
 835                 goto put_err;
 836         f2fs_put_dnode(&dn);
 837
 838         if (unlikely(dn.data_blkaddr == NULL_ADDR)) {
 839                 err = -ENOENT;
 840                 goto put_err;
 841         }
 842         if (dn.data_blkaddr != NEW_ADDR &&
 843                         !f2fs_is_valid_blkaddr(F2FS_I_SB(inode),
 844                                                 dn.data_blkaddr,
 845                                                 DATA_GENERIC_ENHANCE)) {
 846                 err = -EFSCORRUPTED;
 847                 goto put_err;
 848         }
 849 got_it:
 850         if (PageUptodate(page)) {
 851                 unlock_page(page);
 852                 return page;
 853         }
 854
 855         /*
 856          * A new dentry page is allocated but not able to be written, since its
 857          * new inode page couldn't be allocated due to -ENOSPC.
 858          * In such the case, its blkaddr can be remained as NEW_ADDR.
 859          * see, f2fs_add_link -> f2fs_get_new_data_page ->
 860          * f2fs_init_inode_metadata.
 861          */
 862         if (dn.data_blkaddr == NEW_ADDR) {
 863                 zero_user_segment(page, 0, PAGE_SIZE);
 864                 if (!PageUptodate(page))
 865                         SetPageUptodate(page);
 866                 unlock_page(page);
 867                 return page;
 868         }
 869
 870         err = f2fs_submit_page_read(inode, page, dn.data_blkaddr);
 871         if (err)
 872                 goto put_err;
 873         return page;
 874
 875 put_err:
 876         f2fs_put_page(page, 1);
 877         return ERR_PTR(err);
 878 }
 879
 880 struct page *f2fs_find_data_page(struct inode *inode, pgoff_t index)
 881 {
 882         struct address_space *mapping = inode->i_mapping;
 883         struct page *page;
 884
 885         page = find_get_page(mapping, index);
 886         if (page && PageUptodate(page))
 887                 return page;
 888         f2fs_put_page(page, 0);
 889
 890         page = f2fs_get_read_data_page(inode, index, REQ_SYNC, false);
 891         if (IS_ERR(page))
 892                 return page;
 893
 894         if (PageUptodate(page))
 895                 return page;
 896
 897         wait_on_page_locked(page);
 898         if (unlikely(!PageUptodate(page))) {
 899                 f2fs_put_page(page, 0);
 900                 return ERR_PTR(-EIO);
 901         }
 902         return page;
 903 }
 904
 905 /*
 906  * If it tries to access a hole, return an error.
 907  * Because, the callers, functions in dir.c and GC, should be able to know
 908  * whether this page exists or not.
 909  */
 910 struct page *f2fs_get_lock_data_page(struct inode *inode, pgoff_t index,
 911                                                         bool for_write)
 912 {
 913         struct address_space *mapping = inode->i_mapping;
 914         struct page *page;
 915 repeat:
 916         page = f2fs_get_read_data_page(inode, index, REQ_SYNC, for_write);
 917         if (IS_ERR(page))
 918                 return page;
 919
 920         /* wait for read completion */
 921         lock_page(page);
 922         if (unlikely(page->mapping != mapping)) {
 923                 f2fs_put_page(page, 1);
 924                 goto repeat;
 925         }
 926         if (unlikely(!PageUptodate(page))) {
 927                 f2fs_put_page(page, 1);
 928                 return ERR_PTR(-EIO);
 929         }
 930         return page;
 931 }
 932
 933 /*
 934  * Caller ensures that this data page is never allocated.
 935  * A new zero-filled data page is allocated in the page cache.
 936  *
 937  * Also, caller should grab and release a rwsem by calling f2fs_lock_op() and
 938  * f2fs_unlock_op().
 939  * Note that, ipage is set only by make_empty_dir, and if any error occur,
 940  * ipage should be released by this function.
 941  */
 942 struct page *f2fs_get_new_data_page(struct inode *inode,
 943                 struct page *ipage, pgoff_t index, bool new_i_size)
 944 {
 945         struct address_space *mapping = inode->i_mapping;
 946         struct page *page;
 947         struct dnode_of_data dn;
 948         int err;
 949
 950         page = f2fs_grab_cache_page(mapping, index, true);
 951         if (!page) {
 952                 /*
 953                  * before exiting, we should make sure ipage will be released
 954                  * if any error occur.
 955                  */
 956                 f2fs_put_page(ipage, 1);
 957                 return ERR_PTR(-ENOMEM);
 958         }
 959
 960         set_new_dnode(&dn, inode, ipage, NULL, 0);
 961         err = f2fs_reserve_block(&dn, index);
 962         if (err) {
 963                 f2fs_put_page(page, 1);
 964                 return ERR_PTR(err);
 965         }
 966         if (!ipage)
 967                 f2fs_put_dnode(&dn);
 968
 969         if (PageUptodate(page))
 970                 goto got_it;
 971
 972         if (dn.data_blkaddr == NEW_ADDR) {
 973                 zero_user_segment(page, 0, PAGE_SIZE);
 974                 if (!PageUptodate(page))
 975                         SetPageUptodate(page);
 976         } else {
 977                 f2fs_put_page(page, 1);
 978
 979                 /* if ipage exists, blkaddr should be NEW_ADDR */
 980                 f2fs_bug_on(F2FS_I_SB(inode), ipage);
 981                 page = f2fs_get_lock_data_page(inode, index, true);
 982                 if (IS_ERR(page))
 983                         return page;
 984         }
 985 got_it:
 986         if (new_i_size && i_size_read(inode) <
 987                                 ((loff_t)(index + 1) << PAGE_SHIFT))
 988                 f2fs_i_size_write(inode, ((loff_t)(index + 1) << PAGE_SHIFT));
 989         return page;
 990 }
 991
 992 static int __allocate_data_block(struct dnode_of_data *dn, int seg_type)
 993 {
 994         struct f2fs_sb_info *sbi = F2FS_I_SB(dn->inode);
 995         struct f2fs_summary sum;
 996         struct node_info ni;
 997         block_t old_blkaddr;
 998         blkcnt_t count = 1;
 999         int err;
1000
1001         if (unlikely(is_inode_flag_set(dn->inode, FI_NO_ALLOC)))
1002                 return -EPERM;
1003
1004         err = f2fs_get_node_info(sbi, dn->nid, &ni);
1005         if (err)
1006                 return err;
1007
1008         dn->data_blkaddr = datablock_addr(dn->inode,
1009                                 dn->node_page, dn->ofs_in_node);
1010         if (dn->data_blkaddr != NULL_ADDR)
1011                 goto alloc;
1012
1013         if (unlikely((err = inc_valid_block_count(sbi, dn->inode, &count))))
1014                 return err;
1015
1016 alloc:
1017         set_summary(&sum, dn->nid, dn->ofs_in_node, ni.version);
1018         old_blkaddr = dn->data_blkaddr;
1019         f2fs_allocate_data_block(sbi, NULL, old_blkaddr, &dn->data_blkaddr,
1020                                         &sum, seg_type, NULL, false);
1021         if (GET_SEGNO(sbi, old_blkaddr) != NULL_SEGNO)
1022                 invalidate_mapping_pages(META_MAPPING(sbi),
1023                                         old_blkaddr, old_blkaddr);
1024         f2fs_set_data_blkaddr(dn);
1025
1026         /*
1027          * i_size will be updated by direct_IO. Otherwise, we'll get stale
1028          * data from unwritten block via dio_read.
1029          */
1030         return 0;
1031 }
1032
1033 int f2fs_preallocate_blocks(struct kiocb *iocb, struct iov_iter *from)
1034 {
1035         struct inode *inode = file_inode(iocb->ki_filp);
1036         struct f2fs_map_blocks map;
1037         int flag;
1038         int err = 0;
1039         bool direct_io = iocb->ki_flags & IOCB_DIRECT;
1040
1041         /* convert inline data for Direct I/O*/
1042         if (direct_io) {
1043                 err = f2fs_convert_inline_inode(inode);
1044                 if (err)
1045                         return err;
1046         }
1047
1048         if (direct_io && allow_outplace_dio(inode, iocb, from))
1049                 return 0;
1050
1051         if (is_inode_flag_set(inode, FI_NO_PREALLOC))
1052                 return 0;
1053
1054         map.m_lblk = F2FS_BLK_ALIGN(iocb->ki_pos);
1055         map.m_len = F2FS_BYTES_TO_BLK(iocb->ki_pos + iov_iter_count(from));
1056         if (map.m_len > map.m_lblk)
1057                 map.m_len -= map.m_lblk;
1058         else
1059                 map.m_len = 0;
1060
1061         map.m_next_pgofs = NULL;
1062         map.m_next_extent = NULL;
1063         map.m_seg_type = NO_CHECK_TYPE;
1064         map.m_may_create = true;
1065
1066         if (direct_io) {
1067                 map.m_seg_type = f2fs_rw_hint_to_seg_type(iocb->ki_hint);
1068                 flag = f2fs_force_buffered_io(inode, iocb, from) ?
1069                                         F2FS_GET_BLOCK_PRE_AIO :
1070                                         F2FS_GET_BLOCK_PRE_DIO;
1071                 goto map_blocks;
1072         }
1073         if (iocb->ki_pos + iov_iter_count(from) > MAX_INLINE_DATA(inode)) {
1074                 err = f2fs_convert_inline_inode(inode);
1075                 if (err)
1076                         return err;
1077         }
1078         if (f2fs_has_inline_data(inode))
1079                 return err;
1080
1081         flag = F2FS_GET_BLOCK_PRE_AIO;
1082
1083 map_blocks:
1084         err = f2fs_map_blocks(inode, &map, 1, flag);
1085         if (map.m_len > 0 && err == -ENOSPC) {
1086                 if (!direct_io)
1087                         set_inode_flag(inode, FI_NO_PREALLOC);
1088                 err = 0;
1089         }
1090         return err;
1091 }
1092
1093 void __do_map_lock(struct f2fs_sb_info *sbi, int flag, bool lock)
1094 {
1095         if (flag == F2FS_GET_BLOCK_PRE_AIO) {
1096                 if (lock)
1097                         down_read(&sbi->node_change);
1098                 else
1099                         up_read(&sbi->node_change);
1100         } else {
1101                 if (lock)
1102                         f2fs_lock_op(sbi);
1103                 else
1104                         f2fs_unlock_op(sbi);
1105         }
1106 }
1107
1108 /*
1109  * f2fs_map_blocks() now supported readahead/bmap/rw direct_IO with
1110  * f2fs_map_blocks structure.
1111  * If original data blocks are allocated, then give them to blockdev.
1112  * Otherwise,
1113  *     a. preallocate requested block addresses
1114  *     b. do not use extent cache for better performance
1115  *     c. give the block addresses to blockdev
1116  */
1117 int f2fs_map_blocks(struct inode *inode, struct f2fs_map_blocks *map,
1118                                                 int create, int flag)
1119 {
1120         unsigned int maxblocks = map->m_len;
1121         struct dnode_of_data dn;
1122         struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
1123         int mode = map->m_may_create ? ALLOC_NODE : LOOKUP_NODE;
1124         pgoff_t pgofs, end_offset, end;
1125         int err = 0, ofs = 1;
1126         unsigned int ofs_in_node, last_ofs_in_node;
1127         blkcnt_t prealloc;
1128         struct extent_info ei = {0,0,0};
1129         block_t blkaddr;
1130         unsigned int start_pgofs;
1131
1132         if (!maxblocks)
1133                 return 0;
1134
1135         map->m_len = 0;
1136         map->m_flags = 0;
1137
1138         /* it only supports block size == page size */
1139         pgofs = (pgoff_t)map->m_lblk;
1140         end = pgofs + maxblocks;
1141
1142         if (!create && f2fs_lookup_extent_cache(inode, pgofs, &ei)) {
1143                 if (test_opt(sbi, LFS) && flag == F2FS_GET_BLOCK_DIO &&
1144                                                         map->m_may_create)
1145                         goto next_dnode;
1146
1147                 map->m_pblk = ei.blk + pgofs - ei.fofs;
1148                 map->m_len = min((pgoff_t)maxblocks, ei.fofs + ei.len - pgofs);
1149                 map->m_flags = F2FS_MAP_MAPPED;
1150                 if (map->m_next_extent)
1151                         *map->m_next_extent = pgofs + map->m_len;
1152
1153                 /* for hardware encryption, but to avoid potential issue in future */
1154                 if (flag == F2FS_GET_BLOCK_DIO)
1155                         f2fs_wait_on_block_writeback_range(inode,
1156                                                 map->m_pblk, map->m_len);
1157                 goto out;
1158         }
1159
1160 next_dnode:
1161         if (map->m_may_create)
1162                 __do_map_lock(sbi, flag, true);
1163
1164         /* When reading holes, we need its node page */
1165         set_new_dnode(&dn, inode, NULL, NULL, 0);
1166         err = f2fs_get_dnode_of_data(&dn, pgofs, mode);
1167         if (err) {
1168                 if (flag == F2FS_GET_BLOCK_BMAP)
1169                         map->m_pblk = 0;
1170                 if (err == -ENOENT) {
1171                         err = 0;
1172                         if (map->m_next_pgofs)
1173                                 *map->m_next_pgofs =
1174                                         f2fs_get_next_page_offset(&dn, pgofs);
1175                         if (map->m_next_extent)
1176                                 *map->m_next_extent =
1177                                         f2fs_get_next_page_offset(&dn, pgofs);
1178                 }
1179                 goto unlock_out;
1180         }
1181
1182         start_pgofs = pgofs;
1183         prealloc = 0;
1184         last_ofs_in_node = ofs_in_node = dn.ofs_in_node;
1185         end_offset = ADDRS_PER_PAGE(dn.node_page, inode);
1186
1187 next_block:
1188         blkaddr = datablock_addr(dn.inode, dn.node_page, dn.ofs_in_node);
1189
1190         if (__is_valid_data_blkaddr(blkaddr) &&
1191                 !f2fs_is_valid_blkaddr(sbi, blkaddr, DATA_GENERIC_ENHANCE)) {
1192                 err = -EFSCORRUPTED;
1193                 goto sync_out;
1194         }
1195
1196         if (__is_valid_data_blkaddr(blkaddr)) {
1197                 /* use out-place-update for driect IO under LFS mode */
1198                 if (test_opt(sbi, LFS) && flag == F2FS_GET_BLOCK_DIO &&
1199                                                         map->m_may_create) {
1200                         err = __allocate_data_block(&dn, map->m_seg_type);
1201                         if (!err) {
1202                                 blkaddr = dn.data_blkaddr;
1203                                 set_inode_flag(inode, FI_APPEND_WRITE);
1204                         }
1205                 }
1206         } else {
1207                 if (create) {
1208                         if (unlikely(f2fs_cp_error(sbi))) {
1209                                 err = -EIO;
1210                                 goto sync_out;
1211                         }
1212                         if (flag == F2FS_GET_BLOCK_PRE_AIO) {
1213                                 if (blkaddr == NULL_ADDR) {
1214                                         prealloc++;
1215                                         last_ofs_in_node = dn.ofs_in_node;
1216                                 }
1217                         } else {
1218                                 WARN_ON(flag != F2FS_GET_BLOCK_PRE_DIO &&
1219                                         flag != F2FS_GET_BLOCK_DIO);
1220                                 err = __allocate_data_block(&dn,
1221                                                         map->m_seg_type);
1222                                 if (!err)
1223                                         set_inode_flag(inode, FI_APPEND_WRITE);
1224                         }
1225                         if (err)
1226                                 goto sync_out;
1227                         map->m_flags |= F2FS_MAP_NEW;
1228                         blkaddr = dn.data_blkaddr;
1229                 } else {
1230                         if (flag == F2FS_GET_BLOCK_BMAP) {
1231                                 map->m_pblk = 0;
1232                                 goto sync_out;
1233                         }
1234                         if (flag == F2FS_GET_BLOCK_PRECACHE)
1235                                 goto sync_out;
1236                         if (flag == F2FS_GET_BLOCK_FIEMAP &&
1237                                                 blkaddr == NULL_ADDR) {
1238                                 if (map->m_next_pgofs)
1239                                         *map->m_next_pgofs = pgofs + 1;
1240                                 goto sync_out;
1241                         }
1242                         if (flag != F2FS_GET_BLOCK_FIEMAP) {
1243                                 /* for defragment case */
1244                                 if (map->m_next_pgofs)
1245                                         *map->m_next_pgofs = pgofs + 1;
1246                                 goto sync_out;
1247                         }
1248                 }
1249         }
1250
1251         if (flag == F2FS_GET_BLOCK_PRE_AIO)
1252                 goto skip;
1253
1254         if (map->m_len == 0) {
1255                 /* preallocated unwritten block should be mapped for fiemap. */
1256                 if (blkaddr == NEW_ADDR)
1257                         map->m_flags |= F2FS_MAP_UNWRITTEN;
1258                 map->m_flags |= F2FS_MAP_MAPPED;
1259
1260                 map->m_pblk = blkaddr;
1261                 map->m_len = 1;
1262         } else if ((map->m_pblk != NEW_ADDR &&
1263                         blkaddr == (map->m_pblk + ofs)) ||
1264                         (map->m_pblk == NEW_ADDR && blkaddr == NEW_ADDR) ||
1265                         flag == F2FS_GET_BLOCK_PRE_DIO) {
1266                 ofs++;
1267                 map->m_len++;
1268         } else {
1269                 goto sync_out;
1270         }
1271
1272 skip:
1273         dn.ofs_in_node++;
1274         pgofs++;
1275
1276         /* preallocate blocks in batch for one dnode page */
1277         if (flag == F2FS_GET_BLOCK_PRE_AIO &&
1278                         (pgofs == end || dn.ofs_in_node == end_offset)) {
1279
1280                 dn.ofs_in_node = ofs_in_node;
1281                 err = f2fs_reserve_new_blocks(&dn, prealloc);
1282                 if (err)
1283                         goto sync_out;
1284
1285                 map->m_len += dn.ofs_in_node - ofs_in_node;
1286                 if (prealloc && dn.ofs_in_node != last_ofs_in_node + 1) {
1287                         err = -ENOSPC;
1288                         goto sync_out;
1289                 }
1290                 dn.ofs_in_node = end_offset;
1291         }
1292
1293         if (pgofs >= end)
1294                 goto sync_out;
1295         else if (dn.ofs_in_node < end_offset)
1296                 goto next_block;
1297
1298         if (flag == F2FS_GET_BLOCK_PRECACHE) {
1299                 if (map->m_flags & F2FS_MAP_MAPPED) {
1300                         unsigned int ofs = start_pgofs - map->m_lblk;
1301
1302                         f2fs_update_extent_cache_range(&dn,
1303                                 start_pgofs, map->m_pblk + ofs,
1304                                 map->m_len - ofs);
1305                 }
1306         }
1307
1308         f2fs_put_dnode(&dn);
1309
1310         if (map->m_may_create) {
1311                 __do_map_lock(sbi, flag, false);
1312                 f2fs_balance_fs(sbi, dn.node_changed);
1313         }
1314         goto next_dnode;
1315
1316 sync_out:
1317
1318         /* for hardware encryption, but to avoid potential issue in future */
1319         if (flag == F2FS_GET_BLOCK_DIO && map->m_flags & F2FS_MAP_MAPPED)
1320                 f2fs_wait_on_block_writeback_range(inode,
1321                                                 map->m_pblk, map->m_len);
1322
1323         if (flag == F2FS_GET_BLOCK_PRECACHE) {
1324                 if (map->m_flags & F2FS_MAP_MAPPED) {
1325                         unsigned int ofs = start_pgofs - map->m_lblk;
1326
1327                         f2fs_update_extent_cache_range(&dn,
1328                                 start_pgofs, map->m_pblk + ofs,
1329                                 map->m_len - ofs);
1330                 }
1331                 if (map->m_next_extent)
1332                         *map->m_next_extent = pgofs + 1;
1333         }
1334         f2fs_put_dnode(&dn);
1335 unlock_out:
1336         if (map->m_may_create) {
1337                 __do_map_lock(sbi, flag, false);
1338                 f2fs_balance_fs(sbi, dn.node_changed);
1339         }
1340 out:
1341         trace_f2fs_map_blocks(inode, map, err);
1342         return err;
1343 }
1344
1345 bool f2fs_overwrite_io(struct inode *inode, loff_t pos, size_t len)
1346 {
1347         struct f2fs_map_blocks map;
1348         block_t last_lblk;
1349         int err;
1350
1351         if (pos + len > i_size_read(inode))
1352                 return false;
1353
1354         map.m_lblk = F2FS_BYTES_TO_BLK(pos);
1355         map.m_next_pgofs = NULL;
1356         map.m_next_extent = NULL;
1357         map.m_seg_type = NO_CHECK_TYPE;
1358         map.m_may_create = false;
1359         last_lblk = F2FS_BLK_ALIGN(pos + len);
1360
1361         while (map.m_lblk < last_lblk) {
1362                 map.m_len = last_lblk - map.m_lblk;
1363                 err = f2fs_map_blocks(inode, &map, 0, F2FS_GET_BLOCK_DEFAULT);
1364                 if (err || map.m_len == 0)
1365                         return false;
1366                 map.m_lblk += map.m_len;
1367         }
1368         return true;
1369 }
1370
1371 static int __get_data_block(struct inode *inode, sector_t iblock,
1372                         struct buffer_head *bh, int create, int flag,
1373                         pgoff_t *next_pgofs, int seg_type, bool may_write)
1374 {
1375         struct f2fs_map_blocks map;
1376         int err;
1377
1378         map.m_lblk = iblock;
1379         map.m_len = bh->b_size >> inode->i_blkbits;
1380         map.m_next_pgofs = next_pgofs;
1381         map.m_next_extent = NULL;
1382         map.m_seg_type = seg_type;
1383         map.m_may_create = may_write;
1384
1385         err = f2fs_map_blocks(inode, &map, create, flag);
1386         if (!err) {
1387                 map_bh(bh, inode->i_sb, map.m_pblk);
1388                 bh->b_state = (bh->b_state & ~F2FS_MAP_FLAGS) | map.m_flags;
1389                 bh->b_size = (u64)map.m_len << inode->i_blkbits;
1390         }
1391         return err;
1392 }
1393
1394 static int get_data_block(struct inode *inode, sector_t iblock,
1395                         struct buffer_head *bh_result, int create, int flag,
1396                         pgoff_t *next_pgofs)
1397 {
1398         return __get_data_block(inode, iblock, bh_result, create,
1399                                                         flag, next_pgofs,
1400                                                         NO_CHECK_TYPE, create);
1401 }
1402
1403 static int get_data_block_dio_write(struct inode *inode, sector_t iblock,
1404                         struct buffer_head *bh_result, int create)
1405 {
1406         return __get_data_block(inode, iblock, bh_result, create,
1407                                 F2FS_GET_BLOCK_DIO, NULL,
1408                                 f2fs_rw_hint_to_seg_type(inode->i_write_hint),
1409                                 true);
1410 }
1411
1412 static int get_data_block_dio(struct inode *inode, sector_t iblock,
1413                         struct buffer_head *bh_result, int create)
1414 {
1415         return __get_data_block(inode, iblock, bh_result, create,
1416                                 F2FS_GET_BLOCK_DIO, NULL,
1417                                 f2fs_rw_hint_to_seg_type(inode->i_write_hint),
1418                                 false);
1419 }
1420
1421 static int get_data_block_bmap(struct inode *inode, sector_t iblock,
1422                         struct buffer_head *bh_result, int create)
1423 {
1424         /* Block number less than F2FS MAX BLOCKS */
1425         if (unlikely(iblock >= F2FS_I_SB(inode)->max_file_blocks))
1426                 return -EFBIG;
1427
1428         return __get_data_block(inode, iblock, bh_result, create,
1429                                                 F2FS_GET_BLOCK_BMAP, NULL,
1430                                                 NO_CHECK_TYPE, create);
1431 }
1432
1433 static inline sector_t logical_to_blk(struct inode *inode, loff_t offset)
1434 {
1435         return (offset >> inode->i_blkbits);
1436 }
1437
1438 static inline loff_t blk_to_logical(struct inode *inode, sector_t blk)
1439 {
1440         return (blk << inode->i_blkbits);
1441 }
1442
1443 static int f2fs_xattr_fiemap(struct inode *inode,
1444                                 struct fiemap_extent_info *fieinfo)
1445 {
1446         struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
1447         struct page *page;
1448         struct node_info ni;
1449         __u64 phys = 0, len;
1450         __u32 flags;
1451         nid_t xnid = F2FS_I(inode)->i_xattr_nid;
1452         int err = 0;
1453
1454         if (f2fs_has_inline_xattr(inode)) {
1455                 int offset;
1456
1457                 page = f2fs_grab_cache_page(NODE_MAPPING(sbi),
1458                                                 inode->i_ino, false);
1459                 if (!page)
1460                         return -ENOMEM;
1461
1462                 err = f2fs_get_node_info(sbi, inode->i_ino, &ni);
1463                 if (err) {
1464                         f2fs_put_page(page, 1);
1465                         return err;
1466                 }
1467
1468                 phys = (__u64)blk_to_logical(inode, ni.blk_addr);
1469                 offset = offsetof(struct f2fs_inode, i_addr) +
1470                                         sizeof(__le32) * (DEF_ADDRS_PER_INODE -
1471                                         get_inline_xattr_addrs(inode));
1472
1473                 phys += offset;
1474                 len = inline_xattr_size(inode);
1475
1476                 f2fs_put_page(page, 1);
1477
1478                 flags = FIEMAP_EXTENT_DATA_INLINE | FIEMAP_EXTENT_NOT_ALIGNED;
1479
1480                 if (!xnid)
1481                         flags |= FIEMAP_EXTENT_LAST;
1482
1483                 err = fiemap_fill_next_extent(fieinfo, 0, phys, len, flags);
1484                 if (err || err == 1)
1485                         return err;
1486         }
1487
1488         if (xnid) {
1489                 page = f2fs_grab_cache_page(NODE_MAPPING(sbi), xnid, false);
1490                 if (!page)
1491                         return -ENOMEM;
1492
1493                 err = f2fs_get_node_info(sbi, xnid, &ni);
1494                 if (err) {
1495                         f2fs_put_page(page, 1);
1496                         return err;
1497                 }
1498
1499                 phys = (__u64)blk_to_logical(inode, ni.blk_addr);
1500                 len = inode->i_sb->s_blocksize;
1501
1502                 f2fs_put_page(page, 1);
1503
1504                 flags = FIEMAP_EXTENT_LAST;
1505         }
1506
1507         if (phys)
1508                 err = fiemap_fill_next_extent(fieinfo, 0, phys, len, flags);
1509
1510         return (err < 0 ? err : 0);
1511 }
1512
1513 int f2fs_fiemap(struct inode *inode, struct fiemap_extent_info *fieinfo,
1514                 u64 start, u64 len)
1515 {
1516         struct buffer_head map_bh;
1517         sector_t start_blk, last_blk;
1518         pgoff_t next_pgofs;
1519         u64 logical = 0, phys = 0, size = 0;
1520         u32 flags = 0;
1521         int ret = 0;
1522
1523         if (fieinfo->fi_flags & FIEMAP_FLAG_CACHE) {
1524                 ret = f2fs_precache_extents(inode);
1525                 if (ret)
1526                         return ret;
1527         }
1528
1529         ret = fiemap_check_flags(fieinfo, FIEMAP_FLAG_SYNC | FIEMAP_FLAG_XATTR);
1530         if (ret)
1531                 return ret;
1532
1533         inode_lock(inode);
1534
1535         if (fieinfo->fi_flags & FIEMAP_FLAG_XATTR) {
1536                 ret = f2fs_xattr_fiemap(inode, fieinfo);
1537                 goto out;
1538         }
1539
1540         if (f2fs_has_inline_data(inode)) {
1541                 ret = f2fs_inline_data_fiemap(inode, fieinfo, start, len);
1542                 if (ret != -EAGAIN)
1543                         goto out;
1544         }
1545
1546         if (logical_to_blk(inode, len) == 0)
1547                 len = blk_to_logical(inode, 1);
1548
1549         start_blk = logical_to_blk(inode, start);
1550         last_blk = logical_to_blk(inode, start + len - 1);
1551
1552 next:
1553         memset(&map_bh, 0, sizeof(struct buffer_head));
1554         map_bh.b_size = len;
1555
1556         ret = get_data_block(inode, start_blk, &map_bh, 0,
1557                                         F2FS_GET_BLOCK_FIEMAP, &next_pgofs);
1558         if (ret)
1559                 goto out;
1560
1561         /* HOLE */
1562         if (!buffer_mapped(&map_bh)) {
1563                 start_blk = next_pgofs;
1564
1565                 if (blk_to_logical(inode, start_blk) < blk_to_logical(inode,
1566                                         F2FS_I_SB(inode)->max_file_blocks))
1567                         goto prep_next;
1568
1569                 flags |= FIEMAP_EXTENT_LAST;
1570         }
1571
1572         if (size) {
1573                 if (f2fs_encrypted_inode(inode))
1574                         flags |= FIEMAP_EXTENT_DATA_ENCRYPTED;
1575
1576                 ret = fiemap_fill_next_extent(fieinfo, logical,
1577                                 phys, size, flags);
1578         }
1579
1580         if (start_blk > last_blk || ret)
1581                 goto out;
1582
1583         logical = blk_to_logical(inode, start_blk);
1584         phys = blk_to_logical(inode, map_bh.b_blocknr);
1585         size = map_bh.b_size;
1586         flags = 0;
1587         if (buffer_unwritten(&map_bh))
1588                 flags = FIEMAP_EXTENT_UNWRITTEN;
1589
1590         start_blk += logical_to_blk(inode, size);
1591
1592 prep_next:
1593         cond_resched();
1594         if (fatal_signal_pending(current))
1595                 ret = -EINTR;
1596         else
1597                 goto next;
1598 out:
1599         if (ret == 1)
1600                 ret = 0;
1601
1602         inode_unlock(inode);
1603         return ret;
1604 }
1605
1606 static int f2fs_read_single_page(struct inode *inode, struct page *page,
1607                                         unsigned nr_pages,
1608                                         struct f2fs_map_blocks *map,
1609                                         struct bio **bio_ret,
1610                                         sector_t *last_block_in_bio,
1611                                         bool is_readahead)
1612 {
1613         struct bio *bio = *bio_ret;
1614         const unsigned blkbits = inode->i_blkbits;
1615         const unsigned blocksize = 1 << blkbits;
1616         sector_t block_in_file;
1617         sector_t last_block;
1618         sector_t last_block_in_file;
1619         sector_t block_nr;
1620         int ret = 0;
1621
1622         block_in_file = (sector_t)page_index(page);
1623         last_block = block_in_file + nr_pages;
1624         last_block_in_file = (i_size_read(inode) + blocksize - 1) >>
1625                                                         blkbits;
1626         if (last_block > last_block_in_file)
1627                 last_block = last_block_in_file;
1628
1629         /* just zeroing out page which is beyond EOF */
1630         if (block_in_file >= last_block)
1631                 goto zero_out;
1632         /*
1633          * Map blocks using the previous result first.
1634          */
1635         if ((map->m_flags & F2FS_MAP_MAPPED) &&
1636                         block_in_file > map->m_lblk &&
1637                         block_in_file < (map->m_lblk + map->m_len))
1638                 goto got_it;
1639
1640         /*
1641          * Then do more f2fs_map_blocks() calls until we are
1642          * done with this page.
1643          */
1644         map->m_lblk = block_in_file;
1645         map->m_len = last_block - block_in_file;
1646
1647         ret = f2fs_map_blocks(inode, map, 0, F2FS_GET_BLOCK_DEFAULT);
1648         if (ret)
1649                 goto out;
1650 got_it:
1651         if ((map->m_flags & F2FS_MAP_MAPPED)) {
1652                 block_nr = map->m_pblk + block_in_file - map->m_lblk;
1653                 SetPageMappedToDisk(page);
1654
1655                 if (!PageUptodate(page) && (!PageSwapCache(page) &&
1656                                         !cleancache_get_page(page))) {
1657                         SetPageUptodate(page);
1658                         goto confused;
1659                 }
1660
1661                 if (!f2fs_is_valid_blkaddr(F2FS_I_SB(inode), block_nr,
1662                                                 DATA_GENERIC_ENHANCE_READ)) {
1663                         ret = -EFSCORRUPTED;
1664                         goto out;
1665                 }
1666         } else {
1667 zero_out:
1668                 zero_user_segment(page, 0, PAGE_SIZE);
1669                 if (!PageUptodate(page))
1670                         SetPageUptodate(page);
1671                 unlock_page(page);
1672                 goto out;
1673         }
1674
1675         /*
1676          * This page will go to BIO.  Do we need to send this
1677          * BIO off first?
1678          */
1679         if (bio && (*last_block_in_bio != block_nr - 1 ||
1680                 !__same_bdev(F2FS_I_SB(inode), block_nr, bio))) {
1681 submit_and_realloc:
1682                 __f2fs_submit_read_bio(F2FS_I_SB(inode), bio, DATA);
1683                 bio = NULL;
1684         }
1685         if (bio == NULL) {
1686                 bio = f2fs_grab_read_bio(inode, block_nr, nr_pages,
1687                                 is_readahead ? REQ_RAHEAD : 0);
1688                 if (IS_ERR(bio)) {
1689                         ret = PTR_ERR(bio);
1690                         bio = NULL;
1691                         goto out;
1692                 }
1693         }
1694
1695         /*
1696          * If the page is under writeback, we need to wait for
1697          * its completion to see the correct decrypted data.
1698          */
1699         f2fs_wait_on_block_writeback(inode, block_nr);
1700
1701         if (bio_add_page(bio, page, blocksize, 0) < blocksize)
1702                 goto submit_and_realloc;
1703
1704         inc_page_count(F2FS_I_SB(inode), F2FS_RD_DATA);
1705         ClearPageError(page);
1706         *last_block_in_bio = block_nr;
1707         goto out;
1708 confused:
1709         if (bio) {
1710                 __f2fs_submit_read_bio(F2FS_I_SB(inode), bio, DATA);
1711                 bio = NULL;
1712         }
1713         unlock_page(page);
1714 out:
1715         *bio_ret = bio;
1716         return ret;
1717 }
1718
1719 /*
1720  * This function was originally taken from fs/mpage.c, and customized for f2fs.
1721  * Major change was from block_size == page_size in f2fs by default.
1722  *
1723  * Note that the aops->readpages() function is ONLY used for read-ahead. If
1724  * this function ever deviates from doing just read-ahead, it should either
1725  * use ->readpage() or do the necessary surgery to decouple ->readpages()
1726  * from read-ahead.
1727  */
1728 static int f2fs_mpage_readpages(struct address_space *mapping,
1729                         struct list_head *pages, struct page *page,
1730                         unsigned nr_pages, bool is_readahead)
1731 {
1732         struct bio *bio = NULL;
1733         sector_t last_block_in_bio = 0;
1734         struct inode *inode = mapping->host;
1735         struct f2fs_map_blocks map;
1736         int ret = 0;
1737
1738         map.m_pblk = 0;
1739         map.m_lblk = 0;
1740         map.m_len = 0;
1741         map.m_flags = 0;
1742         map.m_next_pgofs = NULL;
1743         map.m_next_extent = NULL;
1744         map.m_seg_type = NO_CHECK_TYPE;
1745         map.m_may_create = false;
1746
1747         for (; nr_pages; nr_pages--) {
1748                 if (pages) {
1749                         page = list_last_entry(pages, struct page, lru);
1750
1751                         prefetchw(&page->flags);
1752                         list_del(&page->lru);
1753                         if (add_to_page_cache_lru(page, mapping,
1754                                                   page_index(page), GFP_KERNEL))
1755                                 goto next_page;
1756                 }
1757
1758                 ret = f2fs_read_single_page(inode, page, nr_pages, &map, &bio,
1759                                         &last_block_in_bio, is_readahead);
1760                 if (ret) {
1761                         SetPageError(page);
1762                         zero_user_segment(page, 0, PAGE_SIZE);
1763                         unlock_page(page);
1764                 }
1765 next_page:
1766                 if (pages)
1767                         put_page(page);
1768         }
1769         BUG_ON(pages && !list_empty(pages));
1770         if (bio)
1771                 __f2fs_submit_read_bio(F2FS_I_SB(inode), bio, DATA);
1772         return pages ? 0 : ret;
1773 }
1774
1775 static int f2fs_read_data_page(struct file *file, struct page *page)
1776 {
1777         struct inode *inode = page_file_mapping(page)->host;
1778         int ret = -EAGAIN;
1779
1780         trace_f2fs_readpage(page, DATA);
1781
1782         /* If the file has inline data, try to read it directly */
1783         if (f2fs_has_inline_data(inode))
1784                 ret = f2fs_read_inline_data(inode, page);
1785         if (ret == -EAGAIN)
1786                 ret = f2fs_mpage_readpages(page_file_mapping(page),
1787                                                 NULL, page, 1, false);
1788         return ret;
1789 }
1790
1791 static int f2fs_read_data_pages(struct file *file,
1792                         struct address_space *mapping,
1793                         struct list_head *pages, unsigned nr_pages)
1794 {
1795         struct inode *inode = mapping->host;
1796         struct page *page = list_last_entry(pages, struct page, lru);
1797
1798         trace_f2fs_readpages(inode, page, nr_pages);
1799
1800         /* If the file has inline data, skip readpages */
1801         if (f2fs_has_inline_data(inode))
1802                 return 0;
1803
1804         return f2fs_mpage_readpages(mapping, pages, NULL, nr_pages, true);
1805 }
1806
1807 static int encrypt_one_page(struct f2fs_io_info *fio)
1808 {
1809         struct inode *inode = fio->page->mapping->host;
1810         struct page *mpage;
1811         gfp_t gfp_flags = GFP_NOFS;
1812
1813         if (!f2fs_encrypted_file(inode))
1814                 return 0;
1815
1816         /* wait for GCed page writeback via META_MAPPING */
1817         f2fs_wait_on_block_writeback(inode, fio->old_blkaddr);
1818
1819 retry_encrypt:
1820         fio->encrypted_page = fscrypt_encrypt_page(inode, fio->page,
1821                         PAGE_SIZE, 0, fio->page->index, gfp_flags);
1822         if (IS_ERR(fio->encrypted_page)) {
1823                 /* flush pending IOs and wait for a while in the ENOMEM case */
1824                 if (PTR_ERR(fio->encrypted_page) == -ENOMEM) {
1825                         f2fs_flush_merged_writes(fio->sbi);
1826                         congestion_wait(BLK_RW_ASYNC, HZ/50);
1827                         gfp_flags |= __GFP_NOFAIL;
1828                         goto retry_encrypt;
1829                 }
1830                 return PTR_ERR(fio->encrypted_page);
1831         }
1832
1833         mpage = find_lock_page(META_MAPPING(fio->sbi), fio->old_blkaddr);
1834         if (mpage) {
1835                 if (PageUptodate(mpage))
1836                         memcpy(page_address(mpage),
1837                                 page_address(fio->encrypted_page), PAGE_SIZE);
1838                 f2fs_put_page(mpage, 1);
1839         }
1840         return 0;
1841 }
1842
1843 static inline bool check_inplace_update_policy(struct inode *inode,
1844                                 struct f2fs_io_info *fio)
1845 {
1846         struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
1847         unsigned int policy = SM_I(sbi)->ipu_policy;
1848
1849         if (policy & (0x1 << F2FS_IPU_FORCE))
1850                 return true;
1851         if (policy & (0x1 << F2FS_IPU_SSR) && f2fs_need_SSR(sbi))
1852                 return true;
1853         if (policy & (0x1 << F2FS_IPU_UTIL) &&
1854                         utilization(sbi) > SM_I(sbi)->min_ipu_util)
1855                 return true;
1856         if (policy & (0x1 << F2FS_IPU_SSR_UTIL) && f2fs_need_SSR(sbi) &&
1857                         utilization(sbi) > SM_I(sbi)->min_ipu_util)
1858                 return true;
1859
1860         /*
1861          * IPU for rewrite async pages
1862          */
1863         if (policy & (0x1 << F2FS_IPU_ASYNC) &&
1864                         fio && fio->op == REQ_OP_WRITE &&
1865                         !(fio->op_flags & REQ_SYNC) &&
1866                         !f2fs_encrypted_inode(inode))
1867                 return true;
1868
1869         /* this is only set during fdatasync */
1870         if (policy & (0x1 << F2FS_IPU_FSYNC) &&
1871                         is_inode_flag_set(inode, FI_NEED_IPU))
1872                 return true;
1873
1874         if (unlikely(fio && is_sbi_flag_set(sbi, SBI_CP_DISABLED) &&
1875                         !f2fs_is_checkpointed_data(sbi, fio->old_blkaddr)))
1876                 return true;
1877
1878         return false;
1879 }
1880
1881 bool f2fs_should_update_inplace(struct inode *inode, struct f2fs_io_info *fio)
1882 {
1883         if (f2fs_is_pinned_file(inode))
1884                 return true;
1885
1886         /* if this is cold file, we should overwrite to avoid fragmentation */
1887         if (file_is_cold(inode))
1888                 return true;
1889
1890         return check_inplace_update_policy(inode, fio);
1891 }
1892
1893 bool f2fs_should_update_outplace(struct inode *inode, struct f2fs_io_info *fio)
1894 {
1895         struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
1896
1897         if (test_opt(sbi, LFS))
1898                 return true;
1899         if (S_ISDIR(inode->i_mode))
1900                 return true;
1901         if (IS_NOQUOTA(inode))
1902                 return true;
1903         if (f2fs_is_atomic_file(inode))
1904                 return true;
1905         if (fio) {
1906                 if (is_cold_data(fio->page))
1907                         return true;
1908                 if (IS_ATOMIC_WRITTEN_PAGE(fio->page))
1909                         return true;
1910                 if (unlikely(is_sbi_flag_set(sbi, SBI_CP_DISABLED) &&
1911                         f2fs_is_checkpointed_data(sbi, fio->old_blkaddr)))
1912                         return true;
1913         }
1914         return false;
1915 }
1916
1917 static inline bool need_inplace_update(struct f2fs_io_info *fio)
1918 {
1919         struct inode *inode = fio->page->mapping->host;
1920
1921         if (f2fs_should_update_outplace(inode, fio))
1922                 return false;
1923
1924         return f2fs_should_update_inplace(inode, fio);
1925 }
1926
1927 int f2fs_do_write_data_page(struct f2fs_io_info *fio)
1928 {
1929         struct page *page = fio->page;
1930         struct inode *inode = page->mapping->host;
1931         struct dnode_of_data dn;
1932         struct extent_info ei = {0,0,0};
1933         struct node_info ni;
1934         bool ipu_force = false;
1935         int err = 0;
1936
1937         set_new_dnode(&dn, inode, NULL, NULL, 0);
1938         if (need_inplace_update(fio) &&
1939                         f2fs_lookup_extent_cache(inode, page->index, &ei)) {
1940                 fio->old_blkaddr = ei.blk + page->index - ei.fofs;
1941
1942                 if (!f2fs_is_valid_blkaddr(fio->sbi, fio->old_blkaddr,
1943                                                 DATA_GENERIC_ENHANCE))
1944                         return -EFSCORRUPTED;
1945
1946                 ipu_force = true;
1947                 fio->need_lock = LOCK_DONE;
1948                 goto got_it;
1949         }
1950
1951         /* Deadlock due to between page->lock and f2fs_lock_op */
1952         if (fio->need_lock == LOCK_REQ && !f2fs_trylock_op(fio->sbi))
1953                 return -EAGAIN;
1954
1955         err = f2fs_get_dnode_of_data(&dn, page->index, LOOKUP_NODE);
1956         if (err)
1957                 goto out;
1958
1959         fio->old_blkaddr = dn.data_blkaddr;
1960
1961         /* This page is already truncated */
1962         if (fio->old_blkaddr == NULL_ADDR) {
1963                 ClearPageUptodate(page);
1964                 clear_cold_data(page);
1965                 goto out_writepage;
1966         }
1967 got_it:
1968         if (__is_valid_data_blkaddr(fio->old_blkaddr) &&
1969                 !f2fs_is_valid_blkaddr(fio->sbi, fio->old_blkaddr,
1970                                                 DATA_GENERIC_ENHANCE)) {
1971                 err = -EFSCORRUPTED;
1972                 goto out_writepage;
1973         }
1974         /*
1975          * If current allocation needs SSR,
1976          * it had better in-place writes for updated data.
1977          */
1978         if (ipu_force ||
1979                 (__is_valid_data_blkaddr(fio->old_blkaddr) &&
1980                                         need_inplace_update(fio))) {
1981                 err = encrypt_one_page(fio);
1982                 if (err)
1983                         goto out_writepage;
1984
1985                 set_page_writeback(page);
1986                 ClearPageError(page);
1987                 f2fs_put_dnode(&dn);
1988                 if (fio->need_lock == LOCK_REQ)
1989                         f2fs_unlock_op(fio->sbi);
1990                 err = f2fs_inplace_write_data(fio);
1991                 if (err) {
1992                         if (f2fs_encrypted_file(inode))
1993                                 fscrypt_pullback_bio_page(&fio->encrypted_page,
1994                                                                         true);
1995                         if (PageWriteback(page))
1996                                 end_page_writeback(page);
1997                 } else {
1998                         set_inode_flag(inode, FI_UPDATE_WRITE);
1999                 }
2000                 trace_f2fs_do_write_data_page(fio->page, IPU);
2001                 return err;
2002         }
2003
2004         if (fio->need_lock == LOCK_RETRY) {
2005                 if (!f2fs_trylock_op(fio->sbi)) {
2006                         err = -EAGAIN;
2007                         goto out_writepage;
2008                 }
2009                 fio->need_lock = LOCK_REQ;
2010         }
2011
2012         err = f2fs_get_node_info(fio->sbi, dn.nid, &ni);
2013         if (err)
2014                 goto out_writepage;
2015
2016         fio->version = ni.version;
2017
2018         err = encrypt_one_page(fio);
2019         if (err)
2020                 goto out_writepage;
2021
2022         set_page_writeback(page);
2023         ClearPageError(page);
2024
2025         /* LFS mode write path */
2026         f2fs_outplace_write_data(&dn, fio);
2027         trace_f2fs_do_write_data_page(page, OPU);
2028         set_inode_flag(inode, FI_APPEND_WRITE);
2029         if (page->index == 0)
2030                 set_inode_flag(inode, FI_FIRST_BLOCK_WRITTEN);
2031 out_writepage:
2032         f2fs_put_dnode(&dn);
2033 out:
2034         if (fio->need_lock == LOCK_REQ)
2035                 f2fs_unlock_op(fio->sbi);
2036         return err;
2037 }
2038
2039 static int __write_data_page(struct page *page, bool *submitted,
2040                                 struct bio **bio,
2041                                 sector_t *last_block,
2042                                 struct writeback_control *wbc,
2043                                 enum iostat_type io_type)
2044 {
2045         struct inode *inode = page->mapping->host;
2046         struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
2047         loff_t i_size = i_size_read(inode);
2048         const pgoff_t end_index = ((unsigned long long) i_size)
2049                                                         >> PAGE_SHIFT;
2050         loff_t psize = (page->index + 1) << PAGE_SHIFT;
2051         unsigned offset = 0;
2052         bool need_balance_fs = false;
2053         int err = 0;
2054         struct f2fs_io_info fio = {
2055                 .sbi = sbi,
2056                 .ino = inode->i_ino,
2057                 .type = DATA,
2058                 .op = REQ_OP_WRITE,
2059                 .op_flags = wbc_to_write_flags(wbc),
2060                 .old_blkaddr = NULL_ADDR,
2061                 .page = page,
2062                 .encrypted_page = NULL,
2063                 .submitted = false,
2064                 .need_lock = LOCK_RETRY,
2065                 .io_type = io_type,
2066                 .io_wbc = wbc,
2067                 .bio = bio,
2068                 .last_block = last_block,
2069         };
2070
2071         trace_f2fs_writepage(page, DATA);
2072
2073         /* we should bypass data pages to proceed the kworkder jobs */
2074         if (unlikely(f2fs_cp_error(sbi))) {
2075                 mapping_set_error(page->mapping, -EIO);
2076                 /*
2077                  * don't drop any dirty dentry pages for keeping lastest
2078                  * directory structure.
2079                  */
2080                 if (S_ISDIR(inode->i_mode))
2081                         goto redirty_out;
2082                 goto out;
2083         }
2084
2085         if (unlikely(is_sbi_flag_set(sbi, SBI_POR_DOING)))
2086                 goto redirty_out;
2087
2088         if (page->index < end_index)
2089                 goto write;
2090
2091         /*
2092          * If the offset is out-of-range of file size,
2093          * this page does not have to be written to disk.
2094          */
2095         offset = i_size & (PAGE_SIZE - 1);
2096         if ((page->index >= end_index + 1) || !offset)
2097                 goto out;
2098
2099         zero_user_segment(page, offset, PAGE_SIZE);
2100 write:
2101         if (f2fs_is_drop_cache(inode))
2102                 goto out;
2103         /* we should not write 0'th page having journal header */
2104         if (f2fs_is_volatile_file(inode) && (!page->index ||
2105                         (!wbc->for_reclaim &&
2106                         f2fs_available_free_memory(sbi, BASE_CHECK))))
2107                 goto redirty_out;
2108
2109         /* Dentry blocks are controlled by checkpoint */
2110         if (S_ISDIR(inode->i_mode)) {
2111                 fio.need_lock = LOCK_DONE;
2112                 err = f2fs_do_write_data_page(&fio);
2113                 goto done;
2114         }
2115
2116         if (!wbc->for_reclaim)
2117                 need_balance_fs = true;
2118         else if (has_not_enough_free_secs(sbi, 0, 0))
2119                 goto redirty_out;
2120         else
2121                 set_inode_flag(inode, FI_HOT_DATA);
2122
2123         err = -EAGAIN;
2124         if (f2fs_has_inline_data(inode)) {
2125                 err = f2fs_write_inline_data(inode, page);
2126                 if (!err)
2127                         goto out;
2128         }
2129
2130         if (err == -EAGAIN) {
2131                 err = f2fs_do_write_data_page(&fio);
2132                 if (err == -EAGAIN) {
2133                         fio.need_lock = LOCK_REQ;
2134                         err = f2fs_do_write_data_page(&fio);
2135                 }
2136         }
2137
2138         if (err) {
2139                 file_set_keep_isize(inode);
2140         } else {
2141                 down_write(&F2FS_I(inode)->i_sem);
2142                 if (F2FS_I(inode)->last_disk_size < psize)
2143                         F2FS_I(inode)->last_disk_size = psize;
2144                 up_write(&F2FS_I(inode)->i_sem);
2145         }
2146
2147 done:
2148         if (err && err != -ENOENT)
2149                 goto redirty_out;
2150
2151 out:
2152         inode_dec_dirty_pages(inode);
2153         if (err) {
2154                 ClearPageUptodate(page);
2155                 clear_cold_data(page);
2156         }
2157
2158         if (wbc->for_reclaim) {
2159                 f2fs_submit_merged_write_cond(sbi, NULL, page, 0, DATA);
2160                 clear_inode_flag(inode, FI_HOT_DATA);
2161                 f2fs_remove_dirty_inode(inode);
2162                 submitted = NULL;
2163         }
2164
2165         unlock_page(page);
2166         if (!S_ISDIR(inode->i_mode) && !IS_NOQUOTA(inode) &&
2167                                         !F2FS_I(inode)->cp_task) {
2168                 f2fs_submit_ipu_bio(sbi, bio, page);
2169                 f2fs_balance_fs(sbi, need_balance_fs);
2170         }
2171
2172         if (unlikely(f2fs_cp_error(sbi))) {
2173                 f2fs_submit_ipu_bio(sbi, bio, page);
2174                 f2fs_submit_merged_write(sbi, DATA);
2175                 submitted = NULL;
2176         }
2177
2178         if (submitted)
2179                 *submitted = fio.submitted;
2180
2181         return 0;
2182
2183 redirty_out:
2184         redirty_page_for_writepage(wbc, page);
2185         /*
2186          * pageout() in MM traslates EAGAIN, so calls handle_write_error()
2187          * -> mapping_set_error() -> set_bit(AS_EIO, ...).
2188          * file_write_and_wait_range() will see EIO error, which is critical
2189          * to return value of fsync() followed by atomic_write failure to user.
2190          */
2191         if (!err || wbc->for_reclaim)
2192                 return AOP_WRITEPAGE_ACTIVATE;
2193         unlock_page(page);
2194         return err;
2195 }
2196
2197 static int f2fs_write_data_page(struct page *page,
2198                                         struct writeback_control *wbc)
2199 {
2200         return __write_data_page(page, NULL, NULL, NULL, wbc, FS_DATA_IO);
2201 }
2202
2203 /*
2204  * This function was copied from write_cche_pages from mm/page-writeback.c.
2205  * The major change is making write step of cold data page separately from
2206  * warm/hot data page.
2207  */
2208 static int f2fs_write_cache_pages(struct address_space *mapping,
2209                                         struct writeback_control *wbc,
2210                                         enum iostat_type io_type)
2211 {
2212         int ret = 0;
2213         int done = 0;
2214         struct pagevec pvec;
2215         struct f2fs_sb_info *sbi = F2FS_M_SB(mapping);
2216         struct bio *bio = NULL;
2217         sector_t last_block;
2218         int nr_pages;
2219         pgoff_t uninitialized_var(writeback_index);
2220         pgoff_t index;
2221         pgoff_t end;            /* Inclusive */
2222         pgoff_t done_index;
2223         int cycled;
2224         int range_whole = 0;
2225         int tag;
2226         int nwritten = 0;
2227
2228         pagevec_init(&pvec, 0);
2229
2230         if (get_dirty_pages(mapping->host) <=
2231                                 SM_I(F2FS_M_SB(mapping))->min_hot_blocks)
2232                 set_inode_flag(mapping->host, FI_HOT_DATA);
2233         else
2234                 clear_inode_flag(mapping->host, FI_HOT_DATA);
2235
2236         if (wbc->range_cyclic) {
2237                 writeback_index = mapping->writeback_index; /* prev offset */
2238                 index = writeback_index;
2239                 if (index == 0)
2240                         cycled = 1;
2241                 else
2242                         cycled = 0;
2243                 end = -1;
2244         } else {
2245                 index = wbc->range_start >> PAGE_SHIFT;
2246                 end = wbc->range_end >> PAGE_SHIFT;
2247                 if (wbc->range_start == 0 && wbc->range_end == LLONG_MAX)
2248                         range_whole = 1;
2249                 cycled = 1; /* ignore range_cyclic tests */
2250         }
2251         if (wbc->sync_mode == WB_SYNC_ALL || wbc->tagged_writepages)
2252                 tag = PAGECACHE_TAG_TOWRITE;
2253         else
2254                 tag = PAGECACHE_TAG_DIRTY;
2255 retry:
2256         if (wbc->sync_mode == WB_SYNC_ALL || wbc->tagged_writepages)
2257                 tag_pages_for_writeback(mapping, index, end);
2258         done_index = index;
2259         while (!done && (index <= end)) {
2260                 int i;
2261
2262                 nr_pages = pagevec_lookup_range_tag(&pvec, mapping, &index, end,
2263                                 tag);
2264                 if (nr_pages == 0)
2265                         break;
2266
2267                 for (i = 0; i < nr_pages; i++) {
2268                         struct page *page = pvec.pages[i];
2269                         bool submitted = false;
2270
2271                         /* give a priority to WB_SYNC threads */
2272                         if (atomic_read(&sbi->wb_sync_req[DATA]) &&
2273                                         wbc->sync_mode == WB_SYNC_NONE) {
2274                                 done = 1;
2275                                 break;
2276                         }
2277
2278                         done_index = page->index;
2279 retry_write:
2280                         lock_page(page);
2281
2282                         if (unlikely(page->mapping != mapping)) {
2283 continue_unlock:
2284                                 unlock_page(page);
2285                                 continue;
2286                         }
2287
2288                         if (!PageDirty(page)) {
2289                                 /* someone wrote it for us */
2290                                 goto continue_unlock;
2291                         }
2292
2293                         if (PageWriteback(page)) {
2294                                 if (wbc->sync_mode != WB_SYNC_NONE) {
2295                                         f2fs_wait_on_page_writeback(page,
2296                                                         DATA, true, true);
2297                                         f2fs_submit_ipu_bio(sbi, &bio, page);
2298                                 } else {
2299                                         goto continue_unlock;
2300                                 }
2301                         }
2302
2303                         if (!clear_page_dirty_for_io(page))
2304                                 goto continue_unlock;
2305
2306                         ret = __write_data_page(page, &submitted, &bio,
2307                                         &last_block, wbc, io_type);
2308                         if (unlikely(ret)) {
2309                                 /*
2310                                  * keep nr_to_write, since vfs uses this to
2311                                  * get # of written pages.
2312                                  */
2313                                 if (ret == AOP_WRITEPAGE_ACTIVATE) {
2314                                         unlock_page(page);
2315                                         ret = 0;
2316                                         continue;
2317                                 } else if (ret == -EAGAIN) {
2318                                         ret = 0;
2319                                         if (wbc->sync_mode == WB_SYNC_ALL) {
2320                                                 cond_resched();
2321                                                 congestion_wait(BLK_RW_ASYNC,
2322                                                                         HZ/50);
2323                                                 goto retry_write;
2324                                         }
2325                                         continue;
2326                                 }
2327                                 done_index = page->index + 1;
2328                                 done = 1;
2329                                 break;
2330                         } else if (submitted) {
2331                                 nwritten++;
2332                         }
2333
2334                         if (--wbc->nr_to_write <= 0 &&
2335                                         wbc->sync_mode == WB_SYNC_NONE) {
2336                                 done = 1;
2337                                 break;
2338                         }
2339                 }
2340                 pagevec_release(&pvec);
2341                 cond_resched();
2342         }
2343
2344         if (!cycled && !done) {
2345                 cycled = 1;
2346                 index = 0;
2347                 end = writeback_index - 1;
2348                 goto retry;
2349         }
2350         if (wbc->range_cyclic || (range_whole && wbc->nr_to_write > 0))
2351                 mapping->writeback_index = done_index;
2352
2353         if (nwritten)
2354                 f2fs_submit_merged_write_cond(F2FS_M_SB(mapping), mapping->host,
2355                                                                 NULL, 0, DATA);
2356         /* submit cached bio of IPU write */
2357         if (bio)
2358                 __submit_bio(sbi, bio, DATA);
2359
2360         return ret;
2361 }
2362
2363 static inline bool __should_serialize_io(struct inode *inode,
2364                                         struct writeback_control *wbc)
2365 {
2366         if (!S_ISREG(inode->i_mode))
2367                 return false;
2368         if (IS_NOQUOTA(inode))
2369                 return false;
2370         /* to avoid deadlock in path of data flush */
2371         if (F2FS_I(inode)->cp_task)
2372                 return false;
2373         if (wbc->sync_mode != WB_SYNC_ALL)
2374                 return true;
2375         if (get_dirty_pages(inode) >= SM_I(F2FS_I_SB(inode))->min_seq_blocks)
2376                 return true;
2377         return false;
2378 }
2379
2380 static int __f2fs_write_data_pages(struct address_space *mapping,
2381                                                 struct writeback_control *wbc,
2382                                                 enum iostat_type io_type)
2383 {
2384         struct inode *inode = mapping->host;
2385         struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
2386         struct blk_plug plug;
2387         int ret;
2388         bool locked = false;
2389
2390         /* deal with chardevs and other special file */
2391         if (!mapping->a_ops->writepage)
2392                 return 0;
2393
2394         /* skip writing if there is no dirty page in this inode */
2395         if (!get_dirty_pages(inode) && wbc->sync_mode == WB_SYNC_NONE)
2396                 return 0;
2397
2398         /* during POR, we don't need to trigger writepage at all. */
2399         if (unlikely(is_sbi_flag_set(sbi, SBI_POR_DOING)))
2400                 goto skip_write;
2401
2402         if ((S_ISDIR(inode->i_mode) || IS_NOQUOTA(inode)) &&
2403                         wbc->sync_mode == WB_SYNC_NONE &&
2404                         get_dirty_pages(inode) < nr_pages_to_skip(sbi, DATA) &&
2405                         f2fs_available_free_memory(sbi, DIRTY_DENTS))
2406                 goto skip_write;
2407
2408         /* skip writing during file defragment */
2409         if (is_inode_flag_set(inode, FI_DO_DEFRAG))
2410                 goto skip_write;
2411
2412         trace_f2fs_writepages(mapping->host, wbc, DATA);
2413
2414         /* to avoid spliting IOs due to mixed WB_SYNC_ALL and WB_SYNC_NONE */
2415         if (wbc->sync_mode == WB_SYNC_ALL)
2416                 atomic_inc(&sbi->wb_sync_req[DATA]);
2417         else if (atomic_read(&sbi->wb_sync_req[DATA]))
2418                 goto skip_write;
2419
2420         if (__should_serialize_io(inode, wbc)) {
2421                 mutex_lock(&sbi->writepages);
2422                 locked = true;
2423         }
2424
2425         blk_start_plug(&plug);
2426         ret = f2fs_write_cache_pages(mapping, wbc, io_type);
2427         blk_finish_plug(&plug);
2428
2429         if (locked)
2430                 mutex_unlock(&sbi->writepages);
2431
2432         if (wbc->sync_mode == WB_SYNC_ALL)
2433                 atomic_dec(&sbi->wb_sync_req[DATA]);
2434         /*
2435          * if some pages were truncated, we cannot guarantee its mapping->host
2436          * to detect pending bios.
2437          */
2438
2439         f2fs_remove_dirty_inode(inode);
2440         return ret;
2441
2442 skip_write:
2443         wbc->pages_skipped += get_dirty_pages(inode);
2444         trace_f2fs_writepages(mapping->host, wbc, DATA);
2445         return 0;
2446 }
2447
2448 static int f2fs_write_data_pages(struct address_space *mapping,
2449                             struct writeback_control *wbc)
2450 {
2451         struct inode *inode = mapping->host;
2452
2453         return __f2fs_write_data_pages(mapping, wbc,
2454                         F2FS_I(inode)->cp_task == current ?
2455                         FS_CP_DATA_IO : FS_DATA_IO);
2456 }
2457
2458 static void f2fs_write_failed(struct address_space *mapping, loff_t to)
2459 {
2460         struct inode *inode = mapping->host;
2461         loff_t i_size = i_size_read(inode);
2462
2463         if (to > i_size) {
2464                 down_write(&F2FS_I(inode)->i_gc_rwsem[WRITE]);
2465                 down_write(&F2FS_I(inode)->i_mmap_sem);
2466
2467                 truncate_pagecache(inode, i_size);
2468                 if (!IS_NOQUOTA(inode))
2469                         f2fs_truncate_blocks(inode, i_size, true);
2470
2471                 up_write(&F2FS_I(inode)->i_mmap_sem);
2472                 up_write(&F2FS_I(inode)->i_gc_rwsem[WRITE]);
2473         }
2474 }
2475
2476 static int prepare_write_begin(struct f2fs_sb_info *sbi,
2477                         struct page *page, loff_t pos, unsigned len,
2478                         block_t *blk_addr, bool *node_changed)
2479 {
2480         struct inode *inode = page->mapping->host;
2481         pgoff_t index = page->index;
2482         struct dnode_of_data dn;
2483         struct page *ipage;
2484         bool locked = false;
2485         struct extent_info ei = {0,0,0};
2486         int err = 0;
2487         int flag;
2488
2489         /*
2490          * we already allocated all the blocks, so we don't need to get
2491          * the block addresses when there is no need to fill the page.
2492          */
2493         if (!f2fs_has_inline_data(inode) && len == PAGE_SIZE &&
2494                         !is_inode_flag_set(inode, FI_NO_PREALLOC))
2495                 return 0;
2496
2497         /* f2fs_lock_op avoids race between write CP and convert_inline_page */
2498         if (f2fs_has_inline_data(inode) && pos + len > MAX_INLINE_DATA(inode))
2499                 flag = F2FS_GET_BLOCK_DEFAULT;
2500         else
2501                 flag = F2FS_GET_BLOCK_PRE_AIO;
2502
2503         if (f2fs_has_inline_data(inode) ||
2504                         (pos & PAGE_MASK) >= i_size_read(inode)) {
2505                 __do_map_lock(sbi, flag, true);
2506                 locked = true;
2507         }
2508 restart:
2509         /* check inline_data */
2510         ipage = f2fs_get_node_page(sbi, inode->i_ino);
2511         if (IS_ERR(ipage)) {
2512                 err = PTR_ERR(ipage);
2513                 goto unlock_out;
2514         }
2515
2516         set_new_dnode(&dn, inode, ipage, ipage, 0);
2517
2518         if (f2fs_has_inline_data(inode)) {
2519                 if (pos + len <= MAX_INLINE_DATA(inode)) {
2520                         f2fs_do_read_inline_data(page, ipage);
2521                         set_inode_flag(inode, FI_DATA_EXIST);
2522                         if (inode->i_nlink)
2523                                 set_inline_node(ipage);
2524                 } else {
2525                         err = f2fs_convert_inline_page(&dn, page);
2526                         if (err)
2527                                 goto out;
2528                         if (dn.data_blkaddr == NULL_ADDR)
2529                                 err = f2fs_get_block(&dn, index);
2530                 }
2531         } else if (locked) {
2532                 err = f2fs_get_block(&dn, index);
2533         } else {
2534                 if (f2fs_lookup_extent_cache(inode, index, &ei)) {
2535                         dn.data_blkaddr = ei.blk + index - ei.fofs;
2536                 } else {
2537                         /* hole case */
2538                         err = f2fs_get_dnode_of_data(&dn, index, LOOKUP_NODE);
2539                         if (err || dn.data_blkaddr == NULL_ADDR) {
2540                                 f2fs_put_dnode(&dn);
2541                                 __do_map_lock(sbi, F2FS_GET_BLOCK_PRE_AIO,
2542                                                                 true);
2543                                 WARN_ON(flag != F2FS_GET_BLOCK_PRE_AIO);
2544                                 locked = true;
2545                                 goto restart;
2546                         }
2547                 }
2548         }
2549
2550         /* convert_inline_page can make node_changed */
2551         *blk_addr = dn.data_blkaddr;
2552         *node_changed = dn.node_changed;
2553 out:
2554         f2fs_put_dnode(&dn);
2555 unlock_out:
2556         if (locked)
2557                 __do_map_lock(sbi, flag, false);
2558         return err;
2559 }
2560
2561 static int f2fs_write_begin(struct file *file, struct address_space *mapping,
2562                 loff_t pos, unsigned len, unsigned flags,
2563                 struct page **pagep, void **fsdata)
2564 {
2565         struct inode *inode = mapping->host;
2566         struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
2567         struct page *page = NULL;
2568         pgoff_t index = ((unsigned long long) pos) >> PAGE_SHIFT;
2569         bool need_balance = false, drop_atomic = false;
2570         block_t blkaddr = NULL_ADDR;
2571         int err = 0;
2572
2573         if (trace_android_fs_datawrite_start_enabled()) {
2574                 char *path, pathbuf[MAX_TRACE_PATHBUF_LEN];
2575
2576                 path = android_fstrace_get_pathname(pathbuf,
2577                                                     MAX_TRACE_PATHBUF_LEN,
2578                                                     inode);
2579                 trace_android_fs_datawrite_start(inode, pos, len,
2580                                                  current->pid, path,
2581                                                  current->comm);
2582         }
2583         trace_f2fs_write_begin(inode, pos, len, flags);
2584
2585         err = f2fs_is_checkpoint_ready(sbi);
2586         if (err)
2587                 goto fail;
2588
2589         if ((f2fs_is_atomic_file(inode) &&
2590                         !f2fs_available_free_memory(sbi, INMEM_PAGES)) ||
2591                         is_inode_flag_set(inode, FI_ATOMIC_REVOKE_REQUEST)) {
2592                 err = -ENOMEM;
2593                 drop_atomic = true;
2594                 goto fail;
2595         }
2596
2597         /*
2598          * We should check this at this moment to avoid deadlock on inode page
2599          * and #0 page. The locking rule for inline_data conversion should be:
2600          * lock_page(page #0) -> lock_page(inode_page)
2601          */
2602         if (index != 0) {
2603                 err = f2fs_convert_inline_inode(inode);
2604                 if (err)
2605                         goto fail;
2606         }
2607 repeat:
2608         /*
2609          * Do not use grab_cache_page_write_begin() to avoid deadlock due to
2610          * wait_for_stable_page. Will wait that below with our IO control.
2611          */
2612         page = f2fs_pagecache_get_page(mapping, index,
2613                                 FGP_LOCK | FGP_WRITE | FGP_CREAT, GFP_NOFS);
2614         if (!page) {
2615                 err = -ENOMEM;
2616                 goto fail;
2617         }
2618
2619         *pagep = page;
2620
2621         err = prepare_write_begin(sbi, page, pos, len,
2622                                         &blkaddr, &need_balance);
2623         if (err)
2624                 goto fail;
2625
2626         if (need_balance && !IS_NOQUOTA(inode) &&
2627                         has_not_enough_free_secs(sbi, 0, 0)) {
2628                 unlock_page(page);
2629                 f2fs_balance_fs(sbi, true);
2630                 lock_page(page);
2631                 if (page->mapping != mapping) {
2632                         /* The page got truncated from under us */
2633                         f2fs_put_page(page, 1);
2634                         goto repeat;
2635                 }
2636         }
2637
2638         f2fs_wait_on_page_writeback(page, DATA, false, true);
2639
2640         if (len == PAGE_SIZE || PageUptodate(page))
2641                 return 0;
2642
2643         if (!(pos & (PAGE_SIZE - 1)) && (pos + len) >= i_size_read(inode)) {
2644                 zero_user_segment(page, len, PAGE_SIZE);
2645                 return 0;
2646         }
2647
2648         if (blkaddr == NEW_ADDR) {
2649                 zero_user_segment(page, 0, PAGE_SIZE);
2650                 SetPageUptodate(page);
2651         } else {
2652                 if (!f2fs_is_valid_blkaddr(sbi, blkaddr,
2653                                 DATA_GENERIC_ENHANCE_READ)) {
2654                         err = -EFSCORRUPTED;
2655                         goto fail;
2656                 }
2657                 err = f2fs_submit_page_read(inode, page, blkaddr);
2658                 if (err)
2659                         goto fail;
2660
2661                 lock_page(page);
2662                 if (unlikely(page->mapping != mapping)) {
2663                         f2fs_put_page(page, 1);
2664                         goto repeat;
2665                 }
2666                 if (unlikely(!PageUptodate(page))) {
2667                         err = -EIO;
2668                         goto fail;
2669                 }
2670         }
2671         return 0;
2672
2673 fail:
2674         f2fs_put_page(page, 1);
2675         f2fs_write_failed(mapping, pos + len);
2676         if (drop_atomic)
2677                 f2fs_drop_inmem_pages_all(sbi, false);
2678         return err;
2679 }
2680
2681 static int f2fs_write_end(struct file *file,
2682                         struct address_space *mapping,
2683                         loff_t pos, unsigned len, unsigned copied,
2684                         struct page *page, void *fsdata)
2685 {
2686         struct inode *inode = page->mapping->host;
2687
2688         trace_android_fs_datawrite_end(inode, pos, len);
2689         trace_f2fs_write_end(inode, pos, len, copied);
2690
2691         /*
2692          * This should be come from len == PAGE_SIZE, and we expect copied
2693          * should be PAGE_SIZE. Otherwise, we treat it with zero copied and
2694          * let generic_perform_write() try to copy data again through copied=0.
2695          */
2696         if (!PageUptodate(page)) {
2697                 if (unlikely(copied != len))
2698                         copied = 0;
2699                 else
2700                         SetPageUptodate(page);
2701         }
2702         if (!copied)
2703                 goto unlock_out;
2704
2705         set_page_dirty(page);
2706
2707         if (pos + copied > i_size_read(inode))
2708                 f2fs_i_size_write(inode, pos + copied);
2709 unlock_out:
2710         f2fs_put_page(page, 1);
2711         f2fs_update_time(F2FS_I_SB(inode), REQ_TIME);
2712         return copied;
2713 }
2714
2715 static int check_direct_IO(struct inode *inode, struct iov_iter *iter,
2716                            loff_t offset)
2717 {
2718         unsigned i_blkbits = READ_ONCE(inode->i_blkbits);
2719         unsigned blkbits = i_blkbits;
2720         unsigned blocksize_mask = (1 << blkbits) - 1;
2721         unsigned long align = offset | iov_iter_alignment(iter);
2722         struct block_device *bdev = inode->i_sb->s_bdev;
2723
2724         if (align & blocksize_mask) {
2725                 if (bdev)
2726                         blkbits = blksize_bits(bdev_logical_block_size(bdev));
2727                 blocksize_mask = (1 << blkbits) - 1;
2728                 if (align & blocksize_mask)
2729                         return -EINVAL;
2730                 return 1;
2731         }
2732         return 0;
2733 }
2734
2735 static void f2fs_dio_end_io(struct bio *bio)
2736 {
2737         struct f2fs_private_dio *dio = bio->bi_private;
2738
2739         dec_page_count(F2FS_I_SB(dio->inode),
2740                         dio->write ? F2FS_DIO_WRITE : F2FS_DIO_READ);
2741
2742         bio->bi_private = dio->orig_private;
2743         bio->bi_end_io = dio->orig_end_io;
2744
2745         kvfree(dio);
2746
2747         bio_endio(bio);
2748 }
2749
2750 static void f2fs_dio_submit_bio(int rw, struct bio *bio, struct inode *inode,
2751                                                         loff_t file_offset)
2752 {
2753         struct f2fs_private_dio *dio;
2754         bool write = (rw == REQ_OP_WRITE);
2755
2756         dio = f2fs_kzalloc(F2FS_I_SB(inode),
2757                         sizeof(struct f2fs_private_dio), GFP_NOFS);
2758         if (!dio)
2759                 goto out;
2760
2761         dio->inode = inode;
2762         dio->orig_end_io = bio->bi_end_io;
2763         dio->orig_private = bio->bi_private;
2764         dio->write = write;
2765
2766         bio->bi_end_io = f2fs_dio_end_io;
2767         bio->bi_private = dio;
2768
2769         inc_page_count(F2FS_I_SB(inode),
2770                         write ? F2FS_DIO_WRITE : F2FS_DIO_READ);
2771
2772         submit_bio(rw, bio);
2773         return;
2774 out:
2775         bio->bi_error = -EIO;
2776         bio_endio(bio);
2777 }
2778
2779 static ssize_t f2fs_direct_IO(struct kiocb *iocb, struct iov_iter *iter,
2780                                 loff_t offset)
2781 {
2782         struct address_space *mapping = iocb->ki_filp->f_mapping;
2783         struct inode *inode = mapping->host;
2784         struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
2785         struct f2fs_inode_info *fi = F2FS_I(inode);
2786         size_t count = iov_iter_count(iter);
2787         int rw = iov_iter_rw(iter);
2788         int err;
2789         enum rw_hint hint = iocb->ki_hint;
2790         int whint_mode = F2FS_OPTION(sbi).whint_mode;
2791         bool do_opu;
2792
2793         err = check_direct_IO(inode, iter, offset);
2794         if (err)
2795                 return err < 0 ? err : 0;
2796
2797         if (f2fs_force_buffered_io(inode, iocb, iter))
2798                 return 0;
2799
2800         if (trace_android_fs_dataread_start_enabled() &&
2801             (iov_iter_rw(iter) == READ)) {
2802                 char *path, pathbuf[MAX_TRACE_PATHBUF_LEN];
2803
2804                 path = android_fstrace_get_pathname(pathbuf,
2805                                                     MAX_TRACE_PATHBUF_LEN,
2806                                                     inode);
2807                 trace_android_fs_dataread_start(inode, offset,
2808                                                 count, current->pid, path,
2809                                                 current->comm);
2810         }
2811         if (trace_android_fs_datawrite_start_enabled() &&
2812             (iov_iter_rw(iter) == WRITE)) {
2813                 char *path, pathbuf[MAX_TRACE_PATHBUF_LEN];
2814
2815                 path = android_fstrace_get_pathname(pathbuf,
2816                                                     MAX_TRACE_PATHBUF_LEN,
2817                                                     inode);
2818                 trace_android_fs_datawrite_start(inode, offset, count,
2819                                                  current->pid, path,
2820                                                  current->comm);
2821         }
2822
2823         do_opu = allow_outplace_dio(inode, iocb, iter);
2824
2825         trace_f2fs_direct_IO_enter(inode, offset, count, rw);
2826
2827         if (rw == WRITE && whint_mode == WHINT_MODE_OFF)
2828                 iocb->ki_hint = WRITE_LIFE_NOT_SET;
2829
2830         if (iocb->ki_flags & IOCB_NOWAIT) {
2831                 if (!down_read_trylock(&fi->i_gc_rwsem[rw])) {
2832                         iocb->ki_hint = hint;
2833                         err = -EAGAIN;
2834                         goto out;
2835                 }
2836                 if (do_opu && !down_read_trylock(&fi->i_gc_rwsem[READ])) {
2837                         up_read(&fi->i_gc_rwsem[rw]);
2838                         iocb->ki_hint = hint;
2839                         err = -EAGAIN;
2840                         goto out;
2841                 }
2842         } else {
2843                 down_read(&fi->i_gc_rwsem[rw]);
2844                 if (do_opu)
2845                         down_read(&fi->i_gc_rwsem[READ]);
2846         }
2847
2848         err = __blockdev_direct_IO(iocb, inode, inode->i_sb->s_bdev,
2849                         iter, offset,
2850                         rw == WRITE ? get_data_block_dio_write :
2851                         get_data_block_dio, NULL, f2fs_dio_submit_bio,
2852                         DIO_LOCKING | DIO_SKIP_HOLES);
2853
2854         if (do_opu)
2855                 up_read(&fi->i_gc_rwsem[READ]);
2856
2857         up_read(&fi->i_gc_rwsem[rw]);
2858
2859         if (rw == WRITE) {
2860                 if (whint_mode == WHINT_MODE_OFF)
2861                         iocb->ki_hint = hint;
2862                 if (err > 0) {
2863                         f2fs_update_iostat(F2FS_I_SB(inode), APP_DIRECT_IO,
2864                                                                         err);
2865                         if (!do_opu)
2866                                 set_inode_flag(inode, FI_UPDATE_WRITE);
2867                 } else if (err < 0) {
2868                         f2fs_write_failed(mapping, offset + count);
2869                 }
2870         }
2871 out:
2872         if (trace_android_fs_dataread_start_enabled() &&
2873             (iov_iter_rw(iter) == READ))
2874                 trace_android_fs_dataread_end(inode, offset, count);
2875         if (trace_android_fs_datawrite_start_enabled() &&
2876             (iov_iter_rw(iter) == WRITE))
2877                 trace_android_fs_datawrite_end(inode, offset, count);
2878
2879         trace_f2fs_direct_IO_exit(inode, offset, count, rw, err);
2880
2881         return err;
2882 }
2883
2884 void f2fs_invalidate_page(struct page *page, unsigned int offset,
2885                                                         unsigned int length)
2886 {
2887         struct inode *inode = page->mapping->host;
2888         struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
2889
2890         if (inode->i_ino >= F2FS_ROOT_INO(sbi) &&
2891                 (offset % PAGE_SIZE || length != PAGE_SIZE))
2892                 return;
2893
2894         if (PageDirty(page)) {
2895                 if (inode->i_ino == F2FS_META_INO(sbi)) {
2896                         dec_page_count(sbi, F2FS_DIRTY_META);
2897                 } else if (inode->i_ino == F2FS_NODE_INO(sbi)) {
2898                         dec_page_count(sbi, F2FS_DIRTY_NODES);
2899                 } else {
2900                         inode_dec_dirty_pages(inode);
2901                         f2fs_remove_dirty_inode(inode);
2902                 }
2903         }
2904
2905         clear_cold_data(page);
2906
2907         if (IS_ATOMIC_WRITTEN_PAGE(page))
2908                 return f2fs_drop_inmem_page(inode, page);
2909
2910         f2fs_clear_page_private(page);
2911 }
2912
2913 int f2fs_release_page(struct page *page, gfp_t wait)
2914 {
2915         /* If this is dirty page, keep PagePrivate */
2916         if (PageDirty(page))
2917                 return 0;
2918
2919         /* This is atomic written page, keep Private */
2920         if (IS_ATOMIC_WRITTEN_PAGE(page))
2921                 return 0;
2922
2923         clear_cold_data(page);
2924         f2fs_clear_page_private(page);
2925         return 1;
2926 }
2927
2928 static int f2fs_set_data_page_dirty(struct page *page)
2929 {
2930         struct inode *inode = page_file_mapping(page)->host;
2931
2932         trace_f2fs_set_page_dirty(page, DATA);
2933
2934         if (!PageUptodate(page))
2935                 SetPageUptodate(page);
2936         if (PageSwapCache(page))
2937                 return __set_page_dirty_nobuffers(page);
2938
2939         if (f2fs_is_atomic_file(inode) && !f2fs_is_commit_atomic_write(inode)) {
2940                 if (!IS_ATOMIC_WRITTEN_PAGE(page)) {
2941                         f2fs_register_inmem_page(inode, page);
2942                         return 1;
2943                 }
2944                 /*
2945                  * Previously, this page has been registered, we just
2946                  * return here.
2947                  */
2948                 return 0;
2949         }
2950
2951         if (!PageDirty(page)) {
2952                 __set_page_dirty_nobuffers(page);
2953                 f2fs_update_dirty_page(inode, page);
2954                 return 1;
2955         }
2956         return 0;
2957 }
2958
2959 static sector_t f2fs_bmap(struct address_space *mapping, sector_t block)
2960 {
2961         struct inode *inode = mapping->host;
2962
2963         if (f2fs_has_inline_data(inode))
2964                 return 0;
2965
2966         /* make sure allocating whole blocks */
2967         if (mapping_tagged(mapping, PAGECACHE_TAG_DIRTY))
2968                 filemap_write_and_wait(mapping);
2969
2970         return generic_block_bmap(mapping, block, get_data_block_bmap);
2971 }
2972
2973 #ifdef CONFIG_MIGRATION
2974 #include <linux/migrate.h>
2975
2976 int f2fs_migrate_page(struct address_space *mapping,
2977                 struct page *newpage, struct page *page, enum migrate_mode mode)
2978 {
2979         int rc, extra_count;
2980         struct f2fs_inode_info *fi = F2FS_I(mapping->host);
2981         bool atomic_written = IS_ATOMIC_WRITTEN_PAGE(page);
2982
2983         BUG_ON(PageWriteback(page));
2984
2985         /* migrating an atomic written page is safe with the inmem_lock hold */
2986         if (atomic_written) {
2987                 if (mode != MIGRATE_SYNC)
2988                         return -EBUSY;
2989                 if (!mutex_trylock(&fi->inmem_lock))
2990                         return -EAGAIN;
2991         }
2992
2993         /* one extra reference was held for atomic_write page */
2994         extra_count = atomic_written ? 1 : 0;
2995         rc = migrate_page_move_mapping(mapping, newpage,
2996                                 page, NULL, mode, extra_count);
2997         if (rc != MIGRATEPAGE_SUCCESS) {
2998                 if (atomic_written)
2999                         mutex_unlock(&fi->inmem_lock);
3000                 return rc;
3001         }
3002
3003         if (atomic_written) {
3004                 struct inmem_pages *cur;
3005                 list_for_each_entry(cur, &fi->inmem_pages, list)
3006                         if (cur->page == page) {
3007                                 cur->page = newpage;
3008                                 break;
3009                         }
3010                 mutex_unlock(&fi->inmem_lock);
3011                 put_page(page);
3012                 get_page(newpage);
3013         }
3014
3015         if (PagePrivate(page)) {
3016                 f2fs_set_page_private(newpage, page_private(page));
3017                 f2fs_clear_page_private(page);
3018         }
3019
3020         migrate_page_copy(newpage, page);
3021
3022         return MIGRATEPAGE_SUCCESS;
3023 }
3024 #endif
3025
3026 #ifdef CONFIG_SWAP
3027 /* Copied from generic_swapfile_activate() to check any holes */
3028 static int check_swap_activate(struct file *swap_file, unsigned int max)
3029 {
3030         struct address_space *mapping = swap_file->f_mapping;
3031         struct inode *inode = mapping->host;
3032         unsigned blocks_per_page;
3033         unsigned long page_no;
3034         unsigned blkbits;
3035         sector_t probe_block;
3036         sector_t last_block;
3037         sector_t lowest_block = -1;
3038         sector_t highest_block = 0;
3039
3040         blkbits = inode->i_blkbits;
3041         blocks_per_page = PAGE_SIZE >> blkbits;
3042
3043         /*
3044          * Map all the blocks into the extent list.  This code doesn't try
3045          * to be very smart.
3046          */
3047         probe_block = 0;
3048         page_no = 0;
3049         last_block = i_size_read(inode) >> blkbits;
3050         while ((probe_block + blocks_per_page) <= last_block && page_no < max) {
3051                 unsigned block_in_page;
3052                 sector_t first_block;
3053
3054                 cond_resched();
3055
3056                 first_block = bmap(inode, probe_block);
3057                 if (first_block == 0)
3058                         goto bad_bmap;
3059
3060                 /*
3061                  * It must be PAGE_SIZE aligned on-disk
3062                  */
3063                 if (first_block & (blocks_per_page - 1)) {
3064                         probe_block++;
3065                         goto reprobe;
3066                 }
3067
3068                 for (block_in_page = 1; block_in_page < blocks_per_page;
3069                                         block_in_page++) {
3070                         sector_t block;
3071
3072                         block = bmap(inode, probe_block + block_in_page);
3073                         if (block == 0)
3074                                 goto bad_bmap;
3075                         if (block != first_block + block_in_page) {
3076                                 /* Discontiguity */
3077                                 probe_block++;
3078                                 goto reprobe;
3079                         }
3080                 }
3081
3082                 first_block >>= (PAGE_SHIFT - blkbits);
3083                 if (page_no) {  /* exclude the header page */
3084                         if (first_block < lowest_block)
3085                                 lowest_block = first_block;
3086                         if (first_block > highest_block)
3087                                 highest_block = first_block;
3088                 }
3089
3090                 page_no++;
3091                 probe_block += blocks_per_page;
3092 reprobe:
3093                 continue;
3094         }
3095         return 0;
3096
3097 bad_bmap:
3098         pr_err("swapon: swapfile has holes\n");
3099         return -EINVAL;
3100 }
3101
3102 static int f2fs_swap_activate(struct swap_info_struct *sis, struct file *file,
3103                                 sector_t *span)
3104 {
3105         struct inode *inode = file_inode(file);
3106         int ret;
3107
3108         if (!S_ISREG(inode->i_mode))
3109                 return -EINVAL;
3110
3111         if (f2fs_readonly(F2FS_I_SB(inode)->sb))
3112                 return -EROFS;
3113
3114         ret = f2fs_convert_inline_inode(inode);
3115         if (ret)
3116                 return ret;
3117
3118         ret = check_swap_activate(file, sis->max);
3119         if (ret)
3120                 return ret;
3121
3122         set_inode_flag(inode, FI_PIN_FILE);
3123         f2fs_precache_extents(inode);
3124         f2fs_update_time(F2FS_I_SB(inode), REQ_TIME);
3125         return 0;
3126 }
3127
3128 static void f2fs_swap_deactivate(struct file *file)
3129 {
3130         struct inode *inode = file_inode(file);
3131
3132         clear_inode_flag(inode, FI_PIN_FILE);
3133 }
3134 #else
3135 static int f2fs_swap_activate(struct swap_info_struct *sis, struct file *file,
3136                                 sector_t *span)
3137 {
3138         return -EOPNOTSUPP;
3139 }
3140
3141 static void f2fs_swap_deactivate(struct file *file)
3142 {
3143 }
3144 #endif
3145
3146 const struct address_space_operations f2fs_dblock_aops = {
3147         .readpage       = f2fs_read_data_page,
3148         .readpages      = f2fs_read_data_pages,
3149         .writepage      = f2fs_write_data_page,
3150         .writepages     = f2fs_write_data_pages,
3151         .write_begin    = f2fs_write_begin,
3152         .write_end      = f2fs_write_end,
3153         .set_page_dirty = f2fs_set_data_page_dirty,
3154         .invalidatepage = f2fs_invalidate_page,
3155         .releasepage    = f2fs_release_page,
3156         .direct_IO      = f2fs_direct_IO,
3157         .bmap           = f2fs_bmap,
3158         .swap_activate  = f2fs_swap_activate,
3159         .swap_deactivate = f2fs_swap_deactivate,
3160 #ifdef CONFIG_MIGRATION
3161         .migratepage    = f2fs_migrate_page,
3162 #endif
3163 };
3164
3165 void f2fs_clear_radix_tree_dirty_tag(struct page *page)
3166 {
3167         struct address_space *mapping = page_mapping(page);
3168         unsigned long flags;
3169
3170         spin_lock_irqsave(&mapping->tree_lock, flags);
3171         radix_tree_tag_clear(&mapping->page_tree, page_index(page),
3172                                         PAGECACHE_TAG_DIRTY);
3173         spin_unlock_irqrestore(&mapping->tree_lock, flags);
3174 }
3175
3176 int __init f2fs_init_post_read_processing(void)
3177 {
3178         bio_post_read_ctx_cache = KMEM_CACHE(bio_post_read_ctx, 0);
3179         if (!bio_post_read_ctx_cache)
3180                 goto fail;
3181         bio_post_read_ctx_pool =
3182                 mempool_create_slab_pool(NUM_PREALLOC_POST_READ_CTXS,
3183                                          bio_post_read_ctx_cache);
3184         if (!bio_post_read_ctx_pool)
3185                 goto fail_free_cache;
3186         return 0;
3187
3188 fail_free_cache:
3189         kmem_cache_destroy(bio_post_read_ctx_cache);
3190 fail:
3191         return -ENOMEM;
3192 }
3193
3194 void __exit f2fs_destroy_post_read_processing(void)
3195 {
3196         mempool_destroy(bio_post_read_ctx_pool);
3197         kmem_cache_destroy(bio_post_read_ctx_cache);
3198 }