drivers/block/zram/zram_drv.c

   1 /*
   2  * Compressed RAM block device
   3  *
   4  * Copyright (C) 2008, 2009, 2010  Nitin Gupta
   5  *               2012, 2013 Minchan Kim
   6  *
   7  * This code is released using a dual license strategy: BSD/GPL
   8  * You can choose the licence that better fits your requirements.
   9  *
  10  * Released under the terms of 3-clause BSD License
  11  * Released under the terms of GNU General Public License Version 2.0
  12  *
  13  */
  14
  15 #define KMSG_COMPONENT "zram"
  16 #define pr_fmt(fmt) KMSG_COMPONENT ": " fmt
  17
  18 #include <linux/module.h>
  19 #include <linux/kernel.h>
  20 #include <linux/bio.h>
  21 #include <linux/bitops.h>
  22 #include <linux/blkdev.h>
  23 #include <linux/buffer_head.h>
  24 #include <linux/device.h>
  25 #include <linux/genhd.h>
  26 #include <linux/highmem.h>
  27 #include <linux/slab.h>
  28 #include <linux/backing-dev.h>
  29 #include <linux/string.h>
  30 #include <linux/vmalloc.h>
  31 #include <linux/err.h>
  32 #include <linux/idr.h>
  33 #include <linux/sysfs.h>
  34 #include <linux/debugfs.h>
  35 #include <linux/cpuhotplug.h>
  36
  37 #include "zram_drv.h"
  38
  39 static DEFINE_IDR(zram_index_idr);
  40 /* idr index must be protected */
  41 static DEFINE_MUTEX(zram_index_mutex);
  42
  43 static int zram_major;
  44 static const char *default_compressor = "lzo";
  45
  46 /* Module params (documentation at end) */
  47 static unsigned int num_devices = 1;
  48 /*
  49  * Pages that compress to sizes equals or greater than this are stored
  50  * uncompressed in memory.
  51  */
  52 static size_t huge_class_size;
  53
  54 static void zram_free_page(struct zram *zram, size_t index);
  55 static int zram_bvec_read(struct zram *zram, struct bio_vec *bvec,
  56                                 u32 index, int offset, struct bio *bio);
  57
  58
  59 static int zram_slot_trylock(struct zram *zram, u32 index)
  60 {
  61         return bit_spin_trylock(ZRAM_LOCK, &zram->table[index].flags);
  62 }
  63
  64 static void zram_slot_lock(struct zram *zram, u32 index)
  65 {
  66         bit_spin_lock(ZRAM_LOCK, &zram->table[index].flags);
  67 }
  68
  69 static void zram_slot_unlock(struct zram *zram, u32 index)
  70 {
  71         bit_spin_unlock(ZRAM_LOCK, &zram->table[index].flags);
  72 }
  73
  74 static inline bool init_done(struct zram *zram)
  75 {
  76         return zram->disksize;
  77 }
  78
  79 static inline struct zram *dev_to_zram(struct device *dev)
  80 {
  81         return (struct zram *)dev_to_disk(dev)->private_data;
  82 }
  83
  84 static unsigned long zram_get_handle(struct zram *zram, u32 index)
  85 {
  86         return zram->table[index].handle;
  87 }
  88
  89 static void zram_set_handle(struct zram *zram, u32 index, unsigned long handle)
  90 {
  91         zram->table[index].handle = handle;
  92 }
  93
  94 /* flag operations require table entry bit_spin_lock() being held */
  95 static bool zram_test_flag(struct zram *zram, u32 index,
  96                         enum zram_pageflags flag)
  97 {
  98         return zram->table[index].flags & BIT(flag);
  99 }
 100
 101 static void zram_set_flag(struct zram *zram, u32 index,
 102                         enum zram_pageflags flag)
 103 {
 104         zram->table[index].flags |= BIT(flag);
 105 }
 106
 107 static void zram_clear_flag(struct zram *zram, u32 index,
 108                         enum zram_pageflags flag)
 109 {
 110         zram->table[index].flags &= ~BIT(flag);
 111 }
 112
 113 static inline void zram_set_element(struct zram *zram, u32 index,
 114                         unsigned long element)
 115 {
 116         zram->table[index].element = element;
 117 }
 118
 119 static unsigned long zram_get_element(struct zram *zram, u32 index)
 120 {
 121         return zram->table[index].element;
 122 }
 123
 124 static size_t zram_get_obj_size(struct zram *zram, u32 index)
 125 {
 126         return zram->table[index].flags & (BIT(ZRAM_FLAG_SHIFT) - 1);
 127 }
 128
 129 static void zram_set_obj_size(struct zram *zram,
 130                                         u32 index, size_t size)
 131 {
 132         unsigned long flags = zram->table[index].flags >> ZRAM_FLAG_SHIFT;
 133
 134         zram->table[index].flags = (flags << ZRAM_FLAG_SHIFT) | size;
 135 }
 136
 137 static inline bool zram_allocated(struct zram *zram, u32 index)
 138 {
 139         return zram_get_obj_size(zram, index) ||
 140                         zram_test_flag(zram, index, ZRAM_SAME) ||
 141                         zram_test_flag(zram, index, ZRAM_WB);
 142 }
 143
 144 #if PAGE_SIZE != 4096
 145 static inline bool is_partial_io(struct bio_vec *bvec)
 146 {
 147         return bvec->bv_len != PAGE_SIZE;
 148 }
 149 #else
 150 static inline bool is_partial_io(struct bio_vec *bvec)
 151 {
 152         return false;
 153 }
 154 #endif
 155
 156 /*
 157  * Check if request is within bounds and aligned on zram logical blocks.
 158  */
 159 static inline bool valid_io_request(struct zram *zram,
 160                 sector_t start, unsigned int size)
 161 {
 162         u64 end, bound;
 163
 164         /* unaligned request */
 165         if (unlikely(start & (ZRAM_SECTOR_PER_LOGICAL_BLOCK - 1)))
 166                 return false;
 167         if (unlikely(size & (ZRAM_LOGICAL_BLOCK_SIZE - 1)))
 168                 return false;
 169
 170         end = start + (size >> SECTOR_SHIFT);
 171         bound = zram->disksize >> SECTOR_SHIFT;
 172         /* out of range range */
 173         if (unlikely(start >= bound || end > bound || start > end))
 174                 return false;
 175
 176         /* I/O request is valid */
 177         return true;
 178 }
 179
 180 static void update_position(u32 *index, int *offset, struct bio_vec *bvec)
 181 {
 182         *index  += (*offset + bvec->bv_len) / PAGE_SIZE;
 183         *offset = (*offset + bvec->bv_len) % PAGE_SIZE;
 184 }
 185
 186 static inline void update_used_max(struct zram *zram,
 187                                         const unsigned long pages)
 188 {
 189         unsigned long old_max, cur_max;
 190
 191         old_max = atomic_long_read(&zram->stats.max_used_pages);
 192
 193         do {
 194                 cur_max = old_max;
 195                 if (pages > cur_max)
 196                         old_max = atomic_long_cmpxchg(
 197                                 &zram->stats.max_used_pages, cur_max, pages);
 198         } while (old_max != cur_max);
 199 }
 200
 201 static inline void zram_fill_page(void *ptr, unsigned long len,
 202                                         unsigned long value)
 203 {
 204         WARN_ON_ONCE(!IS_ALIGNED(len, sizeof(unsigned long)));
 205         memset_l(ptr, value, len / sizeof(unsigned long));
 206 }
 207
 208 static bool page_same_filled(void *ptr, unsigned long *element)
 209 {
 210         unsigned int pos;
 211         unsigned long *page;
 212         unsigned long val;
 213
 214         page = (unsigned long *)ptr;
 215         val = page[0];
 216
 217         for (pos = 1; pos < PAGE_SIZE / sizeof(*page); pos++) {
 218                 if (val != page[pos])
 219                         return false;
 220         }
 221
 222         *element = val;
 223
 224         return true;
 225 }
 226
 227 static ssize_t initstate_show(struct device *dev,
 228                 struct device_attribute *attr, char *buf)
 229 {
 230         u32 val;
 231         struct zram *zram = dev_to_zram(dev);
 232
 233         down_read(&zram->init_lock);
 234         val = init_done(zram);
 235         up_read(&zram->init_lock);
 236
 237         return scnprintf(buf, PAGE_SIZE, "%u\n", val);
 238 }
 239
 240 static ssize_t disksize_show(struct device *dev,
 241                 struct device_attribute *attr, char *buf)
 242 {
 243         struct zram *zram = dev_to_zram(dev);
 244
 245         return scnprintf(buf, PAGE_SIZE, "%llu\n", zram->disksize);
 246 }
 247
 248 static ssize_t mem_limit_store(struct device *dev,
 249                 struct device_attribute *attr, const char *buf, size_t len)
 250 {
 251         u64 limit;
 252         char *tmp;
 253         struct zram *zram = dev_to_zram(dev);
 254
 255         limit = memparse(buf, &tmp);
 256         if (buf == tmp) /* no chars parsed, invalid input */
 257                 return -EINVAL;
 258
 259         down_write(&zram->init_lock);
 260         zram->limit_pages = PAGE_ALIGN(limit) >> PAGE_SHIFT;
 261         up_write(&zram->init_lock);
 262
 263         return len;
 264 }
 265
 266 static ssize_t mem_used_max_store(struct device *dev,
 267                 struct device_attribute *attr, const char *buf, size_t len)
 268 {
 269         int err;
 270         unsigned long val;
 271         struct zram *zram = dev_to_zram(dev);
 272
 273         err = kstrtoul(buf, 10, &val);
 274         if (err || val != 0)
 275                 return -EINVAL;
 276
 277         down_read(&zram->init_lock);
 278         if (init_done(zram)) {
 279                 atomic_long_set(&zram->stats.max_used_pages,
 280                                 zs_get_total_pages(zram->mem_pool));
 281         }
 282         up_read(&zram->init_lock);
 283
 284         return len;
 285 }
 286
 287 static ssize_t idle_store(struct device *dev,
 288                 struct device_attribute *attr, const char *buf, size_t len)
 289 {
 290         struct zram *zram = dev_to_zram(dev);
 291         unsigned long nr_pages = zram->disksize >> PAGE_SHIFT;
 292         int index;
 293         char mode_buf[8];
 294         ssize_t sz;
 295
 296         sz = strscpy(mode_buf, buf, sizeof(mode_buf));
 297         if (sz <= 0)
 298                 return -EINVAL;
 299
 300         /* ignore trailing new line */
 301         if (mode_buf[sz - 1] == '\n')
 302                 mode_buf[sz - 1] = 0x00;
 303
 304         if (strcmp(mode_buf, "all"))
 305                 return -EINVAL;
 306
 307         down_read(&zram->init_lock);
 308         if (!init_done(zram)) {
 309                 up_read(&zram->init_lock);
 310                 return -EINVAL;
 311         }
 312
 313         for (index = 0; index < nr_pages; index++) {
 314                 /*
 315                  * Do not mark ZRAM_UNDER_WB slot as ZRAM_IDLE to close race.
 316                  * See the comment in writeback_store.
 317                  */
 318                 zram_slot_lock(zram, index);
 319                 if (!zram_allocated(zram, index) ||
 320                                 zram_test_flag(zram, index, ZRAM_UNDER_WB))
 321                         goto next;
 322                 zram_set_flag(zram, index, ZRAM_IDLE);
 323 next:
 324                 zram_slot_unlock(zram, index);
 325         }
 326
 327         up_read(&zram->init_lock);
 328
 329         return len;
 330 }
 331
 332 #ifdef CONFIG_ZRAM_WRITEBACK
 333 static ssize_t writeback_limit_store(struct device *dev,
 334                 struct device_attribute *attr, const char *buf, size_t len)
 335 {
 336         struct zram *zram = dev_to_zram(dev);
 337         u64 val;
 338         ssize_t ret = -EINVAL;
 339
 340         if (kstrtoull(buf, 10, &val))
 341                 return ret;
 342
 343         down_read(&zram->init_lock);
 344         atomic64_set(&zram->stats.bd_wb_limit, val);
 345         if (val == 0)
 346                 zram->stop_writeback = false;
 347         up_read(&zram->init_lock);
 348         ret = len;
 349
 350         return ret;
 351 }
 352
 353 static ssize_t writeback_limit_show(struct device *dev,
 354                 struct device_attribute *attr, char *buf)
 355 {
 356         u64 val;
 357         struct zram *zram = dev_to_zram(dev);
 358
 359         down_read(&zram->init_lock);
 360         val = atomic64_read(&zram->stats.bd_wb_limit);
 361         up_read(&zram->init_lock);
 362
 363         return scnprintf(buf, PAGE_SIZE, "%llu\n", val);
 364 }
 365
 366 static void reset_bdev(struct zram *zram)
 367 {
 368         struct block_device *bdev;
 369
 370         if (!zram->backing_dev)
 371                 return;
 372
 373         bdev = zram->bdev;
 374         if (zram->old_block_size)
 375                 set_blocksize(bdev, zram->old_block_size);
 376         blkdev_put(bdev, FMODE_READ|FMODE_WRITE|FMODE_EXCL);
 377         /* hope filp_close flush all of IO */
 378         filp_close(zram->backing_dev, NULL);
 379         zram->backing_dev = NULL;
 380         zram->old_block_size = 0;
 381         zram->bdev = NULL;
 382         zram->disk->queue->backing_dev_info->capabilities |=
 383                                 BDI_CAP_SYNCHRONOUS_IO;
 384         kvfree(zram->bitmap);
 385         zram->bitmap = NULL;
 386 }
 387
 388 static ssize_t backing_dev_show(struct device *dev,
 389                 struct device_attribute *attr, char *buf)
 390 {
 391         struct zram *zram = dev_to_zram(dev);
 392         struct file *file = zram->backing_dev;
 393         char *p;
 394         ssize_t ret;
 395
 396         down_read(&zram->init_lock);
 397         if (!zram->backing_dev) {
 398                 memcpy(buf, "none\n", 5);
 399                 up_read(&zram->init_lock);
 400                 return 5;
 401         }
 402
 403         p = file_path(file, buf, PAGE_SIZE - 1);
 404         if (IS_ERR(p)) {
 405                 ret = PTR_ERR(p);
 406                 goto out;
 407         }
 408
 409         ret = strlen(p);
 410         memmove(buf, p, ret);
 411         buf[ret++] = '\n';
 412 out:
 413         up_read(&zram->init_lock);
 414         return ret;
 415 }
 416
 417 static ssize_t backing_dev_store(struct device *dev,
 418                 struct device_attribute *attr, const char *buf, size_t len)
 419 {
 420         char *file_name;
 421         size_t sz;
 422         struct file *backing_dev = NULL;
 423         struct inode *inode;
 424         struct address_space *mapping;
 425         unsigned int bitmap_sz, old_block_size = 0;
 426         unsigned long nr_pages, *bitmap = NULL;
 427         struct block_device *bdev = NULL;
 428         int err;
 429         struct zram *zram = dev_to_zram(dev);
 430
 431         file_name = kmalloc(PATH_MAX, GFP_KERNEL);
 432         if (!file_name)
 433                 return -ENOMEM;
 434
 435         down_write(&zram->init_lock);
 436         if (init_done(zram)) {
 437                 pr_info("Can't setup backing device for initialized device\n");
 438                 err = -EBUSY;
 439                 goto out;
 440         }
 441
 442         strlcpy(file_name, buf, PATH_MAX);
 443         /* ignore trailing newline */
 444         sz = strlen(file_name);
 445         if (sz > 0 && file_name[sz - 1] == '\n')
 446                 file_name[sz - 1] = 0x00;
 447
 448         backing_dev = filp_open(file_name, O_RDWR|O_LARGEFILE, 0);
 449         if (IS_ERR(backing_dev)) {
 450                 err = PTR_ERR(backing_dev);
 451                 backing_dev = NULL;
 452                 goto out;
 453         }
 454
 455         mapping = backing_dev->f_mapping;
 456         inode = mapping->host;
 457
 458         /* Support only block device in this moment */
 459         if (!S_ISBLK(inode->i_mode)) {
 460                 err = -ENOTBLK;
 461                 goto out;
 462         }
 463
 464         bdev = bdgrab(I_BDEV(inode));
 465         err = blkdev_get(bdev, FMODE_READ | FMODE_WRITE | FMODE_EXCL, zram);
 466         if (err < 0) {
 467                 bdev = NULL;
 468                 goto out;
 469         }
 470
 471         nr_pages = i_size_read(inode) >> PAGE_SHIFT;
 472         bitmap_sz = BITS_TO_LONGS(nr_pages) * sizeof(long);
 473         bitmap = kvzalloc(bitmap_sz, GFP_KERNEL);
 474         if (!bitmap) {
 475                 err = -ENOMEM;
 476                 goto out;
 477         }
 478
 479         old_block_size = block_size(bdev);
 480         err = set_blocksize(bdev, PAGE_SIZE);
 481         if (err)
 482                 goto out;
 483
 484         reset_bdev(zram);
 485
 486         zram->old_block_size = old_block_size;
 487         zram->bdev = bdev;
 488         zram->backing_dev = backing_dev;
 489         zram->bitmap = bitmap;
 490         zram->nr_pages = nr_pages;
 491         /*
 492          * With writeback feature, zram does asynchronous IO so it's no longer
 493          * synchronous device so let's remove synchronous io flag. Othewise,
 494          * upper layer(e.g., swap) could wait IO completion rather than
 495          * (submit and return), which will cause system sluggish.
 496          * Furthermore, when the IO function returns(e.g., swap_readpage),
 497          * upper layer expects IO was done so it could deallocate the page
 498          * freely but in fact, IO is going on so finally could cause
 499          * use-after-free when the IO is really done.
 500          */
 501         zram->disk->queue->backing_dev_info->capabilities &=
 502                         ~BDI_CAP_SYNCHRONOUS_IO;
 503         up_write(&zram->init_lock);
 504
 505         pr_info("setup backing device %s\n", file_name);
 506         kfree(file_name);
 507
 508         return len;
 509 out:
 510         if (bitmap)
 511                 kvfree(bitmap);
 512
 513         if (bdev)
 514                 blkdev_put(bdev, FMODE_READ | FMODE_WRITE | FMODE_EXCL);
 515
 516         if (backing_dev)
 517                 filp_close(backing_dev, NULL);
 518
 519         up_write(&zram->init_lock);
 520
 521         kfree(file_name);
 522
 523         return err;
 524 }
 525
 526 static unsigned long alloc_block_bdev(struct zram *zram)
 527 {
 528         unsigned long blk_idx = 1;
 529 retry:
 530         /* skip 0 bit to confuse zram.handle = 0 */
 531         blk_idx = find_next_zero_bit(zram->bitmap, zram->nr_pages, blk_idx);
 532         if (blk_idx == zram->nr_pages)
 533                 return 0;
 534
 535         if (test_and_set_bit(blk_idx, zram->bitmap))
 536                 goto retry;
 537
 538         atomic64_inc(&zram->stats.bd_count);
 539         return blk_idx;
 540 }
 541
 542 static void free_block_bdev(struct zram *zram, unsigned long blk_idx)
 543 {
 544         int was_set;
 545
 546         was_set = test_and_clear_bit(blk_idx, zram->bitmap);
 547         WARN_ON_ONCE(!was_set);
 548         atomic64_dec(&zram->stats.bd_count);
 549 }
 550
 551 static void zram_page_end_io(struct bio *bio)
 552 {
 553         struct page *page = bio_first_page_all(bio);
 554
 555         page_endio(page, op_is_write(bio_op(bio)),
 556                         blk_status_to_errno(bio->bi_status));
 557         bio_put(bio);
 558 }
 559
 560 /*
 561  * Returns 1 if the submission is successful.
 562  */
 563 static int read_from_bdev_async(struct zram *zram, struct bio_vec *bvec,
 564                         unsigned long entry, struct bio *parent)
 565 {
 566         struct bio *bio;
 567
 568         bio = bio_alloc(GFP_ATOMIC, 1);
 569         if (!bio)
 570                 return -ENOMEM;
 571
 572         bio->bi_iter.bi_sector = entry * (PAGE_SIZE >> 9);
 573         bio_set_dev(bio, zram->bdev);
 574         if (!bio_add_page(bio, bvec->bv_page, bvec->bv_len, bvec->bv_offset)) {
 575                 bio_put(bio);
 576                 return -EIO;
 577         }
 578
 579         if (!parent) {
 580                 bio->bi_opf = REQ_OP_READ;
 581                 bio->bi_end_io = zram_page_end_io;
 582         } else {
 583                 bio->bi_opf = parent->bi_opf;
 584                 bio_chain(bio, parent);
 585         }
 586
 587         submit_bio(bio);
 588         return 1;
 589 }
 590
 591 #define HUGE_WRITEBACK 0x1
 592 #define IDLE_WRITEBACK 0x2
 593
 594 static ssize_t writeback_store(struct device *dev,
 595                 struct device_attribute *attr, const char *buf, size_t len)
 596 {
 597         struct zram *zram = dev_to_zram(dev);
 598         unsigned long nr_pages = zram->disksize >> PAGE_SHIFT;
 599         unsigned long index;
 600         struct bio bio;
 601         struct bio_vec bio_vec;
 602         struct page *page;
 603         ssize_t ret, sz;
 604         char mode_buf[8];
 605         unsigned long mode = -1UL;
 606         unsigned long blk_idx = 0;
 607
 608         sz = strscpy(mode_buf, buf, sizeof(mode_buf));
 609         if (sz <= 0)
 610                 return -EINVAL;
 611
 612         /* ignore trailing newline */
 613         if (mode_buf[sz - 1] == '\n')
 614                 mode_buf[sz - 1] = 0x00;
 615
 616         if (!strcmp(mode_buf, "idle"))
 617                 mode = IDLE_WRITEBACK;
 618         else if (!strcmp(mode_buf, "huge"))
 619                 mode = HUGE_WRITEBACK;
 620
 621         if (mode == -1UL)
 622                 return -EINVAL;
 623
 624         down_read(&zram->init_lock);
 625         if (!init_done(zram)) {
 626                 ret = -EINVAL;
 627                 goto release_init_lock;
 628         }
 629
 630         if (!zram->backing_dev) {
 631                 ret = -ENODEV;
 632                 goto release_init_lock;
 633         }
 634
 635         page = alloc_page(GFP_KERNEL);
 636         if (!page) {
 637                 ret = -ENOMEM;
 638                 goto release_init_lock;
 639         }
 640
 641         for (index = 0; index < nr_pages; index++) {
 642                 struct bio_vec bvec;
 643
 644                 bvec.bv_page = page;
 645                 bvec.bv_len = PAGE_SIZE;
 646                 bvec.bv_offset = 0;
 647
 648                 if (zram->stop_writeback) {
 649                         ret = -EIO;
 650                         break;
 651                 }
 652
 653                 if (!blk_idx) {
 654                         blk_idx = alloc_block_bdev(zram);
 655                         if (!blk_idx) {
 656                                 ret = -ENOSPC;
 657                                 break;
 658                         }
 659                 }
 660
 661                 zram_slot_lock(zram, index);
 662                 if (!zram_allocated(zram, index))
 663                         goto next;
 664
 665                 if (zram_test_flag(zram, index, ZRAM_WB) ||
 666                                 zram_test_flag(zram, index, ZRAM_SAME) ||
 667                                 zram_test_flag(zram, index, ZRAM_UNDER_WB))
 668                         goto next;
 669
 670                 if ((mode & IDLE_WRITEBACK &&
 671                           !zram_test_flag(zram, index, ZRAM_IDLE)) &&
 672                     (mode & HUGE_WRITEBACK &&
 673                           !zram_test_flag(zram, index, ZRAM_HUGE)))
 674                         goto next;
 675                 /*
 676                  * Clearing ZRAM_UNDER_WB is duty of caller.
 677                  * IOW, zram_free_page never clear it.
 678                  */
 679                 zram_set_flag(zram, index, ZRAM_UNDER_WB);
 680                 /* Need for hugepage writeback racing */
 681                 zram_set_flag(zram, index, ZRAM_IDLE);
 682                 zram_slot_unlock(zram, index);
 683                 if (zram_bvec_read(zram, &bvec, index, 0, NULL)) {
 684                         zram_slot_lock(zram, index);
 685                         zram_clear_flag(zram, index, ZRAM_UNDER_WB);
 686                         zram_clear_flag(zram, index, ZRAM_IDLE);
 687                         zram_slot_unlock(zram, index);
 688                         continue;
 689                 }
 690
 691                 bio_init(&bio, &bio_vec, 1);
 692                 bio_set_dev(&bio, zram->bdev);
 693                 bio.bi_iter.bi_sector = blk_idx * (PAGE_SIZE >> 9);
 694                 bio.bi_opf = REQ_OP_WRITE | REQ_SYNC;
 695
 696                 bio_add_page(&bio, bvec.bv_page, bvec.bv_len,
 697                                 bvec.bv_offset);
 698                 /*
 699                  * XXX: A single page IO would be inefficient for write
 700                  * but it would be not bad as starter.
 701                  */
 702                 ret = submit_bio_wait(&bio);
 703                 if (ret) {
 704                         zram_slot_lock(zram, index);
 705                         zram_clear_flag(zram, index, ZRAM_UNDER_WB);
 706                         zram_clear_flag(zram, index, ZRAM_IDLE);
 707                         zram_slot_unlock(zram, index);
 708                         continue;
 709                 }
 710
 711                 atomic64_inc(&zram->stats.bd_writes);
 712                 /*
 713                  * We released zram_slot_lock so need to check if the slot was
 714                  * changed. If there is freeing for the slot, we can catch it
 715                  * easily by zram_allocated.
 716                  * A subtle case is the slot is freed/reallocated/marked as
 717                  * ZRAM_IDLE again. To close the race, idle_store doesn't
 718                  * mark ZRAM_IDLE once it found the slot was ZRAM_UNDER_WB.
 719                  * Thus, we could close the race by checking ZRAM_IDLE bit.
 720                  */
 721                 zram_slot_lock(zram, index);
 722                 if (!zram_allocated(zram, index) ||
 723                           !zram_test_flag(zram, index, ZRAM_IDLE)) {
 724                         zram_clear_flag(zram, index, ZRAM_UNDER_WB);
 725                         zram_clear_flag(zram, index, ZRAM_IDLE);
 726                         goto next;
 727                 }
 728
 729                 zram_free_page(zram, index);
 730                 zram_clear_flag(zram, index, ZRAM_UNDER_WB);
 731                 zram_set_flag(zram, index, ZRAM_WB);
 732                 zram_set_element(zram, index, blk_idx);
 733                 blk_idx = 0;
 734                 atomic64_inc(&zram->stats.pages_stored);
 735                 if (atomic64_add_unless(&zram->stats.bd_wb_limit,
 736                                         -1 << (PAGE_SHIFT - 12), 0)) {
 737                         if (atomic64_read(&zram->stats.bd_wb_limit) == 0)
 738                                 zram->stop_writeback = true;
 739                 }
 740 next:
 741                 zram_slot_unlock(zram, index);
 742         }
 743
 744         if (blk_idx)
 745                 free_block_bdev(zram, blk_idx);
 746         ret = len;
 747         __free_page(page);
 748 release_init_lock:
 749         up_read(&zram->init_lock);
 750
 751         return ret;
 752 }
 753
 754 struct zram_work {
 755         struct work_struct work;
 756         struct zram *zram;
 757         unsigned long entry;
 758         struct bio *bio;
 759 };
 760
 761 #if PAGE_SIZE != 4096
 762 static void zram_sync_read(struct work_struct *work)
 763 {
 764         struct bio_vec bvec;
 765         struct zram_work *zw = container_of(work, struct zram_work, work);
 766         struct zram *zram = zw->zram;
 767         unsigned long entry = zw->entry;
 768         struct bio *bio = zw->bio;
 769
 770         read_from_bdev_async(zram, &bvec, entry, bio);
 771 }
 772
 773 /*
 774  * Block layer want one ->make_request_fn to be active at a time
 775  * so if we use chained IO with parent IO in same context,
 776  * it's a deadlock. To avoid, it, it uses worker thread context.
 777  */
 778 static int read_from_bdev_sync(struct zram *zram, struct bio_vec *bvec,
 779                                 unsigned long entry, struct bio *bio)
 780 {
 781         struct zram_work work;
 782
 783         work.zram = zram;
 784         work.entry = entry;
 785         work.bio = bio;
 786
 787         INIT_WORK_ONSTACK(&work.work, zram_sync_read);
 788         queue_work(system_unbound_wq, &work.work);
 789         flush_work(&work.work);
 790         destroy_work_on_stack(&work.work);
 791
 792         return 1;
 793 }
 794 #else
 795 static int read_from_bdev_sync(struct zram *zram, struct bio_vec *bvec,
 796                                 unsigned long entry, struct bio *bio)
 797 {
 798         WARN_ON(1);
 799         return -EIO;
 800 }
 801 #endif
 802
 803 static int read_from_bdev(struct zram *zram, struct bio_vec *bvec,
 804                         unsigned long entry, struct bio *parent, bool sync)
 805 {
 806         atomic64_inc(&zram->stats.bd_reads);
 807         if (sync)
 808                 return read_from_bdev_sync(zram, bvec, entry, parent);
 809         else
 810                 return read_from_bdev_async(zram, bvec, entry, parent);
 811 }
 812 #else
 813 static inline void reset_bdev(struct zram *zram) {};
 814 static int read_from_bdev(struct zram *zram, struct bio_vec *bvec,
 815                         unsigned long entry, struct bio *parent, bool sync)
 816 {
 817         return -EIO;
 818 }
 819
 820 static void free_block_bdev(struct zram *zram, unsigned long blk_idx) {};
 821 #endif
 822
 823 #ifdef CONFIG_ZRAM_MEMORY_TRACKING
 824
 825 static struct dentry *zram_debugfs_root;
 826
 827 static void zram_debugfs_create(void)
 828 {
 829         zram_debugfs_root = debugfs_create_dir("zram", NULL);
 830 }
 831
 832 static void zram_debugfs_destroy(void)
 833 {
 834         debugfs_remove_recursive(zram_debugfs_root);
 835 }
 836
 837 static void zram_accessed(struct zram *zram, u32 index)
 838 {
 839         zram_clear_flag(zram, index, ZRAM_IDLE);
 840         zram->table[index].ac_time = ktime_get_boottime();
 841 }
 842
 843 static ssize_t read_block_state(struct file *file, char __user *buf,
 844                                 size_t count, loff_t *ppos)
 845 {
 846         char *kbuf;
 847         ssize_t index, written = 0;
 848         struct zram *zram = file->private_data;
 849         unsigned long nr_pages = zram->disksize >> PAGE_SHIFT;
 850         struct timespec64 ts;
 851
 852         kbuf = kvmalloc(count, GFP_KERNEL);
 853         if (!kbuf)
 854                 return -ENOMEM;
 855
 856         down_read(&zram->init_lock);
 857         if (!init_done(zram)) {
 858                 up_read(&zram->init_lock);
 859                 kvfree(kbuf);
 860                 return -EINVAL;
 861         }
 862
 863         for (index = *ppos; index < nr_pages; index++) {
 864                 int copied;
 865
 866                 zram_slot_lock(zram, index);
 867                 if (!zram_allocated(zram, index))
 868                         goto next;
 869
 870                 ts = ktime_to_timespec64(zram->table[index].ac_time);
 871                 copied = snprintf(kbuf + written, count,
 872                         "%12zd %12lld.%06lu %c%c%c%c\n",
 873                         index, (s64)ts.tv_sec,
 874                         ts.tv_nsec / NSEC_PER_USEC,
 875                         zram_test_flag(zram, index, ZRAM_SAME) ? 's' : '.',
 876                         zram_test_flag(zram, index, ZRAM_WB) ? 'w' : '.',
 877                         zram_test_flag(zram, index, ZRAM_HUGE) ? 'h' : '.',
 878                         zram_test_flag(zram, index, ZRAM_IDLE) ? 'i' : '.');
 879
 880                 if (count < copied) {
 881                         zram_slot_unlock(zram, index);
 882                         break;
 883                 }
 884                 written += copied;
 885                 count -= copied;
 886 next:
 887                 zram_slot_unlock(zram, index);
 888                 *ppos += 1;
 889         }
 890
 891         up_read(&zram->init_lock);
 892         if (copy_to_user(buf, kbuf, written))
 893                 written = -EFAULT;
 894         kvfree(kbuf);
 895
 896         return written;
 897 }
 898
 899 static const struct file_operations proc_zram_block_state_op = {
 900         .open = simple_open,
 901         .read = read_block_state,
 902         .llseek = default_llseek,
 903 };
 904
 905 static void zram_debugfs_register(struct zram *zram)
 906 {
 907         if (!zram_debugfs_root)
 908                 return;
 909
 910         zram->debugfs_dir = debugfs_create_dir(zram->disk->disk_name,
 911                                                 zram_debugfs_root);
 912         debugfs_create_file("block_state", 0400, zram->debugfs_dir,
 913                                 zram, &proc_zram_block_state_op);
 914 }
 915
 916 static void zram_debugfs_unregister(struct zram *zram)
 917 {
 918         debugfs_remove_recursive(zram->debugfs_dir);
 919 }
 920 #else
 921 static void zram_debugfs_create(void) {};
 922 static void zram_debugfs_destroy(void) {};
 923 static void zram_accessed(struct zram *zram, u32 index)
 924 {
 925         zram_clear_flag(zram, index, ZRAM_IDLE);
 926 };
 927 static void zram_debugfs_register(struct zram *zram) {};
 928 static void zram_debugfs_unregister(struct zram *zram) {};
 929 #endif
 930
 931 /*
 932  * We switched to per-cpu streams and this attr is not needed anymore.
 933  * However, we will keep it around for some time, because:
 934  * a) we may revert per-cpu streams in the future
 935  * b) it's visible to user space and we need to follow our 2 years
 936  *    retirement rule; but we already have a number of 'soon to be
 937  *    altered' attrs, so max_comp_streams need to wait for the next
 938  *    layoff cycle.
 939  */
 940 static ssize_t max_comp_streams_show(struct device *dev,
 941                 struct device_attribute *attr, char *buf)
 942 {
 943         return scnprintf(buf, PAGE_SIZE, "%d\n", num_online_cpus());
 944 }
 945
 946 static ssize_t max_comp_streams_store(struct device *dev,
 947                 struct device_attribute *attr, const char *buf, size_t len)
 948 {
 949         return len;
 950 }
 951
 952 static ssize_t comp_algorithm_show(struct device *dev,
 953                 struct device_attribute *attr, char *buf)
 954 {
 955         size_t sz;
 956         struct zram *zram = dev_to_zram(dev);
 957
 958         down_read(&zram->init_lock);
 959         sz = zcomp_available_show(zram->compressor, buf);
 960         up_read(&zram->init_lock);
 961
 962         return sz;
 963 }
 964
 965 static ssize_t comp_algorithm_store(struct device *dev,
 966                 struct device_attribute *attr, const char *buf, size_t len)
 967 {
 968         struct zram *zram = dev_to_zram(dev);
 969         char compressor[ARRAY_SIZE(zram->compressor)];
 970         size_t sz;
 971
 972         strlcpy(compressor, buf, sizeof(compressor));
 973         /* ignore trailing newline */
 974         sz = strlen(compressor);
 975         if (sz > 0 && compressor[sz - 1] == '\n')
 976                 compressor[sz - 1] = 0x00;
 977
 978         if (!zcomp_available_algorithm(compressor))
 979                 return -EINVAL;
 980
 981         down_write(&zram->init_lock);
 982         if (init_done(zram)) {
 983                 up_write(&zram->init_lock);
 984                 pr_info("Can't change algorithm for initialized device\n");
 985                 return -EBUSY;
 986         }
 987
 988         strcpy(zram->compressor, compressor);
 989         up_write(&zram->init_lock);
 990         return len;
 991 }
 992
 993 static ssize_t compact_store(struct device *dev,
 994                 struct device_attribute *attr, const char *buf, size_t len)
 995 {
 996         struct zram *zram = dev_to_zram(dev);
 997
 998         down_read(&zram->init_lock);
 999         if (!init_done(zram)) {
1000                 up_read(&zram->init_lock);
1001                 return -EINVAL;
1002         }
1003
1004         zs_compact(zram->mem_pool);
1005         up_read(&zram->init_lock);
1006
1007         return len;
1008 }
1009
1010 static ssize_t io_stat_show(struct device *dev,
1011                 struct device_attribute *attr, char *buf)
1012 {
1013         struct zram *zram = dev_to_zram(dev);
1014         ssize_t ret;
1015
1016         down_read(&zram->init_lock);
1017         ret = scnprintf(buf, PAGE_SIZE,
1018                         "%8llu %8llu %8llu %8llu\n",
1019                         (u64)atomic64_read(&zram->stats.failed_reads),
1020                         (u64)atomic64_read(&zram->stats.failed_writes),
1021                         (u64)atomic64_read(&zram->stats.invalid_io),
1022                         (u64)atomic64_read(&zram->stats.notify_free));
1023         up_read(&zram->init_lock);
1024
1025         return ret;
1026 }
1027
1028 static ssize_t mm_stat_show(struct device *dev,
1029                 struct device_attribute *attr, char *buf)
1030 {
1031         struct zram *zram = dev_to_zram(dev);
1032         struct zs_pool_stats pool_stats;
1033         u64 orig_size, mem_used = 0;
1034         long max_used;
1035         ssize_t ret;
1036
1037         memset(&pool_stats, 0x00, sizeof(struct zs_pool_stats));
1038
1039         down_read(&zram->init_lock);
1040         if (init_done(zram)) {
1041                 mem_used = zs_get_total_pages(zram->mem_pool);
1042                 zs_pool_stats(zram->mem_pool, &pool_stats);
1043         }
1044
1045         orig_size = atomic64_read(&zram->stats.pages_stored);
1046         max_used = atomic_long_read(&zram->stats.max_used_pages);
1047
1048         ret = scnprintf(buf, PAGE_SIZE,
1049                         "%8llu %8llu %8llu %8lu %8ld %8llu %8lu %8llu\n",
1050                         orig_size << PAGE_SHIFT,
1051                         (u64)atomic64_read(&zram->stats.compr_data_size),
1052                         mem_used << PAGE_SHIFT,
1053                         zram->limit_pages << PAGE_SHIFT,
1054                         max_used << PAGE_SHIFT,
1055                         (u64)atomic64_read(&zram->stats.same_pages),
1056                         pool_stats.pages_compacted,
1057                         (u64)atomic64_read(&zram->stats.huge_pages));
1058         up_read(&zram->init_lock);
1059
1060         return ret;
1061 }
1062
1063 #ifdef CONFIG_ZRAM_WRITEBACK
1064 #define FOUR_K(x) ((x) * (1 << (PAGE_SHIFT - 12)))
1065 static ssize_t bd_stat_show(struct device *dev,
1066                 struct device_attribute *attr, char *buf)
1067 {
1068         struct zram *zram = dev_to_zram(dev);
1069         ssize_t ret;
1070
1071         down_read(&zram->init_lock);
1072         ret = scnprintf(buf, PAGE_SIZE,
1073                 "%8llu %8llu %8llu\n",
1074                         FOUR_K((u64)atomic64_read(&zram->stats.bd_count)),
1075                         FOUR_K((u64)atomic64_read(&zram->stats.bd_reads)),
1076                         FOUR_K((u64)atomic64_read(&zram->stats.bd_writes)));
1077         up_read(&zram->init_lock);
1078
1079         return ret;
1080 }
1081 #endif
1082
1083 static ssize_t debug_stat_show(struct device *dev,
1084                 struct device_attribute *attr, char *buf)
1085 {
1086         int version = 1;
1087         struct zram *zram = dev_to_zram(dev);
1088         ssize_t ret;
1089
1090         down_read(&zram->init_lock);
1091         ret = scnprintf(buf, PAGE_SIZE,
1092                         "version: %d\n%8llu %8llu\n",
1093                         version,
1094                         (u64)atomic64_read(&zram->stats.writestall),
1095                         (u64)atomic64_read(&zram->stats.miss_free));
1096         up_read(&zram->init_lock);
1097
1098         return ret;
1099 }
1100
1101 static DEVICE_ATTR_RO(io_stat);
1102 static DEVICE_ATTR_RO(mm_stat);
1103 #ifdef CONFIG_ZRAM_WRITEBACK
1104 static DEVICE_ATTR_RO(bd_stat);
1105 #endif
1106 static DEVICE_ATTR_RO(debug_stat);
1107
1108 static void zram_meta_free(struct zram *zram, u64 disksize)
1109 {
1110         size_t num_pages = disksize >> PAGE_SHIFT;
1111         size_t index;
1112
1113         /* Free all pages that are still in this zram device */
1114         for (index = 0; index < num_pages; index++)
1115                 zram_free_page(zram, index);
1116
1117         zs_destroy_pool(zram->mem_pool);
1118         vfree(zram->table);
1119 }
1120
1121 static bool zram_meta_alloc(struct zram *zram, u64 disksize)
1122 {
1123         size_t num_pages;
1124
1125         num_pages = disksize >> PAGE_SHIFT;
1126         zram->table = vzalloc(array_size(num_pages, sizeof(*zram->table)));
1127         if (!zram->table)
1128                 return false;
1129
1130         zram->mem_pool = zs_create_pool(zram->disk->disk_name);
1131         if (!zram->mem_pool) {
1132                 vfree(zram->table);
1133                 return false;
1134         }
1135
1136         if (!huge_class_size)
1137                 huge_class_size = zs_huge_class_size(zram->mem_pool);
1138         return true;
1139 }
1140
1141 /*
1142  * To protect concurrent access to the same index entry,
1143  * caller should hold this table index entry's bit_spinlock to
1144  * indicate this index entry is accessing.
1145  */
1146 static void zram_free_page(struct zram *zram, size_t index)
1147 {
1148         unsigned long handle;
1149
1150 #ifdef CONFIG_ZRAM_MEMORY_TRACKING
1151         zram->table[index].ac_time = 0;
1152 #endif
1153         if (zram_test_flag(zram, index, ZRAM_IDLE))
1154                 zram_clear_flag(zram, index, ZRAM_IDLE);
1155
1156         if (zram_test_flag(zram, index, ZRAM_HUGE)) {
1157                 zram_clear_flag(zram, index, ZRAM_HUGE);
1158                 atomic64_dec(&zram->stats.huge_pages);
1159         }
1160
1161         if (zram_test_flag(zram, index, ZRAM_WB)) {
1162                 zram_clear_flag(zram, index, ZRAM_WB);
1163                 free_block_bdev(zram, zram_get_element(zram, index));
1164                 goto out;
1165         }
1166
1167         /*
1168          * No memory is allocated for same element filled pages.
1169          * Simply clear same page flag.
1170          */
1171         if (zram_test_flag(zram, index, ZRAM_SAME)) {
1172                 zram_clear_flag(zram, index, ZRAM_SAME);
1173                 atomic64_dec(&zram->stats.same_pages);
1174                 goto out;
1175         }
1176
1177         handle = zram_get_handle(zram, index);
1178         if (!handle)
1179                 return;
1180
1181         zs_free(zram->mem_pool, handle);
1182
1183         atomic64_sub(zram_get_obj_size(zram, index),
1184                         &zram->stats.compr_data_size);
1185 out:
1186         atomic64_dec(&zram->stats.pages_stored);
1187         zram_set_handle(zram, index, 0);
1188         zram_set_obj_size(zram, index, 0);
1189         WARN_ON_ONCE(zram->table[index].flags &
1190                 ~(1UL << ZRAM_LOCK | 1UL << ZRAM_UNDER_WB));
1191 }
1192
1193 static int __zram_bvec_read(struct zram *zram, struct page *page, u32 index,
1194                                 struct bio *bio, bool partial_io)
1195 {
1196         int ret;
1197         unsigned long handle;
1198         unsigned int size;
1199         void *src, *dst;
1200
1201         zram_slot_lock(zram, index);
1202         if (zram_test_flag(zram, index, ZRAM_WB)) {
1203                 struct bio_vec bvec;
1204
1205                 zram_slot_unlock(zram, index);
1206
1207                 bvec.bv_page = page;
1208                 bvec.bv_len = PAGE_SIZE;
1209                 bvec.bv_offset = 0;
1210                 return read_from_bdev(zram, &bvec,
1211                                 zram_get_element(zram, index),
1212                                 bio, partial_io);
1213         }
1214
1215         handle = zram_get_handle(zram, index);
1216         if (!handle || zram_test_flag(zram, index, ZRAM_SAME)) {
1217                 unsigned long value;
1218                 void *mem;
1219
1220                 value = handle ? zram_get_element(zram, index) : 0;
1221                 mem = kmap_atomic(page);
1222                 zram_fill_page(mem, PAGE_SIZE, value);
1223                 kunmap_atomic(mem);
1224                 zram_slot_unlock(zram, index);
1225                 return 0;
1226         }
1227
1228         size = zram_get_obj_size(zram, index);
1229
1230         src = zs_map_object(zram->mem_pool, handle, ZS_MM_RO);
1231         if (size == PAGE_SIZE) {
1232                 dst = kmap_atomic(page);
1233                 memcpy(dst, src, PAGE_SIZE);
1234                 kunmap_atomic(dst);
1235                 ret = 0;
1236         } else {
1237                 struct zcomp_strm *zstrm = zcomp_stream_get(zram->comp);
1238
1239                 dst = kmap_atomic(page);
1240                 ret = zcomp_decompress(zstrm, src, size, dst);
1241                 kunmap_atomic(dst);
1242                 zcomp_stream_put(zram->comp);
1243         }
1244         zs_unmap_object(zram->mem_pool, handle);
1245         zram_slot_unlock(zram, index);
1246
1247         /* Should NEVER happen. Return bio error if it does. */
1248         if (unlikely(ret))
1249                 pr_err("Decompression failed! err=%d, page=%u\n", ret, index);
1250
1251         return ret;
1252 }
1253
1254 static int zram_bvec_read(struct zram *zram, struct bio_vec *bvec,
1255                                 u32 index, int offset, struct bio *bio)
1256 {
1257         int ret;
1258         struct page *page;
1259
1260         page = bvec->bv_page;
1261         if (is_partial_io(bvec)) {
1262                 /* Use a temporary buffer to decompress the page */
1263                 page = alloc_page(GFP_NOIO|__GFP_HIGHMEM);
1264                 if (!page)
1265                         return -ENOMEM;
1266         }
1267
1268         ret = __zram_bvec_read(zram, page, index, bio, is_partial_io(bvec));
1269         if (unlikely(ret))
1270                 goto out;
1271
1272         if (is_partial_io(bvec)) {
1273                 void *dst = kmap_atomic(bvec->bv_page);
1274                 void *src = kmap_atomic(page);
1275
1276                 memcpy(dst + bvec->bv_offset, src + offset, bvec->bv_len);
1277                 kunmap_atomic(src);
1278                 kunmap_atomic(dst);
1279         }
1280 out:
1281         if (is_partial_io(bvec))
1282                 __free_page(page);
1283
1284         return ret;
1285 }
1286
1287 static int __zram_bvec_write(struct zram *zram, struct bio_vec *bvec,
1288                                 u32 index, struct bio *bio)
1289 {
1290         int ret = 0;
1291         unsigned long alloced_pages;
1292         unsigned long handle = 0;
1293         unsigned int comp_len = 0;
1294         void *src, *dst, *mem;
1295         struct zcomp_strm *zstrm;
1296         struct page *page = bvec->bv_page;
1297         unsigned long element = 0;
1298         enum zram_pageflags flags = 0;
1299
1300         mem = kmap_atomic(page);
1301         if (page_same_filled(mem, &element)) {
1302                 kunmap_atomic(mem);
1303                 /* Free memory associated with this sector now. */
1304                 flags = ZRAM_SAME;
1305                 atomic64_inc(&zram->stats.same_pages);
1306                 goto out;
1307         }
1308         kunmap_atomic(mem);
1309
1310 compress_again:
1311         zstrm = zcomp_stream_get(zram->comp);
1312         src = kmap_atomic(page);
1313         ret = zcomp_compress(zstrm, src, &comp_len);
1314         kunmap_atomic(src);
1315
1316         if (unlikely(ret)) {
1317                 zcomp_stream_put(zram->comp);
1318                 pr_err("Compression failed! err=%d\n", ret);
1319                 zs_free(zram->mem_pool, handle);
1320                 return ret;
1321         }
1322
1323         if (comp_len >= huge_class_size)
1324                 comp_len = PAGE_SIZE;
1325         /*
1326          * handle allocation has 2 paths:
1327          * a) fast path is executed with preemption disabled (for
1328          *  per-cpu streams) and has __GFP_DIRECT_RECLAIM bit clear,
1329          *  since we can't sleep;
1330          * b) slow path enables preemption and attempts to allocate
1331          *  the page with __GFP_DIRECT_RECLAIM bit set. we have to
1332          *  put per-cpu compression stream and, thus, to re-do
1333          *  the compression once handle is allocated.
1334          *
1335          * if we have a 'non-null' handle here then we are coming
1336          * from the slow path and handle has already been allocated.
1337          */
1338         if (!handle)
1339                 handle = zs_malloc(zram->mem_pool, comp_len,
1340                                 __GFP_KSWAPD_RECLAIM |
1341                                 __GFP_NOWARN |
1342                                 __GFP_HIGHMEM |
1343                                 __GFP_MOVABLE);
1344         if (!handle) {
1345                 zcomp_stream_put(zram->comp);
1346                 atomic64_inc(&zram->stats.writestall);
1347                 handle = zs_malloc(zram->mem_pool, comp_len,
1348                                 GFP_NOIO | __GFP_HIGHMEM |
1349                                 __GFP_MOVABLE);
1350                 if (handle)
1351                         goto compress_again;
1352                 return -ENOMEM;
1353         }
1354
1355         alloced_pages = zs_get_total_pages(zram->mem_pool);
1356         update_used_max(zram, alloced_pages);
1357
1358         if (zram->limit_pages && alloced_pages > zram->limit_pages) {
1359                 zcomp_stream_put(zram->comp);
1360                 zs_free(zram->mem_pool, handle);
1361                 return -ENOMEM;
1362         }
1363
1364         dst = zs_map_object(zram->mem_pool, handle, ZS_MM_WO);
1365
1366         src = zstrm->buffer;
1367         if (comp_len == PAGE_SIZE)
1368                 src = kmap_atomic(page);
1369         memcpy(dst, src, comp_len);
1370         if (comp_len == PAGE_SIZE)
1371                 kunmap_atomic(src);
1372
1373         zcomp_stream_put(zram->comp);
1374         zs_unmap_object(zram->mem_pool, handle);
1375         atomic64_add(comp_len, &zram->stats.compr_data_size);
1376 out:
1377         /*
1378          * Free memory associated with this sector
1379          * before overwriting unused sectors.
1380          */
1381         zram_slot_lock(zram, index);
1382         zram_free_page(zram, index);
1383
1384         if (comp_len == PAGE_SIZE) {
1385                 zram_set_flag(zram, index, ZRAM_HUGE);
1386                 atomic64_inc(&zram->stats.huge_pages);
1387         }
1388
1389         if (flags) {
1390                 zram_set_flag(zram, index, flags);
1391                 zram_set_element(zram, index, element);
1392         }  else {
1393                 zram_set_handle(zram, index, handle);
1394                 zram_set_obj_size(zram, index, comp_len);
1395         }
1396         zram_slot_unlock(zram, index);
1397
1398         /* Update stats */
1399         atomic64_inc(&zram->stats.pages_stored);
1400         return ret;
1401 }
1402
1403 static int zram_bvec_write(struct zram *zram, struct bio_vec *bvec,
1404                                 u32 index, int offset, struct bio *bio)
1405 {
1406         int ret;
1407         struct page *page = NULL;
1408         void *src;
1409         struct bio_vec vec;
1410
1411         vec = *bvec;
1412         if (is_partial_io(bvec)) {
1413                 void *dst;
1414                 /*
1415                  * This is a partial IO. We need to read the full page
1416                  * before to write the changes.
1417                  */
1418                 page = alloc_page(GFP_NOIO|__GFP_HIGHMEM);
1419                 if (!page)
1420                         return -ENOMEM;
1421
1422                 ret = __zram_bvec_read(zram, page, index, bio, true);
1423                 if (ret)
1424                         goto out;
1425
1426                 src = kmap_atomic(bvec->bv_page);
1427                 dst = kmap_atomic(page);
1428                 memcpy(dst + offset, src + bvec->bv_offset, bvec->bv_len);
1429                 kunmap_atomic(dst);
1430                 kunmap_atomic(src);
1431
1432                 vec.bv_page = page;
1433                 vec.bv_len = PAGE_SIZE;
1434                 vec.bv_offset = 0;
1435         }
1436
1437         ret = __zram_bvec_write(zram, &vec, index, bio);
1438 out:
1439         if (is_partial_io(bvec))
1440                 __free_page(page);
1441         return ret;
1442 }
1443
1444 /*
1445  * zram_bio_discard - handler on discard request
1446  * @index: physical block index in PAGE_SIZE units
1447  * @offset: byte offset within physical block
1448  */
1449 static void zram_bio_discard(struct zram *zram, u32 index,
1450                              int offset, struct bio *bio)
1451 {
1452         size_t n = bio->bi_iter.bi_size;
1453
1454         /*
1455          * zram manages data in physical block size units. Because logical block
1456          * size isn't identical with physical block size on some arch, we
1457          * could get a discard request pointing to a specific offset within a
1458          * certain physical block.  Although we can handle this request by
1459          * reading that physiclal block and decompressing and partially zeroing
1460          * and re-compressing and then re-storing it, this isn't reasonable
1461          * because our intent with a discard request is to save memory.  So
1462          * skipping this logical block is appropriate here.
1463          */
1464         if (offset) {
1465                 if (n <= (PAGE_SIZE - offset))
1466                         return;
1467
1468                 n -= (PAGE_SIZE - offset);
1469                 index++;
1470         }
1471
1472         while (n >= PAGE_SIZE) {
1473                 zram_slot_lock(zram, index);
1474                 zram_free_page(zram, index);
1475                 zram_slot_unlock(zram, index);
1476                 atomic64_inc(&zram->stats.notify_free);
1477                 index++;
1478                 n -= PAGE_SIZE;
1479         }
1480 }
1481
1482 /*
1483  * Returns errno if it has some problem. Otherwise return 0 or 1.
1484  * Returns 0 if IO request was done synchronously
1485  * Returns 1 if IO request was successfully submitted.
1486  */
1487 static int zram_bvec_rw(struct zram *zram, struct bio_vec *bvec, u32 index,
1488                         int offset, unsigned int op, struct bio *bio)
1489 {
1490         unsigned long start_time = jiffies;
1491         struct request_queue *q = zram->disk->queue;
1492         int ret;
1493
1494         generic_start_io_acct(q, op, bvec->bv_len >> SECTOR_SHIFT,
1495                         &zram->disk->part0);
1496
1497         if (!op_is_write(op)) {
1498                 atomic64_inc(&zram->stats.num_reads);
1499                 ret = zram_bvec_read(zram, bvec, index, offset, bio);
1500                 flush_dcache_page(bvec->bv_page);
1501         } else {
1502                 atomic64_inc(&zram->stats.num_writes);
1503                 ret = zram_bvec_write(zram, bvec, index, offset, bio);
1504         }
1505
1506         generic_end_io_acct(q, op, &zram->disk->part0, start_time);
1507
1508         zram_slot_lock(zram, index);
1509         zram_accessed(zram, index);
1510         zram_slot_unlock(zram, index);
1511
1512         if (unlikely(ret < 0)) {
1513                 if (!op_is_write(op))
1514                         atomic64_inc(&zram->stats.failed_reads);
1515                 else
1516                         atomic64_inc(&zram->stats.failed_writes);
1517         }
1518
1519         return ret;
1520 }
1521
1522 static void __zram_make_request(struct zram *zram, struct bio *bio)
1523 {
1524         int offset;
1525         u32 index;
1526         struct bio_vec bvec;
1527         struct bvec_iter iter;
1528
1529         index = bio->bi_iter.bi_sector >> SECTORS_PER_PAGE_SHIFT;
1530         offset = (bio->bi_iter.bi_sector &
1531                   (SECTORS_PER_PAGE - 1)) << SECTOR_SHIFT;
1532
1533         switch (bio_op(bio)) {
1534         case REQ_OP_DISCARD:
1535         case REQ_OP_WRITE_ZEROES:
1536                 zram_bio_discard(zram, index, offset, bio);
1537                 bio_endio(bio);
1538                 return;
1539         default:
1540                 break;
1541         }
1542
1543         bio_for_each_segment(bvec, bio, iter) {
1544                 struct bio_vec bv = bvec;
1545                 unsigned int unwritten = bvec.bv_len;
1546
1547                 do {
1548                         bv.bv_len = min_t(unsigned int, PAGE_SIZE - offset,
1549                                                         unwritten);
1550                         if (zram_bvec_rw(zram, &bv, index, offset,
1551                                          bio_op(bio), bio) < 0)
1552                                 goto out;
1553
1554                         bv.bv_offset += bv.bv_len;
1555                         unwritten -= bv.bv_len;
1556
1557                         update_position(&index, &offset, &bv);
1558                 } while (unwritten);
1559         }
1560
1561         bio_endio(bio);
1562         return;
1563
1564 out:
1565         bio_io_error(bio);
1566 }
1567
1568 /*
1569  * Handler function for all zram I/O requests.
1570  */
1571 static blk_qc_t zram_make_request(struct request_queue *queue, struct bio *bio)
1572 {
1573         struct zram *zram = queue->queuedata;
1574
1575         if (!valid_io_request(zram, bio->bi_iter.bi_sector,
1576                                         bio->bi_iter.bi_size)) {
1577                 atomic64_inc(&zram->stats.invalid_io);
1578                 goto error;
1579         }
1580
1581         __zram_make_request(zram, bio);
1582         return BLK_QC_T_NONE;
1583
1584 error:
1585         bio_io_error(bio);
1586         return BLK_QC_T_NONE;
1587 }
1588
1589 static void zram_slot_free_notify(struct block_device *bdev,
1590                                 unsigned long index)
1591 {
1592         struct zram *zram;
1593
1594         zram = bdev->bd_disk->private_data;
1595
1596         atomic64_inc(&zram->stats.notify_free);
1597         if (!zram_slot_trylock(zram, index)) {
1598                 atomic64_inc(&zram->stats.miss_free);
1599                 return;
1600         }
1601
1602         zram_free_page(zram, index);
1603         zram_slot_unlock(zram, index);
1604 }
1605
1606 static int zram_rw_page(struct block_device *bdev, sector_t sector,
1607                        struct page *page, unsigned int op)
1608 {
1609         int offset, ret;
1610         u32 index;
1611         struct zram *zram;
1612         struct bio_vec bv;
1613
1614         if (PageTransHuge(page))
1615                 return -ENOTSUPP;
1616         zram = bdev->bd_disk->private_data;
1617
1618         if (!valid_io_request(zram, sector, PAGE_SIZE)) {
1619                 atomic64_inc(&zram->stats.invalid_io);
1620                 ret = -EINVAL;
1621                 goto out;
1622         }
1623
1624         index = sector >> SECTORS_PER_PAGE_SHIFT;
1625         offset = (sector & (SECTORS_PER_PAGE - 1)) << SECTOR_SHIFT;
1626
1627         bv.bv_page = page;
1628         bv.bv_len = PAGE_SIZE;
1629         bv.bv_offset = 0;
1630
1631         ret = zram_bvec_rw(zram, &bv, index, offset, op, NULL);
1632 out:
1633         /*
1634          * If I/O fails, just return error(ie, non-zero) without
1635          * calling page_endio.
1636          * It causes resubmit the I/O with bio request by upper functions
1637          * of rw_page(e.g., swap_readpage, __swap_writepage) and
1638          * bio->bi_end_io does things to handle the error
1639          * (e.g., SetPageError, set_page_dirty and extra works).
1640          */
1641         if (unlikely(ret < 0))
1642                 return ret;
1643
1644         switch (ret) {
1645         case 0:
1646                 page_endio(page, op_is_write(op), 0);
1647                 break;
1648         case 1:
1649                 ret = 0;
1650                 break;
1651         default:
1652                 WARN_ON(1);
1653         }
1654         return ret;
1655 }
1656
1657 static void zram_reset_device(struct zram *zram)
1658 {
1659         struct zcomp *comp;
1660         u64 disksize;
1661
1662         down_write(&zram->init_lock);
1663
1664         zram->limit_pages = 0;
1665
1666         if (!init_done(zram)) {
1667                 up_write(&zram->init_lock);
1668                 return;
1669         }
1670
1671         comp = zram->comp;
1672         disksize = zram->disksize;
1673         zram->disksize = 0;
1674
1675         set_capacity(zram->disk, 0);
1676         part_stat_set_all(&zram->disk->part0, 0);
1677
1678         up_write(&zram->init_lock);
1679         /* I/O operation under all of CPU are done so let's free */
1680         zram_meta_free(zram, disksize);
1681         memset(&zram->stats, 0, sizeof(zram->stats));
1682         zcomp_destroy(comp);
1683         reset_bdev(zram);
1684 }
1685
1686 static ssize_t disksize_store(struct device *dev,
1687                 struct device_attribute *attr, const char *buf, size_t len)
1688 {
1689         u64 disksize;
1690         struct zcomp *comp;
1691         struct zram *zram = dev_to_zram(dev);
1692         int err;
1693
1694         disksize = memparse(buf, NULL);
1695         if (!disksize)
1696                 return -EINVAL;
1697
1698         down_write(&zram->init_lock);
1699         if (init_done(zram)) {
1700                 pr_info("Cannot change disksize for initialized device\n");
1701                 err = -EBUSY;
1702                 goto out_unlock;
1703         }
1704
1705         disksize = PAGE_ALIGN(disksize);
1706         if (!zram_meta_alloc(zram, disksize)) {
1707                 err = -ENOMEM;
1708                 goto out_unlock;
1709         }
1710
1711         comp = zcomp_create(zram->compressor);
1712         if (IS_ERR(comp)) {
1713                 pr_err("Cannot initialise %s compressing backend\n",
1714                                 zram->compressor);
1715                 err = PTR_ERR(comp);
1716                 goto out_free_meta;
1717         }
1718
1719         zram->comp = comp;
1720         zram->disksize = disksize;
1721         set_capacity(zram->disk, zram->disksize >> SECTOR_SHIFT);
1722
1723         revalidate_disk(zram->disk);
1724         up_write(&zram->init_lock);
1725
1726         return len;
1727
1728 out_free_meta:
1729         zram_meta_free(zram, disksize);
1730 out_unlock:
1731         up_write(&zram->init_lock);
1732         return err;
1733 }
1734
1735 static ssize_t reset_store(struct device *dev,
1736                 struct device_attribute *attr, const char *buf, size_t len)
1737 {
1738         int ret;
1739         unsigned short do_reset;
1740         struct zram *zram;
1741         struct block_device *bdev;
1742
1743         ret = kstrtou16(buf, 10, &do_reset);
1744         if (ret)
1745                 return ret;
1746
1747         if (!do_reset)
1748                 return -EINVAL;
1749
1750         zram = dev_to_zram(dev);
1751         bdev = bdget_disk(zram->disk, 0);
1752         if (!bdev)
1753                 return -ENOMEM;
1754
1755         mutex_lock(&bdev->bd_mutex);
1756         /* Do not reset an active device or claimed device */
1757         if (bdev->bd_openers || zram->claim) {
1758                 mutex_unlock(&bdev->bd_mutex);
1759                 bdput(bdev);
1760                 return -EBUSY;
1761         }
1762
1763         /* From now on, anyone can't open /dev/zram[0-9] */
1764         zram->claim = true;
1765         mutex_unlock(&bdev->bd_mutex);
1766
1767         /* Make sure all the pending I/O are finished */
1768         fsync_bdev(bdev);
1769         zram_reset_device(zram);
1770         revalidate_disk(zram->disk);
1771         bdput(bdev);
1772
1773         mutex_lock(&bdev->bd_mutex);
1774         zram->claim = false;
1775         mutex_unlock(&bdev->bd_mutex);
1776
1777         return len;
1778 }
1779
1780 static int zram_open(struct block_device *bdev, fmode_t mode)
1781 {
1782         int ret = 0;
1783         struct zram *zram;
1784
1785         WARN_ON(!mutex_is_locked(&bdev->bd_mutex));
1786
1787         zram = bdev->bd_disk->private_data;
1788         /* zram was claimed to reset so open request fails */
1789         if (zram->claim)
1790                 ret = -EBUSY;
1791
1792         return ret;
1793 }
1794
1795 static const struct block_device_operations zram_devops = {
1796         .open = zram_open,
1797         .swap_slot_free_notify = zram_slot_free_notify,
1798         .rw_page = zram_rw_page,
1799         .owner = THIS_MODULE
1800 };
1801
1802 static DEVICE_ATTR_WO(compact);
1803 static DEVICE_ATTR_RW(disksize);
1804 static DEVICE_ATTR_RO(initstate);
1805 static DEVICE_ATTR_WO(reset);
1806 static DEVICE_ATTR_WO(mem_limit);
1807 static DEVICE_ATTR_WO(mem_used_max);
1808 static DEVICE_ATTR_WO(idle);
1809 static DEVICE_ATTR_RW(max_comp_streams);
1810 static DEVICE_ATTR_RW(comp_algorithm);
1811 #ifdef CONFIG_ZRAM_WRITEBACK
1812 static DEVICE_ATTR_RW(backing_dev);
1813 static DEVICE_ATTR_WO(writeback);
1814 static DEVICE_ATTR_RW(writeback_limit);
1815 #endif
1816
1817 static struct attribute *zram_disk_attrs[] = {
1818         &dev_attr_disksize.attr,
1819         &dev_attr_initstate.attr,
1820         &dev_attr_reset.attr,
1821         &dev_attr_compact.attr,
1822         &dev_attr_mem_limit.attr,
1823         &dev_attr_mem_used_max.attr,
1824         &dev_attr_idle.attr,
1825         &dev_attr_max_comp_streams.attr,
1826         &dev_attr_comp_algorithm.attr,
1827 #ifdef CONFIG_ZRAM_WRITEBACK
1828         &dev_attr_backing_dev.attr,
1829         &dev_attr_writeback.attr,
1830         &dev_attr_writeback_limit.attr,
1831 #endif
1832         &dev_attr_io_stat.attr,
1833         &dev_attr_mm_stat.attr,
1834 #ifdef CONFIG_ZRAM_WRITEBACK
1835         &dev_attr_bd_stat.attr,
1836 #endif
1837         &dev_attr_debug_stat.attr,
1838         NULL,
1839 };
1840
1841 static const struct attribute_group zram_disk_attr_group = {
1842         .attrs = zram_disk_attrs,
1843 };
1844
1845 static const struct attribute_group *zram_disk_attr_groups[] = {
1846         &zram_disk_attr_group,
1847         NULL,
1848 };
1849
1850 /*
1851  * Allocate and initialize new zram device. the function returns
1852  * '>= 0' device_id upon success, and negative value otherwise.
1853  */
1854 static int zram_add(void)
1855 {
1856         struct zram *zram;
1857         struct request_queue *queue;
1858         int ret, device_id;
1859
1860         zram = kzalloc(sizeof(struct zram), GFP_KERNEL);
1861         if (!zram)
1862                 return -ENOMEM;
1863
1864         ret = idr_alloc(&zram_index_idr, zram, 0, 0, GFP_KERNEL);
1865         if (ret < 0)
1866                 goto out_free_dev;
1867         device_id = ret;
1868
1869         init_rwsem(&zram->init_lock);
1870
1871         queue = blk_alloc_queue(GFP_KERNEL);
1872         if (!queue) {
1873                 pr_err("Error allocating disk queue for device %d\n",
1874                         device_id);
1875                 ret = -ENOMEM;
1876                 goto out_free_idr;
1877         }
1878
1879         blk_queue_make_request(queue, zram_make_request);
1880
1881         /* gendisk structure */
1882         zram->disk = alloc_disk(1);
1883         if (!zram->disk) {
1884                 pr_err("Error allocating disk structure for device %d\n",
1885                         device_id);
1886                 ret = -ENOMEM;
1887                 goto out_free_queue;
1888         }
1889
1890         zram->disk->major = zram_major;
1891         zram->disk->first_minor = device_id;
1892         zram->disk->fops = &zram_devops;
1893         zram->disk->queue = queue;
1894         zram->disk->queue->queuedata = zram;
1895         zram->disk->private_data = zram;
1896         snprintf(zram->disk->disk_name, 16, "zram%d", device_id);
1897
1898         /* Actual capacity set using syfs (/sys/block/zram<id>/disksize */
1899         set_capacity(zram->disk, 0);
1900         /* zram devices sort of resembles non-rotational disks */
1901         blk_queue_flag_set(QUEUE_FLAG_NONROT, zram->disk->queue);
1902         blk_queue_flag_clear(QUEUE_FLAG_ADD_RANDOM, zram->disk->queue);
1903
1904         /*
1905          * To ensure that we always get PAGE_SIZE aligned
1906          * and n*PAGE_SIZED sized I/O requests.
1907          */
1908         blk_queue_physical_block_size(zram->disk->queue, PAGE_SIZE);
1909         blk_queue_logical_block_size(zram->disk->queue,
1910                                         ZRAM_LOGICAL_BLOCK_SIZE);
1911         blk_queue_io_min(zram->disk->queue, PAGE_SIZE);
1912         blk_queue_io_opt(zram->disk->queue, PAGE_SIZE);
1913         zram->disk->queue->limits.discard_granularity = PAGE_SIZE;
1914         blk_queue_max_discard_sectors(zram->disk->queue, UINT_MAX);
1915         blk_queue_flag_set(QUEUE_FLAG_DISCARD, zram->disk->queue);
1916
1917         /*
1918          * zram_bio_discard() will clear all logical blocks if logical block
1919          * size is identical with physical block size(PAGE_SIZE). But if it is
1920          * different, we will skip discarding some parts of logical blocks in
1921          * the part of the request range which isn't aligned to physical block
1922          * size.  So we can't ensure that all discarded logical blocks are
1923          * zeroed.
1924          */
1925         if (ZRAM_LOGICAL_BLOCK_SIZE == PAGE_SIZE)
1926                 blk_queue_max_write_zeroes_sectors(zram->disk->queue, UINT_MAX);
1927
1928         zram->disk->queue->backing_dev_info->capabilities |=
1929                         (BDI_CAP_STABLE_WRITES | BDI_CAP_SYNCHRONOUS_IO);
1930         device_add_disk(NULL, zram->disk, zram_disk_attr_groups);
1931
1932         strlcpy(zram->compressor, default_compressor, sizeof(zram->compressor));
1933
1934         zram_debugfs_register(zram);
1935         pr_info("Added device: %s\n", zram->disk->disk_name);
1936         return device_id;
1937
1938 out_free_queue:
1939         blk_cleanup_queue(queue);
1940 out_free_idr:
1941         idr_remove(&zram_index_idr, device_id);
1942 out_free_dev:
1943         kfree(zram);
1944         return ret;
1945 }
1946
1947 static int zram_remove(struct zram *zram)
1948 {
1949         struct block_device *bdev;
1950
1951         bdev = bdget_disk(zram->disk, 0);
1952         if (!bdev)
1953                 return -ENOMEM;
1954
1955         mutex_lock(&bdev->bd_mutex);
1956         if (bdev->bd_openers || zram->claim) {
1957                 mutex_unlock(&bdev->bd_mutex);
1958                 bdput(bdev);
1959                 return -EBUSY;
1960         }
1961
1962         zram->claim = true;
1963         mutex_unlock(&bdev->bd_mutex);
1964
1965         zram_debugfs_unregister(zram);
1966
1967         /* Make sure all the pending I/O are finished */
1968         fsync_bdev(bdev);
1969         zram_reset_device(zram);
1970         bdput(bdev);
1971
1972         pr_info("Removed device: %s\n", zram->disk->disk_name);
1973
1974         del_gendisk(zram->disk);
1975         blk_cleanup_queue(zram->disk->queue);
1976         put_disk(zram->disk);
1977         kfree(zram);
1978         return 0;
1979 }
1980
1981 /* zram-control sysfs attributes */
1982
1983 /*
1984  * NOTE: hot_add attribute is not the usual read-only sysfs attribute. In a
1985  * sense that reading from this file does alter the state of your system -- it
1986  * creates a new un-initialized zram device and returns back this device's
1987  * device_id (or an error code if it fails to create a new device).
1988  */
1989 static ssize_t hot_add_show(struct class *class,
1990                         struct class_attribute *attr,
1991                         char *buf)
1992 {
1993         int ret;
1994
1995         mutex_lock(&zram_index_mutex);
1996         ret = zram_add();
1997         mutex_unlock(&zram_index_mutex);
1998
1999         if (ret < 0)
2000                 return ret;
2001         return scnprintf(buf, PAGE_SIZE, "%d\n", ret);
2002 }
2003 static CLASS_ATTR_RO(hot_add);
2004
2005 static ssize_t hot_remove_store(struct class *class,
2006                         struct class_attribute *attr,
2007                         const char *buf,
2008                         size_t count)
2009 {
2010         struct zram *zram;
2011         int ret, dev_id;
2012
2013         /* dev_id is gendisk->first_minor, which is `int' */
2014         ret = kstrtoint(buf, 10, &dev_id);
2015         if (ret)
2016                 return ret;
2017         if (dev_id < 0)
2018                 return -EINVAL;
2019
2020         mutex_lock(&zram_index_mutex);
2021
2022         zram = idr_find(&zram_index_idr, dev_id);
2023         if (zram) {
2024                 ret = zram_remove(zram);
2025                 if (!ret)
2026                         idr_remove(&zram_index_idr, dev_id);
2027         } else {
2028                 ret = -ENODEV;
2029         }
2030
2031         mutex_unlock(&zram_index_mutex);
2032         return ret ? ret : count;
2033 }
2034 static CLASS_ATTR_WO(hot_remove);
2035
2036 static struct attribute *zram_control_class_attrs[] = {
2037         &class_attr_hot_add.attr,
2038         &class_attr_hot_remove.attr,
2039         NULL,
2040 };
2041 ATTRIBUTE_GROUPS(zram_control_class);
2042
2043 static struct class zram_control_class = {
2044         .name           = "zram-control",
2045         .owner          = THIS_MODULE,
2046         .class_groups   = zram_control_class_groups,
2047 };
2048
2049 static int zram_remove_cb(int id, void *ptr, void *data)
2050 {
2051         zram_remove(ptr);
2052         return 0;
2053 }
2054
2055 static void destroy_devices(void)
2056 {
2057         class_unregister(&zram_control_class);
2058         idr_for_each(&zram_index_idr, &zram_remove_cb, NULL);
2059         zram_debugfs_destroy();
2060         idr_destroy(&zram_index_idr);
2061         unregister_blkdev(zram_major, "zram");
2062         cpuhp_remove_multi_state(CPUHP_ZCOMP_PREPARE);
2063 }
2064
2065 static int __init zram_init(void)
2066 {
2067         int ret;
2068
2069         ret = cpuhp_setup_state_multi(CPUHP_ZCOMP_PREPARE, "block/zram:prepare",
2070                                       zcomp_cpu_up_prepare, zcomp_cpu_dead);
2071         if (ret < 0)
2072                 return ret;
2073
2074         ret = class_register(&zram_control_class);
2075         if (ret) {
2076                 pr_err("Unable to register zram-control class\n");
2077                 cpuhp_remove_multi_state(CPUHP_ZCOMP_PREPARE);
2078                 return ret;
2079         }
2080
2081         zram_debugfs_create();
2082         zram_major = register_blkdev(0, "zram");
2083         if (zram_major <= 0) {
2084                 pr_err("Unable to get major number\n");
2085                 class_unregister(&zram_control_class);
2086                 cpuhp_remove_multi_state(CPUHP_ZCOMP_PREPARE);
2087                 return -EBUSY;
2088         }
2089
2090         while (num_devices != 0) {
2091                 mutex_lock(&zram_index_mutex);
2092                 ret = zram_add();
2093                 mutex_unlock(&zram_index_mutex);
2094                 if (ret < 0)
2095                         goto out_error;
2096                 num_devices--;
2097         }
2098
2099         return 0;
2100
2101 out_error:
2102         destroy_devices();
2103         return ret;
2104 }
2105
2106 static void __exit zram_exit(void)
2107 {
2108         destroy_devices();
2109 }
2110
2111 module_init(zram_init);
2112 module_exit(zram_exit);
2113
2114 module_param(num_devices, uint, 0);
2115 MODULE_PARM_DESC(num_devices, "Number of pre-created zram devices");
2116
2117 MODULE_LICENSE("Dual BSD/GPL");
2118 MODULE_AUTHOR("Nitin Gupta <ngupta@vflare.org>");
2119 MODULE_DESCRIPTION("Compressed RAM Block Device");