1 // SPDX-License-Identifier: GPL-2.0-or-later
3 * Copyright (C) International Business Machines Corp., 2000-2005
4 * Portions Copyright (C) Christoph Hellwig, 2001-2002
9 #include <linux/module.h>
10 #include <linux/bio.h>
11 #include <linux/slab.h>
12 #include <linux/init.h>
13 #include <linux/buffer_head.h>
14 #include <linux/mempool.h>
15 #include <linux/seq_file.h>
16 #include <linux/writeback.h>
17 #include "jfs_incore.h"
18 #include "jfs_superblock.h"
19 #include "jfs_filsys.h"
20 #include "jfs_metapage.h"
21 #include "jfs_txnmgr.h"
22 #include "jfs_debug.h"
24 #ifdef CONFIG_JFS_STATISTICS
26 uint pagealloc; /* # of page allocations */
27 uint pagefree; /* # of page frees */
28 uint lockwait; /* # of sleeping lock_metapage() calls */
32 #define metapage_locked(mp) test_bit(META_locked, &(mp)->flag)
33 #define trylock_metapage(mp) test_and_set_bit_lock(META_locked, &(mp)->flag)
35 static inline void unlock_metapage(struct metapage *mp)
37 clear_bit_unlock(META_locked, &mp->flag);
41 static inline void __lock_metapage(struct metapage *mp)
43 DECLARE_WAITQUEUE(wait, current);
44 INCREMENT(mpStat.lockwait);
45 add_wait_queue_exclusive(&mp->wait, &wait);
47 set_current_state(TASK_UNINTERRUPTIBLE);
48 if (metapage_locked(mp)) {
49 unlock_page(mp->page);
53 } while (trylock_metapage(mp));
54 __set_current_state(TASK_RUNNING);
55 remove_wait_queue(&mp->wait, &wait);
59 * Must have mp->page locked
61 static inline void lock_metapage(struct metapage *mp)
63 if (trylock_metapage(mp))
67 #define METAPOOL_MIN_PAGES 32
68 static struct kmem_cache *metapage_cache;
69 static mempool_t *metapage_mempool;
71 #define MPS_PER_PAGE (PAGE_SIZE >> L2PSIZE)
78 struct metapage *mp[MPS_PER_PAGE];
80 #define mp_anchor(page) ((struct meta_anchor *)page_private(page))
82 static inline struct metapage *page_to_mp(struct page *page, int offset)
84 if (!PagePrivate(page))
86 return mp_anchor(page)->mp[offset >> L2PSIZE];
89 static inline int insert_metapage(struct page *page, struct metapage *mp)
91 struct meta_anchor *a;
93 int l2mp_blocks; /* log2 blocks per metapage */
95 if (PagePrivate(page))
98 a = kzalloc(sizeof(struct meta_anchor), GFP_NOFS);
101 set_page_private(page, (unsigned long)a);
102 SetPagePrivate(page);
107 l2mp_blocks = L2PSIZE - page->mapping->host->i_blkbits;
108 index = (mp->index >> l2mp_blocks) & (MPS_PER_PAGE - 1);
116 static inline void remove_metapage(struct page *page, struct metapage *mp)
118 struct meta_anchor *a = mp_anchor(page);
119 int l2mp_blocks = L2PSIZE - page->mapping->host->i_blkbits;
122 index = (mp->index >> l2mp_blocks) & (MPS_PER_PAGE - 1);
124 BUG_ON(a->mp[index] != mp);
127 if (--a->mp_count == 0) {
129 set_page_private(page, 0);
130 ClearPagePrivate(page);
135 static inline void inc_io(struct page *page)
137 atomic_inc(&mp_anchor(page)->io_count);
140 static inline void dec_io(struct page *page, void (*handler) (struct page *))
142 if (atomic_dec_and_test(&mp_anchor(page)->io_count))
147 static inline struct metapage *page_to_mp(struct page *page, int offset)
149 return PagePrivate(page) ? (struct metapage *)page_private(page) : NULL;
152 static inline int insert_metapage(struct page *page, struct metapage *mp)
155 set_page_private(page, (unsigned long)mp);
156 SetPagePrivate(page);
162 static inline void remove_metapage(struct page *page, struct metapage *mp)
164 set_page_private(page, 0);
165 ClearPagePrivate(page);
169 #define inc_io(page) do {} while(0)
170 #define dec_io(page, handler) handler(page)
174 static inline struct metapage *alloc_metapage(gfp_t gfp_mask)
176 struct metapage *mp = mempool_alloc(metapage_mempool, gfp_mask);
184 init_waitqueue_head(&mp->wait);
189 static inline void free_metapage(struct metapage *mp)
191 mempool_free(mp, metapage_mempool);
194 int __init metapage_init(void)
197 * Allocate the metapage structures
199 metapage_cache = kmem_cache_create("jfs_mp", sizeof(struct metapage),
201 if (metapage_cache == NULL)
204 metapage_mempool = mempool_create_slab_pool(METAPOOL_MIN_PAGES,
207 if (metapage_mempool == NULL) {
208 kmem_cache_destroy(metapage_cache);
215 void metapage_exit(void)
217 mempool_destroy(metapage_mempool);
218 kmem_cache_destroy(metapage_cache);
221 static inline void drop_metapage(struct page *page, struct metapage *mp)
223 if (mp->count || mp->nohomeok || test_bit(META_dirty, &mp->flag) ||
224 test_bit(META_io, &mp->flag))
226 remove_metapage(page, mp);
227 INCREMENT(mpStat.pagefree);
232 * Metapage address space operations
235 static sector_t metapage_get_blocks(struct inode *inode, sector_t lblock,
241 sector_t file_blocks = (inode->i_size + inode->i_sb->s_blocksize - 1) >>
244 if (lblock >= file_blocks)
246 if (lblock + *len > file_blocks)
247 *len = file_blocks - lblock;
250 rc = xtLookup(inode, (s64)lblock, *len, &xflag, &xaddr, len, 0);
251 if ((rc == 0) && *len)
252 lblock = (sector_t)xaddr;
255 } /* else no mapping */
260 static void last_read_complete(struct page *page)
262 if (!PageError(page))
263 SetPageUptodate(page);
267 static void metapage_read_end_io(struct bio *bio)
269 struct page *page = bio->bi_private;
271 if (bio->bi_status) {
272 printk(KERN_ERR "metapage_read_end_io: I/O error\n");
276 dec_io(page, last_read_complete);
280 static void remove_from_logsync(struct metapage *mp)
282 struct jfs_log *log = mp->log;
285 * This can race. Recheck that log hasn't been set to null, and after
286 * acquiring logsync lock, recheck lsn
291 LOGSYNC_LOCK(log, flags);
297 list_del(&mp->synclist);
299 LOGSYNC_UNLOCK(log, flags);
302 static void last_write_complete(struct page *page)
307 for (offset = 0; offset < PAGE_SIZE; offset += PSIZE) {
308 mp = page_to_mp(page, offset);
309 if (mp && test_bit(META_io, &mp->flag)) {
311 remove_from_logsync(mp);
312 clear_bit(META_io, &mp->flag);
315 * I'd like to call drop_metapage here, but I don't think it's
316 * safe unless I have the page locked
319 end_page_writeback(page);
322 static void metapage_write_end_io(struct bio *bio)
324 struct page *page = bio->bi_private;
326 BUG_ON(!PagePrivate(page));
328 if (bio->bi_status) {
329 printk(KERN_ERR "metapage_write_end_io: I/O error\n");
332 dec_io(page, last_write_complete);
336 static int metapage_writepage(struct page *page, struct writeback_control *wbc)
338 struct bio *bio = NULL;
339 int block_offset; /* block offset of mp within page */
340 struct inode *inode = page->mapping->host;
341 int blocks_per_mp = JFS_SBI(inode->i_sb)->nbperpage;
349 sector_t next_block = 0;
351 unsigned long bio_bytes = 0;
352 unsigned long bio_offset = 0;
356 page_start = (sector_t)page->index <<
357 (PAGE_SHIFT - inode->i_blkbits);
358 BUG_ON(!PageLocked(page));
359 BUG_ON(PageWriteback(page));
360 set_page_writeback(page);
362 for (offset = 0; offset < PAGE_SIZE; offset += PSIZE) {
363 mp = page_to_mp(page, offset);
365 if (!mp || !test_bit(META_dirty, &mp->flag))
368 if (mp->nohomeok && !test_bit(META_forcewrite, &mp->flag)) {
371 * Make sure this page isn't blocked indefinitely.
372 * If the journal isn't undergoing I/O, push it
374 if (mp->log && !(mp->log->cflag & logGC_PAGEOUT))
375 jfs_flush_journal(mp->log, 0);
379 clear_bit(META_dirty, &mp->flag);
380 set_bit(META_io, &mp->flag);
381 block_offset = offset >> inode->i_blkbits;
382 lblock = page_start + block_offset;
384 if (xlen && lblock == next_block) {
385 /* Contiguous, in memory & on disk */
386 len = min(xlen, blocks_per_mp);
388 bio_bytes += len << inode->i_blkbits;
392 if (bio_add_page(bio, page, bio_bytes, bio_offset) <
396 * Increment counter before submitting i/o to keep
397 * count from hitting zero before we're through
400 if (!bio->bi_iter.bi_size)
407 xlen = (PAGE_SIZE - offset) >> inode->i_blkbits;
408 pblock = metapage_get_blocks(inode, lblock, &xlen);
410 printk(KERN_ERR "JFS: metapage_get_blocks failed\n");
412 * We already called inc_io(), but can't cancel it
413 * with dec_io() until we're done with the page
418 len = min(xlen, (int)JFS_SBI(inode->i_sb)->nbperpage);
420 bio = bio_alloc(inode->i_sb->s_bdev, 1, REQ_OP_WRITE, GFP_NOFS);
421 bio->bi_iter.bi_sector = pblock << (inode->i_blkbits - 9);
422 bio->bi_end_io = metapage_write_end_io;
423 bio->bi_private = page;
425 /* Don't call bio_add_page yet, we may add to this vec */
427 bio_bytes = len << inode->i_blkbits;
430 next_block = lblock + len;
433 if (bio_add_page(bio, page, bio_bytes, bio_offset) < bio_bytes)
435 if (!bio->bi_iter.bi_size)
442 redirty_page_for_writepage(wbc, page);
449 if (nr_underway == 0)
450 end_page_writeback(page);
454 /* We should never reach here, since we're only adding one vec */
455 printk(KERN_ERR "JFS: bio_add_page failed unexpectedly\n");
458 print_hex_dump(KERN_ERR, "JFS: dump of bio: ", DUMP_PREFIX_ADDRESS, 16,
459 4, bio, sizeof(*bio), 0);
463 dec_io(page, last_write_complete);
466 dec_io(page, last_write_complete);
470 static int metapage_readpage(struct file *fp, struct page *page)
472 struct inode *inode = page->mapping->host;
473 struct bio *bio = NULL;
475 int blocks_per_page = i_blocks_per_page(inode, page);
476 sector_t page_start; /* address of page in fs blocks */
482 BUG_ON(!PageLocked(page));
483 page_start = (sector_t)page->index <<
484 (PAGE_SHIFT - inode->i_blkbits);
487 while (block_offset < blocks_per_page) {
488 xlen = blocks_per_page - block_offset;
489 pblock = metapage_get_blocks(inode, page_start + block_offset,
492 if (!PagePrivate(page))
493 insert_metapage(page, NULL);
498 bio = bio_alloc(inode->i_sb->s_bdev, 1, REQ_OP_READ,
500 bio->bi_iter.bi_sector =
501 pblock << (inode->i_blkbits - 9);
502 bio->bi_end_io = metapage_read_end_io;
503 bio->bi_private = page;
504 len = xlen << inode->i_blkbits;
505 offset = block_offset << inode->i_blkbits;
506 if (bio_add_page(bio, page, len, offset) < len)
508 block_offset += xlen;
520 printk(KERN_ERR "JFS: bio_add_page failed unexpectedly\n");
522 dec_io(page, last_read_complete);
526 static int metapage_releasepage(struct page *page, gfp_t gfp_mask)
532 for (offset = 0; offset < PAGE_SIZE; offset += PSIZE) {
533 mp = page_to_mp(page, offset);
538 jfs_info("metapage_releasepage: mp = 0x%p", mp);
539 if (mp->count || mp->nohomeok ||
540 test_bit(META_dirty, &mp->flag)) {
541 jfs_info("count = %ld, nohomeok = %d", mp->count,
547 remove_from_logsync(mp);
548 remove_metapage(page, mp);
549 INCREMENT(mpStat.pagefree);
555 static void metapage_invalidate_folio(struct folio *folio, size_t offset,
558 BUG_ON(offset || length < folio_size(folio));
560 BUG_ON(folio_test_writeback(folio));
562 metapage_releasepage(&folio->page, 0);
565 const struct address_space_operations jfs_metapage_aops = {
566 .readpage = metapage_readpage,
567 .writepage = metapage_writepage,
568 .releasepage = metapage_releasepage,
569 .invalidate_folio = metapage_invalidate_folio,
570 .dirty_folio = filemap_dirty_folio,
573 struct metapage *__get_metapage(struct inode *inode, unsigned long lblock,
574 unsigned int size, int absolute,
579 struct address_space *mapping;
580 struct metapage *mp = NULL;
582 unsigned long page_index;
583 unsigned long page_offset;
585 jfs_info("__get_metapage: ino = %ld, lblock = 0x%lx, abs=%d",
586 inode->i_ino, lblock, absolute);
588 l2bsize = inode->i_blkbits;
589 l2BlocksPerPage = PAGE_SHIFT - l2bsize;
590 page_index = lblock >> l2BlocksPerPage;
591 page_offset = (lblock - (page_index << l2BlocksPerPage)) << l2bsize;
592 if ((page_offset + size) > PAGE_SIZE) {
593 jfs_err("MetaData crosses page boundary!!");
594 jfs_err("lblock = %lx, size = %d", lblock, size);
599 mapping = JFS_SBI(inode->i_sb)->direct_inode->i_mapping;
602 * If an nfs client tries to read an inode that is larger
603 * than any existing inodes, we may try to read past the
604 * end of the inode map
606 if ((lblock << inode->i_blkbits) >= inode->i_size)
608 mapping = inode->i_mapping;
611 if (new && (PSIZE == PAGE_SIZE)) {
612 page = grab_cache_page(mapping, page_index);
614 jfs_err("grab_cache_page failed!");
617 SetPageUptodate(page);
619 page = read_mapping_page(mapping, page_index, NULL);
620 if (IS_ERR(page) || !PageUptodate(page)) {
621 jfs_err("read_mapping_page failed!");
627 mp = page_to_mp(page, page_offset);
629 if (mp->logical_size != size) {
630 jfs_error(inode->i_sb,
631 "get_mp->logical_size != size\n");
632 jfs_err("logical_size = %d, size = %d",
633 mp->logical_size, size);
639 if (test_bit(META_discard, &mp->flag)) {
641 jfs_error(inode->i_sb,
642 "using a discarded metapage\n");
643 discard_metapage(mp);
646 clear_bit(META_discard, &mp->flag);
649 INCREMENT(mpStat.pagealloc);
650 mp = alloc_metapage(GFP_NOFS);
654 mp->sb = inode->i_sb;
656 mp->xflag = COMMIT_PAGE;
659 mp->logical_size = size;
660 mp->data = page_address(page) + page_offset;
662 if (unlikely(insert_metapage(page, mp))) {
670 jfs_info("zeroing mp = 0x%p", mp);
671 memset(mp->data, 0, PSIZE);
675 jfs_info("__get_metapage: returning = 0x%p data = 0x%p", mp, mp->data);
683 void grab_metapage(struct metapage * mp)
685 jfs_info("grab_metapage: mp = 0x%p", mp);
690 unlock_page(mp->page);
693 void force_metapage(struct metapage *mp)
695 struct page *page = mp->page;
696 jfs_info("force_metapage: mp = 0x%p", mp);
697 set_bit(META_forcewrite, &mp->flag);
698 clear_bit(META_sync, &mp->flag);
701 set_page_dirty(page);
702 if (write_one_page(page))
703 jfs_error(mp->sb, "write_one_page() failed\n");
704 clear_bit(META_forcewrite, &mp->flag);
708 void hold_metapage(struct metapage *mp)
713 void put_metapage(struct metapage *mp)
715 if (mp->count || mp->nohomeok) {
716 /* Someone else will release this */
717 unlock_page(mp->page);
723 unlock_page(mp->page);
724 release_metapage(mp);
727 void release_metapage(struct metapage * mp)
729 struct page *page = mp->page;
730 jfs_info("release_metapage: mp = 0x%p, flag = 0x%lx", mp, mp->flag);
738 if (--mp->count || mp->nohomeok) {
744 if (test_bit(META_dirty, &mp->flag)) {
745 set_page_dirty(page);
746 if (test_bit(META_sync, &mp->flag)) {
747 clear_bit(META_sync, &mp->flag);
748 if (write_one_page(page))
749 jfs_error(mp->sb, "write_one_page() failed\n");
750 lock_page(page); /* write_one_page unlocks the page */
752 } else if (mp->lsn) /* discard_metapage doesn't remove it */
753 remove_from_logsync(mp);
755 /* Try to keep metapages from using up too much memory */
756 drop_metapage(page, mp);
762 void __invalidate_metapages(struct inode *ip, s64 addr, int len)
765 int l2BlocksPerPage = PAGE_SHIFT - ip->i_blkbits;
766 int BlocksPerPage = 1 << l2BlocksPerPage;
767 /* All callers are interested in block device's mapping */
768 struct address_space *mapping =
769 JFS_SBI(ip->i_sb)->direct_inode->i_mapping;
775 * Mark metapages to discard. They will eventually be
776 * released, but should not be written.
778 for (lblock = addr & ~(BlocksPerPage - 1); lblock < addr + len;
779 lblock += BlocksPerPage) {
780 page = find_lock_page(mapping, lblock >> l2BlocksPerPage);
783 for (offset = 0; offset < PAGE_SIZE; offset += PSIZE) {
784 mp = page_to_mp(page, offset);
787 if (mp->index < addr)
789 if (mp->index >= addr + len)
792 clear_bit(META_dirty, &mp->flag);
793 set_bit(META_discard, &mp->flag);
795 remove_from_logsync(mp);
802 #ifdef CONFIG_JFS_STATISTICS
803 int jfs_mpstat_proc_show(struct seq_file *m, void *v)
806 "JFS Metapage statistics\n"
807 "=======================\n"
808 "page allocations = %d\n"