mm: perform async writes to SWP_FS_OPS swap-space using ->swap_rw

This patch switches swap-out to SWP_FS_OPS swap-spaces to use ->swap_rw
and makes the writes asynchronous, like they are for other swap spaces.

To make it async we need to allocate the kiocb struct from a mempool.
This may block, but won't block as long as waiting for the write to
complete.  At most it will wait for some previous swap IO to complete.

Link: https://lkml.kernel.org/r/164859778126.29473.12399585304843922231.stgit@noble.brown
Signed-off-by: NeilBrown <neilb@suse.de>
Reviewed-by: Christoph Hellwig <hch@lst.de>
Tested-by: David Howells <dhowells@redhat.com>
Tested-by: Geert Uytterhoeven <geert+renesas@glider.be>
Cc: Hugh Dickins <hughd@google.com>
Cc: Mel Gorman <mgorman@techsingularity.net>
Cc: Trond Myklebust <trond.myklebust@hammerspace.com>
Cc: Miaohe Lin <linmiaohe@huawei.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>

diff --git a/mm/page_io.c b/mm/page_io.c
index 2724d63..fafbda3 100644 (file)
--- a/mm/page_io.c
+++ b/mm/page_io.c
@@ -254,6 +254,57 @@ int sio_pool_init(void)
        return 0;
 }
 
+static void sio_write_complete(struct kiocb *iocb, long ret)
+{
+       struct swap_iocb *sio = container_of(iocb, struct swap_iocb, iocb);
+       struct page *page = sio->bvec.bv_page;
+
+       if (ret != PAGE_SIZE) {
+               /*
+                * In the case of swap-over-nfs, this can be a
+                * temporary failure if the system has limited
+                * memory for allocating transmit buffers.
+                * Mark the page dirty and avoid
+                * folio_rotate_reclaimable but rate-limit the
+                * messages but do not flag PageError like
+                * the normal direct-to-bio case as it could
+                * be temporary.
+                */
+               set_page_dirty(page);
+               ClearPageReclaim(page);
+               pr_err_ratelimited("Write error %ld on dio swapfile (%llu)\n",
+                                  ret, page_file_offset(page));
+       } else
+               count_vm_event(PSWPOUT);
+       end_page_writeback(page);
+       mempool_free(sio, sio_pool);
+}
+
+static int swap_writepage_fs(struct page *page, struct writeback_control *wbc)
+{
+       struct swap_iocb *sio;
+       struct swap_info_struct *sis = page_swap_info(page);
+       struct file *swap_file = sis->swap_file;
+       struct address_space *mapping = swap_file->f_mapping;
+       struct iov_iter from;
+       int ret;
+
+       set_page_writeback(page);
+       unlock_page(page);
+       sio = mempool_alloc(sio_pool, GFP_NOIO);
+       init_sync_kiocb(&sio->iocb, swap_file);
+       sio->iocb.ki_complete = sio_write_complete;
+       sio->iocb.ki_pos = page_file_offset(page);
+       sio->bvec.bv_page = page;
+       sio->bvec.bv_len = PAGE_SIZE;
+       sio->bvec.bv_offset = 0;
+       iov_iter_bvec(&from, WRITE, &sio->bvec, 1, PAGE_SIZE);
+       ret = mapping->a_ops->swap_rw(&sio->iocb, &from);
+       if (ret != -EIOCBQUEUED)
+               sio_write_complete(&sio->iocb, ret);
+       return ret;
+}
+
 int __swap_writepage(struct page *page, struct writeback_control *wbc,
                bio_end_io_t end_write_func)
 {
@@ -262,46 +313,13 @@ int __swap_writepage(struct page *page, struct writeback_control *wbc,
        struct swap_info_struct *sis = page_swap_info(page);
 
        VM_BUG_ON_PAGE(!PageSwapCache(page), page);
-       if (data_race(sis->flags & SWP_FS_OPS)) {
-               struct kiocb kiocb;
-               struct file *swap_file = sis->swap_file;
-               struct address_space *mapping = swap_file->f_mapping;
-               struct bio_vec bv = {
-                       .bv_page = page,
-                       .bv_len  = PAGE_SIZE,
-                       .bv_offset = 0
-               };
-               struct iov_iter from;
-
-               iov_iter_bvec(&from, WRITE, &bv, 1, PAGE_SIZE);
-               init_sync_kiocb(&kiocb, swap_file);
-               kiocb.ki_pos = page_file_offset(page);
-
-               set_page_writeback(page);
-               unlock_page(page);
-               ret = mapping->a_ops->direct_IO(&kiocb, &from);
-               if (ret == PAGE_SIZE) {
-                       count_vm_event(PSWPOUT);
-                       ret = 0;
-               } else {
-                       /*
-                        * In the case of swap-over-nfs, this can be a
-                        * temporary failure if the system has limited
-                        * memory for allocating transmit buffers.
-                        * Mark the page dirty and avoid
-                        * folio_rotate_reclaimable but rate-limit the
-                        * messages but do not flag PageError like
-                        * the normal direct-to-bio case as it could
-                        * be temporary.
-                        */
-                       set_page_dirty(page);
-                       ClearPageReclaim(page);
-                       pr_err_ratelimited("Write error on dio swapfile (%llu)\n",
-                                          page_file_offset(page));
-               }
-               end_page_writeback(page);
-               return ret;
-       }
+       /*
+        * ->flags can be updated non-atomicially (scan_swap_map_slots),
+        * but that will never affect SWP_FS_OPS, so the data_race
+        * is safe.
+        */
+       if (data_race(sis->flags & SWP_FS_OPS))
+               return swap_writepage_fs(page, wbc);
 
        ret = bdev_write_page(sis->bdev, swap_page_sector(page), page, wbc);
        if (!ret) {