mm: introduce get_user_pages_longterm

[tomoyo/tomoyo-test1.git] / mm / huge_memory.c

diff --git a/mm/huge_memory.c b/mm/huge_memory.c
index 86fe697..05b729f 100644 (file)
--- a/mm/huge_memory.c
+++ b/mm/huge_memory.c
@@ -474,10 +474,13 @@ out:
 }
 __setup("transparent_hugepage=", setup_transparent_hugepage);
 
-pmd_t maybe_pmd_mkwrite(pmd_t pmd, struct vm_area_struct *vma)
+pmd_t maybe_pmd_mkwrite(pmd_t pmd, struct vm_area_struct *vma, bool dirty)
 {
-       if (likely(vma->vm_flags & VM_WRITE))
+       if (likely(vma->vm_flags & VM_WRITE)) {
                pmd = pmd_mkwrite(pmd);
+               if (dirty)
+                       pmd = pmd_mkdirty(pmd);
+       }
        return pmd;
 }
 
@@ -599,7 +602,7 @@ static int __do_huge_pmd_anonymous_page(struct vm_fault *vmf, struct page *page,
                }
 
                entry = mk_huge_pmd(page, vma->vm_page_prot);
-               entry = maybe_pmd_mkwrite(pmd_mkdirty(entry), vma);
+               entry = maybe_pmd_mkwrite(entry, vma, true);
                page_add_new_anon_rmap(page, vma, haddr, true);
                mem_cgroup_commit_charge(page, memcg, false, true);
                lru_cache_add_active_or_unevictable(page, vma);
@@ -741,8 +744,8 @@ static void insert_pfn_pmd(struct vm_area_struct *vma, unsigned long addr,
        if (pfn_t_devmap(pfn))
                entry = pmd_mkdevmap(entry);
        if (write) {
-               entry = pmd_mkyoung(pmd_mkdirty(entry));
-               entry = maybe_pmd_mkwrite(entry, vma);
+               entry = pmd_mkyoung(entry);
+               entry = maybe_pmd_mkwrite(entry, vma, true);
        }
 
        if (pgtable) {
@@ -788,10 +791,14 @@ int vmf_insert_pfn_pmd(struct vm_area_struct *vma, unsigned long addr,
 EXPORT_SYMBOL_GPL(vmf_insert_pfn_pmd);
 
 #ifdef CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD
-static pud_t maybe_pud_mkwrite(pud_t pud, struct vm_area_struct *vma)
+static pud_t maybe_pud_mkwrite(pud_t pud, struct vm_area_struct *vma,
+               bool dirty)
 {
-       if (likely(vma->vm_flags & VM_WRITE))
+       if (likely(vma->vm_flags & VM_WRITE)) {
                pud = pud_mkwrite(pud);
+               if (dirty)
+                       pud = pud_mkdirty(pud);
+       }
        return pud;
 }
 
@@ -807,8 +814,8 @@ static void insert_pfn_pud(struct vm_area_struct *vma, unsigned long addr,
        if (pfn_t_devmap(pfn))
                entry = pud_mkdevmap(entry);
        if (write) {
-               entry = pud_mkyoung(pud_mkdirty(entry));
-               entry = maybe_pud_mkwrite(entry, vma);
+               entry = pud_mkyoung(entry);
+               entry = maybe_pud_mkwrite(entry, vma, true);
        }
        set_pud_at(mm, addr, pud, entry);
        update_mmu_cache_pud(vma, addr, pud);
@@ -842,20 +849,15 @@ EXPORT_SYMBOL_GPL(vmf_insert_pfn_pud);
 #endif /* CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD */
 
 static void touch_pmd(struct vm_area_struct *vma, unsigned long addr,
-               pmd_t *pmd)
+               pmd_t *pmd, int flags)
 {
        pmd_t _pmd;
 
-       /*
-        * We should set the dirty bit only for FOLL_WRITE but for now
-        * the dirty bit in the pmd is meaningless.  And if the dirty
-        * bit will become meaningful and we'll only set it with
-        * FOLL_WRITE, an atomic set_bit will be required on the pmd to
-        * set the young bit, instead of the current set_pmd_at.
-        */
-       _pmd = pmd_mkyoung(pmd_mkdirty(*pmd));
+       _pmd = pmd_mkyoung(*pmd);
+       if (flags & FOLL_WRITE)
+               _pmd = pmd_mkdirty(_pmd);
        if (pmdp_set_access_flags(vma, addr & HPAGE_PMD_MASK,
-                               pmd, _pmd,  1))
+                               pmd, _pmd, flags & FOLL_WRITE))
                update_mmu_cache_pmd(vma, addr, pmd);
 }
 
@@ -875,7 +877,7 @@ struct page *follow_devmap_pmd(struct vm_area_struct *vma, unsigned long addr,
         */
        WARN_ONCE(flags & FOLL_COW, "mm: In follow_devmap_pmd with FOLL_COW set");
 
-       if (flags & FOLL_WRITE && !pmd_write(*pmd))
+       if (!pmd_access_permitted(*pmd, flags & FOLL_WRITE))
                return NULL;
 
        if (pmd_present(*pmd) && pmd_devmap(*pmd))
@@ -884,7 +886,7 @@ struct page *follow_devmap_pmd(struct vm_area_struct *vma, unsigned long addr,
                return NULL;
 
        if (flags & FOLL_TOUCH)
-               touch_pmd(vma, addr, pmd);
+               touch_pmd(vma, addr, pmd, flags);
 
        /*
         * device mapped pages can only be returned if the
@@ -995,20 +997,15 @@ out:
 
 #ifdef CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD
 static void touch_pud(struct vm_area_struct *vma, unsigned long addr,
-               pud_t *pud)
+               pud_t *pud, int flags)
 {
        pud_t _pud;
 
-       /*
-        * We should set the dirty bit only for FOLL_WRITE but for now
-        * the dirty bit in the pud is meaningless.  And if the dirty
-        * bit will become meaningful and we'll only set it with
-        * FOLL_WRITE, an atomic set_bit will be required on the pud to
-        * set the young bit, instead of the current set_pud_at.
-        */
-       _pud = pud_mkyoung(pud_mkdirty(*pud));
+       _pud = pud_mkyoung(*pud);
+       if (flags & FOLL_WRITE)
+               _pud = pud_mkdirty(_pud);
        if (pudp_set_access_flags(vma, addr & HPAGE_PUD_MASK,
-                               pud, _pud,  1))
+                               pud, _pud, flags & FOLL_WRITE))
                update_mmu_cache_pud(vma, addr, pud);
 }
 
@@ -1022,7 +1019,7 @@ struct page *follow_devmap_pud(struct vm_area_struct *vma, unsigned long addr,
 
        assert_spin_locked(pud_lockptr(mm, pud));
 
-       if (flags & FOLL_WRITE && !pud_write(*pud))
+       if (!pud_access_permitted(*pud, flags & FOLL_WRITE))
                return NULL;
 
        if (pud_present(*pud) && pud_devmap(*pud))
@@ -1031,7 +1028,7 @@ struct page *follow_devmap_pud(struct vm_area_struct *vma, unsigned long addr,
                return NULL;
 
        if (flags & FOLL_TOUCH)
-               touch_pud(vma, addr, pud);
+               touch_pud(vma, addr, pud, flags);
 
        /*
         * device mapped pages can only be returned if the
@@ -1289,7 +1286,7 @@ int do_huge_pmd_wp_page(struct vm_fault *vmf, pmd_t orig_pmd)
        if (reuse_swap_page(page, NULL)) {
                pmd_t entry;
                entry = pmd_mkyoung(orig_pmd);
-               entry = maybe_pmd_mkwrite(pmd_mkdirty(entry), vma);
+               entry = maybe_pmd_mkwrite(entry, vma, true);
                if (pmdp_set_access_flags(vma, haddr, vmf->pmd, entry,  1))
                        update_mmu_cache_pmd(vma, vmf->address, vmf->pmd);
                ret |= VM_FAULT_WRITE;
@@ -1359,7 +1356,7 @@ alloc:
        } else {
                pmd_t entry;
                entry = mk_huge_pmd(new_page, vma->vm_page_prot);
-               entry = maybe_pmd_mkwrite(pmd_mkdirty(entry), vma);
+               entry = maybe_pmd_mkwrite(entry, vma, true);
                pmdp_huge_clear_flush_notify(vma, haddr, vmf->pmd);
                page_add_new_anon_rmap(new_page, vma, haddr, true);
                mem_cgroup_commit_charge(new_page, memcg, false, true);
@@ -1396,7 +1393,7 @@ out_unlock:
  */
 static inline bool can_follow_write_pmd(pmd_t pmd, unsigned int flags)
 {
-       return pmd_write(pmd) ||
+       return pmd_access_permitted(pmd, WRITE) ||
               ((flags & FOLL_FORCE) && (flags & FOLL_COW) && pmd_dirty(pmd));
 }
 
@@ -1424,7 +1421,7 @@ struct page *follow_trans_huge_pmd(struct vm_area_struct *vma,
        page = pmd_page(*pmd);
        VM_BUG_ON_PAGE(!PageHead(page) && !is_zone_device_page(page), page);
        if (flags & FOLL_TOUCH)
-               touch_pmd(vma, addr, pmd);
+               touch_pmd(vma, addr, pmd, flags);
        if ((flags & FOLL_MLOCK) && (vma->vm_flags & VM_LOCKED)) {
                /*
                 * We don't mlock() pte-mapped THPs. This way we can avoid
@@ -2938,7 +2935,7 @@ void remove_migration_pmd(struct page_vma_mapped_walk *pvmw, struct page *new)
        if (pmd_swp_soft_dirty(*pvmw->pmd))
                pmde = pmd_mksoft_dirty(pmde);
        if (is_write_migration_entry(entry))
-               pmde = maybe_pmd_mkwrite(pmde, vma);
+               pmde = maybe_pmd_mkwrite(pmde, vma, false);
 
        flush_cache_range(vma, mmun_start, mmun_start + HPAGE_PMD_SIZE);
        page_add_anon_rmap(new, vma, mmun_start, true);