Merge tag 'driver-core-5.18-rc1' of git://git.kernel.org/pub/scm/linux/kernel/git...

[uclinux-h8/linux.git] / mm / page_alloc.c

diff --git a/mm/page_alloc.c b/mm/page_alloc.c
index 3da9594..bdc8f60 100644 (file)
--- a/mm/page_alloc.c
+++ b/mm/page_alloc.c
@@ -1297,7 +1297,6 @@ static __always_inline bool free_pages_prepare(struct page *page,
                        unsigned int order, bool check_free, fpi_t fpi_flags)
 {
        int bad = 0;
-       bool skip_kasan_poison = should_skip_kasan_poison(page, fpi_flags);
        bool init = want_init_on_free();
 
        VM_BUG_ON_PAGE(PageTail(page), page);
@@ -1371,7 +1370,7 @@ static __always_inline bool free_pages_prepare(struct page *page,
         * With hardware tag-based KASAN, memory tags must be set before the
         * page becomes unavailable via debug_pagealloc or arch_free_page.
         */
-       if (!skip_kasan_poison) {
+       if (!should_skip_kasan_poison(page, fpi_flags)) {
                kasan_poison_pages(page, order, init);
 
                /* Memory is already initialized if KASAN did it internally. */
@@ -2344,9 +2343,43 @@ static inline bool check_new_pcp(struct page *page, unsigned int order)
 }
 #endif /* CONFIG_DEBUG_VM */
 
+static inline bool should_skip_kasan_unpoison(gfp_t flags, bool init_tags)
+{
+       /* Don't skip if a software KASAN mode is enabled. */
+       if (IS_ENABLED(CONFIG_KASAN_GENERIC) ||
+           IS_ENABLED(CONFIG_KASAN_SW_TAGS))
+               return false;
+
+       /* Skip, if hardware tag-based KASAN is not enabled. */
+       if (!kasan_hw_tags_enabled())
+               return true;
+
+       /*
+        * With hardware tag-based KASAN enabled, skip if either:
+        *
+        * 1. Memory tags have already been cleared via tag_clear_highpage().
+        * 2. Skipping has been requested via __GFP_SKIP_KASAN_UNPOISON.
+        */
+       return init_tags || (flags & __GFP_SKIP_KASAN_UNPOISON);
+}
+
+static inline bool should_skip_init(gfp_t flags)
+{
+       /* Don't skip, if hardware tag-based KASAN is not enabled. */
+       if (!kasan_hw_tags_enabled())
+               return false;
+
+       /* For hardware tag-based KASAN, skip if requested. */
+       return (flags & __GFP_SKIP_ZERO);
+}
+
 inline void post_alloc_hook(struct page *page, unsigned int order,
                                gfp_t gfp_flags)
 {
+       bool init = !want_init_on_free() && want_init_on_alloc(gfp_flags) &&
+                       !should_skip_init(gfp_flags);
+       bool init_tags = init && (gfp_flags & __GFP_ZEROTAGS);
+
        set_page_private(page, 0);
        set_page_refcounted(page);
 
@@ -2362,27 +2395,38 @@ inline void post_alloc_hook(struct page *page, unsigned int order,
 
        /*
         * As memory initialization might be integrated into KASAN,
-        * kasan_alloc_pages and kernel_init_free_pages must be
+        * KASAN unpoisoning and memory initializion code must be
         * kept together to avoid discrepancies in behavior.
         */
-       if (kasan_has_integrated_init()) {
-               kasan_alloc_pages(page, order, gfp_flags);
-       } else {
-               bool init = !want_init_on_free() && want_init_on_alloc(gfp_flags);
 
-               kasan_unpoison_pages(page, order, init);
+       /*
+        * If memory tags should be zeroed (which happens only when memory
+        * should be initialized as well).
+        */
+       if (init_tags) {
+               int i;
 
-               if (init) {
-                       if (gfp_flags & __GFP_ZEROTAGS) {
-                               int i;
+               /* Initialize both memory and tags. */
+               for (i = 0; i != 1 << order; ++i)
+                       tag_clear_highpage(page + i);
 
-                               for (i = 0; i < 1 << order; i++)
-                                       tag_clear_highpage(page + i);
-                       } else {
-                               kernel_init_free_pages(page, 1 << order);
-                       }
-               }
+               /* Note that memory is already initialized by the loop above. */
+               init = false;
        }
+       if (!should_skip_kasan_unpoison(gfp_flags, init_tags)) {
+               /* Unpoison shadow memory or set memory tags. */
+               kasan_unpoison_pages(page, order, init);
+
+               /* Note that memory is already initialized by KASAN. */
+               if (kasan_has_integrated_init())
+                       init = false;
+       }
+       /* If memory is still not initialized, do it now. */
+       if (init)
+               kernel_init_free_pages(page, 1 << order);
+       /* Propagate __GFP_SKIP_KASAN_POISON to page flags. */
+       if (kasan_hw_tags_enabled() && (gfp_flags & __GFP_SKIP_KASAN_POISON))
+               SetPageSkipKASanPoison(page);
 
        set_page_owner(page, order, gfp_flags);
        page_table_check_alloc(page, order);