Merge 4.4.163 into android-4.4

[sagit-ice-cold/kernel_xiaomi_msm8998.git] / arch / x86 / mm / pageattr.c

diff --git a/arch/x86/mm/pageattr.c b/arch/x86/mm/pageattr.c
index a0fe62e..0a5b272 100644 (file)
--- a/arch/x86/mm/pageattr.c
+++ b/arch/x86/mm/pageattr.c
@@ -52,6 +52,7 @@ static DEFINE_SPINLOCK(cpa_lock);
 #define CPA_FLUSHTLB 1
 #define CPA_ARRAY 2
 #define CPA_PAGES_ARRAY 4
+#define CPA_FREE_PAGETABLES 8
 
 #ifdef CONFIG_PROC_FS
 static unsigned long direct_pages_count[PG_LEVEL_NUM];
@@ -278,7 +279,7 @@ static inline pgprot_t static_protections(pgprot_t prot, unsigned long address,
                   __pa_symbol(__end_rodata) >> PAGE_SHIFT))
                pgprot_val(forbidden) |= _PAGE_RW;
 
-#if defined(CONFIG_X86_64) && defined(CONFIG_DEBUG_RODATA)
+#if defined(CONFIG_X86_64)
        /*
         * Once the kernel maps the text as RO (kernel_set_to_readonly is set),
         * kernel text mappings for the large page aligned text, rodata sections
@@ -723,10 +724,13 @@ static int split_large_page(struct cpa_data *cpa, pte_t *kpte,
        return 0;
 }
 
-static bool try_to_free_pte_page(pte_t *pte)
+static bool try_to_free_pte_page(struct cpa_data *cpa, pte_t *pte)
 {
        int i;
 
+       if (!(cpa->flags & CPA_FREE_PAGETABLES))
+               return false;
+
        for (i = 0; i < PTRS_PER_PTE; i++)
                if (!pte_none(pte[i]))
                        return false;
@@ -735,10 +739,13 @@ static bool try_to_free_pte_page(pte_t *pte)
        return true;
 }
 
-static bool try_to_free_pmd_page(pmd_t *pmd)
+static bool try_to_free_pmd_page(struct cpa_data *cpa, pmd_t *pmd)
 {
        int i;
 
+       if (!(cpa->flags & CPA_FREE_PAGETABLES))
+               return false;
+
        for (i = 0; i < PTRS_PER_PMD; i++)
                if (!pmd_none(pmd[i]))
                        return false;
@@ -759,7 +766,9 @@ static bool try_to_free_pud_page(pud_t *pud)
        return true;
 }
 
-static bool unmap_pte_range(pmd_t *pmd, unsigned long start, unsigned long end)
+static bool unmap_pte_range(struct cpa_data *cpa, pmd_t *pmd,
+                           unsigned long start,
+                           unsigned long end)
 {
        pte_t *pte = pte_offset_kernel(pmd, start);
 
@@ -770,22 +779,23 @@ static bool unmap_pte_range(pmd_t *pmd, unsigned long start, unsigned long end)
                pte++;
        }
 
-       if (try_to_free_pte_page((pte_t *)pmd_page_vaddr(*pmd))) {
+       if (try_to_free_pte_page(cpa, (pte_t *)pmd_page_vaddr(*pmd))) {
                pmd_clear(pmd);
                return true;
        }
        return false;
 }
 
-static void __unmap_pmd_range(pud_t *pud, pmd_t *pmd,
+static void __unmap_pmd_range(struct cpa_data *cpa, pud_t *pud, pmd_t *pmd,
                              unsigned long start, unsigned long end)
 {
-       if (unmap_pte_range(pmd, start, end))
-               if (try_to_free_pmd_page((pmd_t *)pud_page_vaddr(*pud)))
+       if (unmap_pte_range(cpa, pmd, start, end))
+               if (try_to_free_pmd_page(cpa, (pmd_t *)pud_page_vaddr(*pud)))
                        pud_clear(pud);
 }
 
-static void unmap_pmd_range(pud_t *pud, unsigned long start, unsigned long end)
+static void unmap_pmd_range(struct cpa_data *cpa, pud_t *pud,
+                           unsigned long start, unsigned long end)
 {
        pmd_t *pmd = pmd_offset(pud, start);
 
@@ -796,7 +806,7 @@ static void unmap_pmd_range(pud_t *pud, unsigned long start, unsigned long end)
                unsigned long next_page = (start + PMD_SIZE) & PMD_MASK;
                unsigned long pre_end = min_t(unsigned long, end, next_page);
 
-               __unmap_pmd_range(pud, pmd, start, pre_end);
+               __unmap_pmd_range(cpa, pud, pmd, start, pre_end);
 
                start = pre_end;
                pmd++;
@@ -809,7 +819,8 @@ static void unmap_pmd_range(pud_t *pud, unsigned long start, unsigned long end)
                if (pmd_large(*pmd))
                        pmd_clear(pmd);
                else
-                       __unmap_pmd_range(pud, pmd, start, start + PMD_SIZE);
+                       __unmap_pmd_range(cpa, pud, pmd,
+                                         start, start + PMD_SIZE);
 
                start += PMD_SIZE;
                pmd++;
@@ -819,17 +830,19 @@ static void unmap_pmd_range(pud_t *pud, unsigned long start, unsigned long end)
         * 4K leftovers?
         */
        if (start < end)
-               return __unmap_pmd_range(pud, pmd, start, end);
+               return __unmap_pmd_range(cpa, pud, pmd, start, end);
 
        /*
         * Try again to free the PMD page if haven't succeeded above.
         */
        if (!pud_none(*pud))
-               if (try_to_free_pmd_page((pmd_t *)pud_page_vaddr(*pud)))
+               if (try_to_free_pmd_page(cpa, (pmd_t *)pud_page_vaddr(*pud)))
                        pud_clear(pud);
 }
 
-static void unmap_pud_range(pgd_t *pgd, unsigned long start, unsigned long end)
+static void __unmap_pud_range(struct cpa_data *cpa, pgd_t *pgd,
+                             unsigned long start,
+                             unsigned long end)
 {
        pud_t *pud = pud_offset(pgd, start);
 
@@ -840,7 +853,7 @@ static void unmap_pud_range(pgd_t *pgd, unsigned long start, unsigned long end)
                unsigned long next_page = (start + PUD_SIZE) & PUD_MASK;
                unsigned long pre_end   = min_t(unsigned long, end, next_page);
 
-               unmap_pmd_range(pud, start, pre_end);
+               unmap_pmd_range(cpa, pud, start, pre_end);
 
                start = pre_end;
                pud++;
@@ -854,7 +867,7 @@ static void unmap_pud_range(pgd_t *pgd, unsigned long start, unsigned long end)
                if (pud_large(*pud))
                        pud_clear(pud);
                else
-                       unmap_pmd_range(pud, start, start + PUD_SIZE);
+                       unmap_pmd_range(cpa, pud, start, start + PUD_SIZE);
 
                start += PUD_SIZE;
                pud++;
@@ -864,7 +877,7 @@ static void unmap_pud_range(pgd_t *pgd, unsigned long start, unsigned long end)
         * 2M leftovers?
         */
        if (start < end)
-               unmap_pmd_range(pud, start, end);
+               unmap_pmd_range(cpa, pud, start, end);
 
        /*
         * No need to try to free the PUD page because we'll free it in
@@ -872,6 +885,24 @@ static void unmap_pud_range(pgd_t *pgd, unsigned long start, unsigned long end)
         */
 }
 
+static void unmap_pud_range(pgd_t *pgd, unsigned long start, unsigned long end)
+{
+       struct cpa_data cpa = {
+               .flags = CPA_FREE_PAGETABLES,
+       };
+
+       __unmap_pud_range(&cpa, pgd, start, end);
+}
+
+void unmap_pud_range_nofree(pgd_t *pgd, unsigned long start, unsigned long end)
+{
+       struct cpa_data cpa = {
+               .flags = 0,
+       };
+
+       __unmap_pud_range(&cpa, pgd, start, end);
+}
+
 static void unmap_pgd_range(pgd_t *root, unsigned long addr, unsigned long end)
 {
        pgd_t *pgd_entry = root + pgd_index(addr);
@@ -911,19 +942,24 @@ static void populate_pte(struct cpa_data *cpa,
        pte = pte_offset_kernel(pmd, start);
 
        while (num_pages-- && start < end) {
-               set_pte(pte, pfn_pte(cpa->pfn, pgprot));
+
+               /* deal with the NX bit */
+               if (!(pgprot_val(pgprot) & _PAGE_NX))
+                       cpa->pfn &= ~_PAGE_NX;
+
+               set_pte(pte, pfn_pte(cpa->pfn >> PAGE_SHIFT, pgprot));
 
                start    += PAGE_SIZE;
-               cpa->pfn++;
+               cpa->pfn += PAGE_SIZE;
                pte++;
        }
 }
 
-static int populate_pmd(struct cpa_data *cpa,
-                       unsigned long start, unsigned long end,
-                       unsigned num_pages, pud_t *pud, pgprot_t pgprot)
+static long populate_pmd(struct cpa_data *cpa,
+                        unsigned long start, unsigned long end,
+                        unsigned num_pages, pud_t *pud, pgprot_t pgprot)
 {
-       unsigned int cur_pages = 0;
+       long cur_pages = 0;
        pmd_t *pmd;
        pgprot_t pmd_pgprot;
 
@@ -970,11 +1006,11 @@ static int populate_pmd(struct cpa_data *cpa,
 
                pmd = pmd_offset(pud, start);
 
-               set_pmd(pmd, __pmd(cpa->pfn << PAGE_SHIFT | _PAGE_PSE |
-                                  massage_pgprot(pmd_pgprot)));
+               set_pmd(pmd, pmd_mkhuge(pfn_pmd(cpa->pfn >> PAGE_SHIFT,
+                                       canon_pgprot(pmd_pgprot))));
 
                start     += PMD_SIZE;
-               cpa->pfn  += PMD_SIZE >> PAGE_SHIFT;
+               cpa->pfn  += PMD_SIZE;
                cur_pages += PMD_SIZE >> PAGE_SHIFT;
        }
 
@@ -993,12 +1029,12 @@ static int populate_pmd(struct cpa_data *cpa,
        return num_pages;
 }
 
-static int populate_pud(struct cpa_data *cpa, unsigned long start, pgd_t *pgd,
-                       pgprot_t pgprot)
+static long populate_pud(struct cpa_data *cpa, unsigned long start, pgd_t *pgd,
+                        pgprot_t pgprot)
 {
        pud_t *pud;
        unsigned long end;
-       int cur_pages = 0;
+       long cur_pages = 0;
        pgprot_t pud_pgprot;
 
        end = start + (cpa->numpages << PAGE_SHIFT);
@@ -1043,18 +1079,18 @@ static int populate_pud(struct cpa_data *cpa, unsigned long start, pgd_t *pgd,
         * Map everything starting from the Gb boundary, possibly with 1G pages
         */
        while (end - start >= PUD_SIZE) {
-               set_pud(pud, __pud(cpa->pfn << PAGE_SHIFT | _PAGE_PSE |
-                                  massage_pgprot(pud_pgprot)));
+               set_pud(pud, pud_mkhuge(pfn_pud(cpa->pfn >> PAGE_SHIFT,
+                                  canon_pgprot(pud_pgprot))));
 
                start     += PUD_SIZE;
-               cpa->pfn  += PUD_SIZE >> PAGE_SHIFT;
+               cpa->pfn  += PUD_SIZE;
                cur_pages += PUD_SIZE >> PAGE_SHIFT;
                pud++;
        }
 
        /* Map trailing leftover */
        if (start < end) {
-               int tmp;
+               long tmp;
 
                pud = pud_offset(pgd, start);
                if (pud_none(*pud))
@@ -1080,7 +1116,7 @@ static int populate_pgd(struct cpa_data *cpa, unsigned long addr)
        pgprot_t pgprot = __pgprot(_KERNPG_TABLE);
        pud_t *pud = NULL;      /* shut up gcc */
        pgd_t *pgd_entry;
-       int ret;
+       long ret;
 
        pgd_entry = cpa->pgd + pgd_index(addr);
 
@@ -1315,7 +1351,8 @@ static int cpa_process_alias(struct cpa_data *cpa)
 
 static int __change_page_attr_set_clr(struct cpa_data *cpa, int checkalias)
 {
-       int ret, numpages = cpa->numpages;
+       unsigned long numpages = cpa->numpages;
+       int ret;
 
        while (numpages) {
                /*