arch/s390/mm/kasan_init.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 #include <linux/kasan.h>
   3 #include <linux/sched/task.h>
   4 #include <linux/pgtable.h>
   5 #include <asm/pgalloc.h>
   6 #include <asm/kasan.h>
   7 #include <asm/mem_detect.h>
   8 #include <asm/processor.h>
   9 #include <asm/sclp.h>
  10 #include <asm/facility.h>
  11 #include <asm/sections.h>
  12 #include <asm/setup.h>
  13 #include <asm/uv.h>
  14
  15 static unsigned long segment_pos __initdata;
  16 static unsigned long segment_low __initdata;
  17 static bool has_edat __initdata;
  18 static bool has_nx __initdata;
  19
  20 #define __sha(x) ((unsigned long)kasan_mem_to_shadow((void *)x))
  21
  22 static void __init kasan_early_panic(const char *reason)
  23 {
  24         sclp_early_printk("The Linux kernel failed to boot with the KernelAddressSanitizer:\n");
  25         sclp_early_printk(reason);
  26         disabled_wait();
  27 }
  28
  29 static void * __init kasan_early_alloc_segment(void)
  30 {
  31         segment_pos -= _SEGMENT_SIZE;
  32
  33         if (segment_pos < segment_low)
  34                 kasan_early_panic("out of memory during initialisation\n");
  35
  36         return __va(segment_pos);
  37 }
  38
  39 static void * __init kasan_early_alloc_pages(unsigned int order)
  40 {
  41         pgalloc_pos -= (PAGE_SIZE << order);
  42
  43         if (pgalloc_pos < pgalloc_low)
  44                 kasan_early_panic("out of memory during initialisation\n");
  45
  46         return __va(pgalloc_pos);
  47 }
  48
  49 static void * __init kasan_early_crst_alloc(unsigned long val)
  50 {
  51         unsigned long *table;
  52
  53         table = kasan_early_alloc_pages(CRST_ALLOC_ORDER);
  54         if (table)
  55                 crst_table_init(table, val);
  56         return table;
  57 }
  58
  59 static pte_t * __init kasan_early_pte_alloc(void)
  60 {
  61         static void *pte_leftover;
  62         pte_t *pte;
  63
  64         BUILD_BUG_ON(_PAGE_TABLE_SIZE * 2 != PAGE_SIZE);
  65
  66         if (!pte_leftover) {
  67                 pte_leftover = kasan_early_alloc_pages(0);
  68                 pte = pte_leftover + _PAGE_TABLE_SIZE;
  69         } else {
  70                 pte = pte_leftover;
  71                 pte_leftover = NULL;
  72         }
  73         memset64((u64 *)pte, _PAGE_INVALID, PTRS_PER_PTE);
  74         return pte;
  75 }
  76
  77 enum populate_mode {
  78         POPULATE_MAP,
  79         POPULATE_ZERO_SHADOW,
  80         POPULATE_SHALLOW
  81 };
  82
  83 static inline pgprot_t pgprot_clear_bit(pgprot_t pgprot, unsigned long bit)
  84 {
  85         return __pgprot(pgprot_val(pgprot) & ~bit);
  86 }
  87
  88 static void __init kasan_early_pgtable_populate(unsigned long address,
  89                                                 unsigned long end,
  90                                                 enum populate_mode mode)
  91 {
  92         pgprot_t pgt_prot_zero = PAGE_KERNEL_RO;
  93         pgprot_t pgt_prot = PAGE_KERNEL;
  94         pgprot_t sgt_prot = SEGMENT_KERNEL;
  95         pgd_t *pg_dir;
  96         p4d_t *p4_dir;
  97         pud_t *pu_dir;
  98         pmd_t *pm_dir;
  99         pte_t *pt_dir;
 100         pmd_t pmd;
 101         pte_t pte;
 102
 103         if (!has_nx) {
 104                 pgt_prot_zero = pgprot_clear_bit(pgt_prot_zero, _PAGE_NOEXEC);
 105                 pgt_prot = pgprot_clear_bit(pgt_prot, _PAGE_NOEXEC);
 106                 sgt_prot = pgprot_clear_bit(sgt_prot, _SEGMENT_ENTRY_NOEXEC);
 107         }
 108
 109         while (address < end) {
 110                 pg_dir = pgd_offset_k(address);
 111                 if (pgd_none(*pg_dir)) {
 112                         if (mode == POPULATE_ZERO_SHADOW &&
 113                             IS_ALIGNED(address, PGDIR_SIZE) &&
 114                             end - address >= PGDIR_SIZE) {
 115                                 pgd_populate(&init_mm, pg_dir,
 116                                                 kasan_early_shadow_p4d);
 117                                 address = (address + PGDIR_SIZE) & PGDIR_MASK;
 118                                 continue;
 119                         }
 120                         p4_dir = kasan_early_crst_alloc(_REGION2_ENTRY_EMPTY);
 121                         pgd_populate(&init_mm, pg_dir, p4_dir);
 122                 }
 123
 124                 if (mode == POPULATE_SHALLOW) {
 125                         address = (address + P4D_SIZE) & P4D_MASK;
 126                         continue;
 127                 }
 128
 129                 p4_dir = p4d_offset(pg_dir, address);
 130                 if (p4d_none(*p4_dir)) {
 131                         if (mode == POPULATE_ZERO_SHADOW &&
 132                             IS_ALIGNED(address, P4D_SIZE) &&
 133                             end - address >= P4D_SIZE) {
 134                                 p4d_populate(&init_mm, p4_dir,
 135                                                 kasan_early_shadow_pud);
 136                                 address = (address + P4D_SIZE) & P4D_MASK;
 137                                 continue;
 138                         }
 139                         pu_dir = kasan_early_crst_alloc(_REGION3_ENTRY_EMPTY);
 140                         p4d_populate(&init_mm, p4_dir, pu_dir);
 141                 }
 142
 143                 pu_dir = pud_offset(p4_dir, address);
 144                 if (pud_none(*pu_dir)) {
 145                         if (mode == POPULATE_ZERO_SHADOW &&
 146                             IS_ALIGNED(address, PUD_SIZE) &&
 147                             end - address >= PUD_SIZE) {
 148                                 pud_populate(&init_mm, pu_dir,
 149                                                 kasan_early_shadow_pmd);
 150                                 address = (address + PUD_SIZE) & PUD_MASK;
 151                                 continue;
 152                         }
 153                         pm_dir = kasan_early_crst_alloc(_SEGMENT_ENTRY_EMPTY);
 154                         pud_populate(&init_mm, pu_dir, pm_dir);
 155                 }
 156
 157                 pm_dir = pmd_offset(pu_dir, address);
 158                 if (pmd_none(*pm_dir)) {
 159                         if (IS_ALIGNED(address, PMD_SIZE) &&
 160                             end - address >= PMD_SIZE) {
 161                                 if (mode == POPULATE_ZERO_SHADOW) {
 162                                         pmd_populate(&init_mm, pm_dir, kasan_early_shadow_pte);
 163                                         address = (address + PMD_SIZE) & PMD_MASK;
 164                                         continue;
 165                                 } else if (has_edat) {
 166                                         void *page = kasan_early_alloc_segment();
 167
 168                                         memset(page, 0, _SEGMENT_SIZE);
 169                                         pmd = __pmd(__pa(page));
 170                                         pmd = set_pmd_bit(pmd, sgt_prot);
 171                                         set_pmd(pm_dir, pmd);
 172                                         address = (address + PMD_SIZE) & PMD_MASK;
 173                                         continue;
 174                                 }
 175                         }
 176                         pt_dir = kasan_early_pte_alloc();
 177                         pmd_populate(&init_mm, pm_dir, pt_dir);
 178                 } else if (pmd_large(*pm_dir)) {
 179                         address = (address + PMD_SIZE) & PMD_MASK;
 180                         continue;
 181                 }
 182
 183                 pt_dir = pte_offset_kernel(pm_dir, address);
 184                 if (pte_none(*pt_dir)) {
 185                         void *page;
 186
 187                         switch (mode) {
 188                         case POPULATE_MAP:
 189                                 page = kasan_early_alloc_pages(0);
 190                                 memset(page, 0, PAGE_SIZE);
 191                                 pte = __pte(__pa(page));
 192                                 pte = set_pte_bit(pte, pgt_prot);
 193                                 set_pte(pt_dir, pte);
 194                                 break;
 195                         case POPULATE_ZERO_SHADOW:
 196                                 page = kasan_early_shadow_page;
 197                                 pte = __pte(__pa(page));
 198                                 pte = set_pte_bit(pte, pgt_prot_zero);
 199                                 set_pte(pt_dir, pte);
 200                                 break;
 201                         case POPULATE_SHALLOW:
 202                                 /* should never happen */
 203                                 break;
 204                         }
 205                 }
 206                 address += PAGE_SIZE;
 207         }
 208 }
 209
 210 static void __init kasan_early_detect_facilities(void)
 211 {
 212         if (test_facility(8)) {
 213                 has_edat = true;
 214                 __ctl_set_bit(0, 23);
 215         }
 216         if (!noexec_disabled && test_facility(130)) {
 217                 has_nx = true;
 218                 __ctl_set_bit(0, 20);
 219         }
 220 }
 221
 222 void __init kasan_early_init(void)
 223 {
 224         pte_t pte_z = __pte(__pa(kasan_early_shadow_page) | pgprot_val(PAGE_KERNEL_RO));
 225         pmd_t pmd_z = __pmd(__pa(kasan_early_shadow_pte) | _SEGMENT_ENTRY);
 226         pud_t pud_z = __pud(__pa(kasan_early_shadow_pmd) | _REGION3_ENTRY);
 227         p4d_t p4d_z = __p4d(__pa(kasan_early_shadow_pud) | _REGION2_ENTRY);
 228         unsigned long untracked_end = MODULES_VADDR;
 229         unsigned long shadow_alloc_size;
 230         unsigned long memsize;
 231
 232         kasan_early_detect_facilities();
 233         if (!has_nx)
 234                 pte_z = clear_pte_bit(pte_z, __pgprot(_PAGE_NOEXEC));
 235
 236         memsize = get_mem_detect_end();
 237         if (!memsize)
 238                 kasan_early_panic("cannot detect physical memory size\n");
 239         /*
 240          * Kasan currently supports standby memory but only if it follows
 241          * online memory (default allocation), i.e. no memory holes.
 242          * - memsize represents end of online memory
 243          * - ident_map_size represents online + standby and memory limits
 244          *   accounted.
 245          * Kasan maps "memsize" right away.
 246          * [__sha(0), __sha(memsize)]   - shadow memory for identity mapping
 247          * The rest [memsize, ident_map_size] if memsize < ident_map_size
 248          * could be mapped/unmapped dynamically later during memory hotplug.
 249          */
 250         memsize = min(memsize, ident_map_size);
 251
 252         BUILD_BUG_ON(!IS_ALIGNED(KASAN_SHADOW_START, P4D_SIZE));
 253         BUILD_BUG_ON(!IS_ALIGNED(KASAN_SHADOW_END, P4D_SIZE));
 254
 255         /* init kasan zero shadow */
 256         crst_table_init((unsigned long *)kasan_early_shadow_p4d, p4d_val(p4d_z));
 257         crst_table_init((unsigned long *)kasan_early_shadow_pud, pud_val(pud_z));
 258         crst_table_init((unsigned long *)kasan_early_shadow_pmd, pmd_val(pmd_z));
 259         memset64((u64 *)kasan_early_shadow_pte, pte_val(pte_z), PTRS_PER_PTE);
 260
 261         shadow_alloc_size = memsize >> KASAN_SHADOW_SCALE_SHIFT;
 262
 263         if (pgalloc_low + shadow_alloc_size > memsize)
 264                 kasan_early_panic("out of memory during initialisation\n");
 265
 266         if (has_edat) {
 267                 segment_pos = round_down(pgalloc_pos, _SEGMENT_SIZE);
 268                 segment_low = segment_pos - shadow_alloc_size;
 269                 segment_low = round_down(segment_low, _SEGMENT_SIZE);
 270                 pgalloc_pos = segment_low;
 271         }
 272         /*
 273          * Current memory layout:
 274          * +- 0 -------------+     +- shadow start -+
 275          * | 1:1 ram mapping |    /| 1/8 ram        |
 276          * |                 |   / |                |
 277          * +- end of ram ----+  /  +----------------+
 278          * | ... gap ...     | /   |                |
 279          * |                 |/    |    kasan       |
 280          * +- shadow start --+     |    zero        |
 281          * | 1/8 addr space  |     |    page        |
 282          * +- shadow end    -+     |    mapping     |
 283          * | ... gap ...     |\    |  (untracked)   |
 284          * +- vmalloc area  -+ \   |                |
 285          * | vmalloc_size    |  \  |                |
 286          * +- modules vaddr -+   \ +----------------+
 287          * | 2Gb             |    \|      unmapped  | allocated per module
 288          * +-----------------+     +- shadow end ---+
 289          *
 290          * Current memory layout (KASAN_VMALLOC):
 291          * +- 0 -------------+     +- shadow start -+
 292          * | 1:1 ram mapping |    /| 1/8 ram        |
 293          * |                 |   / |                |
 294          * +- end of ram ----+  /  +----------------+
 295          * | ... gap ...     | /   |    kasan       |
 296          * |                 |/    |    zero        |
 297          * +- shadow start --+     |    page        |
 298          * | 1/8 addr space  |     |    mapping     |
 299          * +- shadow end    -+     |  (untracked)   |
 300          * | ... gap ...     |\    |                |
 301          * +- vmalloc area  -+ \   +- vmalloc area -+
 302          * | vmalloc_size    |  \  |shallow populate|
 303          * +- modules vaddr -+   \ +- modules area -+
 304          * | 2Gb             |    \|shallow populate|
 305          * +-----------------+     +- shadow end ---+
 306          */
 307         /* populate kasan shadow (for identity mapping and zero page mapping) */
 308         kasan_early_pgtable_populate(__sha(0), __sha(memsize), POPULATE_MAP);
 309         if (IS_ENABLED(CONFIG_KASAN_VMALLOC)) {
 310                 untracked_end = VMALLOC_START;
 311                 /* shallowly populate kasan shadow for vmalloc and modules */
 312                 kasan_early_pgtable_populate(__sha(VMALLOC_START), __sha(MODULES_END),
 313                                              POPULATE_SHALLOW);
 314         }
 315         /* populate kasan shadow for untracked memory */
 316         kasan_early_pgtable_populate(__sha(ident_map_size), __sha(untracked_end),
 317                                      POPULATE_ZERO_SHADOW);
 318         kasan_early_pgtable_populate(__sha(MODULES_END), __sha(_REGION1_SIZE),
 319                                      POPULATE_ZERO_SHADOW);
 320         /* enable kasan */
 321         init_task.kasan_depth = 0;
 322         sclp_early_printk("KernelAddressSanitizer initialized\n");
 323 }