arch/mips/mm/tlb-r4k.c

   1 /*
   2  * This file is subject to the terms and conditions of the GNU General Public
   3  * License.  See the file "COPYING" in the main directory of this archive
   4  * for more details.
   5  *
   6  * Copyright (C) 1996 David S. Miller (davem@davemloft.net)
   7  * Copyright (C) 1997, 1998, 1999, 2000 Ralf Baechle ralf@gnu.org
   8  * Carsten Langgaard, carstenl@mips.com
   9  * Copyright (C) 2002 MIPS Technologies, Inc.  All rights reserved.
  10  */
  11 #include <linux/cpu_pm.h>
  12 #include <linux/init.h>
  13 #include <linux/sched.h>
  14 #include <linux/smp.h>
  15 #include <linux/mm.h>
  16 #include <linux/hugetlb.h>
  17 #include <linux/export.h>
  18
  19 #include <asm/cpu.h>
  20 #include <asm/cpu-type.h>
  21 #include <asm/bootinfo.h>
  22 #include <asm/hazards.h>
  23 #include <asm/mmu_context.h>
  24 #include <asm/pgtable.h>
  25 #include <asm/tlb.h>
  26 #include <asm/tlbmisc.h>
  27
  28 extern void build_tlb_refill_handler(void);
  29
  30 /*
  31  * LOONGSON-2 has a 4 entry itlb which is a subset of jtlb, LOONGSON-3 has
  32  * a 4 entry itlb and a 4 entry dtlb which are subsets of jtlb. Unfortunately,
  33  * itlb/dtlb are not totally transparent to software.
  34  */
  35 static inline void flush_micro_tlb(void)
  36 {
  37         switch (current_cpu_type()) {
  38         case CPU_LOONGSON2EF:
  39                 write_c0_diag(LOONGSON_DIAG_ITLB);
  40                 break;
  41         case CPU_LOONGSON64:
  42                 write_c0_diag(LOONGSON_DIAG_ITLB | LOONGSON_DIAG_DTLB);
  43                 break;
  44         default:
  45                 break;
  46         }
  47 }
  48
  49 static inline void flush_micro_tlb_vm(struct vm_area_struct *vma)
  50 {
  51         if (vma->vm_flags & VM_EXEC)
  52                 flush_micro_tlb();
  53 }
  54
  55 void local_flush_tlb_all(void)
  56 {
  57         unsigned long flags;
  58         unsigned long old_ctx;
  59         int entry, ftlbhighset;
  60
  61         local_irq_save(flags);
  62         /* Save old context and create impossible VPN2 value */
  63         old_ctx = read_c0_entryhi();
  64         htw_stop();
  65         write_c0_entrylo0(0);
  66         write_c0_entrylo1(0);
  67
  68         entry = num_wired_entries();
  69
  70         /*
  71          * Blast 'em all away.
  72          * If there are any wired entries, fall back to iterating
  73          */
  74         if (cpu_has_tlbinv && !entry) {
  75                 if (current_cpu_data.tlbsizevtlb) {
  76                         write_c0_index(0);
  77                         mtc0_tlbw_hazard();
  78                         tlbinvf();  /* invalidate VTLB */
  79                 }
  80                 ftlbhighset = current_cpu_data.tlbsizevtlb +
  81                         current_cpu_data.tlbsizeftlbsets;
  82                 for (entry = current_cpu_data.tlbsizevtlb;
  83                      entry < ftlbhighset;
  84                      entry++) {
  85                         write_c0_index(entry);
  86                         mtc0_tlbw_hazard();
  87                         tlbinvf();  /* invalidate one FTLB set */
  88                 }
  89         } else {
  90                 while (entry < current_cpu_data.tlbsize) {
  91                         /* Make sure all entries differ. */
  92                         write_c0_entryhi(UNIQUE_ENTRYHI(entry));
  93                         write_c0_index(entry);
  94                         mtc0_tlbw_hazard();
  95                         tlb_write_indexed();
  96                         entry++;
  97                 }
  98         }
  99         tlbw_use_hazard();
 100         write_c0_entryhi(old_ctx);
 101         htw_start();
 102         flush_micro_tlb();
 103         local_irq_restore(flags);
 104 }
 105 EXPORT_SYMBOL(local_flush_tlb_all);
 106
 107 void local_flush_tlb_range(struct vm_area_struct *vma, unsigned long start,
 108         unsigned long end)
 109 {
 110         struct mm_struct *mm = vma->vm_mm;
 111         int cpu = smp_processor_id();
 112
 113         if (cpu_context(cpu, mm) != 0) {
 114                 unsigned long size, flags;
 115
 116                 local_irq_save(flags);
 117                 start = round_down(start, PAGE_SIZE << 1);
 118                 end = round_up(end, PAGE_SIZE << 1);
 119                 size = (end - start) >> (PAGE_SHIFT + 1);
 120                 if (size <= (current_cpu_data.tlbsizeftlbsets ?
 121                              current_cpu_data.tlbsize / 8 :
 122                              current_cpu_data.tlbsize / 2)) {
 123                         unsigned long old_entryhi, uninitialized_var(old_mmid);
 124                         int newpid = cpu_asid(cpu, mm);
 125
 126                         old_entryhi = read_c0_entryhi();
 127                         if (cpu_has_mmid) {
 128                                 old_mmid = read_c0_memorymapid();
 129                                 write_c0_memorymapid(newpid);
 130                         }
 131
 132                         htw_stop();
 133                         while (start < end) {
 134                                 int idx;
 135
 136                                 if (cpu_has_mmid)
 137                                         write_c0_entryhi(start);
 138                                 else
 139                                         write_c0_entryhi(start | newpid);
 140                                 start += (PAGE_SIZE << 1);
 141                                 mtc0_tlbw_hazard();
 142                                 tlb_probe();
 143                                 tlb_probe_hazard();
 144                                 idx = read_c0_index();
 145                                 write_c0_entrylo0(0);
 146                                 write_c0_entrylo1(0);
 147                                 if (idx < 0)
 148                                         continue;
 149                                 /* Make sure all entries differ. */
 150                                 write_c0_entryhi(UNIQUE_ENTRYHI(idx));
 151                                 mtc0_tlbw_hazard();
 152                                 tlb_write_indexed();
 153                         }
 154                         tlbw_use_hazard();
 155                         write_c0_entryhi(old_entryhi);
 156                         if (cpu_has_mmid)
 157                                 write_c0_memorymapid(old_mmid);
 158                         htw_start();
 159                 } else {
 160                         drop_mmu_context(mm);
 161                 }
 162                 flush_micro_tlb();
 163                 local_irq_restore(flags);
 164         }
 165 }
 166
 167 void local_flush_tlb_kernel_range(unsigned long start, unsigned long end)
 168 {
 169         unsigned long size, flags;
 170
 171         local_irq_save(flags);
 172         size = (end - start + (PAGE_SIZE - 1)) >> PAGE_SHIFT;
 173         size = (size + 1) >> 1;
 174         if (size <= (current_cpu_data.tlbsizeftlbsets ?
 175                      current_cpu_data.tlbsize / 8 :
 176                      current_cpu_data.tlbsize / 2)) {
 177                 int pid = read_c0_entryhi();
 178
 179                 start &= (PAGE_MASK << 1);
 180                 end += ((PAGE_SIZE << 1) - 1);
 181                 end &= (PAGE_MASK << 1);
 182                 htw_stop();
 183
 184                 while (start < end) {
 185                         int idx;
 186
 187                         write_c0_entryhi(start);
 188                         start += (PAGE_SIZE << 1);
 189                         mtc0_tlbw_hazard();
 190                         tlb_probe();
 191                         tlb_probe_hazard();
 192                         idx = read_c0_index();
 193                         write_c0_entrylo0(0);
 194                         write_c0_entrylo1(0);
 195                         if (idx < 0)
 196                                 continue;
 197                         /* Make sure all entries differ. */
 198                         write_c0_entryhi(UNIQUE_ENTRYHI(idx));
 199                         mtc0_tlbw_hazard();
 200                         tlb_write_indexed();
 201                 }
 202                 tlbw_use_hazard();
 203                 write_c0_entryhi(pid);
 204                 htw_start();
 205         } else {
 206                 local_flush_tlb_all();
 207         }
 208         flush_micro_tlb();
 209         local_irq_restore(flags);
 210 }
 211
 212 void local_flush_tlb_page(struct vm_area_struct *vma, unsigned long page)
 213 {
 214         int cpu = smp_processor_id();
 215
 216         if (cpu_context(cpu, vma->vm_mm) != 0) {
 217                 unsigned long uninitialized_var(old_mmid);
 218                 unsigned long flags, old_entryhi;
 219                 int idx;
 220
 221                 page &= (PAGE_MASK << 1);
 222                 local_irq_save(flags);
 223                 old_entryhi = read_c0_entryhi();
 224                 htw_stop();
 225                 if (cpu_has_mmid) {
 226                         old_mmid = read_c0_memorymapid();
 227                         write_c0_entryhi(page);
 228                         write_c0_memorymapid(cpu_asid(cpu, vma->vm_mm));
 229                 } else {
 230                         write_c0_entryhi(page | cpu_asid(cpu, vma->vm_mm));
 231                 }
 232                 mtc0_tlbw_hazard();
 233                 tlb_probe();
 234                 tlb_probe_hazard();
 235                 idx = read_c0_index();
 236                 write_c0_entrylo0(0);
 237                 write_c0_entrylo1(0);
 238                 if (idx < 0)
 239                         goto finish;
 240                 /* Make sure all entries differ. */
 241                 write_c0_entryhi(UNIQUE_ENTRYHI(idx));
 242                 mtc0_tlbw_hazard();
 243                 tlb_write_indexed();
 244                 tlbw_use_hazard();
 245
 246         finish:
 247                 write_c0_entryhi(old_entryhi);
 248                 if (cpu_has_mmid)
 249                         write_c0_memorymapid(old_mmid);
 250                 htw_start();
 251                 flush_micro_tlb_vm(vma);
 252                 local_irq_restore(flags);
 253         }
 254 }
 255
 256 /*
 257  * This one is only used for pages with the global bit set so we don't care
 258  * much about the ASID.
 259  */
 260 void local_flush_tlb_one(unsigned long page)
 261 {
 262         unsigned long flags;
 263         int oldpid, idx;
 264
 265         local_irq_save(flags);
 266         oldpid = read_c0_entryhi();
 267         htw_stop();
 268         page &= (PAGE_MASK << 1);
 269         write_c0_entryhi(page);
 270         mtc0_tlbw_hazard();
 271         tlb_probe();
 272         tlb_probe_hazard();
 273         idx = read_c0_index();
 274         write_c0_entrylo0(0);
 275         write_c0_entrylo1(0);
 276         if (idx >= 0) {
 277                 /* Make sure all entries differ. */
 278                 write_c0_entryhi(UNIQUE_ENTRYHI(idx));
 279                 mtc0_tlbw_hazard();
 280                 tlb_write_indexed();
 281                 tlbw_use_hazard();
 282         }
 283         write_c0_entryhi(oldpid);
 284         htw_start();
 285         flush_micro_tlb();
 286         local_irq_restore(flags);
 287 }
 288
 289 /*
 290  * We will need multiple versions of update_mmu_cache(), one that just
 291  * updates the TLB with the new pte(s), and another which also checks
 292  * for the R4k "end of page" hardware bug and does the needy.
 293  */
 294 void __update_tlb(struct vm_area_struct * vma, unsigned long address, pte_t pte)
 295 {
 296         unsigned long flags;
 297         pgd_t *pgdp;
 298         p4d_t *p4dp;
 299         pud_t *pudp;
 300         pmd_t *pmdp;
 301         pte_t *ptep;
 302         int idx, pid;
 303
 304         /*
 305          * Handle debugger faulting in for debugee.
 306          */
 307         if (current->active_mm != vma->vm_mm)
 308                 return;
 309
 310         local_irq_save(flags);
 311
 312         htw_stop();
 313         address &= (PAGE_MASK << 1);
 314         if (cpu_has_mmid) {
 315                 write_c0_entryhi(address);
 316         } else {
 317                 pid = read_c0_entryhi() & cpu_asid_mask(&current_cpu_data);
 318                 write_c0_entryhi(address | pid);
 319         }
 320         pgdp = pgd_offset(vma->vm_mm, address);
 321         mtc0_tlbw_hazard();
 322         tlb_probe();
 323         tlb_probe_hazard();
 324         p4dp = p4d_offset(pgdp, address);
 325         pudp = pud_offset(p4dp, address);
 326         pmdp = pmd_offset(pudp, address);
 327         idx = read_c0_index();
 328 #ifdef CONFIG_MIPS_HUGE_TLB_SUPPORT
 329         /* this could be a huge page  */
 330         if (pmd_huge(*pmdp)) {
 331                 unsigned long lo;
 332                 write_c0_pagemask(PM_HUGE_MASK);
 333                 ptep = (pte_t *)pmdp;
 334                 lo = pte_to_entrylo(pte_val(*ptep));
 335                 write_c0_entrylo0(lo);
 336                 write_c0_entrylo1(lo + (HPAGE_SIZE >> 7));
 337
 338                 mtc0_tlbw_hazard();
 339                 if (idx < 0)
 340                         tlb_write_random();
 341                 else
 342                         tlb_write_indexed();
 343                 tlbw_use_hazard();
 344                 write_c0_pagemask(PM_DEFAULT_MASK);
 345         } else
 346 #endif
 347         {
 348                 ptep = pte_offset_map(pmdp, address);
 349
 350 #if defined(CONFIG_PHYS_ADDR_T_64BIT) && defined(CONFIG_CPU_MIPS32)
 351 #ifdef CONFIG_XPA
 352                 write_c0_entrylo0(pte_to_entrylo(ptep->pte_high));
 353                 if (cpu_has_xpa)
 354                         writex_c0_entrylo0(ptep->pte_low & _PFNX_MASK);
 355                 ptep++;
 356                 write_c0_entrylo1(pte_to_entrylo(ptep->pte_high));
 357                 if (cpu_has_xpa)
 358                         writex_c0_entrylo1(ptep->pte_low & _PFNX_MASK);
 359 #else
 360                 write_c0_entrylo0(ptep->pte_high);
 361                 ptep++;
 362                 write_c0_entrylo1(ptep->pte_high);
 363 #endif
 364 #else
 365                 write_c0_entrylo0(pte_to_entrylo(pte_val(*ptep++)));
 366                 write_c0_entrylo1(pte_to_entrylo(pte_val(*ptep)));
 367 #endif
 368                 mtc0_tlbw_hazard();
 369                 if (idx < 0)
 370                         tlb_write_random();
 371                 else
 372                         tlb_write_indexed();
 373         }
 374         tlbw_use_hazard();
 375         htw_start();
 376         flush_micro_tlb_vm(vma);
 377         local_irq_restore(flags);
 378 }
 379
 380 void add_wired_entry(unsigned long entrylo0, unsigned long entrylo1,
 381                      unsigned long entryhi, unsigned long pagemask)
 382 {
 383 #ifdef CONFIG_XPA
 384         panic("Broken for XPA kernels");
 385 #else
 386         unsigned int uninitialized_var(old_mmid);
 387         unsigned long flags;
 388         unsigned long wired;
 389         unsigned long old_pagemask;
 390         unsigned long old_ctx;
 391
 392         local_irq_save(flags);
 393         if (cpu_has_mmid) {
 394                 old_mmid = read_c0_memorymapid();
 395                 write_c0_memorymapid(MMID_KERNEL_WIRED);
 396         }
 397         /* Save old context and create impossible VPN2 value */
 398         old_ctx = read_c0_entryhi();
 399         htw_stop();
 400         old_pagemask = read_c0_pagemask();
 401         wired = num_wired_entries();
 402         write_c0_wired(wired + 1);
 403         write_c0_index(wired);
 404         tlbw_use_hazard();      /* What is the hazard here? */
 405         write_c0_pagemask(pagemask);
 406         write_c0_entryhi(entryhi);
 407         write_c0_entrylo0(entrylo0);
 408         write_c0_entrylo1(entrylo1);
 409         mtc0_tlbw_hazard();
 410         tlb_write_indexed();
 411         tlbw_use_hazard();
 412
 413         write_c0_entryhi(old_ctx);
 414         if (cpu_has_mmid)
 415                 write_c0_memorymapid(old_mmid);
 416         tlbw_use_hazard();      /* What is the hazard here? */
 417         htw_start();
 418         write_c0_pagemask(old_pagemask);
 419         local_flush_tlb_all();
 420         local_irq_restore(flags);
 421 #endif
 422 }
 423
 424 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
 425
 426 int has_transparent_hugepage(void)
 427 {
 428         static unsigned int mask = -1;
 429
 430         if (mask == -1) {       /* first call comes during __init */
 431                 unsigned long flags;
 432
 433                 local_irq_save(flags);
 434                 write_c0_pagemask(PM_HUGE_MASK);
 435                 back_to_back_c0_hazard();
 436                 mask = read_c0_pagemask();
 437                 write_c0_pagemask(PM_DEFAULT_MASK);
 438                 local_irq_restore(flags);
 439         }
 440         return mask == PM_HUGE_MASK;
 441 }
 442
 443 #endif /* CONFIG_TRANSPARENT_HUGEPAGE  */
 444
 445 /*
 446  * Used for loading TLB entries before trap_init() has started, when we
 447  * don't actually want to add a wired entry which remains throughout the
 448  * lifetime of the system
 449  */
 450
 451 int temp_tlb_entry;
 452
 453 __init int add_temporary_entry(unsigned long entrylo0, unsigned long entrylo1,
 454                                unsigned long entryhi, unsigned long pagemask)
 455 {
 456         int ret = 0;
 457         unsigned long flags;
 458         unsigned long wired;
 459         unsigned long old_pagemask;
 460         unsigned long old_ctx;
 461
 462         local_irq_save(flags);
 463         /* Save old context and create impossible VPN2 value */
 464         htw_stop();
 465         old_ctx = read_c0_entryhi();
 466         old_pagemask = read_c0_pagemask();
 467         wired = num_wired_entries();
 468         if (--temp_tlb_entry < wired) {
 469                 printk(KERN_WARNING
 470                        "No TLB space left for add_temporary_entry\n");
 471                 ret = -ENOSPC;
 472                 goto out;
 473         }
 474
 475         write_c0_index(temp_tlb_entry);
 476         write_c0_pagemask(pagemask);
 477         write_c0_entryhi(entryhi);
 478         write_c0_entrylo0(entrylo0);
 479         write_c0_entrylo1(entrylo1);
 480         mtc0_tlbw_hazard();
 481         tlb_write_indexed();
 482         tlbw_use_hazard();
 483
 484         write_c0_entryhi(old_ctx);
 485         write_c0_pagemask(old_pagemask);
 486         htw_start();
 487 out:
 488         local_irq_restore(flags);
 489         return ret;
 490 }
 491
 492 static int ntlb;
 493 static int __init set_ntlb(char *str)
 494 {
 495         get_option(&str, &ntlb);
 496         return 1;
 497 }
 498
 499 __setup("ntlb=", set_ntlb);
 500
 501 /*
 502  * Configure TLB (for init or after a CPU has been powered off).
 503  */
 504 static void r4k_tlb_configure(void)
 505 {
 506         /*
 507          * You should never change this register:
 508          *   - On R4600 1.7 the tlbp never hits for pages smaller than
 509          *     the value in the c0_pagemask register.
 510          *   - The entire mm handling assumes the c0_pagemask register to
 511          *     be set to fixed-size pages.
 512          */
 513         write_c0_pagemask(PM_DEFAULT_MASK);
 514         back_to_back_c0_hazard();
 515         if (read_c0_pagemask() != PM_DEFAULT_MASK)
 516                 panic("MMU doesn't support PAGE_SIZE=0x%lx", PAGE_SIZE);
 517
 518         write_c0_wired(0);
 519         if (current_cpu_type() == CPU_R10000 ||
 520             current_cpu_type() == CPU_R12000 ||
 521             current_cpu_type() == CPU_R14000 ||
 522             current_cpu_type() == CPU_R16000)
 523                 write_c0_framemask(0);
 524
 525         if (cpu_has_rixi) {
 526                 /*
 527                  * Enable the no read, no exec bits, and enable large physical
 528                  * address.
 529                  */
 530 #ifdef CONFIG_64BIT
 531                 set_c0_pagegrain(PG_RIE | PG_XIE | PG_ELPA);
 532 #else
 533                 set_c0_pagegrain(PG_RIE | PG_XIE);
 534 #endif
 535         }
 536
 537         temp_tlb_entry = current_cpu_data.tlbsize - 1;
 538
 539         /* From this point on the ARC firmware is dead.  */
 540         local_flush_tlb_all();
 541
 542         /* Did I tell you that ARC SUCKS?  */
 543 }
 544
 545 void tlb_init(void)
 546 {
 547         r4k_tlb_configure();
 548
 549         if (ntlb) {
 550                 if (ntlb > 1 && ntlb <= current_cpu_data.tlbsize) {
 551                         int wired = current_cpu_data.tlbsize - ntlb;
 552                         write_c0_wired(wired);
 553                         write_c0_index(wired-1);
 554                         printk("Restricting TLB to %d entries\n", ntlb);
 555                 } else
 556                         printk("Ignoring invalid argument ntlb=%d\n", ntlb);
 557         }
 558
 559         build_tlb_refill_handler();
 560 }
 561
 562 static int r4k_tlb_pm_notifier(struct notifier_block *self, unsigned long cmd,
 563                                void *v)
 564 {
 565         switch (cmd) {
 566         case CPU_PM_ENTER_FAILED:
 567         case CPU_PM_EXIT:
 568                 r4k_tlb_configure();
 569                 break;
 570         }
 571
 572         return NOTIFY_OK;
 573 }
 574
 575 static struct notifier_block r4k_tlb_pm_notifier_block = {
 576         .notifier_call = r4k_tlb_pm_notifier,
 577 };
 578
 579 static int __init r4k_tlb_init_pm(void)
 580 {
 581         return cpu_pm_register_notifier(&r4k_tlb_pm_notifier_block);
 582 }
 583 arch_initcall(r4k_tlb_init_pm);