mflr r9; /* Get LR, later save to stack */ \
ld r2,PACATOC(r13); /* get kernel TOC into r2 */ \
std r9,_LINK(r1); \
- lbz r10,PACASOFTIRQEN(r13); \
+ lbz r10,PACAIRQSOFTMASK(r13); \
mfspr r11,SPRN_XER; /* save XER in stackframe */ \
std r10,SOFTE(r1); \
std r11,_XER(r1); \
#define SOFTEN_VALUE_0xea0 PACA_IRQ_EE
#define __SOFTEN_TEST(h, vec) \
- lbz r10,PACASOFTIRQEN(r13); \
+ lbz r10,PACAIRQSOFTMASK(r13); \
andi. r10,r10,IRQS_DISABLED; \
li r10,SOFTEN_VALUE_##vec; \
bne masked_##h##interrupt
#define PACA_IRQ_HMI 0x20
/*
- * flags for paca->soft_enabled
+ * flags for paca->irq_soft_mask
*/
#define IRQS_ENABLED 0
#define IRQS_DISABLED 1
#ifdef CONFIG_PPC64
#include <asm/paca.h>
-static inline notrace unsigned long soft_enabled_return(void)
+static inline notrace unsigned long irq_soft_mask_return(void)
{
unsigned long flags;
asm volatile(
"lbz %0,%1(13)"
: "=r" (flags)
- : "i" (offsetof(struct paca_struct, soft_enabled)));
+ : "i" (offsetof(struct paca_struct, irq_soft_mask)));
return flags;
}
/*
* The "memory" clobber acts as both a compiler barrier
* for the critical section and as a clobber because
- * we changed paca->soft_enabled
+ * we changed paca->irq_soft_mask
*/
-static inline notrace void soft_enabled_set(unsigned long enable)
+static inline notrace void irq_soft_mask_set(unsigned long mask)
{
#ifdef CONFIG_TRACE_IRQFLAGS
/*
- * mask must always include LINUX bit if any are set, and
- * interrupts don't get replayed until the Linux interrupt is
- * unmasked. This could be changed to replay partial unmasks
- * in future, which would allow Linux masks to nest inside
- * other masks, among other things. For now, be very dumb and
- * simple.
+ * The irq mask must always include the STD bit if any are set.
+ *
+ * and interrupts don't get replayed until the standard
+ * interrupt (local_irq_disable()) is unmasked.
+ *
+ * Other masks must only provide additional masking beyond
+ * the standard, and they are also not replayed until the
+ * standard interrupt becomes unmasked.
+ *
+ * This could be changed, but it will require partial
+ * unmasks to be replayed, among other things. For now, take
+ * the simple approach.
*/
WARN_ON(mask && !(mask & IRQS_DISABLED));
#endif
asm volatile(
"stb %0,%1(13)"
:
- : "r" (enable),
- "i" (offsetof(struct paca_struct, soft_enabled))
+ : "r" (mask),
+ "i" (offsetof(struct paca_struct, irq_soft_mask))
: "memory");
}
-static inline notrace unsigned long soft_enabled_set_return(unsigned long mask)
+static inline notrace unsigned long irq_soft_mask_set_return(unsigned long mask)
{
unsigned long flags;
asm volatile(
"lbz %0,%1(13); stb %2,%1(13)"
: "=&r" (flags)
- : "i" (offsetof(struct paca_struct, soft_enabled)),
+ : "i" (offsetof(struct paca_struct, irq_soft_mask)),
"r" (mask)
: "memory");
static inline unsigned long arch_local_save_flags(void)
{
- return soft_enabled_return();
+ return irq_soft_mask_return();
}
static inline void arch_local_irq_disable(void)
{
- soft_enabled_set(IRQS_DISABLED);
+ irq_soft_mask_set(IRQS_DISABLED);
}
extern void arch_local_irq_restore(unsigned long);
static inline unsigned long arch_local_irq_save(void)
{
- return soft_enabled_set_return(IRQS_DISABLED);
+ return irq_soft_mask_set_return(IRQS_DISABLED);
}
static inline bool arch_irqs_disabled_flags(unsigned long flags)
#define __hard_irq_disable() __mtmsrd(local_paca->kernel_msr, 1)
#endif
-#define hard_irq_disable() do { \
- unsigned long flags; \
- __hard_irq_disable(); \
- flags = soft_enabled_set_return(IRQS_DISABLED);\
- local_paca->irq_happened |= PACA_IRQ_HARD_DIS; \
- if (!arch_irqs_disabled_flags(flags)) \
- trace_hardirqs_off(); \
+#define hard_irq_disable() do { \
+ unsigned long flags; \
+ __hard_irq_disable(); \
+ flags = irq_soft_mask_set_return(IRQS_DISABLED); \
+ local_paca->irq_happened |= PACA_IRQ_HARD_DIS; \
+ if (!arch_irqs_disabled_flags(flags)) \
+ trace_hardirqs_off(); \
} while(0)
static inline bool lazy_irq_pending(void)
* be clobbered.
*/
#define RECONCILE_IRQ_STATE(__rA, __rB) \
- lbz __rA,PACASOFTIRQEN(r13); \
+ lbz __rA,PACAIRQSOFTMASK(r13); \
lbz __rB,PACAIRQHAPPENED(r13); \
- andi. __rA,__rA,IRQS_DISABLED;\
- li __rA,IRQS_DISABLED; \
+ andi. __rA,__rA,IRQS_DISABLED; \
+ li __rA,IRQS_DISABLED; \
ori __rB,__rB,PACA_IRQ_HARD_DIS; \
stb __rB,PACAIRQHAPPENED(r13); \
bne 44f; \
- stb __rA,PACASOFTIRQEN(r13); \
+ stb __rA,PACAIRQSOFTMASK(r13); \
TRACE_DISABLE_INTS; \
44:
#define RECONCILE_IRQ_STATE(__rA, __rB) \
lbz __rA,PACAIRQHAPPENED(r13); \
- li __rB,IRQS_DISABLED; \
+ li __rB,IRQS_DISABLED; \
ori __rA,__rA,PACA_IRQ_HARD_DIS; \
- stb __rB,PACASOFTIRQEN(r13); \
+ stb __rB,PACAIRQSOFTMASK(r13); \
stb __rA,PACAIRQHAPPENED(r13)
#endif
#endif
/* Only need to enable IRQs by hard enabling them after this */
local_paca->irq_happened = 0;
- soft_enabled_set(IRQS_ENABLED);
+ irq_soft_mask_set(IRQS_ENABLED);
#endif
}
u64 saved_r1; /* r1 save for RTAS calls or PM */
u64 saved_msr; /* MSR saved here by enter_rtas */
u16 trap_save; /* Used when bad stack is encountered */
- u8 soft_enabled; /* irq soft-enable flag */
+ u8 irq_soft_mask; /* mask for irq soft masking */
u8 irq_happened; /* irq happened while soft-disabled */
u8 io_sync; /* writel() needs spin_unlock sync */
u8 irq_work_pending; /* IRQ_WORK interrupt while soft-disable */
OFFSET(PACATOC, paca_struct, kernel_toc);
OFFSET(PACAKBASE, paca_struct, kernelbase);
OFFSET(PACAKMSR, paca_struct, kernel_msr);
- OFFSET(PACASOFTIRQEN, paca_struct, soft_enabled);
+ OFFSET(PACAIRQSOFTMASK, paca_struct, irq_soft_mask);
OFFSET(PACAIRQHAPPENED, paca_struct, irq_happened);
#ifdef CONFIG_PPC_BOOK3S
OFFSET(PACACONTEXTID, paca_struct, mm_ctx_id);
* is correct
*/
#if defined(CONFIG_TRACE_IRQFLAGS) && defined(CONFIG_BUG)
- lbz r10,PACASOFTIRQEN(r13)
+ lbz r10,PACAIRQSOFTMASK(r13)
1: tdnei r10,IRQS_ENABLED
EMIT_BUG_ENTRY 1b,__FILE__,__LINE__,BUGFLAG_WARNING
#endif
* are about to re-enable interrupts
*/
ld r5,SOFTE(r1)
- lbz r6,PACASOFTIRQEN(r13)
+ lbz r6,PACAIRQSOFTMASK(r13)
andi. r5,r5,IRQS_DISABLED
bne .Lrestore_irq_off
.Lrestore_no_replay:
TRACE_ENABLE_INTS
li r0,IRQS_ENABLED
- stb r0,PACASOFTIRQEN(r13);
+ stb r0,PACAIRQSOFTMASK(r13);
/*
* Final return path. BookE is handled in a different file
1:
#if defined(CONFIG_TRACE_IRQFLAGS) && defined(CONFIG_BUG)
/* The interrupt should not have soft enabled. */
- lbz r7,PACASOFTIRQEN(r13)
-1: tdnei r7,IRQS_DISABLED
+ lbz r7,PACAIRQSOFTMASK(r13)
+1: tdeqi r7,IRQS_ENABLED
EMIT_BUG_ENTRY 1b,__FILE__,__LINE__,BUGFLAG_WARNING
#endif
b .Ldo_restore
/* There is no way it is acceptable to get here with interrupts enabled,
* check it with the asm equivalent of WARN_ON
*/
- lbz r0,PACASOFTIRQEN(r13)
+ lbz r0,PACAIRQSOFTMASK(r13)
1: tdeqi r0,IRQS_ENABLED
EMIT_BUG_ENTRY 1b,__FILE__,__LINE__,BUGFLAG_WARNING
#endif
mfspr r10,SPRN_ESR
SPECIAL_EXC_STORE(r10,ESR)
- lbz r10,PACASOFTIRQEN(r13)
+ lbz r10,PACAIRQSOFTMASK(r13)
SPECIAL_EXC_STORE(r10,SOFTE)
ld r10,_NIP(r1)
SPECIAL_EXC_STORE(r10,CSRR0)
mtspr SPRN_MAS8,r10
END_FTR_SECTION_IFSET(CPU_FTR_EMB_HV)
- lbz r6,PACASOFTIRQEN(r13)
+ lbz r6,PACAIRQSOFTMASK(r13)
ld r5,SOFTE(r1)
/* Interrupts had better not already be enabled... */
bne 1f
TRACE_ENABLE_INTS
- stb r5,PACASOFTIRQEN(r13)
+ stb r5,PACAIRQSOFTMASK(r13)
1:
/*
* Restore PACAIRQHAPPENED rather than setting it based on
#define PROLOG_ADDITION_NONE_MC(n)
#define PROLOG_ADDITION_MASKABLE_GEN(n) \
- lbz r10,PACASOFTIRQEN(r13); /* are irqs soft-disabled ? */ \
+ lbz r10,PACAIRQSOFTMASK(r13); /* are irqs soft-masked? */ \
andi. r10,r10,IRQS_DISABLED; /* yes -> go out of line */ \
bne masked_interrupt_book3e_##n
mfspr r8,SPRN_XER; /* save XER in stackframe */ \
ld r9,excf+EX_R1(r13); /* load orig r1 back from PACA */ \
lwz r10,excf+EX_CR(r13); /* load orig CR back from PACA */ \
- lbz r11,PACASOFTIRQEN(r13); /* get current IRQ softe */ \
+ lbz r11,PACAIRQSOFTMASK(r13); /* get current IRQ softe */ \
ld r12,exception_marker@toc(r2); \
li r0,0; \
std r3,GPR10(r1); /* save r10 to stackframe */ \
* in the PACA when doing hotplug)
*/
li r0,IRQS_DISABLED
- stb r0,PACASOFTIRQEN(r13)
+ stb r0,PACAIRQSOFTMASK(r13)
li r0,PACA_IRQ_HARD_DIS
stb r0,PACAIRQHAPPENED(r13)
* in the PACA when doing hotplug)
*/
li r7,IRQS_DISABLED
- stb r7,PACASOFTIRQEN(r13)
+ stb r7,PACAIRQSOFTMASK(r13)
li r0,PACA_IRQ_HARD_DIS
stb r0,PACAIRQHAPPENED(r13)
* in the PACA when doing hotplug)
*/
li r0,IRQS_DISABLED
- stb r0,PACASOFTIRQEN(r13)
+ stb r0,PACAIRQSOFTMASK(r13)
li r0,PACA_IRQ_HARD_DIS
stb r0,PACAIRQHAPPENED(r13)
addi r1,r1,128
#endif
li r0,IRQS_ENABLED
- stb r0,PACASOFTIRQEN(r13)
+ stb r0,PACAIRQSOFTMASK(r13)
/* Interrupts will make use return to LR, so get something we want
* in there
#endif /* CONFIG_TRACE_IRQFLAGS */
li r0,IRQS_ENABLED
- stb r0,PACASOFTIRQEN(r13) /* we'll hard-enable shortly */
+ stb r0,PACAIRQSOFTMASK(r13) /* we'll hard-enable shortly */
BEGIN_FTR_SECTION
DSSALL
sync
unsigned int replay;
/* Write the new soft-enabled value */
- soft_enabled_set(mask);
- if (mask) {
-#ifdef CONFIG_TRACE_IRQFLAGS
- /*
- * mask must always include LINUX bit if any
- * are set, and interrupts don't get replayed until
- * the Linux interrupt is unmasked. This could be
- * changed to replay partial unmasks in future,
- * which would allow Linux masks to nest inside
- * other masks, among other things. For now, be very
- * dumb and simple.
- */
- WARN_ON(!(mask & IRQS_DISABLED));
-#endif
+ irq_soft_mask_set(mask);
+ if (mask)
return;
- }
/*
* From this point onward, we can take interrupts, preempt,
}
#endif /* CONFIG_TRACE_IRQFLAGS */
- soft_enabled_set(IRQS_DISABLED);
+ irq_soft_mask_set(IRQS_DISABLED);
trace_hardirqs_off();
/*
/* We can soft-enable now */
trace_hardirqs_on();
- soft_enabled_set(IRQS_ENABLED);
+ irq_soft_mask_set(IRQS_ENABLED);
/*
* And replay if we have to. This will return with interrupts
* of entering the low power state.
*/
local_paca->irq_happened &= ~PACA_IRQ_HARD_DIS;
- soft_enabled_set(IRQS_ENABLED);
+ irq_soft_mask_set(IRQS_ENABLED);
/* Tell the caller to enter the low power state */
return true;
std r5,_XER(r1)
mfcr r5
std r5,_CCR(r1)
- lbz r5,PACASOFTIRQEN(r13)
+ lbz r5,PACAIRQSOFTMASK(r13)
std r5,SOFTE(r1)
/*
#ifdef CONFIG_PPC64
/*
- * softe copies paca->soft_enabled variable state. Since soft_enabled is
+ * softe copies paca->irq_soft_mask variable state. Since irq_soft_mask is
* no more used as a flag, lets force usr to alway see the softe value as 1
* which means interrupts are not soft disabled.
*/
/* Allow percpu accesses to work until we setup percpu data */
get_paca()->data_offset = 0;
/* Mark interrupts disabled in PACA */
- soft_enabled_set(IRQS_DISABLED);
+ irq_soft_mask_set(IRQS_DISABLED);
}
static void __init configure_exceptions(void)
void early_setup_secondary(void)
{
/* Mark interrupts disabled in PACA */
- soft_enabled_set(IRQS_DISABLED);
+ irq_soft_mask_set(IRQS_DISABLED);
/* Initialize the hash table or TLB handling */
early_init_mmu_secondary();
void accumulate_stolen_time(void)
{
u64 sst, ust;
- unsigned long save_soft_enabled = soft_enabled_return();
+ unsigned long save_irq_soft_mask = irq_soft_mask_return();
struct cpu_accounting_data *acct = &local_paca->accounting;
/* We are called early in the exception entry, before
* needs to reflect that so various debug stuff doesn't
* complain
*/
- soft_enabled_set(IRQS_DISABLED);
+ irq_soft_mask_set(IRQS_DISABLED);
sst = scan_dispatch_log(acct->starttime_user);
ust = scan_dispatch_log(acct->starttime);
acct->utime -= ust;
acct->steal_time += ust + sst;
- soft_enabled_set(save_soft_enabled);
+ irq_soft_mask_set(save_irq_soft_mask);
}
static inline u64 calculate_stolen_time(u64 stop_tb)
mfctr r4
#endif
mfxer r5
- lbz r6, PACASOFTIRQEN(r13)
+ lbz r6, PACAIRQSOFTMASK(r13)
std r3, _LINK(r1)
std r4, _CTR(r1)
std r5, _XER(r1)
* So long as we atomically load page table pointers we are safe against teardown,
* we can follow the address down to the the page and take a ref on it.
* This function need to be called with interrupts disabled. We use this variant
- * when we have MSR[EE] = 0 but the paca->soft_enabled = IRQS_ENABLED
+ * when we have MSR[EE] = 0 but the paca->irq_soft_mask = IRQS_ENABLED
*/
pte_t *__find_linux_pte(pgd_t *pgdir, unsigned long ea,
bool *is_thp, unsigned *hpage_shift)
printf(" current = 0x%lx\n", current);
#ifdef CONFIG_PPC64
printf(" paca = 0x%lx\t softe: %d\t irq_happened: 0x%02x\n",
- local_paca, local_paca->soft_enabled, local_paca->irq_happened);
+ local_paca, local_paca->irq_soft_mask, local_paca->irq_happened);
#endif
if (current) {
printf(" pid = %ld, comm = %s\n",
DUMP(p, stab_rr, "lx");
DUMP(p, saved_r1, "lx");
DUMP(p, trap_save, "x");
- DUMP(p, soft_enabled, "x");
+ DUMP(p, irq_soft_mask, "x");
DUMP(p, irq_happened, "x");
DUMP(p, io_sync, "x");
DUMP(p, irq_work_pending, "x");
OSDN Git Service
