drivers/gpu/drm/i915/i915_irq.c

   1 /* i915_irq.c -- IRQ support for the I915 -*- linux-c -*-
   2  */
   3 /*
   4  * Copyright 2003 Tungsten Graphics, Inc., Cedar Park, Texas.
   5  * All Rights Reserved.
   6  *
   7  * Permission is hereby granted, free of charge, to any person obtaining a
   8  * copy of this software and associated documentation files (the
   9  * "Software"), to deal in the Software without restriction, including
  10  * without limitation the rights to use, copy, modify, merge, publish,
  11  * distribute, sub license, and/or sell copies of the Software, and to
  12  * permit persons to whom the Software is furnished to do so, subject to
  13  * the following conditions:
  14  *
  15  * The above copyright notice and this permission notice (including the
  16  * next paragraph) shall be included in all copies or substantial portions
  17  * of the Software.
  18  *
  19  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
  20  * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  21  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT.
  22  * IN NO EVENT SHALL TUNGSTEN GRAPHICS AND/OR ITS SUPPLIERS BE LIABLE FOR
  23  * ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
  24  * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
  25  * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  26  *
  27  */
  28
  29 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
  30
  31 #include <linux/sysrq.h>
  32 #include <linux/slab.h>
  33 #include <linux/circ_buf.h>
  34 #include <drm/drmP.h>
  35 #include <drm/i915_drm.h>
  36 #include "i915_drv.h"
  37 #include "i915_trace.h"
  38 #include "intel_drv.h"
  39
  40 /**
  41  * DOC: interrupt handling
  42  *
  43  * These functions provide the basic support for enabling and disabling the
  44  * interrupt handling support. There's a lot more functionality in i915_irq.c
  45  * and related files, but that will be described in separate chapters.
  46  */
  47
  48 static const u32 hpd_ibx[HPD_NUM_PINS] = {
  49         [HPD_CRT] = SDE_CRT_HOTPLUG,
  50         [HPD_SDVO_B] = SDE_SDVOB_HOTPLUG,
  51         [HPD_PORT_B] = SDE_PORTB_HOTPLUG,
  52         [HPD_PORT_C] = SDE_PORTC_HOTPLUG,
  53         [HPD_PORT_D] = SDE_PORTD_HOTPLUG
  54 };
  55
  56 static const u32 hpd_cpt[HPD_NUM_PINS] = {
  57         [HPD_CRT] = SDE_CRT_HOTPLUG_CPT,
  58         [HPD_SDVO_B] = SDE_SDVOB_HOTPLUG_CPT,
  59         [HPD_PORT_B] = SDE_PORTB_HOTPLUG_CPT,
  60         [HPD_PORT_C] = SDE_PORTC_HOTPLUG_CPT,
  61         [HPD_PORT_D] = SDE_PORTD_HOTPLUG_CPT
  62 };
  63
  64 static const u32 hpd_spt[HPD_NUM_PINS] = {
  65         [HPD_PORT_B] = SDE_PORTB_HOTPLUG_CPT,
  66         [HPD_PORT_C] = SDE_PORTC_HOTPLUG_CPT,
  67         [HPD_PORT_D] = SDE_PORTD_HOTPLUG_CPT,
  68         [HPD_PORT_E] = SDE_PORTE_HOTPLUG_SPT
  69 };
  70
  71 static const u32 hpd_mask_i915[HPD_NUM_PINS] = {
  72         [HPD_CRT] = CRT_HOTPLUG_INT_EN,
  73         [HPD_SDVO_B] = SDVOB_HOTPLUG_INT_EN,
  74         [HPD_SDVO_C] = SDVOC_HOTPLUG_INT_EN,
  75         [HPD_PORT_B] = PORTB_HOTPLUG_INT_EN,
  76         [HPD_PORT_C] = PORTC_HOTPLUG_INT_EN,
  77         [HPD_PORT_D] = PORTD_HOTPLUG_INT_EN
  78 };
  79
  80 static const u32 hpd_status_g4x[HPD_NUM_PINS] = {
  81         [HPD_CRT] = CRT_HOTPLUG_INT_STATUS,
  82         [HPD_SDVO_B] = SDVOB_HOTPLUG_INT_STATUS_G4X,
  83         [HPD_SDVO_C] = SDVOC_HOTPLUG_INT_STATUS_G4X,
  84         [HPD_PORT_B] = PORTB_HOTPLUG_INT_STATUS,
  85         [HPD_PORT_C] = PORTC_HOTPLUG_INT_STATUS,
  86         [HPD_PORT_D] = PORTD_HOTPLUG_INT_STATUS
  87 };
  88
  89 static const u32 hpd_status_i915[HPD_NUM_PINS] = {
  90         [HPD_CRT] = CRT_HOTPLUG_INT_STATUS,
  91         [HPD_SDVO_B] = SDVOB_HOTPLUG_INT_STATUS_I915,
  92         [HPD_SDVO_C] = SDVOC_HOTPLUG_INT_STATUS_I915,
  93         [HPD_PORT_B] = PORTB_HOTPLUG_INT_STATUS,
  94         [HPD_PORT_C] = PORTC_HOTPLUG_INT_STATUS,
  95         [HPD_PORT_D] = PORTD_HOTPLUG_INT_STATUS
  96 };
  97
  98 /* BXT hpd list */
  99 static const u32 hpd_bxt[HPD_NUM_PINS] = {
 100         [HPD_PORT_B] = BXT_DE_PORT_HP_DDIB,
 101         [HPD_PORT_C] = BXT_DE_PORT_HP_DDIC
 102 };
 103
 104 /* IIR can theoretically queue up two events. Be paranoid. */
 105 #define GEN8_IRQ_RESET_NDX(type, which) do { \
 106         I915_WRITE(GEN8_##type##_IMR(which), 0xffffffff); \
 107         POSTING_READ(GEN8_##type##_IMR(which)); \
 108         I915_WRITE(GEN8_##type##_IER(which), 0); \
 109         I915_WRITE(GEN8_##type##_IIR(which), 0xffffffff); \
 110         POSTING_READ(GEN8_##type##_IIR(which)); \
 111         I915_WRITE(GEN8_##type##_IIR(which), 0xffffffff); \
 112         POSTING_READ(GEN8_##type##_IIR(which)); \
 113 } while (0)
 114
 115 #define GEN5_IRQ_RESET(type) do { \
 116         I915_WRITE(type##IMR, 0xffffffff); \
 117         POSTING_READ(type##IMR); \
 118         I915_WRITE(type##IER, 0); \
 119         I915_WRITE(type##IIR, 0xffffffff); \
 120         POSTING_READ(type##IIR); \
 121         I915_WRITE(type##IIR, 0xffffffff); \
 122         POSTING_READ(type##IIR); \
 123 } while (0)
 124
 125 /*
 126  * We should clear IMR at preinstall/uninstall, and just check at postinstall.
 127  */
 128 #define GEN5_ASSERT_IIR_IS_ZERO(reg) do { \
 129         u32 val = I915_READ(reg); \
 130         if (val) { \
 131                 WARN(1, "Interrupt register 0x%x is not zero: 0x%08x\n", \
 132                      (reg), val); \
 133                 I915_WRITE((reg), 0xffffffff); \
 134                 POSTING_READ(reg); \
 135                 I915_WRITE((reg), 0xffffffff); \
 136                 POSTING_READ(reg); \
 137         } \
 138 } while (0)
 139
 140 #define GEN8_IRQ_INIT_NDX(type, which, imr_val, ier_val) do { \
 141         GEN5_ASSERT_IIR_IS_ZERO(GEN8_##type##_IIR(which)); \
 142         I915_WRITE(GEN8_##type##_IER(which), (ier_val)); \
 143         I915_WRITE(GEN8_##type##_IMR(which), (imr_val)); \
 144         POSTING_READ(GEN8_##type##_IMR(which)); \
 145 } while (0)
 146
 147 #define GEN5_IRQ_INIT(type, imr_val, ier_val) do { \
 148         GEN5_ASSERT_IIR_IS_ZERO(type##IIR); \
 149         I915_WRITE(type##IER, (ier_val)); \
 150         I915_WRITE(type##IMR, (imr_val)); \
 151         POSTING_READ(type##IMR); \
 152 } while (0)
 153
 154 static void gen6_rps_irq_handler(struct drm_i915_private *dev_priv, u32 pm_iir);
 155
 156 /* For display hotplug interrupt */
 157 void
 158 ironlake_enable_display_irq(struct drm_i915_private *dev_priv, u32 mask)
 159 {
 160         assert_spin_locked(&dev_priv->irq_lock);
 161
 162         if (WARN_ON(!intel_irqs_enabled(dev_priv)))
 163                 return;
 164
 165         if ((dev_priv->irq_mask & mask) != 0) {
 166                 dev_priv->irq_mask &= ~mask;
 167                 I915_WRITE(DEIMR, dev_priv->irq_mask);
 168                 POSTING_READ(DEIMR);
 169         }
 170 }
 171
 172 void
 173 ironlake_disable_display_irq(struct drm_i915_private *dev_priv, u32 mask)
 174 {
 175         assert_spin_locked(&dev_priv->irq_lock);
 176
 177         if (WARN_ON(!intel_irqs_enabled(dev_priv)))
 178                 return;
 179
 180         if ((dev_priv->irq_mask & mask) != mask) {
 181                 dev_priv->irq_mask |= mask;
 182                 I915_WRITE(DEIMR, dev_priv->irq_mask);
 183                 POSTING_READ(DEIMR);
 184         }
 185 }
 186
 187 /**
 188  * ilk_update_gt_irq - update GTIMR
 189  * @dev_priv: driver private
 190  * @interrupt_mask: mask of interrupt bits to update
 191  * @enabled_irq_mask: mask of interrupt bits to enable
 192  */
 193 static void ilk_update_gt_irq(struct drm_i915_private *dev_priv,
 194                               uint32_t interrupt_mask,
 195                               uint32_t enabled_irq_mask)
 196 {
 197         assert_spin_locked(&dev_priv->irq_lock);
 198
 199         WARN_ON(enabled_irq_mask & ~interrupt_mask);
 200
 201         if (WARN_ON(!intel_irqs_enabled(dev_priv)))
 202                 return;
 203
 204         dev_priv->gt_irq_mask &= ~interrupt_mask;
 205         dev_priv->gt_irq_mask |= (~enabled_irq_mask & interrupt_mask);
 206         I915_WRITE(GTIMR, dev_priv->gt_irq_mask);
 207         POSTING_READ(GTIMR);
 208 }
 209
 210 void gen5_enable_gt_irq(struct drm_i915_private *dev_priv, uint32_t mask)
 211 {
 212         ilk_update_gt_irq(dev_priv, mask, mask);
 213 }
 214
 215 void gen5_disable_gt_irq(struct drm_i915_private *dev_priv, uint32_t mask)
 216 {
 217         ilk_update_gt_irq(dev_priv, mask, 0);
 218 }
 219
 220 static u32 gen6_pm_iir(struct drm_i915_private *dev_priv)
 221 {
 222         return INTEL_INFO(dev_priv)->gen >= 8 ? GEN8_GT_IIR(2) : GEN6_PMIIR;
 223 }
 224
 225 static u32 gen6_pm_imr(struct drm_i915_private *dev_priv)
 226 {
 227         return INTEL_INFO(dev_priv)->gen >= 8 ? GEN8_GT_IMR(2) : GEN6_PMIMR;
 228 }
 229
 230 static u32 gen6_pm_ier(struct drm_i915_private *dev_priv)
 231 {
 232         return INTEL_INFO(dev_priv)->gen >= 8 ? GEN8_GT_IER(2) : GEN6_PMIER;
 233 }
 234
 235 /**
 236   * snb_update_pm_irq - update GEN6_PMIMR
 237   * @dev_priv: driver private
 238   * @interrupt_mask: mask of interrupt bits to update
 239   * @enabled_irq_mask: mask of interrupt bits to enable
 240   */
 241 static void snb_update_pm_irq(struct drm_i915_private *dev_priv,
 242                               uint32_t interrupt_mask,
 243                               uint32_t enabled_irq_mask)
 244 {
 245         uint32_t new_val;
 246
 247         WARN_ON(enabled_irq_mask & ~interrupt_mask);
 248
 249         assert_spin_locked(&dev_priv->irq_lock);
 250
 251         new_val = dev_priv->pm_irq_mask;
 252         new_val &= ~interrupt_mask;
 253         new_val |= (~enabled_irq_mask & interrupt_mask);
 254
 255         if (new_val != dev_priv->pm_irq_mask) {
 256                 dev_priv->pm_irq_mask = new_val;
 257                 I915_WRITE(gen6_pm_imr(dev_priv), dev_priv->pm_irq_mask);
 258                 POSTING_READ(gen6_pm_imr(dev_priv));
 259         }
 260 }
 261
 262 void gen6_enable_pm_irq(struct drm_i915_private *dev_priv, uint32_t mask)
 263 {
 264         if (WARN_ON(!intel_irqs_enabled(dev_priv)))
 265                 return;
 266
 267         snb_update_pm_irq(dev_priv, mask, mask);
 268 }
 269
 270 static void __gen6_disable_pm_irq(struct drm_i915_private *dev_priv,
 271                                   uint32_t mask)
 272 {
 273         snb_update_pm_irq(dev_priv, mask, 0);
 274 }
 275
 276 void gen6_disable_pm_irq(struct drm_i915_private *dev_priv, uint32_t mask)
 277 {
 278         if (WARN_ON(!intel_irqs_enabled(dev_priv)))
 279                 return;
 280
 281         __gen6_disable_pm_irq(dev_priv, mask);
 282 }
 283
 284 void gen6_reset_rps_interrupts(struct drm_device *dev)
 285 {
 286         struct drm_i915_private *dev_priv = dev->dev_private;
 287         uint32_t reg = gen6_pm_iir(dev_priv);
 288
 289         spin_lock_irq(&dev_priv->irq_lock);
 290         I915_WRITE(reg, dev_priv->pm_rps_events);
 291         I915_WRITE(reg, dev_priv->pm_rps_events);
 292         POSTING_READ(reg);
 293         dev_priv->rps.pm_iir = 0;
 294         spin_unlock_irq(&dev_priv->irq_lock);
 295 }
 296
 297 void gen6_enable_rps_interrupts(struct drm_device *dev)
 298 {
 299         struct drm_i915_private *dev_priv = dev->dev_private;
 300
 301         spin_lock_irq(&dev_priv->irq_lock);
 302
 303         WARN_ON(dev_priv->rps.pm_iir);
 304         WARN_ON(I915_READ(gen6_pm_iir(dev_priv)) & dev_priv->pm_rps_events);
 305         dev_priv->rps.interrupts_enabled = true;
 306         I915_WRITE(gen6_pm_ier(dev_priv), I915_READ(gen6_pm_ier(dev_priv)) |
 307                                 dev_priv->pm_rps_events);
 308         gen6_enable_pm_irq(dev_priv, dev_priv->pm_rps_events);
 309
 310         spin_unlock_irq(&dev_priv->irq_lock);
 311 }
 312
 313 u32 gen6_sanitize_rps_pm_mask(struct drm_i915_private *dev_priv, u32 mask)
 314 {
 315         /*
 316          * SNB,IVB can while VLV,CHV may hard hang on looping batchbuffer
 317          * if GEN6_PM_UP_EI_EXPIRED is masked.
 318          *
 319          * TODO: verify if this can be reproduced on VLV,CHV.
 320          */
 321         if (INTEL_INFO(dev_priv)->gen <= 7 && !IS_HASWELL(dev_priv))
 322                 mask &= ~GEN6_PM_RP_UP_EI_EXPIRED;
 323
 324         if (INTEL_INFO(dev_priv)->gen >= 8)
 325                 mask &= ~GEN8_PMINTR_REDIRECT_TO_NON_DISP;
 326
 327         return mask;
 328 }
 329
 330 void gen6_disable_rps_interrupts(struct drm_device *dev)
 331 {
 332         struct drm_i915_private *dev_priv = dev->dev_private;
 333
 334         spin_lock_irq(&dev_priv->irq_lock);
 335         dev_priv->rps.interrupts_enabled = false;
 336         spin_unlock_irq(&dev_priv->irq_lock);
 337
 338         cancel_work_sync(&dev_priv->rps.work);
 339
 340         spin_lock_irq(&dev_priv->irq_lock);
 341
 342         I915_WRITE(GEN6_PMINTRMSK, gen6_sanitize_rps_pm_mask(dev_priv, ~0));
 343
 344         __gen6_disable_pm_irq(dev_priv, dev_priv->pm_rps_events);
 345         I915_WRITE(gen6_pm_ier(dev_priv), I915_READ(gen6_pm_ier(dev_priv)) &
 346                                 ~dev_priv->pm_rps_events);
 347
 348         spin_unlock_irq(&dev_priv->irq_lock);
 349
 350         synchronize_irq(dev->irq);
 351 }
 352
 353 /**
 354  * ibx_display_interrupt_update - update SDEIMR
 355  * @dev_priv: driver private
 356  * @interrupt_mask: mask of interrupt bits to update
 357  * @enabled_irq_mask: mask of interrupt bits to enable
 358  */
 359 void ibx_display_interrupt_update(struct drm_i915_private *dev_priv,
 360                                   uint32_t interrupt_mask,
 361                                   uint32_t enabled_irq_mask)
 362 {
 363         uint32_t sdeimr = I915_READ(SDEIMR);
 364         sdeimr &= ~interrupt_mask;
 365         sdeimr |= (~enabled_irq_mask & interrupt_mask);
 366
 367         WARN_ON(enabled_irq_mask & ~interrupt_mask);
 368
 369         assert_spin_locked(&dev_priv->irq_lock);
 370
 371         if (WARN_ON(!intel_irqs_enabled(dev_priv)))
 372                 return;
 373
 374         I915_WRITE(SDEIMR, sdeimr);
 375         POSTING_READ(SDEIMR);
 376 }
 377
 378 static void
 379 __i915_enable_pipestat(struct drm_i915_private *dev_priv, enum pipe pipe,
 380                        u32 enable_mask, u32 status_mask)
 381 {
 382         u32 reg = PIPESTAT(pipe);
 383         u32 pipestat = I915_READ(reg) & PIPESTAT_INT_ENABLE_MASK;
 384
 385         assert_spin_locked(&dev_priv->irq_lock);
 386         WARN_ON(!intel_irqs_enabled(dev_priv));
 387
 388         if (WARN_ONCE(enable_mask & ~PIPESTAT_INT_ENABLE_MASK ||
 389                       status_mask & ~PIPESTAT_INT_STATUS_MASK,
 390                       "pipe %c: enable_mask=0x%x, status_mask=0x%x\n",
 391                       pipe_name(pipe), enable_mask, status_mask))
 392                 return;
 393
 394         if ((pipestat & enable_mask) == enable_mask)
 395                 return;
 396
 397         dev_priv->pipestat_irq_mask[pipe] |= status_mask;
 398
 399         /* Enable the interrupt, clear any pending status */
 400         pipestat |= enable_mask | status_mask;
 401         I915_WRITE(reg, pipestat);
 402         POSTING_READ(reg);
 403 }
 404
 405 static void
 406 __i915_disable_pipestat(struct drm_i915_private *dev_priv, enum pipe pipe,
 407                         u32 enable_mask, u32 status_mask)
 408 {
 409         u32 reg = PIPESTAT(pipe);
 410         u32 pipestat = I915_READ(reg) & PIPESTAT_INT_ENABLE_MASK;
 411
 412         assert_spin_locked(&dev_priv->irq_lock);
 413         WARN_ON(!intel_irqs_enabled(dev_priv));
 414
 415         if (WARN_ONCE(enable_mask & ~PIPESTAT_INT_ENABLE_MASK ||
 416                       status_mask & ~PIPESTAT_INT_STATUS_MASK,
 417                       "pipe %c: enable_mask=0x%x, status_mask=0x%x\n",
 418                       pipe_name(pipe), enable_mask, status_mask))
 419                 return;
 420
 421         if ((pipestat & enable_mask) == 0)
 422                 return;
 423
 424         dev_priv->pipestat_irq_mask[pipe] &= ~status_mask;
 425
 426         pipestat &= ~enable_mask;
 427         I915_WRITE(reg, pipestat);
 428         POSTING_READ(reg);
 429 }
 430
 431 static u32 vlv_get_pipestat_enable_mask(struct drm_device *dev, u32 status_mask)
 432 {
 433         u32 enable_mask = status_mask << 16;
 434
 435         /*
 436          * On pipe A we don't support the PSR interrupt yet,
 437          * on pipe B and C the same bit MBZ.
 438          */
 439         if (WARN_ON_ONCE(status_mask & PIPE_A_PSR_STATUS_VLV))
 440                 return 0;
 441         /*
 442          * On pipe B and C we don't support the PSR interrupt yet, on pipe
 443          * A the same bit is for perf counters which we don't use either.
 444          */
 445         if (WARN_ON_ONCE(status_mask & PIPE_B_PSR_STATUS_VLV))
 446                 return 0;
 447
 448         enable_mask &= ~(PIPE_FIFO_UNDERRUN_STATUS |
 449                          SPRITE0_FLIP_DONE_INT_EN_VLV |
 450                          SPRITE1_FLIP_DONE_INT_EN_VLV);
 451         if (status_mask & SPRITE0_FLIP_DONE_INT_STATUS_VLV)
 452                 enable_mask |= SPRITE0_FLIP_DONE_INT_EN_VLV;
 453         if (status_mask & SPRITE1_FLIP_DONE_INT_STATUS_VLV)
 454                 enable_mask |= SPRITE1_FLIP_DONE_INT_EN_VLV;
 455
 456         return enable_mask;
 457 }
 458
 459 void
 460 i915_enable_pipestat(struct drm_i915_private *dev_priv, enum pipe pipe,
 461                      u32 status_mask)
 462 {
 463         u32 enable_mask;
 464
 465         if (IS_VALLEYVIEW(dev_priv->dev))
 466                 enable_mask = vlv_get_pipestat_enable_mask(dev_priv->dev,
 467                                                            status_mask);
 468         else
 469                 enable_mask = status_mask << 16;
 470         __i915_enable_pipestat(dev_priv, pipe, enable_mask, status_mask);
 471 }
 472
 473 void
 474 i915_disable_pipestat(struct drm_i915_private *dev_priv, enum pipe pipe,
 475                       u32 status_mask)
 476 {
 477         u32 enable_mask;
 478
 479         if (IS_VALLEYVIEW(dev_priv->dev))
 480                 enable_mask = vlv_get_pipestat_enable_mask(dev_priv->dev,
 481                                                            status_mask);
 482         else
 483                 enable_mask = status_mask << 16;
 484         __i915_disable_pipestat(dev_priv, pipe, enable_mask, status_mask);
 485 }
 486
 487 /**
 488  * i915_enable_asle_pipestat - enable ASLE pipestat for OpRegion
 489  */
 490 static void i915_enable_asle_pipestat(struct drm_device *dev)
 491 {
 492         struct drm_i915_private *dev_priv = dev->dev_private;
 493
 494         if (!dev_priv->opregion.asle || !IS_MOBILE(dev))
 495                 return;
 496
 497         spin_lock_irq(&dev_priv->irq_lock);
 498
 499         i915_enable_pipestat(dev_priv, PIPE_B, PIPE_LEGACY_BLC_EVENT_STATUS);
 500         if (INTEL_INFO(dev)->gen >= 4)
 501                 i915_enable_pipestat(dev_priv, PIPE_A,
 502                                      PIPE_LEGACY_BLC_EVENT_STATUS);
 503
 504         spin_unlock_irq(&dev_priv->irq_lock);
 505 }
 506
 507 /*
 508  * This timing diagram depicts the video signal in and
 509  * around the vertical blanking period.
 510  *
 511  * Assumptions about the fictitious mode used in this example:
 512  *  vblank_start >= 3
 513  *  vsync_start = vblank_start + 1
 514  *  vsync_end = vblank_start + 2
 515  *  vtotal = vblank_start + 3
 516  *
 517  *           start of vblank:
 518  *           latch double buffered registers
 519  *           increment frame counter (ctg+)
 520  *           generate start of vblank interrupt (gen4+)
 521  *           |
 522  *           |          frame start:
 523  *           |          generate frame start interrupt (aka. vblank interrupt) (gmch)
 524  *           |          may be shifted forward 1-3 extra lines via PIPECONF
 525  *           |          |
 526  *           |          |  start of vsync:
 527  *           |          |  generate vsync interrupt
 528  *           |          |  |
 529  * ___xxxx___    ___xxxx___    ___xxxx___    ___xxxx___    ___xxxx___    ___xxxx
 530  *       .   \hs/   .      \hs/          \hs/          \hs/   .      \hs/
 531  * ----va---> <-----------------vb--------------------> <--------va-------------
 532  *       |          |       <----vs----->                     |
 533  * -vbs-----> <---vbs+1---> <---vbs+2---> <-----0-----> <-----1-----> <-----2--- (scanline counter gen2)
 534  * -vbs-2---> <---vbs-1---> <---vbs-----> <---vbs+1---> <---vbs+2---> <-----0--- (scanline counter gen3+)
 535  * -vbs-2---> <---vbs-2---> <---vbs-1---> <---vbs-----> <---vbs+1---> <---vbs+2- (scanline counter hsw+ hdmi)
 536  *       |          |                                         |
 537  *       last visible pixel                                   first visible pixel
 538  *                  |                                         increment frame counter (gen3/4)
 539  *                  pixel counter = vblank_start * htotal     pixel counter = 0 (gen3/4)
 540  *
 541  * x  = horizontal active
 542  * _  = horizontal blanking
 543  * hs = horizontal sync
 544  * va = vertical active
 545  * vb = vertical blanking
 546  * vs = vertical sync
 547  * vbs = vblank_start (number)
 548  *
 549  * Summary:
 550  * - most events happen at the start of horizontal sync
 551  * - frame start happens at the start of horizontal blank, 1-4 lines
 552  *   (depending on PIPECONF settings) after the start of vblank
 553  * - gen3/4 pixel and frame counter are synchronized with the start
 554  *   of horizontal active on the first line of vertical active
 555  */
 556
 557 static u32 i8xx_get_vblank_counter(struct drm_device *dev, int pipe)
 558 {
 559         /* Gen2 doesn't have a hardware frame counter */
 560         return 0;
 561 }
 562
 563 /* Called from drm generic code, passed a 'crtc', which
 564  * we use as a pipe index
 565  */
 566 static u32 i915_get_vblank_counter(struct drm_device *dev, int pipe)
 567 {
 568         struct drm_i915_private *dev_priv = dev->dev_private;
 569         unsigned long high_frame;
 570         unsigned long low_frame;
 571         u32 high1, high2, low, pixel, vbl_start, hsync_start, htotal;
 572         struct intel_crtc *intel_crtc =
 573                 to_intel_crtc(dev_priv->pipe_to_crtc_mapping[pipe]);
 574         const struct drm_display_mode *mode = &intel_crtc->base.hwmode;
 575
 576         htotal = mode->crtc_htotal;
 577         hsync_start = mode->crtc_hsync_start;
 578         vbl_start = mode->crtc_vblank_start;
 579         if (mode->flags & DRM_MODE_FLAG_INTERLACE)
 580                 vbl_start = DIV_ROUND_UP(vbl_start, 2);
 581
 582         /* Convert to pixel count */
 583         vbl_start *= htotal;
 584
 585         /* Start of vblank event occurs at start of hsync */
 586         vbl_start -= htotal - hsync_start;
 587
 588         high_frame = PIPEFRAME(pipe);
 589         low_frame = PIPEFRAMEPIXEL(pipe);
 590
 591         /*
 592          * High & low register fields aren't synchronized, so make sure
 593          * we get a low value that's stable across two reads of the high
 594          * register.
 595          */
 596         do {
 597                 high1 = I915_READ(high_frame) & PIPE_FRAME_HIGH_MASK;
 598                 low   = I915_READ(low_frame);
 599                 high2 = I915_READ(high_frame) & PIPE_FRAME_HIGH_MASK;
 600         } while (high1 != high2);
 601
 602         high1 >>= PIPE_FRAME_HIGH_SHIFT;
 603         pixel = low & PIPE_PIXEL_MASK;
 604         low >>= PIPE_FRAME_LOW_SHIFT;
 605
 606         /*
 607          * The frame counter increments at beginning of active.
 608          * Cook up a vblank counter by also checking the pixel
 609          * counter against vblank start.
 610          */
 611         return (((high1 << 8) | low) + (pixel >= vbl_start)) & 0xffffff;
 612 }
 613
 614 static u32 gm45_get_vblank_counter(struct drm_device *dev, int pipe)
 615 {
 616         struct drm_i915_private *dev_priv = dev->dev_private;
 617         int reg = PIPE_FRMCOUNT_GM45(pipe);
 618
 619         return I915_READ(reg);
 620 }
 621
 622 /* raw reads, only for fast reads of display block, no need for forcewake etc. */
 623 #define __raw_i915_read32(dev_priv__, reg__) readl((dev_priv__)->regs + (reg__))
 624
 625 static int __intel_get_crtc_scanline(struct intel_crtc *crtc)
 626 {
 627         struct drm_device *dev = crtc->base.dev;
 628         struct drm_i915_private *dev_priv = dev->dev_private;
 629         const struct drm_display_mode *mode = &crtc->base.hwmode;
 630         enum pipe pipe = crtc->pipe;
 631         int position, vtotal;
 632
 633         vtotal = mode->crtc_vtotal;
 634         if (mode->flags & DRM_MODE_FLAG_INTERLACE)
 635                 vtotal /= 2;
 636
 637         if (IS_GEN2(dev))
 638                 position = __raw_i915_read32(dev_priv, PIPEDSL(pipe)) & DSL_LINEMASK_GEN2;
 639         else
 640                 position = __raw_i915_read32(dev_priv, PIPEDSL(pipe)) & DSL_LINEMASK_GEN3;
 641
 642         /*
 643          * See update_scanline_offset() for the details on the
 644          * scanline_offset adjustment.
 645          */
 646         return (position + crtc->scanline_offset) % vtotal;
 647 }
 648
 649 static int i915_get_crtc_scanoutpos(struct drm_device *dev, int pipe,
 650                                     unsigned int flags, int *vpos, int *hpos,
 651                                     ktime_t *stime, ktime_t *etime,
 652                                     const struct drm_display_mode *mode)
 653 {
 654         struct drm_i915_private *dev_priv = dev->dev_private;
 655         struct drm_crtc *crtc = dev_priv->pipe_to_crtc_mapping[pipe];
 656         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
 657         int position;
 658         int vbl_start, vbl_end, hsync_start, htotal, vtotal;
 659         bool in_vbl = true;
 660         int ret = 0;
 661         unsigned long irqflags;
 662
 663         if (WARN_ON(!mode->crtc_clock)) {
 664                 DRM_DEBUG_DRIVER("trying to get scanoutpos for disabled "
 665                                  "pipe %c\n", pipe_name(pipe));
 666                 return 0;
 667         }
 668
 669         htotal = mode->crtc_htotal;
 670         hsync_start = mode->crtc_hsync_start;
 671         vtotal = mode->crtc_vtotal;
 672         vbl_start = mode->crtc_vblank_start;
 673         vbl_end = mode->crtc_vblank_end;
 674
 675         if (mode->flags & DRM_MODE_FLAG_INTERLACE) {
 676                 vbl_start = DIV_ROUND_UP(vbl_start, 2);
 677                 vbl_end /= 2;
 678                 vtotal /= 2;
 679         }
 680
 681         ret |= DRM_SCANOUTPOS_VALID | DRM_SCANOUTPOS_ACCURATE;
 682
 683         /*
 684          * Lock uncore.lock, as we will do multiple timing critical raw
 685          * register reads, potentially with preemption disabled, so the
 686          * following code must not block on uncore.lock.
 687          */
 688         spin_lock_irqsave(&dev_priv->uncore.lock, irqflags);
 689
 690         /* preempt_disable_rt() should go right here in PREEMPT_RT patchset. */
 691
 692         /* Get optional system timestamp before query. */
 693         if (stime)
 694                 *stime = ktime_get();
 695
 696         if (IS_GEN2(dev) || IS_G4X(dev) || INTEL_INFO(dev)->gen >= 5) {
 697                 /* No obvious pixelcount register. Only query vertical
 698                  * scanout position from Display scan line register.
 699                  */
 700                 position = __intel_get_crtc_scanline(intel_crtc);
 701         } else {
 702                 /* Have access to pixelcount since start of frame.
 703                  * We can split this into vertical and horizontal
 704                  * scanout position.
 705                  */
 706                 position = (__raw_i915_read32(dev_priv, PIPEFRAMEPIXEL(pipe)) & PIPE_PIXEL_MASK) >> PIPE_PIXEL_SHIFT;
 707
 708                 /* convert to pixel counts */
 709                 vbl_start *= htotal;
 710                 vbl_end *= htotal;
 711                 vtotal *= htotal;
 712
 713                 /*
 714                  * In interlaced modes, the pixel counter counts all pixels,
 715                  * so one field will have htotal more pixels. In order to avoid
 716                  * the reported position from jumping backwards when the pixel
 717                  * counter is beyond the length of the shorter field, just
 718                  * clamp the position the length of the shorter field. This
 719                  * matches how the scanline counter based position works since
 720                  * the scanline counter doesn't count the two half lines.
 721                  */
 722                 if (position >= vtotal)
 723                         position = vtotal - 1;
 724
 725                 /*
 726                  * Start of vblank interrupt is triggered at start of hsync,
 727                  * just prior to the first active line of vblank. However we
 728                  * consider lines to start at the leading edge of horizontal
 729                  * active. So, should we get here before we've crossed into
 730                  * the horizontal active of the first line in vblank, we would
 731                  * not set the DRM_SCANOUTPOS_INVBL flag. In order to fix that,
 732                  * always add htotal-hsync_start to the current pixel position.
 733                  */
 734                 position = (position + htotal - hsync_start) % vtotal;
 735         }
 736
 737         /* Get optional system timestamp after query. */
 738         if (etime)
 739                 *etime = ktime_get();
 740
 741         /* preempt_enable_rt() should go right here in PREEMPT_RT patchset. */
 742
 743         spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags);
 744
 745         in_vbl = position >= vbl_start && position < vbl_end;
 746
 747         /*
 748          * While in vblank, position will be negative
 749          * counting up towards 0 at vbl_end. And outside
 750          * vblank, position will be positive counting
 751          * up since vbl_end.
 752          */
 753         if (position >= vbl_start)
 754                 position -= vbl_end;
 755         else
 756                 position += vtotal - vbl_end;
 757
 758         if (IS_GEN2(dev) || IS_G4X(dev) || INTEL_INFO(dev)->gen >= 5) {
 759                 *vpos = position;
 760                 *hpos = 0;
 761         } else {
 762                 *vpos = position / htotal;
 763                 *hpos = position - (*vpos * htotal);
 764         }
 765
 766         /* In vblank? */
 767         if (in_vbl)
 768                 ret |= DRM_SCANOUTPOS_IN_VBLANK;
 769
 770         return ret;
 771 }
 772
 773 int intel_get_crtc_scanline(struct intel_crtc *crtc)
 774 {
 775         struct drm_i915_private *dev_priv = crtc->base.dev->dev_private;
 776         unsigned long irqflags;
 777         int position;
 778
 779         spin_lock_irqsave(&dev_priv->uncore.lock, irqflags);
 780         position = __intel_get_crtc_scanline(crtc);
 781         spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags);
 782
 783         return position;
 784 }
 785
 786 static int i915_get_vblank_timestamp(struct drm_device *dev, int pipe,
 787                               int *max_error,
 788                               struct timeval *vblank_time,
 789                               unsigned flags)
 790 {
 791         struct drm_crtc *crtc;
 792
 793         if (pipe < 0 || pipe >= INTEL_INFO(dev)->num_pipes) {
 794                 DRM_ERROR("Invalid crtc %d\n", pipe);
 795                 return -EINVAL;
 796         }
 797
 798         /* Get drm_crtc to timestamp: */
 799         crtc = intel_get_crtc_for_pipe(dev, pipe);
 800         if (crtc == NULL) {
 801                 DRM_ERROR("Invalid crtc %d\n", pipe);
 802                 return -EINVAL;
 803         }
 804
 805         if (!crtc->hwmode.crtc_clock) {
 806                 DRM_DEBUG_KMS("crtc %d is disabled\n", pipe);
 807                 return -EBUSY;
 808         }
 809
 810         /* Helper routine in DRM core does all the work: */
 811         return drm_calc_vbltimestamp_from_scanoutpos(dev, pipe, max_error,
 812                                                      vblank_time, flags,
 813                                                      &crtc->hwmode);
 814 }
 815
 816 static void ironlake_rps_change_irq_handler(struct drm_device *dev)
 817 {
 818         struct drm_i915_private *dev_priv = dev->dev_private;
 819         u32 busy_up, busy_down, max_avg, min_avg;
 820         u8 new_delay;
 821
 822         spin_lock(&mchdev_lock);
 823
 824         I915_WRITE16(MEMINTRSTS, I915_READ(MEMINTRSTS));
 825
 826         new_delay = dev_priv->ips.cur_delay;
 827
 828         I915_WRITE16(MEMINTRSTS, MEMINT_EVAL_CHG);
 829         busy_up = I915_READ(RCPREVBSYTUPAVG);
 830         busy_down = I915_READ(RCPREVBSYTDNAVG);
 831         max_avg = I915_READ(RCBMAXAVG);
 832         min_avg = I915_READ(RCBMINAVG);
 833
 834         /* Handle RCS change request from hw */
 835         if (busy_up > max_avg) {
 836                 if (dev_priv->ips.cur_delay != dev_priv->ips.max_delay)
 837                         new_delay = dev_priv->ips.cur_delay - 1;
 838                 if (new_delay < dev_priv->ips.max_delay)
 839                         new_delay = dev_priv->ips.max_delay;
 840         } else if (busy_down < min_avg) {
 841                 if (dev_priv->ips.cur_delay != dev_priv->ips.min_delay)
 842                         new_delay = dev_priv->ips.cur_delay + 1;
 843                 if (new_delay > dev_priv->ips.min_delay)
 844                         new_delay = dev_priv->ips.min_delay;
 845         }
 846
 847         if (ironlake_set_drps(dev, new_delay))
 848                 dev_priv->ips.cur_delay = new_delay;
 849
 850         spin_unlock(&mchdev_lock);
 851
 852         return;
 853 }
 854
 855 static void notify_ring(struct intel_engine_cs *ring)
 856 {
 857         if (!intel_ring_initialized(ring))
 858                 return;
 859
 860         trace_i915_gem_request_notify(ring);
 861
 862         wake_up_all(&ring->irq_queue);
 863 }
 864
 865 static void vlv_c0_read(struct drm_i915_private *dev_priv,
 866                         struct intel_rps_ei *ei)
 867 {
 868         ei->cz_clock = vlv_punit_read(dev_priv, PUNIT_REG_CZ_TIMESTAMP);
 869         ei->render_c0 = I915_READ(VLV_RENDER_C0_COUNT);
 870         ei->media_c0 = I915_READ(VLV_MEDIA_C0_COUNT);
 871 }
 872
 873 static bool vlv_c0_above(struct drm_i915_private *dev_priv,
 874                          const struct intel_rps_ei *old,
 875                          const struct intel_rps_ei *now,
 876                          int threshold)
 877 {
 878         u64 time, c0;
 879
 880         if (old->cz_clock == 0)
 881                 return false;
 882
 883         time = now->cz_clock - old->cz_clock;
 884         time *= threshold * dev_priv->mem_freq;
 885
 886         /* Workload can be split between render + media, e.g. SwapBuffers
 887          * being blitted in X after being rendered in mesa. To account for
 888          * this we need to combine both engines into our activity counter.
 889          */
 890         c0 = now->render_c0 - old->render_c0;
 891         c0 += now->media_c0 - old->media_c0;
 892         c0 *= 100 * VLV_CZ_CLOCK_TO_MILLI_SEC * 4 / 1000;
 893
 894         return c0 >= time;
 895 }
 896
 897 void gen6_rps_reset_ei(struct drm_i915_private *dev_priv)
 898 {
 899         vlv_c0_read(dev_priv, &dev_priv->rps.down_ei);
 900         dev_priv->rps.up_ei = dev_priv->rps.down_ei;
 901 }
 902
 903 static u32 vlv_wa_c0_ei(struct drm_i915_private *dev_priv, u32 pm_iir)
 904 {
 905         struct intel_rps_ei now;
 906         u32 events = 0;
 907
 908         if ((pm_iir & (GEN6_PM_RP_DOWN_EI_EXPIRED | GEN6_PM_RP_UP_EI_EXPIRED)) == 0)
 909                 return 0;
 910
 911         vlv_c0_read(dev_priv, &now);
 912         if (now.cz_clock == 0)
 913                 return 0;
 914
 915         if (pm_iir & GEN6_PM_RP_DOWN_EI_EXPIRED) {
 916                 if (!vlv_c0_above(dev_priv,
 917                                   &dev_priv->rps.down_ei, &now,
 918                                   dev_priv->rps.down_threshold))
 919                         events |= GEN6_PM_RP_DOWN_THRESHOLD;
 920                 dev_priv->rps.down_ei = now;
 921         }
 922
 923         if (pm_iir & GEN6_PM_RP_UP_EI_EXPIRED) {
 924                 if (vlv_c0_above(dev_priv,
 925                                  &dev_priv->rps.up_ei, &now,
 926                                  dev_priv->rps.up_threshold))
 927                         events |= GEN6_PM_RP_UP_THRESHOLD;
 928                 dev_priv->rps.up_ei = now;
 929         }
 930
 931         return events;
 932 }
 933
 934 static bool any_waiters(struct drm_i915_private *dev_priv)
 935 {
 936         struct intel_engine_cs *ring;
 937         int i;
 938
 939         for_each_ring(ring, dev_priv, i)
 940                 if (ring->irq_refcount)
 941                         return true;
 942
 943         return false;
 944 }
 945
 946 static void gen6_pm_rps_work(struct work_struct *work)
 947 {
 948         struct drm_i915_private *dev_priv =
 949                 container_of(work, struct drm_i915_private, rps.work);
 950         bool client_boost;
 951         int new_delay, adj, min, max;
 952         u32 pm_iir;
 953
 954         spin_lock_irq(&dev_priv->irq_lock);
 955         /* Speed up work cancelation during disabling rps interrupts. */
 956         if (!dev_priv->rps.interrupts_enabled) {
 957                 spin_unlock_irq(&dev_priv->irq_lock);
 958                 return;
 959         }
 960         pm_iir = dev_priv->rps.pm_iir;
 961         dev_priv->rps.pm_iir = 0;
 962         /* Make sure not to corrupt PMIMR state used by ringbuffer on GEN6 */
 963         gen6_enable_pm_irq(dev_priv, dev_priv->pm_rps_events);
 964         client_boost = dev_priv->rps.client_boost;
 965         dev_priv->rps.client_boost = false;
 966         spin_unlock_irq(&dev_priv->irq_lock);
 967
 968         /* Make sure we didn't queue anything we're not going to process. */
 969         WARN_ON(pm_iir & ~dev_priv->pm_rps_events);
 970
 971         if ((pm_iir & dev_priv->pm_rps_events) == 0 && !client_boost)
 972                 return;
 973
 974         mutex_lock(&dev_priv->rps.hw_lock);
 975
 976         pm_iir |= vlv_wa_c0_ei(dev_priv, pm_iir);
 977
 978         adj = dev_priv->rps.last_adj;
 979         new_delay = dev_priv->rps.cur_freq;
 980         min = dev_priv->rps.min_freq_softlimit;
 981         max = dev_priv->rps.max_freq_softlimit;
 982
 983         if (client_boost) {
 984                 new_delay = dev_priv->rps.max_freq_softlimit;
 985                 adj = 0;
 986         } else if (pm_iir & GEN6_PM_RP_UP_THRESHOLD) {
 987                 if (adj > 0)
 988                         adj *= 2;
 989                 else /* CHV needs even encode values */
 990                         adj = IS_CHERRYVIEW(dev_priv) ? 2 : 1;
 991                 /*
 992                  * For better performance, jump directly
 993                  * to RPe if we're below it.
 994                  */
 995                 if (new_delay < dev_priv->rps.efficient_freq - adj) {
 996                         new_delay = dev_priv->rps.efficient_freq;
 997                         adj = 0;
 998                 }
 999         } else if (any_waiters(dev_priv)) {
1000                 adj = 0;
1001         } else if (pm_iir & GEN6_PM_RP_DOWN_TIMEOUT) {
1002                 if (dev_priv->rps.cur_freq > dev_priv->rps.efficient_freq)
1003                         new_delay = dev_priv->rps.efficient_freq;
1004                 else
1005                         new_delay = dev_priv->rps.min_freq_softlimit;
1006                 adj = 0;
1007         } else if (pm_iir & GEN6_PM_RP_DOWN_THRESHOLD) {
1008                 if (adj < 0)
1009                         adj *= 2;
1010                 else /* CHV needs even encode values */
1011                         adj = IS_CHERRYVIEW(dev_priv) ? -2 : -1;
1012         } else { /* unknown event */
1013                 adj = 0;
1014         }
1015
1016         dev_priv->rps.last_adj = adj;
1017
1018         /* sysfs frequency interfaces may have snuck in while servicing the
1019          * interrupt
1020          */
1021         new_delay += adj;
1022         new_delay = clamp_t(int, new_delay, min, max);
1023
1024         intel_set_rps(dev_priv->dev, new_delay);
1025
1026         mutex_unlock(&dev_priv->rps.hw_lock);
1027 }
1028
1029
1030 /**
1031  * ivybridge_parity_work - Workqueue called when a parity error interrupt
1032  * occurred.
1033  * @work: workqueue struct
1034  *
1035  * Doesn't actually do anything except notify userspace. As a consequence of
1036  * this event, userspace should try to remap the bad rows since statistically
1037  * it is likely the same row is more likely to go bad again.
1038  */
1039 static void ivybridge_parity_work(struct work_struct *work)
1040 {
1041         struct drm_i915_private *dev_priv =
1042                 container_of(work, struct drm_i915_private, l3_parity.error_work);
1043         u32 error_status, row, bank, subbank;
1044         char *parity_event[6];
1045         uint32_t misccpctl;
1046         uint8_t slice = 0;
1047
1048         /* We must turn off DOP level clock gating to access the L3 registers.
1049          * In order to prevent a get/put style interface, acquire struct mutex
1050          * any time we access those registers.
1051          */
1052         mutex_lock(&dev_priv->dev->struct_mutex);
1053
1054         /* If we've screwed up tracking, just let the interrupt fire again */
1055         if (WARN_ON(!dev_priv->l3_parity.which_slice))
1056                 goto out;
1057
1058         misccpctl = I915_READ(GEN7_MISCCPCTL);
1059         I915_WRITE(GEN7_MISCCPCTL, misccpctl & ~GEN7_DOP_CLOCK_GATE_ENABLE);
1060         POSTING_READ(GEN7_MISCCPCTL);
1061
1062         while ((slice = ffs(dev_priv->l3_parity.which_slice)) != 0) {
1063                 u32 reg;
1064
1065                 slice--;
1066                 if (WARN_ON_ONCE(slice >= NUM_L3_SLICES(dev_priv->dev)))
1067                         break;
1068
1069                 dev_priv->l3_parity.which_slice &= ~(1<<slice);
1070
1071                 reg = GEN7_L3CDERRST1 + (slice * 0x200);
1072
1073                 error_status = I915_READ(reg);
1074                 row = GEN7_PARITY_ERROR_ROW(error_status);
1075                 bank = GEN7_PARITY_ERROR_BANK(error_status);
1076                 subbank = GEN7_PARITY_ERROR_SUBBANK(error_status);
1077
1078                 I915_WRITE(reg, GEN7_PARITY_ERROR_VALID | GEN7_L3CDERRST1_ENABLE);
1079                 POSTING_READ(reg);
1080
1081                 parity_event[0] = I915_L3_PARITY_UEVENT "=1";
1082                 parity_event[1] = kasprintf(GFP_KERNEL, "ROW=%d", row);
1083                 parity_event[2] = kasprintf(GFP_KERNEL, "BANK=%d", bank);
1084                 parity_event[3] = kasprintf(GFP_KERNEL, "SUBBANK=%d", subbank);
1085                 parity_event[4] = kasprintf(GFP_KERNEL, "SLICE=%d", slice);
1086                 parity_event[5] = NULL;
1087
1088                 kobject_uevent_env(&dev_priv->dev->primary->kdev->kobj,
1089                                    KOBJ_CHANGE, parity_event);
1090
1091                 DRM_DEBUG("Parity error: Slice = %d, Row = %d, Bank = %d, Sub bank = %d.\n",
1092                           slice, row, bank, subbank);
1093
1094                 kfree(parity_event[4]);
1095                 kfree(parity_event[3]);
1096                 kfree(parity_event[2]);
1097                 kfree(parity_event[1]);
1098         }
1099
1100         I915_WRITE(GEN7_MISCCPCTL, misccpctl);
1101
1102 out:
1103         WARN_ON(dev_priv->l3_parity.which_slice);
1104         spin_lock_irq(&dev_priv->irq_lock);
1105         gen5_enable_gt_irq(dev_priv, GT_PARITY_ERROR(dev_priv->dev));
1106         spin_unlock_irq(&dev_priv->irq_lock);
1107
1108         mutex_unlock(&dev_priv->dev->struct_mutex);
1109 }
1110
1111 static void ivybridge_parity_error_irq_handler(struct drm_device *dev, u32 iir)
1112 {
1113         struct drm_i915_private *dev_priv = dev->dev_private;
1114
1115         if (!HAS_L3_DPF(dev))
1116                 return;
1117
1118         spin_lock(&dev_priv->irq_lock);
1119         gen5_disable_gt_irq(dev_priv, GT_PARITY_ERROR(dev));
1120         spin_unlock(&dev_priv->irq_lock);
1121
1122         iir &= GT_PARITY_ERROR(dev);
1123         if (iir & GT_RENDER_L3_PARITY_ERROR_INTERRUPT_S1)
1124                 dev_priv->l3_parity.which_slice |= 1 << 1;
1125
1126         if (iir & GT_RENDER_L3_PARITY_ERROR_INTERRUPT)
1127                 dev_priv->l3_parity.which_slice |= 1 << 0;
1128
1129         queue_work(dev_priv->wq, &dev_priv->l3_parity.error_work);
1130 }
1131
1132 static void ilk_gt_irq_handler(struct drm_device *dev,
1133                                struct drm_i915_private *dev_priv,
1134                                u32 gt_iir)
1135 {
1136         if (gt_iir &
1137             (GT_RENDER_USER_INTERRUPT | GT_RENDER_PIPECTL_NOTIFY_INTERRUPT))
1138                 notify_ring(&dev_priv->ring[RCS]);
1139         if (gt_iir & ILK_BSD_USER_INTERRUPT)
1140                 notify_ring(&dev_priv->ring[VCS]);
1141 }
1142
1143 static void snb_gt_irq_handler(struct drm_device *dev,
1144                                struct drm_i915_private *dev_priv,
1145                                u32 gt_iir)
1146 {
1147
1148         if (gt_iir &
1149             (GT_RENDER_USER_INTERRUPT | GT_RENDER_PIPECTL_NOTIFY_INTERRUPT))
1150                 notify_ring(&dev_priv->ring[RCS]);
1151         if (gt_iir & GT_BSD_USER_INTERRUPT)
1152                 notify_ring(&dev_priv->ring[VCS]);
1153         if (gt_iir & GT_BLT_USER_INTERRUPT)
1154                 notify_ring(&dev_priv->ring[BCS]);
1155
1156         if (gt_iir & (GT_BLT_CS_ERROR_INTERRUPT |
1157                       GT_BSD_CS_ERROR_INTERRUPT |
1158                       GT_RENDER_CS_MASTER_ERROR_INTERRUPT))
1159                 DRM_DEBUG("Command parser error, gt_iir 0x%08x\n", gt_iir);
1160
1161         if (gt_iir & GT_PARITY_ERROR(dev))
1162                 ivybridge_parity_error_irq_handler(dev, gt_iir);
1163 }
1164
1165 static irqreturn_t gen8_gt_irq_handler(struct drm_i915_private *dev_priv,
1166                                        u32 master_ctl)
1167 {
1168         irqreturn_t ret = IRQ_NONE;
1169
1170         if (master_ctl & (GEN8_GT_RCS_IRQ | GEN8_GT_BCS_IRQ)) {
1171                 u32 tmp = I915_READ_FW(GEN8_GT_IIR(0));
1172                 if (tmp) {
1173                         I915_WRITE_FW(GEN8_GT_IIR(0), tmp);
1174                         ret = IRQ_HANDLED;
1175
1176                         if (tmp & (GT_CONTEXT_SWITCH_INTERRUPT << GEN8_RCS_IRQ_SHIFT))
1177                                 intel_lrc_irq_handler(&dev_priv->ring[RCS]);
1178                         if (tmp & (GT_RENDER_USER_INTERRUPT << GEN8_RCS_IRQ_SHIFT))
1179                                 notify_ring(&dev_priv->ring[RCS]);
1180
1181                         if (tmp & (GT_CONTEXT_SWITCH_INTERRUPT << GEN8_BCS_IRQ_SHIFT))
1182                                 intel_lrc_irq_handler(&dev_priv->ring[BCS]);
1183                         if (tmp & (GT_RENDER_USER_INTERRUPT << GEN8_BCS_IRQ_SHIFT))
1184                                 notify_ring(&dev_priv->ring[BCS]);
1185                 } else
1186                         DRM_ERROR("The master control interrupt lied (GT0)!\n");
1187         }
1188
1189         if (master_ctl & (GEN8_GT_VCS1_IRQ | GEN8_GT_VCS2_IRQ)) {
1190                 u32 tmp = I915_READ_FW(GEN8_GT_IIR(1));
1191                 if (tmp) {
1192                         I915_WRITE_FW(GEN8_GT_IIR(1), tmp);
1193                         ret = IRQ_HANDLED;
1194
1195                         if (tmp & (GT_CONTEXT_SWITCH_INTERRUPT << GEN8_VCS1_IRQ_SHIFT))
1196                                 intel_lrc_irq_handler(&dev_priv->ring[VCS]);
1197                         if (tmp & (GT_RENDER_USER_INTERRUPT << GEN8_VCS1_IRQ_SHIFT))
1198                                 notify_ring(&dev_priv->ring[VCS]);
1199
1200                         if (tmp & (GT_CONTEXT_SWITCH_INTERRUPT << GEN8_VCS2_IRQ_SHIFT))
1201                                 intel_lrc_irq_handler(&dev_priv->ring[VCS2]);
1202                         if (tmp & (GT_RENDER_USER_INTERRUPT << GEN8_VCS2_IRQ_SHIFT))
1203                                 notify_ring(&dev_priv->ring[VCS2]);
1204                 } else
1205                         DRM_ERROR("The master control interrupt lied (GT1)!\n");
1206         }
1207
1208         if (master_ctl & GEN8_GT_VECS_IRQ) {
1209                 u32 tmp = I915_READ_FW(GEN8_GT_IIR(3));
1210                 if (tmp) {
1211                         I915_WRITE_FW(GEN8_GT_IIR(3), tmp);
1212                         ret = IRQ_HANDLED;
1213
1214                         if (tmp & (GT_CONTEXT_SWITCH_INTERRUPT << GEN8_VECS_IRQ_SHIFT))
1215                                 intel_lrc_irq_handler(&dev_priv->ring[VECS]);
1216                         if (tmp & (GT_RENDER_USER_INTERRUPT << GEN8_VECS_IRQ_SHIFT))
1217                                 notify_ring(&dev_priv->ring[VECS]);
1218                 } else
1219                         DRM_ERROR("The master control interrupt lied (GT3)!\n");
1220         }
1221
1222         if (master_ctl & GEN8_GT_PM_IRQ) {
1223                 u32 tmp = I915_READ_FW(GEN8_GT_IIR(2));
1224                 if (tmp & dev_priv->pm_rps_events) {
1225                         I915_WRITE_FW(GEN8_GT_IIR(2),
1226                                       tmp & dev_priv->pm_rps_events);
1227                         ret = IRQ_HANDLED;
1228                         gen6_rps_irq_handler(dev_priv, tmp);
1229                 } else
1230                         DRM_ERROR("The master control interrupt lied (PM)!\n");
1231         }
1232
1233         return ret;
1234 }
1235
1236 static bool bxt_port_hotplug_long_detect(enum port port, u32 val)
1237 {
1238         switch (port) {
1239         case PORT_A:
1240                 return val & BXT_PORTA_HOTPLUG_LONG_DETECT;
1241         case PORT_B:
1242                 return val & PORTB_HOTPLUG_LONG_DETECT;
1243         case PORT_C:
1244                 return val & PORTC_HOTPLUG_LONG_DETECT;
1245         case PORT_D:
1246                 return val & PORTD_HOTPLUG_LONG_DETECT;
1247         default:
1248                 return false;
1249         }
1250 }
1251
1252 static bool pch_port_hotplug_long_detect(enum port port, u32 val)
1253 {
1254         switch (port) {
1255         case PORT_B:
1256                 return val & PORTB_HOTPLUG_LONG_DETECT;
1257         case PORT_C:
1258                 return val & PORTC_HOTPLUG_LONG_DETECT;
1259         case PORT_D:
1260                 return val & PORTD_HOTPLUG_LONG_DETECT;
1261         case PORT_E:
1262                 return val & PORTE_HOTPLUG_LONG_DETECT;
1263         default:
1264                 return false;
1265         }
1266 }
1267
1268 static bool i9xx_port_hotplug_long_detect(enum port port, u32 val)
1269 {
1270         switch (port) {
1271         case PORT_B:
1272                 return val & PORTB_HOTPLUG_INT_LONG_PULSE;
1273         case PORT_C:
1274                 return val & PORTC_HOTPLUG_INT_LONG_PULSE;
1275         case PORT_D:
1276                 return val & PORTD_HOTPLUG_INT_LONG_PULSE;
1277         default:
1278                 return false;
1279         }
1280 }
1281
1282 /* Get a bit mask of pins that have triggered, and which ones may be long. */
1283 static void intel_get_hpd_pins(u32 *pin_mask, u32 *long_mask,
1284                              u32 hotplug_trigger, u32 dig_hotplug_reg,
1285                              const u32 hpd[HPD_NUM_PINS],
1286                              bool long_pulse_detect(enum port port, u32 val))
1287 {
1288         enum port port;
1289         int i;
1290
1291         *pin_mask = 0;
1292         *long_mask = 0;
1293
1294         for_each_hpd_pin(i) {
1295                 if ((hpd[i] & hotplug_trigger) == 0)
1296                         continue;
1297
1298                 *pin_mask |= BIT(i);
1299
1300                 if (!intel_hpd_pin_to_port(i, &port))
1301                         continue;
1302
1303                 if (long_pulse_detect(port, dig_hotplug_reg))
1304                         *long_mask |= BIT(i);
1305         }
1306
1307         DRM_DEBUG_DRIVER("hotplug event received, stat 0x%08x, dig 0x%08x, pins 0x%08x\n",
1308                          hotplug_trigger, dig_hotplug_reg, *pin_mask);
1309
1310 }
1311
1312 static void gmbus_irq_handler(struct drm_device *dev)
1313 {
1314         struct drm_i915_private *dev_priv = dev->dev_private;
1315
1316         wake_up_all(&dev_priv->gmbus_wait_queue);
1317 }
1318
1319 static void dp_aux_irq_handler(struct drm_device *dev)
1320 {
1321         struct drm_i915_private *dev_priv = dev->dev_private;
1322
1323         wake_up_all(&dev_priv->gmbus_wait_queue);
1324 }
1325
1326 #if defined(CONFIG_DEBUG_FS)
1327 static void display_pipe_crc_irq_handler(struct drm_device *dev, enum pipe pipe,
1328                                          uint32_t crc0, uint32_t crc1,
1329                                          uint32_t crc2, uint32_t crc3,
1330                                          uint32_t crc4)
1331 {
1332         struct drm_i915_private *dev_priv = dev->dev_private;
1333         struct intel_pipe_crc *pipe_crc = &dev_priv->pipe_crc[pipe];
1334         struct intel_pipe_crc_entry *entry;
1335         int head, tail;
1336
1337         spin_lock(&pipe_crc->lock);
1338
1339         if (!pipe_crc->entries) {
1340                 spin_unlock(&pipe_crc->lock);
1341                 DRM_DEBUG_KMS("spurious interrupt\n");
1342                 return;
1343         }
1344
1345         head = pipe_crc->head;
1346         tail = pipe_crc->tail;
1347
1348         if (CIRC_SPACE(head, tail, INTEL_PIPE_CRC_ENTRIES_NR) < 1) {
1349                 spin_unlock(&pipe_crc->lock);
1350                 DRM_ERROR("CRC buffer overflowing\n");
1351                 return;
1352         }
1353
1354         entry = &pipe_crc->entries[head];
1355
1356         entry->frame = dev->driver->get_vblank_counter(dev, pipe);
1357         entry->crc[0] = crc0;
1358         entry->crc[1] = crc1;
1359         entry->crc[2] = crc2;
1360         entry->crc[3] = crc3;
1361         entry->crc[4] = crc4;
1362
1363         head = (head + 1) & (INTEL_PIPE_CRC_ENTRIES_NR - 1);
1364         pipe_crc->head = head;
1365
1366         spin_unlock(&pipe_crc->lock);
1367
1368         wake_up_interruptible(&pipe_crc->wq);
1369 }
1370 #else
1371 static inline void
1372 display_pipe_crc_irq_handler(struct drm_device *dev, enum pipe pipe,
1373                              uint32_t crc0, uint32_t crc1,
1374                              uint32_t crc2, uint32_t crc3,
1375                              uint32_t crc4) {}
1376 #endif
1377
1378
1379 static void hsw_pipe_crc_irq_handler(struct drm_device *dev, enum pipe pipe)
1380 {
1381         struct drm_i915_private *dev_priv = dev->dev_private;
1382
1383         display_pipe_crc_irq_handler(dev, pipe,
1384                                      I915_READ(PIPE_CRC_RES_1_IVB(pipe)),
1385                                      0, 0, 0, 0);
1386 }
1387
1388 static void ivb_pipe_crc_irq_handler(struct drm_device *dev, enum pipe pipe)
1389 {
1390         struct drm_i915_private *dev_priv = dev->dev_private;
1391
1392         display_pipe_crc_irq_handler(dev, pipe,
1393                                      I915_READ(PIPE_CRC_RES_1_IVB(pipe)),
1394                                      I915_READ(PIPE_CRC_RES_2_IVB(pipe)),
1395                                      I915_READ(PIPE_CRC_RES_3_IVB(pipe)),
1396                                      I915_READ(PIPE_CRC_RES_4_IVB(pipe)),
1397                                      I915_READ(PIPE_CRC_RES_5_IVB(pipe)));
1398 }
1399
1400 static void i9xx_pipe_crc_irq_handler(struct drm_device *dev, enum pipe pipe)
1401 {
1402         struct drm_i915_private *dev_priv = dev->dev_private;
1403         uint32_t res1, res2;
1404
1405         if (INTEL_INFO(dev)->gen >= 3)
1406                 res1 = I915_READ(PIPE_CRC_RES_RES1_I915(pipe));
1407         else
1408                 res1 = 0;
1409
1410         if (INTEL_INFO(dev)->gen >= 5 || IS_G4X(dev))
1411                 res2 = I915_READ(PIPE_CRC_RES_RES2_G4X(pipe));
1412         else
1413                 res2 = 0;
1414
1415         display_pipe_crc_irq_handler(dev, pipe,
1416                                      I915_READ(PIPE_CRC_RES_RED(pipe)),
1417                                      I915_READ(PIPE_CRC_RES_GREEN(pipe)),
1418                                      I915_READ(PIPE_CRC_RES_BLUE(pipe)),
1419                                      res1, res2);
1420 }
1421
1422 /* The RPS events need forcewake, so we add them to a work queue and mask their
1423  * IMR bits until the work is done. Other interrupts can be processed without
1424  * the work queue. */
1425 static void gen6_rps_irq_handler(struct drm_i915_private *dev_priv, u32 pm_iir)
1426 {
1427         if (pm_iir & dev_priv->pm_rps_events) {
1428                 spin_lock(&dev_priv->irq_lock);
1429                 gen6_disable_pm_irq(dev_priv, pm_iir & dev_priv->pm_rps_events);
1430                 if (dev_priv->rps.interrupts_enabled) {
1431                         dev_priv->rps.pm_iir |= pm_iir & dev_priv->pm_rps_events;
1432                         queue_work(dev_priv->wq, &dev_priv->rps.work);
1433                 }
1434                 spin_unlock(&dev_priv->irq_lock);
1435         }
1436
1437         if (INTEL_INFO(dev_priv)->gen >= 8)
1438                 return;
1439
1440         if (HAS_VEBOX(dev_priv->dev)) {
1441                 if (pm_iir & PM_VEBOX_USER_INTERRUPT)
1442                         notify_ring(&dev_priv->ring[VECS]);
1443
1444                 if (pm_iir & PM_VEBOX_CS_ERROR_INTERRUPT)
1445                         DRM_DEBUG("Command parser error, pm_iir 0x%08x\n", pm_iir);
1446         }
1447 }
1448
1449 static bool intel_pipe_handle_vblank(struct drm_device *dev, enum pipe pipe)
1450 {
1451         if (!drm_handle_vblank(dev, pipe))
1452                 return false;
1453
1454         return true;
1455 }
1456
1457 static void valleyview_pipestat_irq_handler(struct drm_device *dev, u32 iir)
1458 {
1459         struct drm_i915_private *dev_priv = dev->dev_private;
1460         u32 pipe_stats[I915_MAX_PIPES] = { };
1461         int pipe;
1462
1463         spin_lock(&dev_priv->irq_lock);
1464         for_each_pipe(dev_priv, pipe) {
1465                 int reg;
1466                 u32 mask, iir_bit = 0;
1467
1468                 /*
1469                  * PIPESTAT bits get signalled even when the interrupt is
1470                  * disabled with the mask bits, and some of the status bits do
1471                  * not generate interrupts at all (like the underrun bit). Hence
1472                  * we need to be careful that we only handle what we want to
1473                  * handle.
1474                  */
1475
1476                 /* fifo underruns are filterered in the underrun handler. */
1477                 mask = PIPE_FIFO_UNDERRUN_STATUS;
1478
1479                 switch (pipe) {
1480                 case PIPE_A:
1481                         iir_bit = I915_DISPLAY_PIPE_A_EVENT_INTERRUPT;
1482                         break;
1483                 case PIPE_B:
1484                         iir_bit = I915_DISPLAY_PIPE_B_EVENT_INTERRUPT;
1485                         break;
1486                 case PIPE_C:
1487                         iir_bit = I915_DISPLAY_PIPE_C_EVENT_INTERRUPT;
1488                         break;
1489                 }
1490                 if (iir & iir_bit)
1491                         mask |= dev_priv->pipestat_irq_mask[pipe];
1492
1493                 if (!mask)
1494                         continue;
1495
1496                 reg = PIPESTAT(pipe);
1497                 mask |= PIPESTAT_INT_ENABLE_MASK;
1498                 pipe_stats[pipe] = I915_READ(reg) & mask;
1499
1500                 /*
1501                  * Clear the PIPE*STAT regs before the IIR
1502                  */
1503                 if (pipe_stats[pipe] & (PIPE_FIFO_UNDERRUN_STATUS |
1504                                         PIPESTAT_INT_STATUS_MASK))
1505                         I915_WRITE(reg, pipe_stats[pipe]);
1506         }
1507         spin_unlock(&dev_priv->irq_lock);
1508
1509         for_each_pipe(dev_priv, pipe) {
1510                 if (pipe_stats[pipe] & PIPE_START_VBLANK_INTERRUPT_STATUS &&
1511                     intel_pipe_handle_vblank(dev, pipe))
1512                         intel_check_page_flip(dev, pipe);
1513
1514                 if (pipe_stats[pipe] & PLANE_FLIP_DONE_INT_STATUS_VLV) {
1515                         intel_prepare_page_flip(dev, pipe);
1516                         intel_finish_page_flip(dev, pipe);
1517                 }
1518
1519                 if (pipe_stats[pipe] & PIPE_CRC_DONE_INTERRUPT_STATUS)
1520                         i9xx_pipe_crc_irq_handler(dev, pipe);
1521
1522                 if (pipe_stats[pipe] & PIPE_FIFO_UNDERRUN_STATUS)
1523                         intel_cpu_fifo_underrun_irq_handler(dev_priv, pipe);
1524         }
1525
1526         if (pipe_stats[0] & PIPE_GMBUS_INTERRUPT_STATUS)
1527                 gmbus_irq_handler(dev);
1528 }
1529
1530 static void i9xx_hpd_irq_handler(struct drm_device *dev)
1531 {
1532         struct drm_i915_private *dev_priv = dev->dev_private;
1533         u32 hotplug_status = I915_READ(PORT_HOTPLUG_STAT);
1534         u32 pin_mask, long_mask;
1535
1536         if (!hotplug_status)
1537                 return;
1538
1539         I915_WRITE(PORT_HOTPLUG_STAT, hotplug_status);
1540         /*
1541          * Make sure hotplug status is cleared before we clear IIR, or else we
1542          * may miss hotplug events.
1543          */
1544         POSTING_READ(PORT_HOTPLUG_STAT);
1545
1546         if (IS_G4X(dev) || IS_VALLEYVIEW(dev)) {
1547                 u32 hotplug_trigger = hotplug_status & HOTPLUG_INT_STATUS_G4X;
1548
1549                 intel_get_hpd_pins(&pin_mask, &long_mask, hotplug_trigger,
1550                                    hotplug_trigger, hpd_status_g4x,
1551                                    i9xx_port_hotplug_long_detect);
1552                 intel_hpd_irq_handler(dev, pin_mask, long_mask);
1553
1554                 if (hotplug_status & DP_AUX_CHANNEL_MASK_INT_STATUS_G4X)
1555                         dp_aux_irq_handler(dev);
1556         } else {
1557                 u32 hotplug_trigger = hotplug_status & HOTPLUG_INT_STATUS_I915;
1558
1559                 intel_get_hpd_pins(&pin_mask, &long_mask, hotplug_trigger,
1560                                    hotplug_trigger, hpd_status_i915,
1561                                    i9xx_port_hotplug_long_detect);
1562                 intel_hpd_irq_handler(dev, pin_mask, long_mask);
1563         }
1564 }
1565
1566 static irqreturn_t valleyview_irq_handler(int irq, void *arg)
1567 {
1568         struct drm_device *dev = arg;
1569         struct drm_i915_private *dev_priv = dev->dev_private;
1570         u32 iir, gt_iir, pm_iir;
1571         irqreturn_t ret = IRQ_NONE;
1572
1573         if (!intel_irqs_enabled(dev_priv))
1574                 return IRQ_NONE;
1575
1576         while (true) {
1577                 /* Find, clear, then process each source of interrupt */
1578
1579                 gt_iir = I915_READ(GTIIR);
1580                 if (gt_iir)
1581                         I915_WRITE(GTIIR, gt_iir);
1582
1583                 pm_iir = I915_READ(GEN6_PMIIR);
1584                 if (pm_iir)
1585                         I915_WRITE(GEN6_PMIIR, pm_iir);
1586
1587                 iir = I915_READ(VLV_IIR);
1588                 if (iir) {
1589                         /* Consume port before clearing IIR or we'll miss events */
1590                         if (iir & I915_DISPLAY_PORT_INTERRUPT)
1591                                 i9xx_hpd_irq_handler(dev);
1592                         I915_WRITE(VLV_IIR, iir);
1593                 }
1594
1595                 if (gt_iir == 0 && pm_iir == 0 && iir == 0)
1596                         goto out;
1597
1598                 ret = IRQ_HANDLED;
1599
1600                 if (gt_iir)
1601                         snb_gt_irq_handler(dev, dev_priv, gt_iir);
1602                 if (pm_iir)
1603                         gen6_rps_irq_handler(dev_priv, pm_iir);
1604                 /* Call regardless, as some status bits might not be
1605                  * signalled in iir */
1606                 valleyview_pipestat_irq_handler(dev, iir);
1607         }
1608
1609 out:
1610         return ret;
1611 }
1612
1613 static irqreturn_t cherryview_irq_handler(int irq, void *arg)
1614 {
1615         struct drm_device *dev = arg;
1616         struct drm_i915_private *dev_priv = dev->dev_private;
1617         u32 master_ctl, iir;
1618         irqreturn_t ret = IRQ_NONE;
1619
1620         if (!intel_irqs_enabled(dev_priv))
1621                 return IRQ_NONE;
1622
1623         for (;;) {
1624                 master_ctl = I915_READ(GEN8_MASTER_IRQ) & ~GEN8_MASTER_IRQ_CONTROL;
1625                 iir = I915_READ(VLV_IIR);
1626
1627                 if (master_ctl == 0 && iir == 0)
1628                         break;
1629
1630                 ret = IRQ_HANDLED;
1631
1632                 I915_WRITE(GEN8_MASTER_IRQ, 0);
1633
1634                 /* Find, clear, then process each source of interrupt */
1635
1636                 if (iir) {
1637                         /* Consume port before clearing IIR or we'll miss events */
1638                         if (iir & I915_DISPLAY_PORT_INTERRUPT)
1639                                 i9xx_hpd_irq_handler(dev);
1640                         I915_WRITE(VLV_IIR, iir);
1641                 }
1642
1643                 gen8_gt_irq_handler(dev_priv, master_ctl);
1644
1645                 /* Call regardless, as some status bits might not be
1646                  * signalled in iir */
1647                 valleyview_pipestat_irq_handler(dev, iir);
1648
1649                 I915_WRITE(GEN8_MASTER_IRQ, DE_MASTER_IRQ_CONTROL);
1650                 POSTING_READ(GEN8_MASTER_IRQ);
1651         }
1652
1653         return ret;
1654 }
1655
1656 static void ibx_irq_handler(struct drm_device *dev, u32 pch_iir)
1657 {
1658         struct drm_i915_private *dev_priv = dev->dev_private;
1659         int pipe;
1660         u32 hotplug_trigger = pch_iir & SDE_HOTPLUG_MASK;
1661
1662         if (hotplug_trigger) {
1663                 u32 dig_hotplug_reg, pin_mask, long_mask;
1664
1665                 dig_hotplug_reg = I915_READ(PCH_PORT_HOTPLUG);
1666                 I915_WRITE(PCH_PORT_HOTPLUG, dig_hotplug_reg);
1667
1668                 intel_get_hpd_pins(&pin_mask, &long_mask, hotplug_trigger,
1669                                    dig_hotplug_reg, hpd_ibx,
1670                                    pch_port_hotplug_long_detect);
1671                 intel_hpd_irq_handler(dev, pin_mask, long_mask);
1672         }
1673
1674         if (pch_iir & SDE_AUDIO_POWER_MASK) {
1675                 int port = ffs((pch_iir & SDE_AUDIO_POWER_MASK) >>
1676                                SDE_AUDIO_POWER_SHIFT);
1677                 DRM_DEBUG_DRIVER("PCH audio power change on port %d\n",
1678                                  port_name(port));
1679         }
1680
1681         if (pch_iir & SDE_AUX_MASK)
1682                 dp_aux_irq_handler(dev);
1683
1684         if (pch_iir & SDE_GMBUS)
1685                 gmbus_irq_handler(dev);
1686
1687         if (pch_iir & SDE_AUDIO_HDCP_MASK)
1688                 DRM_DEBUG_DRIVER("PCH HDCP audio interrupt\n");
1689
1690         if (pch_iir & SDE_AUDIO_TRANS_MASK)
1691                 DRM_DEBUG_DRIVER("PCH transcoder audio interrupt\n");
1692
1693         if (pch_iir & SDE_POISON)
1694                 DRM_ERROR("PCH poison interrupt\n");
1695
1696         if (pch_iir & SDE_FDI_MASK)
1697                 for_each_pipe(dev_priv, pipe)
1698                         DRM_DEBUG_DRIVER("  pipe %c FDI IIR: 0x%08x\n",
1699                                          pipe_name(pipe),
1700                                          I915_READ(FDI_RX_IIR(pipe)));
1701
1702         if (pch_iir & (SDE_TRANSB_CRC_DONE | SDE_TRANSA_CRC_DONE))
1703                 DRM_DEBUG_DRIVER("PCH transcoder CRC done interrupt\n");
1704
1705         if (pch_iir & (SDE_TRANSB_CRC_ERR | SDE_TRANSA_CRC_ERR))
1706                 DRM_DEBUG_DRIVER("PCH transcoder CRC error interrupt\n");
1707
1708         if (pch_iir & SDE_TRANSA_FIFO_UNDER)
1709                 intel_pch_fifo_underrun_irq_handler(dev_priv, TRANSCODER_A);
1710
1711         if (pch_iir & SDE_TRANSB_FIFO_UNDER)
1712                 intel_pch_fifo_underrun_irq_handler(dev_priv, TRANSCODER_B);
1713 }
1714
1715 static void ivb_err_int_handler(struct drm_device *dev)
1716 {
1717         struct drm_i915_private *dev_priv = dev->dev_private;
1718         u32 err_int = I915_READ(GEN7_ERR_INT);
1719         enum pipe pipe;
1720
1721         if (err_int & ERR_INT_POISON)
1722                 DRM_ERROR("Poison interrupt\n");
1723
1724         for_each_pipe(dev_priv, pipe) {
1725                 if (err_int & ERR_INT_FIFO_UNDERRUN(pipe))
1726                         intel_cpu_fifo_underrun_irq_handler(dev_priv, pipe);
1727
1728                 if (err_int & ERR_INT_PIPE_CRC_DONE(pipe)) {
1729                         if (IS_IVYBRIDGE(dev))
1730                                 ivb_pipe_crc_irq_handler(dev, pipe);
1731                         else
1732                                 hsw_pipe_crc_irq_handler(dev, pipe);
1733                 }
1734         }
1735
1736         I915_WRITE(GEN7_ERR_INT, err_int);
1737 }
1738
1739 static void cpt_serr_int_handler(struct drm_device *dev)
1740 {
1741         struct drm_i915_private *dev_priv = dev->dev_private;
1742         u32 serr_int = I915_READ(SERR_INT);
1743
1744         if (serr_int & SERR_INT_POISON)
1745                 DRM_ERROR("PCH poison interrupt\n");
1746
1747         if (serr_int & SERR_INT_TRANS_A_FIFO_UNDERRUN)
1748                 intel_pch_fifo_underrun_irq_handler(dev_priv, TRANSCODER_A);
1749
1750         if (serr_int & SERR_INT_TRANS_B_FIFO_UNDERRUN)
1751                 intel_pch_fifo_underrun_irq_handler(dev_priv, TRANSCODER_B);
1752
1753         if (serr_int & SERR_INT_TRANS_C_FIFO_UNDERRUN)
1754                 intel_pch_fifo_underrun_irq_handler(dev_priv, TRANSCODER_C);
1755
1756         I915_WRITE(SERR_INT, serr_int);
1757 }
1758
1759 static void cpt_irq_handler(struct drm_device *dev, u32 pch_iir)
1760 {
1761         struct drm_i915_private *dev_priv = dev->dev_private;
1762         int pipe;
1763         u32 hotplug_trigger;
1764
1765         if (HAS_PCH_SPT(dev))
1766                 hotplug_trigger = pch_iir & SDE_HOTPLUG_MASK_SPT;
1767         else
1768                 hotplug_trigger = pch_iir & SDE_HOTPLUG_MASK_CPT;
1769
1770         if (hotplug_trigger) {
1771                 u32 dig_hotplug_reg, pin_mask, long_mask;
1772
1773                 dig_hotplug_reg = I915_READ(PCH_PORT_HOTPLUG);
1774                 I915_WRITE(PCH_PORT_HOTPLUG, dig_hotplug_reg);
1775
1776                 if (HAS_PCH_SPT(dev)) {
1777                         intel_get_hpd_pins(&pin_mask, &long_mask,
1778                                            hotplug_trigger,
1779                                            dig_hotplug_reg, hpd_spt,
1780                                            pch_port_hotplug_long_detect);
1781
1782                         /* detect PORTE HP event */
1783                         dig_hotplug_reg = I915_READ(PCH_PORT_HOTPLUG2);
1784                         if (pch_port_hotplug_long_detect(PORT_E,
1785                                                          dig_hotplug_reg))
1786                                 long_mask |= 1 << HPD_PORT_E;
1787                 } else
1788                         intel_get_hpd_pins(&pin_mask, &long_mask,
1789                                            hotplug_trigger,
1790                                            dig_hotplug_reg, hpd_cpt,
1791                                            pch_port_hotplug_long_detect);
1792
1793                 intel_hpd_irq_handler(dev, pin_mask, long_mask);
1794         }
1795
1796         if (pch_iir & SDE_AUDIO_POWER_MASK_CPT) {
1797                 int port = ffs((pch_iir & SDE_AUDIO_POWER_MASK_CPT) >>
1798                                SDE_AUDIO_POWER_SHIFT_CPT);
1799                 DRM_DEBUG_DRIVER("PCH audio power change on port %c\n",
1800                                  port_name(port));
1801         }
1802
1803         if (pch_iir & SDE_AUX_MASK_CPT)
1804                 dp_aux_irq_handler(dev);
1805
1806         if (pch_iir & SDE_GMBUS_CPT)
1807                 gmbus_irq_handler(dev);
1808
1809         if (pch_iir & SDE_AUDIO_CP_REQ_CPT)
1810                 DRM_DEBUG_DRIVER("Audio CP request interrupt\n");
1811
1812         if (pch_iir & SDE_AUDIO_CP_CHG_CPT)
1813                 DRM_DEBUG_DRIVER("Audio CP change interrupt\n");
1814
1815         if (pch_iir & SDE_FDI_MASK_CPT)
1816                 for_each_pipe(dev_priv, pipe)
1817                         DRM_DEBUG_DRIVER("  pipe %c FDI IIR: 0x%08x\n",
1818                                          pipe_name(pipe),
1819                                          I915_READ(FDI_RX_IIR(pipe)));
1820
1821         if (pch_iir & SDE_ERROR_CPT)
1822                 cpt_serr_int_handler(dev);
1823 }
1824
1825 static void ilk_display_irq_handler(struct drm_device *dev, u32 de_iir)
1826 {
1827         struct drm_i915_private *dev_priv = dev->dev_private;
1828         enum pipe pipe;
1829
1830         if (de_iir & DE_AUX_CHANNEL_A)
1831                 dp_aux_irq_handler(dev);
1832
1833         if (de_iir & DE_GSE)
1834                 intel_opregion_asle_intr(dev);
1835
1836         if (de_iir & DE_POISON)
1837                 DRM_ERROR("Poison interrupt\n");
1838
1839         for_each_pipe(dev_priv, pipe) {
1840                 if (de_iir & DE_PIPE_VBLANK(pipe) &&
1841                     intel_pipe_handle_vblank(dev, pipe))
1842                         intel_check_page_flip(dev, pipe);
1843
1844                 if (de_iir & DE_PIPE_FIFO_UNDERRUN(pipe))
1845                         intel_cpu_fifo_underrun_irq_handler(dev_priv, pipe);
1846
1847                 if (de_iir & DE_PIPE_CRC_DONE(pipe))
1848                         i9xx_pipe_crc_irq_handler(dev, pipe);
1849
1850                 /* plane/pipes map 1:1 on ilk+ */
1851                 if (de_iir & DE_PLANE_FLIP_DONE(pipe)) {
1852                         intel_prepare_page_flip(dev, pipe);
1853                         intel_finish_page_flip_plane(dev, pipe);
1854                 }
1855         }
1856
1857         /* check event from PCH */
1858         if (de_iir & DE_PCH_EVENT) {
1859                 u32 pch_iir = I915_READ(SDEIIR);
1860
1861                 if (HAS_PCH_CPT(dev))
1862                         cpt_irq_handler(dev, pch_iir);
1863                 else
1864                         ibx_irq_handler(dev, pch_iir);
1865
1866                 /* should clear PCH hotplug event before clear CPU irq */
1867                 I915_WRITE(SDEIIR, pch_iir);
1868         }
1869
1870         if (IS_GEN5(dev) && de_iir & DE_PCU_EVENT)
1871                 ironlake_rps_change_irq_handler(dev);
1872 }
1873
1874 static void ivb_display_irq_handler(struct drm_device *dev, u32 de_iir)
1875 {
1876         struct drm_i915_private *dev_priv = dev->dev_private;
1877         enum pipe pipe;
1878
1879         if (de_iir & DE_ERR_INT_IVB)
1880                 ivb_err_int_handler(dev);
1881
1882         if (de_iir & DE_AUX_CHANNEL_A_IVB)
1883                 dp_aux_irq_handler(dev);
1884
1885         if (de_iir & DE_GSE_IVB)
1886                 intel_opregion_asle_intr(dev);
1887
1888         for_each_pipe(dev_priv, pipe) {
1889                 if (de_iir & (DE_PIPE_VBLANK_IVB(pipe)) &&
1890                     intel_pipe_handle_vblank(dev, pipe))
1891                         intel_check_page_flip(dev, pipe);
1892
1893                 /* plane/pipes map 1:1 on ilk+ */
1894                 if (de_iir & DE_PLANE_FLIP_DONE_IVB(pipe)) {
1895                         intel_prepare_page_flip(dev, pipe);
1896                         intel_finish_page_flip_plane(dev, pipe);
1897                 }
1898         }
1899
1900         /* check event from PCH */
1901         if (!HAS_PCH_NOP(dev) && (de_iir & DE_PCH_EVENT_IVB)) {
1902                 u32 pch_iir = I915_READ(SDEIIR);
1903
1904                 cpt_irq_handler(dev, pch_iir);
1905
1906                 /* clear PCH hotplug event before clear CPU irq */
1907                 I915_WRITE(SDEIIR, pch_iir);
1908         }
1909 }
1910
1911 /*
1912  * To handle irqs with the minimum potential races with fresh interrupts, we:
1913  * 1 - Disable Master Interrupt Control.
1914  * 2 - Find the source(s) of the interrupt.
1915  * 3 - Clear the Interrupt Identity bits (IIR).
1916  * 4 - Process the interrupt(s) that had bits set in the IIRs.
1917  * 5 - Re-enable Master Interrupt Control.
1918  */
1919 static irqreturn_t ironlake_irq_handler(int irq, void *arg)
1920 {
1921         struct drm_device *dev = arg;
1922         struct drm_i915_private *dev_priv = dev->dev_private;
1923         u32 de_iir, gt_iir, de_ier, sde_ier = 0;
1924         irqreturn_t ret = IRQ_NONE;
1925
1926         if (!intel_irqs_enabled(dev_priv))
1927                 return IRQ_NONE;
1928
1929         /* We get interrupts on unclaimed registers, so check for this before we
1930          * do any I915_{READ,WRITE}. */
1931         intel_uncore_check_errors(dev);
1932
1933         /* disable master interrupt before clearing iir  */
1934         de_ier = I915_READ(DEIER);
1935         I915_WRITE(DEIER, de_ier & ~DE_MASTER_IRQ_CONTROL);
1936         POSTING_READ(DEIER);
1937
1938         /* Disable south interrupts. We'll only write to SDEIIR once, so further
1939          * interrupts will will be stored on its back queue, and then we'll be
1940          * able to process them after we restore SDEIER (as soon as we restore
1941          * it, we'll get an interrupt if SDEIIR still has something to process
1942          * due to its back queue). */
1943         if (!HAS_PCH_NOP(dev)) {
1944                 sde_ier = I915_READ(SDEIER);
1945                 I915_WRITE(SDEIER, 0);
1946                 POSTING_READ(SDEIER);
1947         }
1948
1949         /* Find, clear, then process each source of interrupt */
1950
1951         gt_iir = I915_READ(GTIIR);
1952         if (gt_iir) {
1953                 I915_WRITE(GTIIR, gt_iir);
1954                 ret = IRQ_HANDLED;
1955                 if (INTEL_INFO(dev)->gen >= 6)
1956                         snb_gt_irq_handler(dev, dev_priv, gt_iir);
1957                 else
1958                         ilk_gt_irq_handler(dev, dev_priv, gt_iir);
1959         }
1960
1961         de_iir = I915_READ(DEIIR);
1962         if (de_iir) {
1963                 I915_WRITE(DEIIR, de_iir);
1964                 ret = IRQ_HANDLED;
1965                 if (INTEL_INFO(dev)->gen >= 7)
1966                         ivb_display_irq_handler(dev, de_iir);
1967                 else
1968                         ilk_display_irq_handler(dev, de_iir);
1969         }
1970
1971         if (INTEL_INFO(dev)->gen >= 6) {
1972                 u32 pm_iir = I915_READ(GEN6_PMIIR);
1973                 if (pm_iir) {
1974                         I915_WRITE(GEN6_PMIIR, pm_iir);
1975                         ret = IRQ_HANDLED;
1976                         gen6_rps_irq_handler(dev_priv, pm_iir);
1977                 }
1978         }
1979
1980         I915_WRITE(DEIER, de_ier);
1981         POSTING_READ(DEIER);
1982         if (!HAS_PCH_NOP(dev)) {
1983                 I915_WRITE(SDEIER, sde_ier);
1984                 POSTING_READ(SDEIER);
1985         }
1986
1987         return ret;
1988 }
1989
1990 static void bxt_hpd_handler(struct drm_device *dev, uint32_t iir_status)
1991 {
1992         struct drm_i915_private *dev_priv = dev->dev_private;
1993         u32 hp_control, hp_trigger;
1994         u32 pin_mask, long_mask;
1995
1996         /* Get the status */
1997         hp_trigger = iir_status & BXT_DE_PORT_HOTPLUG_MASK;
1998         hp_control = I915_READ(BXT_HOTPLUG_CTL);
1999
2000         /* Hotplug not enabled ? */
2001         if (!(hp_control & BXT_HOTPLUG_CTL_MASK)) {
2002                 DRM_ERROR("Interrupt when HPD disabled\n");
2003                 return;
2004         }
2005
2006         /* Clear sticky bits in hpd status */
2007         I915_WRITE(BXT_HOTPLUG_CTL, hp_control);
2008
2009         intel_get_hpd_pins(&pin_mask, &long_mask, hp_trigger, hp_control,
2010                            hpd_bxt, bxt_port_hotplug_long_detect);
2011         intel_hpd_irq_handler(dev, pin_mask, long_mask);
2012 }
2013
2014 static irqreturn_t gen8_irq_handler(int irq, void *arg)
2015 {
2016         struct drm_device *dev = arg;
2017         struct drm_i915_private *dev_priv = dev->dev_private;
2018         u32 master_ctl;
2019         irqreturn_t ret = IRQ_NONE;
2020         uint32_t tmp = 0;
2021         enum pipe pipe;
2022         u32 aux_mask = GEN8_AUX_CHANNEL_A;
2023
2024         if (!intel_irqs_enabled(dev_priv))
2025                 return IRQ_NONE;
2026
2027         if (IS_GEN9(dev))
2028                 aux_mask |=  GEN9_AUX_CHANNEL_B | GEN9_AUX_CHANNEL_C |
2029                         GEN9_AUX_CHANNEL_D;
2030
2031         master_ctl = I915_READ_FW(GEN8_MASTER_IRQ);
2032         master_ctl &= ~GEN8_MASTER_IRQ_CONTROL;
2033         if (!master_ctl)
2034                 return IRQ_NONE;
2035
2036         I915_WRITE_FW(GEN8_MASTER_IRQ, 0);
2037
2038         /* Find, clear, then process each source of interrupt */
2039
2040         ret = gen8_gt_irq_handler(dev_priv, master_ctl);
2041
2042         if (master_ctl & GEN8_DE_MISC_IRQ) {
2043                 tmp = I915_READ(GEN8_DE_MISC_IIR);
2044                 if (tmp) {
2045                         I915_WRITE(GEN8_DE_MISC_IIR, tmp);
2046                         ret = IRQ_HANDLED;
2047                         if (tmp & GEN8_DE_MISC_GSE)
2048                                 intel_opregion_asle_intr(dev);
2049                         else
2050                                 DRM_ERROR("Unexpected DE Misc interrupt\n");
2051                 }
2052                 else
2053                         DRM_ERROR("The master control interrupt lied (DE MISC)!\n");
2054         }
2055
2056         if (master_ctl & GEN8_DE_PORT_IRQ) {
2057                 tmp = I915_READ(GEN8_DE_PORT_IIR);
2058                 if (tmp) {
2059                         bool found = false;
2060
2061                         I915_WRITE(GEN8_DE_PORT_IIR, tmp);
2062                         ret = IRQ_HANDLED;
2063
2064                         if (tmp & aux_mask) {
2065                                 dp_aux_irq_handler(dev);
2066                                 found = true;
2067                         }
2068
2069                         if (IS_BROXTON(dev) && tmp & BXT_DE_PORT_HOTPLUG_MASK) {
2070                                 bxt_hpd_handler(dev, tmp);
2071                                 found = true;
2072                         }
2073
2074                         if (IS_BROXTON(dev) && (tmp & BXT_DE_PORT_GMBUS)) {
2075                                 gmbus_irq_handler(dev);
2076                                 found = true;
2077                         }
2078
2079                         if (!found)
2080                                 DRM_ERROR("Unexpected DE Port interrupt\n");
2081                 }
2082                 else
2083                         DRM_ERROR("The master control interrupt lied (DE PORT)!\n");
2084         }
2085
2086         for_each_pipe(dev_priv, pipe) {
2087                 uint32_t pipe_iir, flip_done = 0, fault_errors = 0;
2088
2089                 if (!(master_ctl & GEN8_DE_PIPE_IRQ(pipe)))
2090                         continue;
2091
2092                 pipe_iir = I915_READ(GEN8_DE_PIPE_IIR(pipe));
2093                 if (pipe_iir) {
2094                         ret = IRQ_HANDLED;
2095                         I915_WRITE(GEN8_DE_PIPE_IIR(pipe), pipe_iir);
2096
2097                         if (pipe_iir & GEN8_PIPE_VBLANK &&
2098                             intel_pipe_handle_vblank(dev, pipe))
2099                                 intel_check_page_flip(dev, pipe);
2100
2101                         if (IS_GEN9(dev))
2102                                 flip_done = pipe_iir & GEN9_PIPE_PLANE1_FLIP_DONE;
2103                         else
2104                                 flip_done = pipe_iir & GEN8_PIPE_PRIMARY_FLIP_DONE;
2105
2106                         if (flip_done) {
2107                                 intel_prepare_page_flip(dev, pipe);
2108                                 intel_finish_page_flip_plane(dev, pipe);
2109                         }
2110
2111                         if (pipe_iir & GEN8_PIPE_CDCLK_CRC_DONE)
2112                                 hsw_pipe_crc_irq_handler(dev, pipe);
2113
2114                         if (pipe_iir & GEN8_PIPE_FIFO_UNDERRUN)
2115                                 intel_cpu_fifo_underrun_irq_handler(dev_priv,
2116                                                                     pipe);
2117
2118
2119                         if (IS_GEN9(dev))
2120                                 fault_errors = pipe_iir & GEN9_DE_PIPE_IRQ_FAULT_ERRORS;
2121                         else
2122                                 fault_errors = pipe_iir & GEN8_DE_PIPE_IRQ_FAULT_ERRORS;
2123
2124                         if (fault_errors)
2125                                 DRM_ERROR("Fault errors on pipe %c\n: 0x%08x",
2126                                           pipe_name(pipe),
2127                                           pipe_iir & GEN8_DE_PIPE_IRQ_FAULT_ERRORS);
2128                 } else
2129                         DRM_ERROR("The master control interrupt lied (DE PIPE)!\n");
2130         }
2131
2132         if (HAS_PCH_SPLIT(dev) && !HAS_PCH_NOP(dev) &&
2133             master_ctl & GEN8_DE_PCH_IRQ) {
2134                 /*
2135                  * FIXME(BDW): Assume for now that the new interrupt handling
2136                  * scheme also closed the SDE interrupt handling race we've seen
2137                  * on older pch-split platforms. But this needs testing.
2138                  */
2139                 u32 pch_iir = I915_READ(SDEIIR);
2140                 if (pch_iir) {
2141                         I915_WRITE(SDEIIR, pch_iir);
2142                         ret = IRQ_HANDLED;
2143                         cpt_irq_handler(dev, pch_iir);
2144                 } else
2145                         DRM_ERROR("The master control interrupt lied (SDE)!\n");
2146
2147         }
2148
2149         I915_WRITE_FW(GEN8_MASTER_IRQ, GEN8_MASTER_IRQ_CONTROL);
2150         POSTING_READ_FW(GEN8_MASTER_IRQ);
2151
2152         return ret;
2153 }
2154
2155 static void i915_error_wake_up(struct drm_i915_private *dev_priv,
2156                                bool reset_completed)
2157 {
2158         struct intel_engine_cs *ring;
2159         int i;
2160
2161         /*
2162          * Notify all waiters for GPU completion events that reset state has
2163          * been changed, and that they need to restart their wait after
2164          * checking for potential errors (and bail out to drop locks if there is
2165          * a gpu reset pending so that i915_error_work_func can acquire them).
2166          */
2167
2168         /* Wake up __wait_seqno, potentially holding dev->struct_mutex. */
2169         for_each_ring(ring, dev_priv, i)
2170                 wake_up_all(&ring->irq_queue);
2171
2172         /* Wake up intel_crtc_wait_for_pending_flips, holding crtc->mutex. */
2173         wake_up_all(&dev_priv->pending_flip_queue);
2174
2175         /*
2176          * Signal tasks blocked in i915_gem_wait_for_error that the pending
2177          * reset state is cleared.
2178          */
2179         if (reset_completed)
2180                 wake_up_all(&dev_priv->gpu_error.reset_queue);
2181 }
2182
2183 /**
2184  * i915_reset_and_wakeup - do process context error handling work
2185  *
2186  * Fire an error uevent so userspace can see that a hang or error
2187  * was detected.
2188  */
2189 static void i915_reset_and_wakeup(struct drm_device *dev)
2190 {
2191         struct drm_i915_private *dev_priv = to_i915(dev);
2192         struct i915_gpu_error *error = &dev_priv->gpu_error;
2193         char *error_event[] = { I915_ERROR_UEVENT "=1", NULL };
2194         char *reset_event[] = { I915_RESET_UEVENT "=1", NULL };
2195         char *reset_done_event[] = { I915_ERROR_UEVENT "=0", NULL };
2196         int ret;
2197
2198         kobject_uevent_env(&dev->primary->kdev->kobj, KOBJ_CHANGE, error_event);
2199
2200         /*
2201          * Note that there's only one work item which does gpu resets, so we
2202          * need not worry about concurrent gpu resets potentially incrementing
2203          * error->reset_counter twice. We only need to take care of another
2204          * racing irq/hangcheck declaring the gpu dead for a second time. A
2205          * quick check for that is good enough: schedule_work ensures the
2206          * correct ordering between hang detection and this work item, and since
2207          * the reset in-progress bit is only ever set by code outside of this
2208          * work we don't need to worry about any other races.
2209          */
2210         if (i915_reset_in_progress(error) && !i915_terminally_wedged(error)) {
2211                 DRM_DEBUG_DRIVER("resetting chip\n");
2212                 kobject_uevent_env(&dev->primary->kdev->kobj, KOBJ_CHANGE,
2213                                    reset_event);
2214
2215                 /*
2216                  * In most cases it's guaranteed that we get here with an RPM
2217                  * reference held, for example because there is a pending GPU
2218                  * request that won't finish until the reset is done. This
2219                  * isn't the case at least when we get here by doing a
2220                  * simulated reset via debugs, so get an RPM reference.
2221                  */
2222                 intel_runtime_pm_get(dev_priv);
2223
2224                 intel_prepare_reset(dev);
2225
2226                 /*
2227                  * All state reset _must_ be completed before we update the
2228                  * reset counter, for otherwise waiters might miss the reset
2229                  * pending state and not properly drop locks, resulting in
2230                  * deadlocks with the reset work.
2231                  */
2232                 ret = i915_reset(dev);
2233
2234                 intel_finish_reset(dev);
2235
2236                 intel_runtime_pm_put(dev_priv);
2237
2238                 if (ret == 0) {
2239                         /*
2240                          * After all the gem state is reset, increment the reset
2241                          * counter and wake up everyone waiting for the reset to
2242                          * complete.
2243                          *
2244                          * Since unlock operations are a one-sided barrier only,
2245                          * we need to insert a barrier here to order any seqno
2246                          * updates before
2247                          * the counter increment.
2248                          */
2249                         smp_mb__before_atomic();
2250                         atomic_inc(&dev_priv->gpu_error.reset_counter);
2251
2252                         kobject_uevent_env(&dev->primary->kdev->kobj,
2253                                            KOBJ_CHANGE, reset_done_event);
2254                 } else {
2255                         atomic_or(I915_WEDGED, &error->reset_counter);
2256                 }
2257
2258                 /*
2259                  * Note: The wake_up also serves as a memory barrier so that
2260                  * waiters see the update value of the reset counter atomic_t.
2261                  */
2262                 i915_error_wake_up(dev_priv, true);
2263         }
2264 }
2265
2266 static void i915_report_and_clear_eir(struct drm_device *dev)
2267 {
2268         struct drm_i915_private *dev_priv = dev->dev_private;
2269         uint32_t instdone[I915_NUM_INSTDONE_REG];
2270         u32 eir = I915_READ(EIR);
2271         int pipe, i;
2272
2273         if (!eir)
2274                 return;
2275
2276         pr_err("render error detected, EIR: 0x%08x\n", eir);
2277
2278         i915_get_extra_instdone(dev, instdone);
2279
2280         if (IS_G4X(dev)) {
2281                 if (eir & (GM45_ERROR_MEM_PRIV | GM45_ERROR_CP_PRIV)) {
2282                         u32 ipeir = I915_READ(IPEIR_I965);
2283
2284                         pr_err("  IPEIR: 0x%08x\n", I915_READ(IPEIR_I965));
2285                         pr_err("  IPEHR: 0x%08x\n", I915_READ(IPEHR_I965));
2286                         for (i = 0; i < ARRAY_SIZE(instdone); i++)
2287                                 pr_err("  INSTDONE_%d: 0x%08x\n", i, instdone[i]);
2288                         pr_err("  INSTPS: 0x%08x\n", I915_READ(INSTPS));
2289                         pr_err("  ACTHD: 0x%08x\n", I915_READ(ACTHD_I965));
2290                         I915_WRITE(IPEIR_I965, ipeir);
2291                         POSTING_READ(IPEIR_I965);
2292                 }
2293                 if (eir & GM45_ERROR_PAGE_TABLE) {
2294                         u32 pgtbl_err = I915_READ(PGTBL_ER);
2295                         pr_err("page table error\n");
2296                         pr_err("  PGTBL_ER: 0x%08x\n", pgtbl_err);
2297                         I915_WRITE(PGTBL_ER, pgtbl_err);
2298                         POSTING_READ(PGTBL_ER);
2299                 }
2300         }
2301
2302         if (!IS_GEN2(dev)) {
2303                 if (eir & I915_ERROR_PAGE_TABLE) {
2304                         u32 pgtbl_err = I915_READ(PGTBL_ER);
2305                         pr_err("page table error\n");
2306                         pr_err("  PGTBL_ER: 0x%08x\n", pgtbl_err);
2307                         I915_WRITE(PGTBL_ER, pgtbl_err);
2308                         POSTING_READ(PGTBL_ER);
2309                 }
2310         }
2311
2312         if (eir & I915_ERROR_MEMORY_REFRESH) {
2313                 pr_err("memory refresh error:\n");
2314                 for_each_pipe(dev_priv, pipe)
2315                         pr_err("pipe %c stat: 0x%08x\n",
2316                                pipe_name(pipe), I915_READ(PIPESTAT(pipe)));
2317                 /* pipestat has already been acked */
2318         }
2319         if (eir & I915_ERROR_INSTRUCTION) {
2320                 pr_err("instruction error\n");
2321                 pr_err("  INSTPM: 0x%08x\n", I915_READ(INSTPM));
2322                 for (i = 0; i < ARRAY_SIZE(instdone); i++)
2323                         pr_err("  INSTDONE_%d: 0x%08x\n", i, instdone[i]);
2324                 if (INTEL_INFO(dev)->gen < 4) {
2325                         u32 ipeir = I915_READ(IPEIR);
2326
2327                         pr_err("  IPEIR: 0x%08x\n", I915_READ(IPEIR));
2328                         pr_err("  IPEHR: 0x%08x\n", I915_READ(IPEHR));
2329                         pr_err("  ACTHD: 0x%08x\n", I915_READ(ACTHD));
2330                         I915_WRITE(IPEIR, ipeir);
2331                         POSTING_READ(IPEIR);
2332                 } else {
2333                         u32 ipeir = I915_READ(IPEIR_I965);
2334
2335                         pr_err("  IPEIR: 0x%08x\n", I915_READ(IPEIR_I965));
2336                         pr_err("  IPEHR: 0x%08x\n", I915_READ(IPEHR_I965));
2337                         pr_err("  INSTPS: 0x%08x\n", I915_READ(INSTPS));
2338                         pr_err("  ACTHD: 0x%08x\n", I915_READ(ACTHD_I965));
2339                         I915_WRITE(IPEIR_I965, ipeir);
2340                         POSTING_READ(IPEIR_I965);
2341                 }
2342         }
2343
2344         I915_WRITE(EIR, eir);
2345         POSTING_READ(EIR);
2346         eir = I915_READ(EIR);
2347         if (eir) {
2348                 /*
2349                  * some errors might have become stuck,
2350                  * mask them.
2351                  */
2352                 DRM_ERROR("EIR stuck: 0x%08x, masking\n", eir);
2353                 I915_WRITE(EMR, I915_READ(EMR) | eir);
2354                 I915_WRITE(IIR, I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT);
2355         }
2356 }
2357
2358 /**
2359  * i915_handle_error - handle a gpu error
2360  * @dev: drm device
2361  *
2362  * Do some basic checking of regsiter state at error time and
2363  * dump it to the syslog.  Also call i915_capture_error_state() to make
2364  * sure we get a record and make it available in debugfs.  Fire a uevent
2365  * so userspace knows something bad happened (should trigger collection
2366  * of a ring dump etc.).
2367  */
2368 void i915_handle_error(struct drm_device *dev, bool wedged,
2369                        const char *fmt, ...)
2370 {
2371         struct drm_i915_private *dev_priv = dev->dev_private;
2372         va_list args;
2373         char error_msg[80];
2374
2375         va_start(args, fmt);
2376         vscnprintf(error_msg, sizeof(error_msg), fmt, args);
2377         va_end(args);
2378
2379         i915_capture_error_state(dev, wedged, error_msg);
2380         i915_report_and_clear_eir(dev);
2381
2382         if (wedged) {
2383                 atomic_or(I915_RESET_IN_PROGRESS_FLAG,
2384                                 &dev_priv->gpu_error.reset_counter);
2385
2386                 /*
2387                  * Wakeup waiting processes so that the reset function
2388                  * i915_reset_and_wakeup doesn't deadlock trying to grab
2389                  * various locks. By bumping the reset counter first, the woken
2390                  * processes will see a reset in progress and back off,
2391                  * releasing their locks and then wait for the reset completion.
2392                  * We must do this for _all_ gpu waiters that might hold locks
2393                  * that the reset work needs to acquire.
2394                  *
2395                  * Note: The wake_up serves as the required memory barrier to
2396                  * ensure that the waiters see the updated value of the reset
2397                  * counter atomic_t.
2398                  */
2399                 i915_error_wake_up(dev_priv, false);
2400         }
2401
2402         i915_reset_and_wakeup(dev);
2403 }
2404
2405 /* Called from drm generic code, passed 'crtc' which
2406  * we use as a pipe index
2407  */
2408 static int i915_enable_vblank(struct drm_device *dev, int pipe)
2409 {
2410         struct drm_i915_private *dev_priv = dev->dev_private;
2411         unsigned long irqflags;
2412
2413         spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
2414         if (INTEL_INFO(dev)->gen >= 4)
2415                 i915_enable_pipestat(dev_priv, pipe,
2416                                      PIPE_START_VBLANK_INTERRUPT_STATUS);
2417         else
2418                 i915_enable_pipestat(dev_priv, pipe,
2419                                      PIPE_VBLANK_INTERRUPT_STATUS);
2420         spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
2421
2422         return 0;
2423 }
2424
2425 static int ironlake_enable_vblank(struct drm_device *dev, int pipe)
2426 {
2427         struct drm_i915_private *dev_priv = dev->dev_private;
2428         unsigned long irqflags;
2429         uint32_t bit = (INTEL_INFO(dev)->gen >= 7) ? DE_PIPE_VBLANK_IVB(pipe) :
2430                                                      DE_PIPE_VBLANK(pipe);
2431
2432         spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
2433         ironlake_enable_display_irq(dev_priv, bit);
2434         spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
2435
2436         return 0;
2437 }
2438
2439 static int valleyview_enable_vblank(struct drm_device *dev, int pipe)
2440 {
2441         struct drm_i915_private *dev_priv = dev->dev_private;
2442         unsigned long irqflags;
2443
2444         spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
2445         i915_enable_pipestat(dev_priv, pipe,
2446                              PIPE_START_VBLANK_INTERRUPT_STATUS);
2447         spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
2448
2449         return 0;
2450 }
2451
2452 static int gen8_enable_vblank(struct drm_device *dev, int pipe)
2453 {
2454         struct drm_i915_private *dev_priv = dev->dev_private;
2455         unsigned long irqflags;
2456
2457         spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
2458         dev_priv->de_irq_mask[pipe] &= ~GEN8_PIPE_VBLANK;
2459         I915_WRITE(GEN8_DE_PIPE_IMR(pipe), dev_priv->de_irq_mask[pipe]);
2460         POSTING_READ(GEN8_DE_PIPE_IMR(pipe));
2461         spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
2462         return 0;
2463 }
2464
2465 /* Called from drm generic code, passed 'crtc' which
2466  * we use as a pipe index
2467  */
2468 static void i915_disable_vblank(struct drm_device *dev, int pipe)
2469 {
2470         struct drm_i915_private *dev_priv = dev->dev_private;
2471         unsigned long irqflags;
2472
2473         spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
2474         i915_disable_pipestat(dev_priv, pipe,
2475                               PIPE_VBLANK_INTERRUPT_STATUS |
2476                               PIPE_START_VBLANK_INTERRUPT_STATUS);
2477         spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
2478 }
2479
2480 static void ironlake_disable_vblank(struct drm_device *dev, int pipe)
2481 {
2482         struct drm_i915_private *dev_priv = dev->dev_private;
2483         unsigned long irqflags;
2484         uint32_t bit = (INTEL_INFO(dev)->gen >= 7) ? DE_PIPE_VBLANK_IVB(pipe) :
2485                                                      DE_PIPE_VBLANK(pipe);
2486
2487         spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
2488         ironlake_disable_display_irq(dev_priv, bit);
2489         spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
2490 }
2491
2492 static void valleyview_disable_vblank(struct drm_device *dev, int pipe)
2493 {
2494         struct drm_i915_private *dev_priv = dev->dev_private;
2495         unsigned long irqflags;
2496
2497         spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
2498         i915_disable_pipestat(dev_priv, pipe,
2499                               PIPE_START_VBLANK_INTERRUPT_STATUS);
2500         spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
2501 }
2502
2503 static void gen8_disable_vblank(struct drm_device *dev, int pipe)
2504 {
2505         struct drm_i915_private *dev_priv = dev->dev_private;
2506         unsigned long irqflags;
2507
2508         spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
2509         dev_priv->de_irq_mask[pipe] |= GEN8_PIPE_VBLANK;
2510         I915_WRITE(GEN8_DE_PIPE_IMR(pipe), dev_priv->de_irq_mask[pipe]);
2511         POSTING_READ(GEN8_DE_PIPE_IMR(pipe));
2512         spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
2513 }
2514
2515 static bool
2516 ring_idle(struct intel_engine_cs *ring, u32 seqno)
2517 {
2518         return (list_empty(&ring->request_list) ||
2519                 i915_seqno_passed(seqno, ring->last_submitted_seqno));
2520 }
2521
2522 static bool
2523 ipehr_is_semaphore_wait(struct drm_device *dev, u32 ipehr)
2524 {
2525         if (INTEL_INFO(dev)->gen >= 8) {
2526                 return (ipehr >> 23) == 0x1c;
2527         } else {
2528                 ipehr &= ~MI_SEMAPHORE_SYNC_MASK;
2529                 return ipehr == (MI_SEMAPHORE_MBOX | MI_SEMAPHORE_COMPARE |
2530                                  MI_SEMAPHORE_REGISTER);
2531         }
2532 }
2533
2534 static struct intel_engine_cs *
2535 semaphore_wait_to_signaller_ring(struct intel_engine_cs *ring, u32 ipehr, u64 offset)
2536 {
2537         struct drm_i915_private *dev_priv = ring->dev->dev_private;
2538         struct intel_engine_cs *signaller;
2539         int i;
2540
2541         if (INTEL_INFO(dev_priv->dev)->gen >= 8) {
2542                 for_each_ring(signaller, dev_priv, i) {
2543                         if (ring == signaller)
2544                                 continue;
2545
2546                         if (offset == signaller->semaphore.signal_ggtt[ring->id])
2547                                 return signaller;
2548                 }
2549         } else {
2550                 u32 sync_bits = ipehr & MI_SEMAPHORE_SYNC_MASK;
2551
2552                 for_each_ring(signaller, dev_priv, i) {
2553                         if(ring == signaller)
2554                                 continue;
2555
2556                         if (sync_bits == signaller->semaphore.mbox.wait[ring->id])
2557                                 return signaller;
2558                 }
2559         }
2560
2561         DRM_ERROR("No signaller ring found for ring %i, ipehr 0x%08x, offset 0x%016llx\n",
2562                   ring->id, ipehr, offset);
2563
2564         return NULL;
2565 }
2566
2567 static struct intel_engine_cs *
2568 semaphore_waits_for(struct intel_engine_cs *ring, u32 *seqno)
2569 {
2570         struct drm_i915_private *dev_priv = ring->dev->dev_private;
2571         u32 cmd, ipehr, head;
2572         u64 offset = 0;
2573         int i, backwards;
2574
2575         ipehr = I915_READ(RING_IPEHR(ring->mmio_base));
2576         if (!ipehr_is_semaphore_wait(ring->dev, ipehr))
2577                 return NULL;
2578
2579         /*
2580          * HEAD is likely pointing to the dword after the actual command,
2581          * so scan backwards until we find the MBOX. But limit it to just 3
2582          * or 4 dwords depending on the semaphore wait command size.
2583          * Note that we don't care about ACTHD here since that might
2584          * point at at batch, and semaphores are always emitted into the
2585          * ringbuffer itself.
2586          */
2587         head = I915_READ_HEAD(ring) & HEAD_ADDR;
2588         backwards = (INTEL_INFO(ring->dev)->gen >= 8) ? 5 : 4;
2589
2590         for (i = backwards; i; --i) {
2591                 /*
2592                  * Be paranoid and presume the hw has gone off into the wild -
2593                  * our ring is smaller than what the hardware (and hence
2594                  * HEAD_ADDR) allows. Also handles wrap-around.
2595                  */
2596                 head &= ring->buffer->size - 1;
2597
2598                 /* This here seems to blow up */
2599                 cmd = ioread32(ring->buffer->virtual_start + head);
2600                 if (cmd == ipehr)
2601                         break;
2602
2603                 head -= 4;
2604         }
2605
2606         if (!i)
2607                 return NULL;
2608
2609         *seqno = ioread32(ring->buffer->virtual_start + head + 4) + 1;
2610         if (INTEL_INFO(ring->dev)->gen >= 8) {
2611                 offset = ioread32(ring->buffer->virtual_start + head + 12);
2612                 offset <<= 32;
2613                 offset = ioread32(ring->buffer->virtual_start + head + 8);
2614         }
2615         return semaphore_wait_to_signaller_ring(ring, ipehr, offset);
2616 }
2617
2618 static int semaphore_passed(struct intel_engine_cs *ring)
2619 {
2620         struct drm_i915_private *dev_priv = ring->dev->dev_private;
2621         struct intel_engine_cs *signaller;
2622         u32 seqno;
2623
2624         ring->hangcheck.deadlock++;
2625
2626         signaller = semaphore_waits_for(ring, &seqno);
2627         if (signaller == NULL)
2628                 return -1;
2629
2630         /* Prevent pathological recursion due to driver bugs */
2631         if (signaller->hangcheck.deadlock >= I915_NUM_RINGS)
2632                 return -1;
2633
2634         if (i915_seqno_passed(signaller->get_seqno(signaller, false), seqno))
2635                 return 1;
2636
2637         /* cursory check for an unkickable deadlock */
2638         if (I915_READ_CTL(signaller) & RING_WAIT_SEMAPHORE &&
2639             semaphore_passed(signaller) < 0)
2640                 return -1;
2641
2642         return 0;
2643 }
2644
2645 static void semaphore_clear_deadlocks(struct drm_i915_private *dev_priv)
2646 {
2647         struct intel_engine_cs *ring;
2648         int i;
2649
2650         for_each_ring(ring, dev_priv, i)
2651                 ring->hangcheck.deadlock = 0;
2652 }
2653
2654 static enum intel_ring_hangcheck_action
2655 ring_stuck(struct intel_engine_cs *ring, u64 acthd)
2656 {
2657         struct drm_device *dev = ring->dev;
2658         struct drm_i915_private *dev_priv = dev->dev_private;
2659         u32 tmp;
2660
2661         if (acthd != ring->hangcheck.acthd) {
2662                 if (acthd > ring->hangcheck.max_acthd) {
2663                         ring->hangcheck.max_acthd = acthd;
2664                         return HANGCHECK_ACTIVE;
2665                 }
2666
2667                 return HANGCHECK_ACTIVE_LOOP;
2668         }
2669
2670         if (IS_GEN2(dev))
2671                 return HANGCHECK_HUNG;
2672
2673         /* Is the chip hanging on a WAIT_FOR_EVENT?
2674          * If so we can simply poke the RB_WAIT bit
2675          * and break the hang. This should work on
2676          * all but the second generation chipsets.
2677          */
2678         tmp = I915_READ_CTL(ring);
2679         if (tmp & RING_WAIT) {
2680                 i915_handle_error(dev, false,
2681                                   "Kicking stuck wait on %s",
2682                                   ring->name);
2683                 I915_WRITE_CTL(ring, tmp);
2684                 return HANGCHECK_KICK;
2685         }
2686
2687         if (INTEL_INFO(dev)->gen >= 6 && tmp & RING_WAIT_SEMAPHORE) {
2688                 switch (semaphore_passed(ring)) {
2689                 default:
2690                         return HANGCHECK_HUNG;
2691                 case 1:
2692                         i915_handle_error(dev, false,
2693                                           "Kicking stuck semaphore on %s",
2694                                           ring->name);
2695                         I915_WRITE_CTL(ring, tmp);
2696                         return HANGCHECK_KICK;
2697                 case 0:
2698                         return HANGCHECK_WAIT;
2699                 }
2700         }
2701
2702         return HANGCHECK_HUNG;
2703 }
2704
2705 /*
2706  * This is called when the chip hasn't reported back with completed
2707  * batchbuffers in a long time. We keep track per ring seqno progress and
2708  * if there are no progress, hangcheck score for that ring is increased.
2709  * Further, acthd is inspected to see if the ring is stuck. On stuck case
2710  * we kick the ring. If we see no progress on three subsequent calls
2711  * we assume chip is wedged and try to fix it by resetting the chip.
2712  */
2713 static void i915_hangcheck_elapsed(struct work_struct *work)
2714 {
2715         struct drm_i915_private *dev_priv =
2716                 container_of(work, typeof(*dev_priv),
2717                              gpu_error.hangcheck_work.work);
2718         struct drm_device *dev = dev_priv->dev;
2719         struct intel_engine_cs *ring;
2720         int i;
2721         int busy_count = 0, rings_hung = 0;
2722         bool stuck[I915_NUM_RINGS] = { 0 };
2723 #define BUSY 1
2724 #define KICK 5
2725 #define HUNG 20
2726
2727         if (!i915.enable_hangcheck)
2728                 return;
2729
2730         for_each_ring(ring, dev_priv, i) {
2731                 u64 acthd;
2732                 u32 seqno;
2733                 bool busy = true;
2734
2735                 semaphore_clear_deadlocks(dev_priv);
2736
2737                 seqno = ring->get_seqno(ring, false);
2738                 acthd = intel_ring_get_active_head(ring);
2739
2740                 if (ring->hangcheck.seqno == seqno) {
2741                         if (ring_idle(ring, seqno)) {
2742                                 ring->hangcheck.action = HANGCHECK_IDLE;
2743
2744                                 if (waitqueue_active(&ring->irq_queue)) {
2745                                         /* Issue a wake-up to catch stuck h/w. */
2746                                         if (!test_and_set_bit(ring->id, &dev_priv->gpu_error.missed_irq_rings)) {
2747                                                 if (!(dev_priv->gpu_error.test_irq_rings & intel_ring_flag(ring)))
2748                                                         DRM_ERROR("Hangcheck timer elapsed... %s idle\n",
2749                                                                   ring->name);
2750                                                 else
2751                                                         DRM_INFO("Fake missed irq on %s\n",
2752                                                                  ring->name);
2753                                                 wake_up_all(&ring->irq_queue);
2754                                         }
2755                                         /* Safeguard against driver failure */
2756                                         ring->hangcheck.score += BUSY;
2757                                 } else
2758                                         busy = false;
2759                         } else {
2760                                 /* We always increment the hangcheck score
2761                                  * if the ring is busy and still processing
2762                                  * the same request, so that no single request
2763                                  * can run indefinitely (such as a chain of
2764                                  * batches). The only time we do not increment
2765                                  * the hangcheck score on this ring, if this
2766                                  * ring is in a legitimate wait for another
2767                                  * ring. In that case the waiting ring is a
2768                                  * victim and we want to be sure we catch the
2769                                  * right culprit. Then every time we do kick
2770                                  * the ring, add a small increment to the
2771                                  * score so that we can catch a batch that is
2772                                  * being repeatedly kicked and so responsible
2773                                  * for stalling the machine.
2774                                  */
2775                                 ring->hangcheck.action = ring_stuck(ring,
2776                                                                     acthd);
2777
2778                                 switch (ring->hangcheck.action) {
2779                                 case HANGCHECK_IDLE:
2780                                 case HANGCHECK_WAIT:
2781                                 case HANGCHECK_ACTIVE:
2782                                         break;
2783                                 case HANGCHECK_ACTIVE_LOOP:
2784                                         ring->hangcheck.score += BUSY;
2785                                         break;
2786                                 case HANGCHECK_KICK:
2787                                         ring->hangcheck.score += KICK;
2788                                         break;
2789                                 case HANGCHECK_HUNG:
2790                                         ring->hangcheck.score += HUNG;
2791                                         stuck[i] = true;
2792                                         break;
2793                                 }
2794                         }
2795                 } else {
2796                         ring->hangcheck.action = HANGCHECK_ACTIVE;
2797
2798                         /* Gradually reduce the count so that we catch DoS
2799                          * attempts across multiple batches.
2800                          */
2801                         if (ring->hangcheck.score > 0)
2802                                 ring->hangcheck.score--;
2803
2804                         ring->hangcheck.acthd = ring->hangcheck.max_acthd = 0;
2805                 }
2806
2807                 ring->hangcheck.seqno = seqno;
2808                 ring->hangcheck.acthd = acthd;
2809                 busy_count += busy;
2810         }
2811
2812         for_each_ring(ring, dev_priv, i) {
2813                 if (ring->hangcheck.score >= HANGCHECK_SCORE_RING_HUNG) {
2814                         DRM_INFO("%s on %s\n",
2815                                  stuck[i] ? "stuck" : "no progress",
2816                                  ring->name);
2817                         rings_hung++;
2818                 }
2819         }
2820
2821         if (rings_hung)
2822                 return i915_handle_error(dev, true, "Ring hung");
2823
2824         if (busy_count)
2825                 /* Reset timer case chip hangs without another request
2826                  * being added */
2827                 i915_queue_hangcheck(dev);
2828 }
2829
2830 void i915_queue_hangcheck(struct drm_device *dev)
2831 {
2832         struct i915_gpu_error *e = &to_i915(dev)->gpu_error;
2833
2834         if (!i915.enable_hangcheck)
2835                 return;
2836
2837         /* Don't continually defer the hangcheck so that it is always run at
2838          * least once after work has been scheduled on any ring. Otherwise,
2839          * we will ignore a hung ring if a second ring is kept busy.
2840          */
2841
2842         queue_delayed_work(e->hangcheck_wq, &e->hangcheck_work,
2843                            round_jiffies_up_relative(DRM_I915_HANGCHECK_JIFFIES));
2844 }
2845
2846 static void ibx_irq_reset(struct drm_device *dev)
2847 {
2848         struct drm_i915_private *dev_priv = dev->dev_private;
2849
2850         if (HAS_PCH_NOP(dev))
2851                 return;
2852
2853         GEN5_IRQ_RESET(SDE);
2854
2855         if (HAS_PCH_CPT(dev) || HAS_PCH_LPT(dev))
2856                 I915_WRITE(SERR_INT, 0xffffffff);
2857 }
2858
2859 /*
2860  * SDEIER is also touched by the interrupt handler to work around missed PCH
2861  * interrupts. Hence we can't update it after the interrupt handler is enabled -
2862  * instead we unconditionally enable all PCH interrupt sources here, but then
2863  * only unmask them as needed with SDEIMR.
2864  *
2865  * This function needs to be called before interrupts are enabled.
2866  */
2867 static void ibx_irq_pre_postinstall(struct drm_device *dev)
2868 {
2869         struct drm_i915_private *dev_priv = dev->dev_private;
2870
2871         if (HAS_PCH_NOP(dev))
2872                 return;
2873
2874         WARN_ON(I915_READ(SDEIER) != 0);
2875         I915_WRITE(SDEIER, 0xffffffff);
2876         POSTING_READ(SDEIER);
2877 }
2878
2879 static void gen5_gt_irq_reset(struct drm_device *dev)
2880 {
2881         struct drm_i915_private *dev_priv = dev->dev_private;
2882
2883         GEN5_IRQ_RESET(GT);
2884         if (INTEL_INFO(dev)->gen >= 6)
2885                 GEN5_IRQ_RESET(GEN6_PM);
2886 }
2887
2888 /* drm_dma.h hooks
2889 */
2890 static void ironlake_irq_reset(struct drm_device *dev)
2891 {
2892         struct drm_i915_private *dev_priv = dev->dev_private;
2893
2894         I915_WRITE(HWSTAM, 0xffffffff);
2895
2896         GEN5_IRQ_RESET(DE);
2897         if (IS_GEN7(dev))
2898                 I915_WRITE(GEN7_ERR_INT, 0xffffffff);
2899
2900         gen5_gt_irq_reset(dev);
2901
2902         ibx_irq_reset(dev);
2903 }
2904
2905 static void vlv_display_irq_reset(struct drm_i915_private *dev_priv)
2906 {
2907         enum pipe pipe;
2908
2909         I915_WRITE(PORT_HOTPLUG_EN, 0);
2910         I915_WRITE(PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT));
2911
2912         for_each_pipe(dev_priv, pipe)
2913                 I915_WRITE(PIPESTAT(pipe), 0xffff);
2914
2915         GEN5_IRQ_RESET(VLV_);
2916 }
2917
2918 static void valleyview_irq_preinstall(struct drm_device *dev)
2919 {
2920         struct drm_i915_private *dev_priv = dev->dev_private;
2921
2922         /* VLV magic */
2923         I915_WRITE(VLV_IMR, 0);
2924         I915_WRITE(RING_IMR(RENDER_RING_BASE), 0);
2925         I915_WRITE(RING_IMR(GEN6_BSD_RING_BASE), 0);
2926         I915_WRITE(RING_IMR(BLT_RING_BASE), 0);
2927
2928         gen5_gt_irq_reset(dev);
2929
2930         I915_WRITE(DPINVGTT, DPINVGTT_STATUS_MASK);
2931
2932         vlv_display_irq_reset(dev_priv);
2933 }
2934
2935 static void gen8_gt_irq_reset(struct drm_i915_private *dev_priv)
2936 {
2937         GEN8_IRQ_RESET_NDX(GT, 0);
2938         GEN8_IRQ_RESET_NDX(GT, 1);
2939         GEN8_IRQ_RESET_NDX(GT, 2);
2940         GEN8_IRQ_RESET_NDX(GT, 3);
2941 }
2942
2943 static void gen8_irq_reset(struct drm_device *dev)
2944 {
2945         struct drm_i915_private *dev_priv = dev->dev_private;
2946         int pipe;
2947
2948         I915_WRITE(GEN8_MASTER_IRQ, 0);
2949         POSTING_READ(GEN8_MASTER_IRQ);
2950
2951         gen8_gt_irq_reset(dev_priv);
2952
2953         for_each_pipe(dev_priv, pipe)
2954                 if (intel_display_power_is_enabled(dev_priv,
2955                                                    POWER_DOMAIN_PIPE(pipe)))
2956                         GEN8_IRQ_RESET_NDX(DE_PIPE, pipe);
2957
2958         GEN5_IRQ_RESET(GEN8_DE_PORT_);
2959         GEN5_IRQ_RESET(GEN8_DE_MISC_);
2960         GEN5_IRQ_RESET(GEN8_PCU_);
2961
2962         if (HAS_PCH_SPLIT(dev))
2963                 ibx_irq_reset(dev);
2964 }
2965
2966 void gen8_irq_power_well_post_enable(struct drm_i915_private *dev_priv,
2967                                      unsigned int pipe_mask)
2968 {
2969         uint32_t extra_ier = GEN8_PIPE_VBLANK | GEN8_PIPE_FIFO_UNDERRUN;
2970
2971         spin_lock_irq(&dev_priv->irq_lock);
2972         if (pipe_mask & 1 << PIPE_A)
2973                 GEN8_IRQ_INIT_NDX(DE_PIPE, PIPE_A,
2974                                   dev_priv->de_irq_mask[PIPE_A],
2975                                   ~dev_priv->de_irq_mask[PIPE_A] | extra_ier);
2976         if (pipe_mask & 1 << PIPE_B)
2977                 GEN8_IRQ_INIT_NDX(DE_PIPE, PIPE_B,
2978                                   dev_priv->de_irq_mask[PIPE_B],
2979                                   ~dev_priv->de_irq_mask[PIPE_B] | extra_ier);
2980         if (pipe_mask & 1 << PIPE_C)
2981                 GEN8_IRQ_INIT_NDX(DE_PIPE, PIPE_C,
2982                                   dev_priv->de_irq_mask[PIPE_C],
2983                                   ~dev_priv->de_irq_mask[PIPE_C] | extra_ier);
2984         spin_unlock_irq(&dev_priv->irq_lock);
2985 }
2986
2987 static void cherryview_irq_preinstall(struct drm_device *dev)
2988 {
2989         struct drm_i915_private *dev_priv = dev->dev_private;
2990
2991         I915_WRITE(GEN8_MASTER_IRQ, 0);
2992         POSTING_READ(GEN8_MASTER_IRQ);
2993
2994         gen8_gt_irq_reset(dev_priv);
2995
2996         GEN5_IRQ_RESET(GEN8_PCU_);
2997
2998         I915_WRITE(DPINVGTT, DPINVGTT_STATUS_MASK_CHV);
2999
3000         vlv_display_irq_reset(dev_priv);
3001 }
3002
3003 static void ibx_hpd_irq_setup(struct drm_device *dev)
3004 {
3005         struct drm_i915_private *dev_priv = dev->dev_private;
3006         struct intel_encoder *intel_encoder;
3007         u32 hotplug_irqs, hotplug, enabled_irqs = 0;
3008
3009         if (HAS_PCH_IBX(dev)) {
3010                 hotplug_irqs = SDE_HOTPLUG_MASK;
3011                 for_each_intel_encoder(dev, intel_encoder)
3012                         if (dev_priv->hotplug.stats[intel_encoder->hpd_pin].state == HPD_ENABLED)
3013                                 enabled_irqs |= hpd_ibx[intel_encoder->hpd_pin];
3014         } else if (HAS_PCH_SPT(dev)) {
3015                 hotplug_irqs = SDE_HOTPLUG_MASK_SPT;
3016                 for_each_intel_encoder(dev, intel_encoder)
3017                         if (dev_priv->hotplug.stats[intel_encoder->hpd_pin].state == HPD_ENABLED)
3018                                 enabled_irqs |= hpd_spt[intel_encoder->hpd_pin];
3019         } else {
3020                 hotplug_irqs = SDE_HOTPLUG_MASK_CPT;
3021                 for_each_intel_encoder(dev, intel_encoder)
3022                         if (dev_priv->hotplug.stats[intel_encoder->hpd_pin].state == HPD_ENABLED)
3023                                 enabled_irqs |= hpd_cpt[intel_encoder->hpd_pin];
3024         }
3025
3026         ibx_display_interrupt_update(dev_priv, hotplug_irqs, enabled_irqs);
3027
3028         /*
3029          * Enable digital hotplug on the PCH, and configure the DP short pulse
3030          * duration to 2ms (which is the minimum in the Display Port spec)
3031          *
3032          * This register is the same on all known PCH chips.
3033          */
3034         hotplug = I915_READ(PCH_PORT_HOTPLUG);
3035         hotplug &= ~(PORTD_PULSE_DURATION_MASK|PORTC_PULSE_DURATION_MASK|PORTB_PULSE_DURATION_MASK);
3036         hotplug |= PORTD_HOTPLUG_ENABLE | PORTD_PULSE_DURATION_2ms;
3037         hotplug |= PORTC_HOTPLUG_ENABLE | PORTC_PULSE_DURATION_2ms;
3038         hotplug |= PORTB_HOTPLUG_ENABLE | PORTB_PULSE_DURATION_2ms;
3039         I915_WRITE(PCH_PORT_HOTPLUG, hotplug);
3040
3041         /* enable SPT PORTE hot plug */
3042         if (HAS_PCH_SPT(dev)) {
3043                 hotplug = I915_READ(PCH_PORT_HOTPLUG2);
3044                 hotplug |= PORTE_HOTPLUG_ENABLE;
3045                 I915_WRITE(PCH_PORT_HOTPLUG2, hotplug);
3046         }
3047 }
3048
3049 static void bxt_hpd_irq_setup(struct drm_device *dev)
3050 {
3051         struct drm_i915_private *dev_priv = dev->dev_private;
3052         struct intel_encoder *intel_encoder;
3053         u32 hotplug_port = 0;
3054         u32 hotplug_ctrl;
3055
3056         /* Now, enable HPD */
3057         for_each_intel_encoder(dev, intel_encoder) {
3058                 if (dev_priv->hotplug.stats[intel_encoder->hpd_pin].state
3059                                 == HPD_ENABLED)
3060                         hotplug_port |= hpd_bxt[intel_encoder->hpd_pin];
3061         }
3062
3063         /* Mask all HPD control bits */
3064         hotplug_ctrl = I915_READ(BXT_HOTPLUG_CTL) & ~BXT_HOTPLUG_CTL_MASK;
3065
3066         /* Enable requested port in hotplug control */
3067         /* TODO: implement (short) HPD support on port A */
3068         WARN_ON_ONCE(hotplug_port & BXT_DE_PORT_HP_DDIA);
3069         if (hotplug_port & BXT_DE_PORT_HP_DDIB)
3070                 hotplug_ctrl |= BXT_DDIB_HPD_ENABLE;
3071         if (hotplug_port & BXT_DE_PORT_HP_DDIC)
3072                 hotplug_ctrl |= BXT_DDIC_HPD_ENABLE;
3073         I915_WRITE(BXT_HOTPLUG_CTL, hotplug_ctrl);
3074
3075         /* Unmask DDI hotplug in IMR */
3076         hotplug_ctrl = I915_READ(GEN8_DE_PORT_IMR) & ~hotplug_port;
3077         I915_WRITE(GEN8_DE_PORT_IMR, hotplug_ctrl);
3078
3079         /* Enable DDI hotplug in IER */
3080         hotplug_ctrl = I915_READ(GEN8_DE_PORT_IER) | hotplug_port;
3081         I915_WRITE(GEN8_DE_PORT_IER, hotplug_ctrl);
3082         POSTING_READ(GEN8_DE_PORT_IER);
3083 }
3084
3085 static void ibx_irq_postinstall(struct drm_device *dev)
3086 {
3087         struct drm_i915_private *dev_priv = dev->dev_private;
3088         u32 mask;
3089
3090         if (HAS_PCH_NOP(dev))
3091                 return;
3092
3093         if (HAS_PCH_IBX(dev))
3094                 mask = SDE_GMBUS | SDE_AUX_MASK | SDE_POISON;
3095         else
3096                 mask = SDE_GMBUS_CPT | SDE_AUX_MASK_CPT;
3097
3098         GEN5_ASSERT_IIR_IS_ZERO(SDEIIR);
3099         I915_WRITE(SDEIMR, ~mask);
3100 }
3101
3102 static void gen5_gt_irq_postinstall(struct drm_device *dev)
3103 {
3104         struct drm_i915_private *dev_priv = dev->dev_private;
3105         u32 pm_irqs, gt_irqs;
3106
3107         pm_irqs = gt_irqs = 0;
3108
3109         dev_priv->gt_irq_mask = ~0;
3110         if (HAS_L3_DPF(dev)) {
3111                 /* L3 parity interrupt is always unmasked. */
3112                 dev_priv->gt_irq_mask = ~GT_PARITY_ERROR(dev);
3113                 gt_irqs |= GT_PARITY_ERROR(dev);
3114         }
3115
3116         gt_irqs |= GT_RENDER_USER_INTERRUPT;
3117         if (IS_GEN5(dev)) {
3118                 gt_irqs |= GT_RENDER_PIPECTL_NOTIFY_INTERRUPT |
3119                            ILK_BSD_USER_INTERRUPT;
3120         } else {
3121                 gt_irqs |= GT_BLT_USER_INTERRUPT | GT_BSD_USER_INTERRUPT;
3122         }
3123
3124         GEN5_IRQ_INIT(GT, dev_priv->gt_irq_mask, gt_irqs);
3125
3126         if (INTEL_INFO(dev)->gen >= 6) {
3127                 /*
3128                  * RPS interrupts will get enabled/disabled on demand when RPS
3129                  * itself is enabled/disabled.
3130                  */
3131                 if (HAS_VEBOX(dev))
3132                         pm_irqs |= PM_VEBOX_USER_INTERRUPT;
3133
3134                 dev_priv->pm_irq_mask = 0xffffffff;
3135                 GEN5_IRQ_INIT(GEN6_PM, dev_priv->pm_irq_mask, pm_irqs);
3136         }
3137 }
3138
3139 static int ironlake_irq_postinstall(struct drm_device *dev)
3140 {
3141         struct drm_i915_private *dev_priv = dev->dev_private;
3142         u32 display_mask, extra_mask;
3143
3144         if (INTEL_INFO(dev)->gen >= 7) {
3145                 display_mask = (DE_MASTER_IRQ_CONTROL | DE_GSE_IVB |
3146                                 DE_PCH_EVENT_IVB | DE_PLANEC_FLIP_DONE_IVB |
3147                                 DE_PLANEB_FLIP_DONE_IVB |
3148                                 DE_PLANEA_FLIP_DONE_IVB | DE_AUX_CHANNEL_A_IVB);
3149                 extra_mask = (DE_PIPEC_VBLANK_IVB | DE_PIPEB_VBLANK_IVB |
3150                               DE_PIPEA_VBLANK_IVB | DE_ERR_INT_IVB);
3151         } else {
3152                 display_mask = (DE_MASTER_IRQ_CONTROL | DE_GSE | DE_PCH_EVENT |
3153                                 DE_PLANEA_FLIP_DONE | DE_PLANEB_FLIP_DONE |
3154                                 DE_AUX_CHANNEL_A |
3155                                 DE_PIPEB_CRC_DONE | DE_PIPEA_CRC_DONE |
3156                                 DE_POISON);
3157                 extra_mask = DE_PIPEA_VBLANK | DE_PIPEB_VBLANK | DE_PCU_EVENT |
3158                                 DE_PIPEB_FIFO_UNDERRUN | DE_PIPEA_FIFO_UNDERRUN;
3159         }
3160
3161         dev_priv->irq_mask = ~display_mask;
3162
3163         I915_WRITE(HWSTAM, 0xeffe);
3164
3165         ibx_irq_pre_postinstall(dev);
3166
3167         GEN5_IRQ_INIT(DE, dev_priv->irq_mask, display_mask | extra_mask);
3168
3169         gen5_gt_irq_postinstall(dev);
3170
3171         ibx_irq_postinstall(dev);
3172
3173         if (IS_IRONLAKE_M(dev)) {
3174                 /* Enable PCU event interrupts
3175                  *
3176                  * spinlocking not required here for correctness since interrupt
3177                  * setup is guaranteed to run in single-threaded context. But we
3178                  * need it to make the assert_spin_locked happy. */
3179                 spin_lock_irq(&dev_priv->irq_lock);
3180                 ironlake_enable_display_irq(dev_priv, DE_PCU_EVENT);
3181                 spin_unlock_irq(&dev_priv->irq_lock);
3182         }
3183
3184         return 0;
3185 }
3186
3187 static void valleyview_display_irqs_install(struct drm_i915_private *dev_priv)
3188 {
3189         u32 pipestat_mask;
3190         u32 iir_mask;
3191         enum pipe pipe;
3192
3193         pipestat_mask = PIPESTAT_INT_STATUS_MASK |
3194                         PIPE_FIFO_UNDERRUN_STATUS;
3195
3196         for_each_pipe(dev_priv, pipe)
3197                 I915_WRITE(PIPESTAT(pipe), pipestat_mask);
3198         POSTING_READ(PIPESTAT(PIPE_A));
3199
3200         pipestat_mask = PLANE_FLIP_DONE_INT_STATUS_VLV |
3201                         PIPE_CRC_DONE_INTERRUPT_STATUS;
3202
3203         i915_enable_pipestat(dev_priv, PIPE_A, PIPE_GMBUS_INTERRUPT_STATUS);
3204         for_each_pipe(dev_priv, pipe)
3205                       i915_enable_pipestat(dev_priv, pipe, pipestat_mask);
3206
3207         iir_mask = I915_DISPLAY_PORT_INTERRUPT |
3208                    I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
3209                    I915_DISPLAY_PIPE_B_EVENT_INTERRUPT;
3210         if (IS_CHERRYVIEW(dev_priv))
3211                 iir_mask |= I915_DISPLAY_PIPE_C_EVENT_INTERRUPT;
3212         dev_priv->irq_mask &= ~iir_mask;
3213
3214         I915_WRITE(VLV_IIR, iir_mask);
3215         I915_WRITE(VLV_IIR, iir_mask);
3216         I915_WRITE(VLV_IER, ~dev_priv->irq_mask);
3217         I915_WRITE(VLV_IMR, dev_priv->irq_mask);
3218         POSTING_READ(VLV_IMR);
3219 }
3220
3221 static void valleyview_display_irqs_uninstall(struct drm_i915_private *dev_priv)
3222 {
3223         u32 pipestat_mask;
3224         u32 iir_mask;
3225         enum pipe pipe;
3226
3227         iir_mask = I915_DISPLAY_PORT_INTERRUPT |
3228                    I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
3229                    I915_DISPLAY_PIPE_B_EVENT_INTERRUPT;
3230         if (IS_CHERRYVIEW(dev_priv))
3231                 iir_mask |= I915_DISPLAY_PIPE_C_EVENT_INTERRUPT;
3232
3233         dev_priv->irq_mask |= iir_mask;
3234         I915_WRITE(VLV_IMR, dev_priv->irq_mask);
3235         I915_WRITE(VLV_IER, ~dev_priv->irq_mask);
3236         I915_WRITE(VLV_IIR, iir_mask);
3237         I915_WRITE(VLV_IIR, iir_mask);
3238         POSTING_READ(VLV_IIR);
3239
3240         pipestat_mask = PLANE_FLIP_DONE_INT_STATUS_VLV |
3241                         PIPE_CRC_DONE_INTERRUPT_STATUS;
3242
3243         i915_disable_pipestat(dev_priv, PIPE_A, PIPE_GMBUS_INTERRUPT_STATUS);
3244         for_each_pipe(dev_priv, pipe)
3245                 i915_disable_pipestat(dev_priv, pipe, pipestat_mask);
3246
3247         pipestat_mask = PIPESTAT_INT_STATUS_MASK |
3248                         PIPE_FIFO_UNDERRUN_STATUS;
3249
3250         for_each_pipe(dev_priv, pipe)
3251                 I915_WRITE(PIPESTAT(pipe), pipestat_mask);
3252         POSTING_READ(PIPESTAT(PIPE_A));
3253 }
3254
3255 void valleyview_enable_display_irqs(struct drm_i915_private *dev_priv)
3256 {
3257         assert_spin_locked(&dev_priv->irq_lock);
3258
3259         if (dev_priv->display_irqs_enabled)
3260                 return;
3261
3262         dev_priv->display_irqs_enabled = true;
3263
3264         if (intel_irqs_enabled(dev_priv))
3265                 valleyview_display_irqs_install(dev_priv);
3266 }
3267
3268 void valleyview_disable_display_irqs(struct drm_i915_private *dev_priv)
3269 {
3270         assert_spin_locked(&dev_priv->irq_lock);
3271
3272         if (!dev_priv->display_irqs_enabled)
3273                 return;
3274
3275         dev_priv->display_irqs_enabled = false;
3276
3277         if (intel_irqs_enabled(dev_priv))
3278                 valleyview_display_irqs_uninstall(dev_priv);
3279 }
3280
3281 static void vlv_display_irq_postinstall(struct drm_i915_private *dev_priv)
3282 {
3283         dev_priv->irq_mask = ~0;
3284
3285         I915_WRITE(PORT_HOTPLUG_EN, 0);
3286         POSTING_READ(PORT_HOTPLUG_EN);
3287
3288         I915_WRITE(VLV_IIR, 0xffffffff);
3289         I915_WRITE(VLV_IIR, 0xffffffff);
3290         I915_WRITE(VLV_IER, ~dev_priv->irq_mask);
3291         I915_WRITE(VLV_IMR, dev_priv->irq_mask);
3292         POSTING_READ(VLV_IMR);
3293
3294         /* Interrupt setup is already guaranteed to be single-threaded, this is
3295          * just to make the assert_spin_locked check happy. */
3296         spin_lock_irq(&dev_priv->irq_lock);
3297         if (dev_priv->display_irqs_enabled)
3298                 valleyview_display_irqs_install(dev_priv);
3299         spin_unlock_irq(&dev_priv->irq_lock);
3300 }
3301
3302 static int valleyview_irq_postinstall(struct drm_device *dev)
3303 {
3304         struct drm_i915_private *dev_priv = dev->dev_private;
3305
3306         vlv_display_irq_postinstall(dev_priv);
3307
3308         gen5_gt_irq_postinstall(dev);
3309
3310         /* ack & enable invalid PTE error interrupts */
3311 #if 0 /* FIXME: add support to irq handler for checking these bits */
3312         I915_WRITE(DPINVGTT, DPINVGTT_STATUS_MASK);
3313         I915_WRITE(DPINVGTT, DPINVGTT_EN_MASK);
3314 #endif
3315
3316         I915_WRITE(VLV_MASTER_IER, MASTER_INTERRUPT_ENABLE);
3317
3318         return 0;
3319 }
3320
3321 static void gen8_gt_irq_postinstall(struct drm_i915_private *dev_priv)
3322 {
3323         /* These are interrupts we'll toggle with the ring mask register */
3324         uint32_t gt_interrupts[] = {
3325                 GT_RENDER_USER_INTERRUPT << GEN8_RCS_IRQ_SHIFT |
3326                         GT_CONTEXT_SWITCH_INTERRUPT << GEN8_RCS_IRQ_SHIFT |
3327                         GT_RENDER_L3_PARITY_ERROR_INTERRUPT |
3328                         GT_RENDER_USER_INTERRUPT << GEN8_BCS_IRQ_SHIFT |
3329                         GT_CONTEXT_SWITCH_INTERRUPT << GEN8_BCS_IRQ_SHIFT,
3330                 GT_RENDER_USER_INTERRUPT << GEN8_VCS1_IRQ_SHIFT |
3331                         GT_CONTEXT_SWITCH_INTERRUPT << GEN8_VCS1_IRQ_SHIFT |
3332                         GT_RENDER_USER_INTERRUPT << GEN8_VCS2_IRQ_SHIFT |
3333                         GT_CONTEXT_SWITCH_INTERRUPT << GEN8_VCS2_IRQ_SHIFT,
3334                 0,
3335                 GT_RENDER_USER_INTERRUPT << GEN8_VECS_IRQ_SHIFT |
3336                         GT_CONTEXT_SWITCH_INTERRUPT << GEN8_VECS_IRQ_SHIFT
3337                 };
3338
3339         dev_priv->pm_irq_mask = 0xffffffff;
3340         GEN8_IRQ_INIT_NDX(GT, 0, ~gt_interrupts[0], gt_interrupts[0]);
3341         GEN8_IRQ_INIT_NDX(GT, 1, ~gt_interrupts[1], gt_interrupts[1]);
3342         /*
3343          * RPS interrupts will get enabled/disabled on demand when RPS itself
3344          * is enabled/disabled.
3345          */
3346         GEN8_IRQ_INIT_NDX(GT, 2, dev_priv->pm_irq_mask, 0);
3347         GEN8_IRQ_INIT_NDX(GT, 3, ~gt_interrupts[3], gt_interrupts[3]);
3348 }
3349
3350 static void gen8_de_irq_postinstall(struct drm_i915_private *dev_priv)
3351 {
3352         uint32_t de_pipe_masked = GEN8_PIPE_CDCLK_CRC_DONE;
3353         uint32_t de_pipe_enables;
3354         int pipe;
3355         u32 de_port_en = GEN8_AUX_CHANNEL_A;
3356
3357         if (IS_GEN9(dev_priv)) {
3358                 de_pipe_masked |= GEN9_PIPE_PLANE1_FLIP_DONE |
3359                                   GEN9_DE_PIPE_IRQ_FAULT_ERRORS;
3360                 de_port_en |= GEN9_AUX_CHANNEL_B | GEN9_AUX_CHANNEL_C |
3361                         GEN9_AUX_CHANNEL_D;
3362
3363                 if (IS_BROXTON(dev_priv))
3364                         de_port_en |= BXT_DE_PORT_GMBUS;
3365         } else
3366                 de_pipe_masked |= GEN8_PIPE_PRIMARY_FLIP_DONE |
3367                                   GEN8_DE_PIPE_IRQ_FAULT_ERRORS;
3368
3369         de_pipe_enables = de_pipe_masked | GEN8_PIPE_VBLANK |
3370                                            GEN8_PIPE_FIFO_UNDERRUN;
3371
3372         dev_priv->de_irq_mask[PIPE_A] = ~de_pipe_masked;
3373         dev_priv->de_irq_mask[PIPE_B] = ~de_pipe_masked;
3374         dev_priv->de_irq_mask[PIPE_C] = ~de_pipe_masked;
3375
3376         for_each_pipe(dev_priv, pipe)
3377                 if (intel_display_power_is_enabled(dev_priv,
3378                                 POWER_DOMAIN_PIPE(pipe)))
3379                         GEN8_IRQ_INIT_NDX(DE_PIPE, pipe,
3380                                           dev_priv->de_irq_mask[pipe],
3381                                           de_pipe_enables);
3382
3383         GEN5_IRQ_INIT(GEN8_DE_PORT_, ~de_port_en, de_port_en);
3384 }
3385
3386 static int gen8_irq_postinstall(struct drm_device *dev)
3387 {
3388         struct drm_i915_private *dev_priv = dev->dev_private;
3389
3390         if (HAS_PCH_SPLIT(dev))
3391                 ibx_irq_pre_postinstall(dev);
3392
3393         gen8_gt_irq_postinstall(dev_priv);
3394         gen8_de_irq_postinstall(dev_priv);
3395
3396         if (HAS_PCH_SPLIT(dev))
3397                 ibx_irq_postinstall(dev);
3398
3399         I915_WRITE(GEN8_MASTER_IRQ, DE_MASTER_IRQ_CONTROL);
3400         POSTING_READ(GEN8_MASTER_IRQ);
3401
3402         return 0;
3403 }
3404
3405 static int cherryview_irq_postinstall(struct drm_device *dev)
3406 {
3407         struct drm_i915_private *dev_priv = dev->dev_private;
3408
3409         vlv_display_irq_postinstall(dev_priv);
3410
3411         gen8_gt_irq_postinstall(dev_priv);
3412
3413         I915_WRITE(GEN8_MASTER_IRQ, MASTER_INTERRUPT_ENABLE);
3414         POSTING_READ(GEN8_MASTER_IRQ);
3415
3416         return 0;
3417 }
3418
3419 static void gen8_irq_uninstall(struct drm_device *dev)
3420 {
3421         struct drm_i915_private *dev_priv = dev->dev_private;
3422
3423         if (!dev_priv)
3424                 return;
3425
3426         gen8_irq_reset(dev);
3427 }
3428
3429 static void vlv_display_irq_uninstall(struct drm_i915_private *dev_priv)
3430 {
3431         /* Interrupt setup is already guaranteed to be single-threaded, this is
3432          * just to make the assert_spin_locked check happy. */
3433         spin_lock_irq(&dev_priv->irq_lock);
3434         if (dev_priv->display_irqs_enabled)
3435                 valleyview_display_irqs_uninstall(dev_priv);
3436         spin_unlock_irq(&dev_priv->irq_lock);
3437
3438         vlv_display_irq_reset(dev_priv);
3439
3440         dev_priv->irq_mask = ~0;
3441 }
3442
3443 static void valleyview_irq_uninstall(struct drm_device *dev)
3444 {
3445         struct drm_i915_private *dev_priv = dev->dev_private;
3446
3447         if (!dev_priv)
3448                 return;
3449
3450         I915_WRITE(VLV_MASTER_IER, 0);
3451
3452         gen5_gt_irq_reset(dev);
3453
3454         I915_WRITE(HWSTAM, 0xffffffff);
3455
3456         vlv_display_irq_uninstall(dev_priv);
3457 }
3458
3459 static void cherryview_irq_uninstall(struct drm_device *dev)
3460 {
3461         struct drm_i915_private *dev_priv = dev->dev_private;
3462
3463         if (!dev_priv)
3464                 return;
3465
3466         I915_WRITE(GEN8_MASTER_IRQ, 0);
3467         POSTING_READ(GEN8_MASTER_IRQ);
3468
3469         gen8_gt_irq_reset(dev_priv);
3470
3471         GEN5_IRQ_RESET(GEN8_PCU_);
3472
3473         vlv_display_irq_uninstall(dev_priv);
3474 }
3475
3476 static void ironlake_irq_uninstall(struct drm_device *dev)
3477 {
3478         struct drm_i915_private *dev_priv = dev->dev_private;
3479
3480         if (!dev_priv)
3481                 return;
3482
3483         ironlake_irq_reset(dev);
3484 }
3485
3486 static void i8xx_irq_preinstall(struct drm_device * dev)
3487 {
3488         struct drm_i915_private *dev_priv = dev->dev_private;
3489         int pipe;
3490
3491         for_each_pipe(dev_priv, pipe)
3492                 I915_WRITE(PIPESTAT(pipe), 0);
3493         I915_WRITE16(IMR, 0xffff);
3494         I915_WRITE16(IER, 0x0);
3495         POSTING_READ16(IER);
3496 }
3497
3498 static int i8xx_irq_postinstall(struct drm_device *dev)
3499 {
3500         struct drm_i915_private *dev_priv = dev->dev_private;
3501
3502         I915_WRITE16(EMR,
3503                      ~(I915_ERROR_PAGE_TABLE | I915_ERROR_MEMORY_REFRESH));
3504
3505         /* Unmask the interrupts that we always want on. */
3506         dev_priv->irq_mask =
3507                 ~(I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
3508                   I915_DISPLAY_PIPE_B_EVENT_INTERRUPT |
3509                   I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT |
3510                   I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT);
3511         I915_WRITE16(IMR, dev_priv->irq_mask);
3512
3513         I915_WRITE16(IER,
3514                      I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
3515                      I915_DISPLAY_PIPE_B_EVENT_INTERRUPT |
3516                      I915_USER_INTERRUPT);
3517         POSTING_READ16(IER);
3518
3519         /* Interrupt setup is already guaranteed to be single-threaded, this is
3520          * just to make the assert_spin_locked check happy. */
3521         spin_lock_irq(&dev_priv->irq_lock);
3522         i915_enable_pipestat(dev_priv, PIPE_A, PIPE_CRC_DONE_INTERRUPT_STATUS);
3523         i915_enable_pipestat(dev_priv, PIPE_B, PIPE_CRC_DONE_INTERRUPT_STATUS);
3524         spin_unlock_irq(&dev_priv->irq_lock);
3525
3526         return 0;
3527 }
3528
3529 /*
3530  * Returns true when a page flip has completed.
3531  */
3532 static bool i8xx_handle_vblank(struct drm_device *dev,
3533                                int plane, int pipe, u32 iir)
3534 {
3535         struct drm_i915_private *dev_priv = dev->dev_private;
3536         u16 flip_pending = DISPLAY_PLANE_FLIP_PENDING(plane);
3537
3538         if (!intel_pipe_handle_vblank(dev, pipe))
3539                 return false;
3540
3541         if ((iir & flip_pending) == 0)
3542                 goto check_page_flip;
3543
3544         /* We detect FlipDone by looking for the change in PendingFlip from '1'
3545          * to '0' on the following vblank, i.e. IIR has the Pendingflip
3546          * asserted following the MI_DISPLAY_FLIP, but ISR is deasserted, hence
3547          * the flip is completed (no longer pending). Since this doesn't raise
3548          * an interrupt per se, we watch for the change at vblank.
3549          */
3550         if (I915_READ16(ISR) & flip_pending)
3551                 goto check_page_flip;
3552
3553         intel_prepare_page_flip(dev, plane);
3554         intel_finish_page_flip(dev, pipe);
3555         return true;
3556
3557 check_page_flip:
3558         intel_check_page_flip(dev, pipe);
3559         return false;
3560 }
3561
3562 static irqreturn_t i8xx_irq_handler(int irq, void *arg)
3563 {
3564         struct drm_device *dev = arg;
3565         struct drm_i915_private *dev_priv = dev->dev_private;
3566         u16 iir, new_iir;
3567         u32 pipe_stats[2];
3568         int pipe;
3569         u16 flip_mask =
3570                 I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT |
3571                 I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT;
3572
3573         if (!intel_irqs_enabled(dev_priv))
3574                 return IRQ_NONE;
3575
3576         iir = I915_READ16(IIR);
3577         if (iir == 0)
3578                 return IRQ_NONE;
3579
3580         while (iir & ~flip_mask) {
3581                 /* Can't rely on pipestat interrupt bit in iir as it might
3582                  * have been cleared after the pipestat interrupt was received.
3583                  * It doesn't set the bit in iir again, but it still produces
3584                  * interrupts (for non-MSI).
3585                  */
3586                 spin_lock(&dev_priv->irq_lock);
3587                 if (iir & I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT)
3588                         DRM_DEBUG("Command parser error, iir 0x%08x\n", iir);
3589
3590                 for_each_pipe(dev_priv, pipe) {
3591                         int reg = PIPESTAT(pipe);
3592                         pipe_stats[pipe] = I915_READ(reg);
3593
3594                         /*
3595                          * Clear the PIPE*STAT regs before the IIR
3596                          */
3597                         if (pipe_stats[pipe] & 0x8000ffff)
3598                                 I915_WRITE(reg, pipe_stats[pipe]);
3599                 }
3600                 spin_unlock(&dev_priv->irq_lock);
3601
3602                 I915_WRITE16(IIR, iir & ~flip_mask);
3603                 new_iir = I915_READ16(IIR); /* Flush posted writes */
3604
3605                 if (iir & I915_USER_INTERRUPT)
3606                         notify_ring(&dev_priv->ring[RCS]);
3607
3608                 for_each_pipe(dev_priv, pipe) {
3609                         int plane = pipe;
3610                         if (HAS_FBC(dev))
3611                                 plane = !plane;
3612
3613                         if (pipe_stats[pipe] & PIPE_VBLANK_INTERRUPT_STATUS &&
3614                             i8xx_handle_vblank(dev, plane, pipe, iir))
3615                                 flip_mask &= ~DISPLAY_PLANE_FLIP_PENDING(plane);
3616
3617                         if (pipe_stats[pipe] & PIPE_CRC_DONE_INTERRUPT_STATUS)
3618                                 i9xx_pipe_crc_irq_handler(dev, pipe);
3619
3620                         if (pipe_stats[pipe] & PIPE_FIFO_UNDERRUN_STATUS)
3621                                 intel_cpu_fifo_underrun_irq_handler(dev_priv,
3622                                                                     pipe);
3623                 }
3624
3625                 iir = new_iir;
3626         }
3627
3628         return IRQ_HANDLED;
3629 }
3630
3631 static void i8xx_irq_uninstall(struct drm_device * dev)
3632 {
3633         struct drm_i915_private *dev_priv = dev->dev_private;
3634         int pipe;
3635
3636         for_each_pipe(dev_priv, pipe) {
3637                 /* Clear enable bits; then clear status bits */
3638                 I915_WRITE(PIPESTAT(pipe), 0);
3639                 I915_WRITE(PIPESTAT(pipe), I915_READ(PIPESTAT(pipe)));
3640         }
3641         I915_WRITE16(IMR, 0xffff);
3642         I915_WRITE16(IER, 0x0);
3643         I915_WRITE16(IIR, I915_READ16(IIR));
3644 }
3645
3646 static void i915_irq_preinstall(struct drm_device * dev)
3647 {
3648         struct drm_i915_private *dev_priv = dev->dev_private;
3649         int pipe;
3650
3651         if (I915_HAS_HOTPLUG(dev)) {
3652                 I915_WRITE(PORT_HOTPLUG_EN, 0);
3653                 I915_WRITE(PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT));
3654         }
3655
3656         I915_WRITE16(HWSTAM, 0xeffe);
3657         for_each_pipe(dev_priv, pipe)
3658                 I915_WRITE(PIPESTAT(pipe), 0);
3659         I915_WRITE(IMR, 0xffffffff);
3660         I915_WRITE(IER, 0x0);
3661         POSTING_READ(IER);
3662 }
3663
3664 static int i915_irq_postinstall(struct drm_device *dev)
3665 {
3666         struct drm_i915_private *dev_priv = dev->dev_private;
3667         u32 enable_mask;
3668
3669         I915_WRITE(EMR, ~(I915_ERROR_PAGE_TABLE | I915_ERROR_MEMORY_REFRESH));
3670
3671         /* Unmask the interrupts that we always want on. */
3672         dev_priv->irq_mask =
3673                 ~(I915_ASLE_INTERRUPT |
3674                   I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
3675                   I915_DISPLAY_PIPE_B_EVENT_INTERRUPT |
3676                   I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT |
3677                   I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT);
3678
3679         enable_mask =
3680                 I915_ASLE_INTERRUPT |
3681                 I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
3682                 I915_DISPLAY_PIPE_B_EVENT_INTERRUPT |
3683                 I915_USER_INTERRUPT;
3684
3685         if (I915_HAS_HOTPLUG(dev)) {
3686                 I915_WRITE(PORT_HOTPLUG_EN, 0);
3687                 POSTING_READ(PORT_HOTPLUG_EN);
3688
3689                 /* Enable in IER... */
3690                 enable_mask |= I915_DISPLAY_PORT_INTERRUPT;
3691                 /* and unmask in IMR */
3692                 dev_priv->irq_mask &= ~I915_DISPLAY_PORT_INTERRUPT;
3693         }
3694
3695         I915_WRITE(IMR, dev_priv->irq_mask);
3696         I915_WRITE(IER, enable_mask);
3697         POSTING_READ(IER);
3698
3699         i915_enable_asle_pipestat(dev);
3700
3701         /* Interrupt setup is already guaranteed to be single-threaded, this is
3702          * just to make the assert_spin_locked check happy. */
3703         spin_lock_irq(&dev_priv->irq_lock);
3704         i915_enable_pipestat(dev_priv, PIPE_A, PIPE_CRC_DONE_INTERRUPT_STATUS);
3705         i915_enable_pipestat(dev_priv, PIPE_B, PIPE_CRC_DONE_INTERRUPT_STATUS);
3706         spin_unlock_irq(&dev_priv->irq_lock);
3707
3708         return 0;
3709 }
3710
3711 /*
3712  * Returns true when a page flip has completed.
3713  */
3714 static bool i915_handle_vblank(struct drm_device *dev,
3715                                int plane, int pipe, u32 iir)
3716 {
3717         struct drm_i915_private *dev_priv = dev->dev_private;
3718         u32 flip_pending = DISPLAY_PLANE_FLIP_PENDING(plane);
3719
3720         if (!intel_pipe_handle_vblank(dev, pipe))
3721                 return false;
3722
3723         if ((iir & flip_pending) == 0)
3724                 goto check_page_flip;
3725
3726         /* We detect FlipDone by looking for the change in PendingFlip from '1'
3727          * to '0' on the following vblank, i.e. IIR has the Pendingflip
3728          * asserted following the MI_DISPLAY_FLIP, but ISR is deasserted, hence
3729          * the flip is completed (no longer pending). Since this doesn't raise
3730          * an interrupt per se, we watch for the change at vblank.
3731          */
3732         if (I915_READ(ISR) & flip_pending)
3733                 goto check_page_flip;
3734
3735         intel_prepare_page_flip(dev, plane);
3736         intel_finish_page_flip(dev, pipe);
3737         return true;
3738
3739 check_page_flip:
3740         intel_check_page_flip(dev, pipe);
3741         return false;
3742 }
3743
3744 static irqreturn_t i915_irq_handler(int irq, void *arg)
3745 {
3746         struct drm_device *dev = arg;
3747         struct drm_i915_private *dev_priv = dev->dev_private;
3748         u32 iir, new_iir, pipe_stats[I915_MAX_PIPES];
3749         u32 flip_mask =
3750                 I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT |
3751                 I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT;
3752         int pipe, ret = IRQ_NONE;
3753
3754         if (!intel_irqs_enabled(dev_priv))
3755                 return IRQ_NONE;
3756
3757         iir = I915_READ(IIR);
3758         do {
3759                 bool irq_received = (iir & ~flip_mask) != 0;
3760                 bool blc_event = false;
3761
3762                 /* Can't rely on pipestat interrupt bit in iir as it might
3763                  * have been cleared after the pipestat interrupt was received.
3764                  * It doesn't set the bit in iir again, but it still produces
3765                  * interrupts (for non-MSI).
3766                  */
3767                 spin_lock(&dev_priv->irq_lock);
3768                 if (iir & I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT)
3769                         DRM_DEBUG("Command parser error, iir 0x%08x\n", iir);
3770
3771                 for_each_pipe(dev_priv, pipe) {
3772                         int reg = PIPESTAT(pipe);
3773                         pipe_stats[pipe] = I915_READ(reg);
3774
3775                         /* Clear the PIPE*STAT regs before the IIR */
3776                         if (pipe_stats[pipe] & 0x8000ffff) {
3777                                 I915_WRITE(reg, pipe_stats[pipe]);
3778                                 irq_received = true;
3779                         }
3780                 }
3781                 spin_unlock(&dev_priv->irq_lock);
3782
3783                 if (!irq_received)
3784                         break;
3785
3786                 /* Consume port.  Then clear IIR or we'll miss events */
3787                 if (I915_HAS_HOTPLUG(dev) &&
3788                     iir & I915_DISPLAY_PORT_INTERRUPT)
3789                         i9xx_hpd_irq_handler(dev);
3790
3791                 I915_WRITE(IIR, iir & ~flip_mask);
3792                 new_iir = I915_READ(IIR); /* Flush posted writes */
3793
3794                 if (iir & I915_USER_INTERRUPT)
3795                         notify_ring(&dev_priv->ring[RCS]);
3796
3797                 for_each_pipe(dev_priv, pipe) {
3798                         int plane = pipe;
3799                         if (HAS_FBC(dev))
3800                                 plane = !plane;
3801
3802                         if (pipe_stats[pipe] & PIPE_VBLANK_INTERRUPT_STATUS &&
3803                             i915_handle_vblank(dev, plane, pipe, iir))
3804                                 flip_mask &= ~DISPLAY_PLANE_FLIP_PENDING(plane);
3805
3806                         if (pipe_stats[pipe] & PIPE_LEGACY_BLC_EVENT_STATUS)
3807                                 blc_event = true;
3808
3809                         if (pipe_stats[pipe] & PIPE_CRC_DONE_INTERRUPT_STATUS)
3810                                 i9xx_pipe_crc_irq_handler(dev, pipe);
3811
3812                         if (pipe_stats[pipe] & PIPE_FIFO_UNDERRUN_STATUS)
3813                                 intel_cpu_fifo_underrun_irq_handler(dev_priv,
3814                                                                     pipe);
3815                 }
3816
3817                 if (blc_event || (iir & I915_ASLE_INTERRUPT))
3818                         intel_opregion_asle_intr(dev);
3819
3820                 /* With MSI, interrupts are only generated when iir
3821                  * transitions from zero to nonzero.  If another bit got
3822                  * set while we were handling the existing iir bits, then
3823                  * we would never get another interrupt.
3824                  *
3825                  * This is fine on non-MSI as well, as if we hit this path
3826                  * we avoid exiting the interrupt handler only to generate
3827                  * another one.
3828                  *
3829                  * Note that for MSI this could cause a stray interrupt report
3830                  * if an interrupt landed in the time between writing IIR and
3831                  * the posting read.  This should be rare enough to never
3832                  * trigger the 99% of 100,000 interrupts test for disabling
3833                  * stray interrupts.
3834                  */
3835                 ret = IRQ_HANDLED;
3836                 iir = new_iir;
3837         } while (iir & ~flip_mask);
3838
3839         return ret;
3840 }
3841
3842 static void i915_irq_uninstall(struct drm_device * dev)
3843 {
3844         struct drm_i915_private *dev_priv = dev->dev_private;
3845         int pipe;
3846
3847         if (I915_HAS_HOTPLUG(dev)) {
3848                 I915_WRITE(PORT_HOTPLUG_EN, 0);
3849                 I915_WRITE(PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT));
3850         }
3851
3852         I915_WRITE16(HWSTAM, 0xffff);
3853         for_each_pipe(dev_priv, pipe) {
3854                 /* Clear enable bits; then clear status bits */
3855                 I915_WRITE(PIPESTAT(pipe), 0);
3856                 I915_WRITE(PIPESTAT(pipe), I915_READ(PIPESTAT(pipe)));
3857         }
3858         I915_WRITE(IMR, 0xffffffff);
3859         I915_WRITE(IER, 0x0);
3860
3861         I915_WRITE(IIR, I915_READ(IIR));
3862 }
3863
3864 static void i965_irq_preinstall(struct drm_device * dev)
3865 {
3866         struct drm_i915_private *dev_priv = dev->dev_private;
3867         int pipe;
3868
3869         I915_WRITE(PORT_HOTPLUG_EN, 0);
3870         I915_WRITE(PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT));
3871
3872         I915_WRITE(HWSTAM, 0xeffe);
3873         for_each_pipe(dev_priv, pipe)
3874                 I915_WRITE(PIPESTAT(pipe), 0);
3875         I915_WRITE(IMR, 0xffffffff);
3876         I915_WRITE(IER, 0x0);
3877         POSTING_READ(IER);
3878 }
3879
3880 static int i965_irq_postinstall(struct drm_device *dev)
3881 {
3882         struct drm_i915_private *dev_priv = dev->dev_private;
3883         u32 enable_mask;
3884         u32 error_mask;
3885
3886         /* Unmask the interrupts that we always want on. */
3887         dev_priv->irq_mask = ~(I915_ASLE_INTERRUPT |
3888                                I915_DISPLAY_PORT_INTERRUPT |
3889                                I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
3890                                I915_DISPLAY_PIPE_B_EVENT_INTERRUPT |
3891                                I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT |
3892                                I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT |
3893                                I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT);
3894
3895         enable_mask = ~dev_priv->irq_mask;
3896         enable_mask &= ~(I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT |
3897                          I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT);
3898         enable_mask |= I915_USER_INTERRUPT;
3899
3900         if (IS_G4X(dev))
3901                 enable_mask |= I915_BSD_USER_INTERRUPT;
3902
3903         /* Interrupt setup is already guaranteed to be single-threaded, this is
3904          * just to make the assert_spin_locked check happy. */
3905         spin_lock_irq(&dev_priv->irq_lock);
3906         i915_enable_pipestat(dev_priv, PIPE_A, PIPE_GMBUS_INTERRUPT_STATUS);
3907         i915_enable_pipestat(dev_priv, PIPE_A, PIPE_CRC_DONE_INTERRUPT_STATUS);
3908         i915_enable_pipestat(dev_priv, PIPE_B, PIPE_CRC_DONE_INTERRUPT_STATUS);
3909         spin_unlock_irq(&dev_priv->irq_lock);
3910
3911         /*
3912          * Enable some error detection, note the instruction error mask
3913          * bit is reserved, so we leave it masked.
3914          */
3915         if (IS_G4X(dev)) {
3916                 error_mask = ~(GM45_ERROR_PAGE_TABLE |
3917                                GM45_ERROR_MEM_PRIV |
3918                                GM45_ERROR_CP_PRIV |
3919                                I915_ERROR_MEMORY_REFRESH);
3920         } else {
3921                 error_mask = ~(I915_ERROR_PAGE_TABLE |
3922                                I915_ERROR_MEMORY_REFRESH);
3923         }
3924         I915_WRITE(EMR, error_mask);
3925
3926         I915_WRITE(IMR, dev_priv->irq_mask);
3927         I915_WRITE(IER, enable_mask);
3928         POSTING_READ(IER);
3929
3930         I915_WRITE(PORT_HOTPLUG_EN, 0);
3931         POSTING_READ(PORT_HOTPLUG_EN);
3932
3933         i915_enable_asle_pipestat(dev);
3934
3935         return 0;
3936 }
3937
3938 static void i915_hpd_irq_setup(struct drm_device *dev)
3939 {
3940         struct drm_i915_private *dev_priv = dev->dev_private;
3941         struct intel_encoder *intel_encoder;
3942         u32 hotplug_en;
3943
3944         assert_spin_locked(&dev_priv->irq_lock);
3945
3946         hotplug_en = I915_READ(PORT_HOTPLUG_EN);
3947         hotplug_en &= ~HOTPLUG_INT_EN_MASK;
3948         /* Note HDMI and DP share hotplug bits */
3949         /* enable bits are the same for all generations */
3950         for_each_intel_encoder(dev, intel_encoder)
3951                 if (dev_priv->hotplug.stats[intel_encoder->hpd_pin].state == HPD_ENABLED)
3952                         hotplug_en |= hpd_mask_i915[intel_encoder->hpd_pin];
3953         /* Programming the CRT detection parameters tends
3954            to generate a spurious hotplug event about three
3955            seconds later.  So just do it once.
3956         */
3957         if (IS_G4X(dev))
3958                 hotplug_en |= CRT_HOTPLUG_ACTIVATION_PERIOD_64;
3959         hotplug_en &= ~CRT_HOTPLUG_VOLTAGE_COMPARE_MASK;
3960         hotplug_en |= CRT_HOTPLUG_VOLTAGE_COMPARE_50;
3961
3962         /* Ignore TV since it's buggy */
3963         I915_WRITE(PORT_HOTPLUG_EN, hotplug_en);
3964 }
3965
3966 static irqreturn_t i965_irq_handler(int irq, void *arg)
3967 {
3968         struct drm_device *dev = arg;
3969         struct drm_i915_private *dev_priv = dev->dev_private;
3970         u32 iir, new_iir;
3971         u32 pipe_stats[I915_MAX_PIPES];
3972         int ret = IRQ_NONE, pipe;
3973         u32 flip_mask =
3974                 I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT |
3975                 I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT;
3976
3977         if (!intel_irqs_enabled(dev_priv))
3978                 return IRQ_NONE;
3979
3980         iir = I915_READ(IIR);
3981
3982         for (;;) {
3983                 bool irq_received = (iir & ~flip_mask) != 0;
3984                 bool blc_event = false;
3985
3986                 /* Can't rely on pipestat interrupt bit in iir as it might
3987                  * have been cleared after the pipestat interrupt was received.
3988                  * It doesn't set the bit in iir again, but it still produces
3989                  * interrupts (for non-MSI).
3990                  */
3991                 spin_lock(&dev_priv->irq_lock);
3992                 if (iir & I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT)
3993                         DRM_DEBUG("Command parser error, iir 0x%08x\n", iir);
3994
3995                 for_each_pipe(dev_priv, pipe) {
3996                         int reg = PIPESTAT(pipe);
3997                         pipe_stats[pipe] = I915_READ(reg);
3998
3999                         /*
4000                          * Clear the PIPE*STAT regs before the IIR
4001                          */
4002                         if (pipe_stats[pipe] & 0x8000ffff) {
4003                                 I915_WRITE(reg, pipe_stats[pipe]);
4004                                 irq_received = true;
4005                         }
4006                 }
4007                 spin_unlock(&dev_priv->irq_lock);
4008
4009                 if (!irq_received)
4010                         break;
4011
4012                 ret = IRQ_HANDLED;
4013
4014                 /* Consume port.  Then clear IIR or we'll miss events */
4015                 if (iir & I915_DISPLAY_PORT_INTERRUPT)
4016                         i9xx_hpd_irq_handler(dev);
4017
4018                 I915_WRITE(IIR, iir & ~flip_mask);
4019                 new_iir = I915_READ(IIR); /* Flush posted writes */
4020
4021                 if (iir & I915_USER_INTERRUPT)
4022                         notify_ring(&dev_priv->ring[RCS]);
4023                 if (iir & I915_BSD_USER_INTERRUPT)
4024                         notify_ring(&dev_priv->ring[VCS]);
4025
4026                 for_each_pipe(dev_priv, pipe) {
4027                         if (pipe_stats[pipe] & PIPE_START_VBLANK_INTERRUPT_STATUS &&
4028                             i915_handle_vblank(dev, pipe, pipe, iir))
4029                                 flip_mask &= ~DISPLAY_PLANE_FLIP_PENDING(pipe);
4030
4031                         if (pipe_stats[pipe] & PIPE_LEGACY_BLC_EVENT_STATUS)
4032                                 blc_event = true;
4033
4034                         if (pipe_stats[pipe] & PIPE_CRC_DONE_INTERRUPT_STATUS)
4035                                 i9xx_pipe_crc_irq_handler(dev, pipe);
4036
4037                         if (pipe_stats[pipe] & PIPE_FIFO_UNDERRUN_STATUS)
4038                                 intel_cpu_fifo_underrun_irq_handler(dev_priv, pipe);
4039                 }
4040
4041                 if (blc_event || (iir & I915_ASLE_INTERRUPT))
4042                         intel_opregion_asle_intr(dev);
4043
4044                 if (pipe_stats[0] & PIPE_GMBUS_INTERRUPT_STATUS)
4045                         gmbus_irq_handler(dev);
4046
4047                 /* With MSI, interrupts are only generated when iir
4048                  * transitions from zero to nonzero.  If another bit got
4049                  * set while we were handling the existing iir bits, then
4050                  * we would never get another interrupt.
4051                  *
4052                  * This is fine on non-MSI as well, as if we hit this path
4053                  * we avoid exiting the interrupt handler only to generate
4054                  * another one.
4055                  *
4056                  * Note that for MSI this could cause a stray interrupt report
4057                  * if an interrupt landed in the time between writing IIR and
4058                  * the posting read.  This should be rare enough to never
4059                  * trigger the 99% of 100,000 interrupts test for disabling
4060                  * stray interrupts.
4061                  */
4062                 iir = new_iir;
4063         }
4064
4065         return ret;
4066 }
4067
4068 static void i965_irq_uninstall(struct drm_device * dev)
4069 {
4070         struct drm_i915_private *dev_priv = dev->dev_private;
4071         int pipe;
4072
4073         if (!dev_priv)
4074                 return;
4075
4076         I915_WRITE(PORT_HOTPLUG_EN, 0);
4077         I915_WRITE(PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT));
4078
4079         I915_WRITE(HWSTAM, 0xffffffff);
4080         for_each_pipe(dev_priv, pipe)
4081                 I915_WRITE(PIPESTAT(pipe), 0);
4082         I915_WRITE(IMR, 0xffffffff);
4083         I915_WRITE(IER, 0x0);
4084
4085         for_each_pipe(dev_priv, pipe)
4086                 I915_WRITE(PIPESTAT(pipe),
4087                            I915_READ(PIPESTAT(pipe)) & 0x8000ffff);
4088         I915_WRITE(IIR, I915_READ(IIR));
4089 }
4090
4091 /**
4092  * intel_irq_init - initializes irq support
4093  * @dev_priv: i915 device instance
4094  *
4095  * This function initializes all the irq support including work items, timers
4096  * and all the vtables. It does not setup the interrupt itself though.
4097  */
4098 void intel_irq_init(struct drm_i915_private *dev_priv)
4099 {
4100         struct drm_device *dev = dev_priv->dev;
4101
4102         intel_hpd_init_work(dev_priv);
4103
4104         INIT_WORK(&dev_priv->rps.work, gen6_pm_rps_work);
4105         INIT_WORK(&dev_priv->l3_parity.error_work, ivybridge_parity_work);
4106
4107         /* Let's track the enabled rps events */
4108         if (IS_VALLEYVIEW(dev_priv) && !IS_CHERRYVIEW(dev_priv))
4109                 /* WaGsvRC0ResidencyMethod:vlv */
4110                 dev_priv->pm_rps_events = GEN6_PM_RP_DOWN_EI_EXPIRED | GEN6_PM_RP_UP_EI_EXPIRED;
4111         else
4112                 dev_priv->pm_rps_events = GEN6_PM_RPS_EVENTS;
4113
4114         INIT_DELAYED_WORK(&dev_priv->gpu_error.hangcheck_work,
4115                           i915_hangcheck_elapsed);
4116
4117         pm_qos_add_request(&dev_priv->pm_qos, PM_QOS_CPU_DMA_LATENCY, PM_QOS_DEFAULT_VALUE);
4118
4119         if (IS_GEN2(dev_priv)) {
4120                 dev->max_vblank_count = 0;
4121                 dev->driver->get_vblank_counter = i8xx_get_vblank_counter;
4122         } else if (IS_G4X(dev_priv) || INTEL_INFO(dev_priv)->gen >= 5) {
4123                 dev->max_vblank_count = 0xffffffff; /* full 32 bit counter */
4124                 dev->driver->get_vblank_counter = gm45_get_vblank_counter;
4125         } else {
4126                 dev->driver->get_vblank_counter = i915_get_vblank_counter;
4127                 dev->max_vblank_count = 0xffffff; /* only 24 bits of frame count */
4128         }
4129
4130         /*
4131          * Opt out of the vblank disable timer on everything except gen2.
4132          * Gen2 doesn't have a hardware frame counter and so depends on
4133          * vblank interrupts to produce sane vblank seuquence numbers.
4134          */
4135         if (!IS_GEN2(dev_priv))
4136                 dev->vblank_disable_immediate = true;
4137
4138         dev->driver->get_vblank_timestamp = i915_get_vblank_timestamp;
4139         dev->driver->get_scanout_position = i915_get_crtc_scanoutpos;
4140
4141         if (IS_CHERRYVIEW(dev_priv)) {
4142                 dev->driver->irq_handler = cherryview_irq_handler;
4143                 dev->driver->irq_preinstall = cherryview_irq_preinstall;
4144                 dev->driver->irq_postinstall = cherryview_irq_postinstall;
4145                 dev->driver->irq_uninstall = cherryview_irq_uninstall;
4146                 dev->driver->enable_vblank = valleyview_enable_vblank;
4147                 dev->driver->disable_vblank = valleyview_disable_vblank;
4148                 dev_priv->display.hpd_irq_setup = i915_hpd_irq_setup;
4149         } else if (IS_VALLEYVIEW(dev_priv)) {
4150                 dev->driver->irq_handler = valleyview_irq_handler;
4151                 dev->driver->irq_preinstall = valleyview_irq_preinstall;
4152                 dev->driver->irq_postinstall = valleyview_irq_postinstall;
4153                 dev->driver->irq_uninstall = valleyview_irq_uninstall;
4154                 dev->driver->enable_vblank = valleyview_enable_vblank;
4155                 dev->driver->disable_vblank = valleyview_disable_vblank;
4156                 dev_priv->display.hpd_irq_setup = i915_hpd_irq_setup;
4157         } else if (INTEL_INFO(dev_priv)->gen >= 8) {
4158                 dev->driver->irq_handler = gen8_irq_handler;
4159                 dev->driver->irq_preinstall = gen8_irq_reset;
4160                 dev->driver->irq_postinstall = gen8_irq_postinstall;
4161                 dev->driver->irq_uninstall = gen8_irq_uninstall;
4162                 dev->driver->enable_vblank = gen8_enable_vblank;
4163                 dev->driver->disable_vblank = gen8_disable_vblank;
4164                 if (HAS_PCH_SPLIT(dev))
4165                         dev_priv->display.hpd_irq_setup = ibx_hpd_irq_setup;
4166                 else
4167                         dev_priv->display.hpd_irq_setup = bxt_hpd_irq_setup;
4168         } else if (HAS_PCH_SPLIT(dev)) {
4169                 dev->driver->irq_handler = ironlake_irq_handler;
4170                 dev->driver->irq_preinstall = ironlake_irq_reset;
4171                 dev->driver->irq_postinstall = ironlake_irq_postinstall;
4172                 dev->driver->irq_uninstall = ironlake_irq_uninstall;
4173                 dev->driver->enable_vblank = ironlake_enable_vblank;
4174                 dev->driver->disable_vblank = ironlake_disable_vblank;
4175                 dev_priv->display.hpd_irq_setup = ibx_hpd_irq_setup;
4176         } else {
4177                 if (INTEL_INFO(dev_priv)->gen == 2) {
4178                         dev->driver->irq_preinstall = i8xx_irq_preinstall;
4179                         dev->driver->irq_postinstall = i8xx_irq_postinstall;
4180                         dev->driver->irq_handler = i8xx_irq_handler;
4181                         dev->driver->irq_uninstall = i8xx_irq_uninstall;
4182                 } else if (INTEL_INFO(dev_priv)->gen == 3) {
4183                         dev->driver->irq_preinstall = i915_irq_preinstall;
4184                         dev->driver->irq_postinstall = i915_irq_postinstall;
4185                         dev->driver->irq_uninstall = i915_irq_uninstall;
4186                         dev->driver->irq_handler = i915_irq_handler;
4187                 } else {
4188                         dev->driver->irq_preinstall = i965_irq_preinstall;
4189                         dev->driver->irq_postinstall = i965_irq_postinstall;
4190                         dev->driver->irq_uninstall = i965_irq_uninstall;
4191                         dev->driver->irq_handler = i965_irq_handler;
4192                 }
4193                 if (I915_HAS_HOTPLUG(dev_priv))
4194                         dev_priv->display.hpd_irq_setup = i915_hpd_irq_setup;
4195                 dev->driver->enable_vblank = i915_enable_vblank;
4196                 dev->driver->disable_vblank = i915_disable_vblank;
4197         }
4198 }
4199
4200 /**
4201  * intel_irq_install - enables the hardware interrupt
4202  * @dev_priv: i915 device instance
4203  *
4204  * This function enables the hardware interrupt handling, but leaves the hotplug
4205  * handling still disabled. It is called after intel_irq_init().
4206  *
4207  * In the driver load and resume code we need working interrupts in a few places
4208  * but don't want to deal with the hassle of concurrent probe and hotplug
4209  * workers. Hence the split into this two-stage approach.
4210  */
4211 int intel_irq_install(struct drm_i915_private *dev_priv)
4212 {
4213         /*
4214          * We enable some interrupt sources in our postinstall hooks, so mark
4215          * interrupts as enabled _before_ actually enabling them to avoid
4216          * special cases in our ordering checks.
4217          */
4218         dev_priv->pm.irqs_enabled = true;
4219
4220         return drm_irq_install(dev_priv->dev, dev_priv->dev->pdev->irq);
4221 }
4222
4223 /**
4224  * intel_irq_uninstall - finilizes all irq handling
4225  * @dev_priv: i915 device instance
4226  *
4227  * This stops interrupt and hotplug handling and unregisters and frees all
4228  * resources acquired in the init functions.
4229  */
4230 void intel_irq_uninstall(struct drm_i915_private *dev_priv)
4231 {
4232         drm_irq_uninstall(dev_priv->dev);
4233         intel_hpd_cancel_work(dev_priv);
4234         dev_priv->pm.irqs_enabled = false;
4235 }
4236
4237 /**
4238  * intel_runtime_pm_disable_interrupts - runtime interrupt disabling
4239  * @dev_priv: i915 device instance
4240  *
4241  * This function is used to disable interrupts at runtime, both in the runtime
4242  * pm and the system suspend/resume code.
4243  */
4244 void intel_runtime_pm_disable_interrupts(struct drm_i915_private *dev_priv)
4245 {
4246         dev_priv->dev->driver->irq_uninstall(dev_priv->dev);
4247         dev_priv->pm.irqs_enabled = false;
4248         synchronize_irq(dev_priv->dev->irq);
4249 }
4250
4251 /**
4252  * intel_runtime_pm_enable_interrupts - runtime interrupt enabling
4253  * @dev_priv: i915 device instance
4254  *
4255  * This function is used to enable interrupts at runtime, both in the runtime
4256  * pm and the system suspend/resume code.
4257  */
4258 void intel_runtime_pm_enable_interrupts(struct drm_i915_private *dev_priv)
4259 {
4260         dev_priv->pm.irqs_enabled = true;
4261         dev_priv->dev->driver->irq_preinstall(dev_priv->dev);
4262         dev_priv->dev->driver->irq_postinstall(dev_priv->dev);
4263 }