drivers/gpu/drm/i915/intel_pm.c

   1 /*
   2  * Copyright © 2012 Intel Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21  * IN THE SOFTWARE.
  22  *
  23  * Authors:
  24  *    Eugeni Dodonov <eugeni.dodonov@intel.com>
  25  *
  26  */
  27
  28 #include <linux/module.h>
  29 #include <linux/pm_runtime.h>
  30
  31 #include <drm/drm_atomic_helper.h>
  32 #include <drm/drm_fourcc.h>
  33 #include <drm/drm_plane_helper.h>
  34
  35 #include "display/intel_atomic.h"
  36 #include "display/intel_display_types.h"
  37 #include "display/intel_fbc.h"
  38 #include "display/intel_sprite.h"
  39
  40 #include "gt/intel_llc.h"
  41
  42 #include "i915_drv.h"
  43 #include "i915_irq.h"
  44 #include "i915_trace.h"
  45 #include "intel_pm.h"
  46 #include "intel_sideband.h"
  47 #include "../../../platform/x86/intel_ips.h"
  48
  49 static void gen9_init_clock_gating(struct drm_i915_private *dev_priv)
  50 {
  51         if (HAS_LLC(dev_priv)) {
  52                 /*
  53                  * WaCompressedResourceDisplayNewHashMode:skl,kbl
  54                  * Display WA #0390: skl,kbl
  55                  *
  56                  * Must match Sampler, Pixel Back End, and Media. See
  57                  * WaCompressedResourceSamplerPbeMediaNewHashMode.
  58                  */
  59                 I915_WRITE(CHICKEN_PAR1_1,
  60                            I915_READ(CHICKEN_PAR1_1) |
  61                            SKL_DE_COMPRESSED_HASH_MODE);
  62         }
  63
  64         /* See Bspec note for PSR2_CTL bit 31, Wa#828:skl,bxt,kbl,cfl */
  65         I915_WRITE(CHICKEN_PAR1_1,
  66                    I915_READ(CHICKEN_PAR1_1) | SKL_EDP_PSR_FIX_RDWRAP);
  67
  68         /* WaEnableChickenDCPR:skl,bxt,kbl,glk,cfl */
  69         I915_WRITE(GEN8_CHICKEN_DCPR_1,
  70                    I915_READ(GEN8_CHICKEN_DCPR_1) | MASK_WAKEMEM);
  71
  72         /* WaFbcTurnOffFbcWatermark:skl,bxt,kbl,cfl */
  73         /* WaFbcWakeMemOn:skl,bxt,kbl,glk,cfl */
  74         I915_WRITE(DISP_ARB_CTL, I915_READ(DISP_ARB_CTL) |
  75                    DISP_FBC_WM_DIS |
  76                    DISP_FBC_MEMORY_WAKE);
  77
  78         /* WaFbcHighMemBwCorruptionAvoidance:skl,bxt,kbl,cfl */
  79         I915_WRITE(ILK_DPFC_CHICKEN, I915_READ(ILK_DPFC_CHICKEN) |
  80                    ILK_DPFC_DISABLE_DUMMY0);
  81
  82         if (IS_SKYLAKE(dev_priv)) {
  83                 /* WaDisableDopClockGating */
  84                 I915_WRITE(GEN7_MISCCPCTL, I915_READ(GEN7_MISCCPCTL)
  85                            & ~GEN7_DOP_CLOCK_GATE_ENABLE);
  86         }
  87 }
  88
  89 static void bxt_init_clock_gating(struct drm_i915_private *dev_priv)
  90 {
  91         gen9_init_clock_gating(dev_priv);
  92
  93         /* WaDisableSDEUnitClockGating:bxt */
  94         I915_WRITE(GEN8_UCGCTL6, I915_READ(GEN8_UCGCTL6) |
  95                    GEN8_SDEUNIT_CLOCK_GATE_DISABLE);
  96
  97         /*
  98          * FIXME:
  99          * GEN8_HDCUNIT_CLOCK_GATE_DISABLE_HDCREQ applies on 3x6 GT SKUs only.
 100          */
 101         I915_WRITE(GEN8_UCGCTL6, I915_READ(GEN8_UCGCTL6) |
 102                    GEN8_HDCUNIT_CLOCK_GATE_DISABLE_HDCREQ);
 103
 104         /*
 105          * Wa: Backlight PWM may stop in the asserted state, causing backlight
 106          * to stay fully on.
 107          */
 108         I915_WRITE(GEN9_CLKGATE_DIS_0, I915_READ(GEN9_CLKGATE_DIS_0) |
 109                    PWM1_GATING_DIS | PWM2_GATING_DIS);
 110
 111         /*
 112          * Lower the display internal timeout.
 113          * This is needed to avoid any hard hangs when DSI port PLL
 114          * is off and a MMIO access is attempted by any privilege
 115          * application, using batch buffers or any other means.
 116          */
 117         I915_WRITE(RM_TIMEOUT, MMIO_TIMEOUT_US(950));
 118 }
 119
 120 static void glk_init_clock_gating(struct drm_i915_private *dev_priv)
 121 {
 122         gen9_init_clock_gating(dev_priv);
 123
 124         /*
 125          * WaDisablePWMClockGating:glk
 126          * Backlight PWM may stop in the asserted state, causing backlight
 127          * to stay fully on.
 128          */
 129         I915_WRITE(GEN9_CLKGATE_DIS_0, I915_READ(GEN9_CLKGATE_DIS_0) |
 130                    PWM1_GATING_DIS | PWM2_GATING_DIS);
 131
 132         /* WaDDIIOTimeout:glk */
 133         if (IS_GLK_REVID(dev_priv, 0, GLK_REVID_A1)) {
 134                 u32 val = I915_READ(CHICKEN_MISC_2);
 135                 val &= ~(GLK_CL0_PWR_DOWN |
 136                          GLK_CL1_PWR_DOWN |
 137                          GLK_CL2_PWR_DOWN);
 138                 I915_WRITE(CHICKEN_MISC_2, val);
 139         }
 140
 141 }
 142
 143 static void pnv_get_mem_freq(struct drm_i915_private *dev_priv)
 144 {
 145         u32 tmp;
 146
 147         tmp = I915_READ(CLKCFG);
 148
 149         switch (tmp & CLKCFG_FSB_MASK) {
 150         case CLKCFG_FSB_533:
 151                 dev_priv->fsb_freq = 533; /* 133*4 */
 152                 break;
 153         case CLKCFG_FSB_800:
 154                 dev_priv->fsb_freq = 800; /* 200*4 */
 155                 break;
 156         case CLKCFG_FSB_667:
 157                 dev_priv->fsb_freq =  667; /* 167*4 */
 158                 break;
 159         case CLKCFG_FSB_400:
 160                 dev_priv->fsb_freq = 400; /* 100*4 */
 161                 break;
 162         }
 163
 164         switch (tmp & CLKCFG_MEM_MASK) {
 165         case CLKCFG_MEM_533:
 166                 dev_priv->mem_freq = 533;
 167                 break;
 168         case CLKCFG_MEM_667:
 169                 dev_priv->mem_freq = 667;
 170                 break;
 171         case CLKCFG_MEM_800:
 172                 dev_priv->mem_freq = 800;
 173                 break;
 174         }
 175
 176         /* detect pineview DDR3 setting */
 177         tmp = I915_READ(CSHRDDR3CTL);
 178         dev_priv->is_ddr3 = (tmp & CSHRDDR3CTL_DDR3) ? 1 : 0;
 179 }
 180
 181 static void ilk_get_mem_freq(struct drm_i915_private *dev_priv)
 182 {
 183         u16 ddrpll, csipll;
 184
 185         ddrpll = intel_uncore_read16(&dev_priv->uncore, DDRMPLL1);
 186         csipll = intel_uncore_read16(&dev_priv->uncore, CSIPLL0);
 187
 188         switch (ddrpll & 0xff) {
 189         case 0xc:
 190                 dev_priv->mem_freq = 800;
 191                 break;
 192         case 0x10:
 193                 dev_priv->mem_freq = 1066;
 194                 break;
 195         case 0x14:
 196                 dev_priv->mem_freq = 1333;
 197                 break;
 198         case 0x18:
 199                 dev_priv->mem_freq = 1600;
 200                 break;
 201         default:
 202                 drm_dbg(&dev_priv->drm, "unknown memory frequency 0x%02x\n",
 203                         ddrpll & 0xff);
 204                 dev_priv->mem_freq = 0;
 205                 break;
 206         }
 207
 208         switch (csipll & 0x3ff) {
 209         case 0x00c:
 210                 dev_priv->fsb_freq = 3200;
 211                 break;
 212         case 0x00e:
 213                 dev_priv->fsb_freq = 3733;
 214                 break;
 215         case 0x010:
 216                 dev_priv->fsb_freq = 4266;
 217                 break;
 218         case 0x012:
 219                 dev_priv->fsb_freq = 4800;
 220                 break;
 221         case 0x014:
 222                 dev_priv->fsb_freq = 5333;
 223                 break;
 224         case 0x016:
 225                 dev_priv->fsb_freq = 5866;
 226                 break;
 227         case 0x018:
 228                 dev_priv->fsb_freq = 6400;
 229                 break;
 230         default:
 231                 drm_dbg(&dev_priv->drm, "unknown fsb frequency 0x%04x\n",
 232                         csipll & 0x3ff);
 233                 dev_priv->fsb_freq = 0;
 234                 break;
 235         }
 236 }
 237
 238 static const struct cxsr_latency cxsr_latency_table[] = {
 239         {1, 0, 800, 400, 3382, 33382, 3983, 33983},    /* DDR2-400 SC */
 240         {1, 0, 800, 667, 3354, 33354, 3807, 33807},    /* DDR2-667 SC */
 241         {1, 0, 800, 800, 3347, 33347, 3763, 33763},    /* DDR2-800 SC */
 242         {1, 1, 800, 667, 6420, 36420, 6873, 36873},    /* DDR3-667 SC */
 243         {1, 1, 800, 800, 5902, 35902, 6318, 36318},    /* DDR3-800 SC */
 244
 245         {1, 0, 667, 400, 3400, 33400, 4021, 34021},    /* DDR2-400 SC */
 246         {1, 0, 667, 667, 3372, 33372, 3845, 33845},    /* DDR2-667 SC */
 247         {1, 0, 667, 800, 3386, 33386, 3822, 33822},    /* DDR2-800 SC */
 248         {1, 1, 667, 667, 6438, 36438, 6911, 36911},    /* DDR3-667 SC */
 249         {1, 1, 667, 800, 5941, 35941, 6377, 36377},    /* DDR3-800 SC */
 250
 251         {1, 0, 400, 400, 3472, 33472, 4173, 34173},    /* DDR2-400 SC */
 252         {1, 0, 400, 667, 3443, 33443, 3996, 33996},    /* DDR2-667 SC */
 253         {1, 0, 400, 800, 3430, 33430, 3946, 33946},    /* DDR2-800 SC */
 254         {1, 1, 400, 667, 6509, 36509, 7062, 37062},    /* DDR3-667 SC */
 255         {1, 1, 400, 800, 5985, 35985, 6501, 36501},    /* DDR3-800 SC */
 256
 257         {0, 0, 800, 400, 3438, 33438, 4065, 34065},    /* DDR2-400 SC */
 258         {0, 0, 800, 667, 3410, 33410, 3889, 33889},    /* DDR2-667 SC */
 259         {0, 0, 800, 800, 3403, 33403, 3845, 33845},    /* DDR2-800 SC */
 260         {0, 1, 800, 667, 6476, 36476, 6955, 36955},    /* DDR3-667 SC */
 261         {0, 1, 800, 800, 5958, 35958, 6400, 36400},    /* DDR3-800 SC */
 262
 263         {0, 0, 667, 400, 3456, 33456, 4103, 34106},    /* DDR2-400 SC */
 264         {0, 0, 667, 667, 3428, 33428, 3927, 33927},    /* DDR2-667 SC */
 265         {0, 0, 667, 800, 3443, 33443, 3905, 33905},    /* DDR2-800 SC */
 266         {0, 1, 667, 667, 6494, 36494, 6993, 36993},    /* DDR3-667 SC */
 267         {0, 1, 667, 800, 5998, 35998, 6460, 36460},    /* DDR3-800 SC */
 268
 269         {0, 0, 400, 400, 3528, 33528, 4255, 34255},    /* DDR2-400 SC */
 270         {0, 0, 400, 667, 3500, 33500, 4079, 34079},    /* DDR2-667 SC */
 271         {0, 0, 400, 800, 3487, 33487, 4029, 34029},    /* DDR2-800 SC */
 272         {0, 1, 400, 667, 6566, 36566, 7145, 37145},    /* DDR3-667 SC */
 273         {0, 1, 400, 800, 6042, 36042, 6584, 36584},    /* DDR3-800 SC */
 274 };
 275
 276 static const struct cxsr_latency *intel_get_cxsr_latency(bool is_desktop,
 277                                                          bool is_ddr3,
 278                                                          int fsb,
 279                                                          int mem)
 280 {
 281         const struct cxsr_latency *latency;
 282         int i;
 283
 284         if (fsb == 0 || mem == 0)
 285                 return NULL;
 286
 287         for (i = 0; i < ARRAY_SIZE(cxsr_latency_table); i++) {
 288                 latency = &cxsr_latency_table[i];
 289                 if (is_desktop == latency->is_desktop &&
 290                     is_ddr3 == latency->is_ddr3 &&
 291                     fsb == latency->fsb_freq && mem == latency->mem_freq)
 292                         return latency;
 293         }
 294
 295         DRM_DEBUG_KMS("Unknown FSB/MEM found, disable CxSR\n");
 296
 297         return NULL;
 298 }
 299
 300 static void chv_set_memory_dvfs(struct drm_i915_private *dev_priv, bool enable)
 301 {
 302         u32 val;
 303
 304         vlv_punit_get(dev_priv);
 305
 306         val = vlv_punit_read(dev_priv, PUNIT_REG_DDR_SETUP2);
 307         if (enable)
 308                 val &= ~FORCE_DDR_HIGH_FREQ;
 309         else
 310                 val |= FORCE_DDR_HIGH_FREQ;
 311         val &= ~FORCE_DDR_LOW_FREQ;
 312         val |= FORCE_DDR_FREQ_REQ_ACK;
 313         vlv_punit_write(dev_priv, PUNIT_REG_DDR_SETUP2, val);
 314
 315         if (wait_for((vlv_punit_read(dev_priv, PUNIT_REG_DDR_SETUP2) &
 316                       FORCE_DDR_FREQ_REQ_ACK) == 0, 3))
 317                 drm_err(&dev_priv->drm,
 318                         "timed out waiting for Punit DDR DVFS request\n");
 319
 320         vlv_punit_put(dev_priv);
 321 }
 322
 323 static void chv_set_memory_pm5(struct drm_i915_private *dev_priv, bool enable)
 324 {
 325         u32 val;
 326
 327         vlv_punit_get(dev_priv);
 328
 329         val = vlv_punit_read(dev_priv, PUNIT_REG_DSPSSPM);
 330         if (enable)
 331                 val |= DSP_MAXFIFO_PM5_ENABLE;
 332         else
 333                 val &= ~DSP_MAXFIFO_PM5_ENABLE;
 334         vlv_punit_write(dev_priv, PUNIT_REG_DSPSSPM, val);
 335
 336         vlv_punit_put(dev_priv);
 337 }
 338
 339 #define FW_WM(value, plane) \
 340         (((value) << DSPFW_ ## plane ## _SHIFT) & DSPFW_ ## plane ## _MASK)
 341
 342 static bool _intel_set_memory_cxsr(struct drm_i915_private *dev_priv, bool enable)
 343 {
 344         bool was_enabled;
 345         u32 val;
 346
 347         if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) {
 348                 was_enabled = I915_READ(FW_BLC_SELF_VLV) & FW_CSPWRDWNEN;
 349                 I915_WRITE(FW_BLC_SELF_VLV, enable ? FW_CSPWRDWNEN : 0);
 350                 POSTING_READ(FW_BLC_SELF_VLV);
 351         } else if (IS_G4X(dev_priv) || IS_I965GM(dev_priv)) {
 352                 was_enabled = I915_READ(FW_BLC_SELF) & FW_BLC_SELF_EN;
 353                 I915_WRITE(FW_BLC_SELF, enable ? FW_BLC_SELF_EN : 0);
 354                 POSTING_READ(FW_BLC_SELF);
 355         } else if (IS_PINEVIEW(dev_priv)) {
 356                 val = I915_READ(DSPFW3);
 357                 was_enabled = val & PINEVIEW_SELF_REFRESH_EN;
 358                 if (enable)
 359                         val |= PINEVIEW_SELF_REFRESH_EN;
 360                 else
 361                         val &= ~PINEVIEW_SELF_REFRESH_EN;
 362                 I915_WRITE(DSPFW3, val);
 363                 POSTING_READ(DSPFW3);
 364         } else if (IS_I945G(dev_priv) || IS_I945GM(dev_priv)) {
 365                 was_enabled = I915_READ(FW_BLC_SELF) & FW_BLC_SELF_EN;
 366                 val = enable ? _MASKED_BIT_ENABLE(FW_BLC_SELF_EN) :
 367                                _MASKED_BIT_DISABLE(FW_BLC_SELF_EN);
 368                 I915_WRITE(FW_BLC_SELF, val);
 369                 POSTING_READ(FW_BLC_SELF);
 370         } else if (IS_I915GM(dev_priv)) {
 371                 /*
 372                  * FIXME can't find a bit like this for 915G, and
 373                  * and yet it does have the related watermark in
 374                  * FW_BLC_SELF. What's going on?
 375                  */
 376                 was_enabled = I915_READ(INSTPM) & INSTPM_SELF_EN;
 377                 val = enable ? _MASKED_BIT_ENABLE(INSTPM_SELF_EN) :
 378                                _MASKED_BIT_DISABLE(INSTPM_SELF_EN);
 379                 I915_WRITE(INSTPM, val);
 380                 POSTING_READ(INSTPM);
 381         } else {
 382                 return false;
 383         }
 384
 385         trace_intel_memory_cxsr(dev_priv, was_enabled, enable);
 386
 387         drm_dbg_kms(&dev_priv->drm, "memory self-refresh is %s (was %s)\n",
 388                     enableddisabled(enable),
 389                     enableddisabled(was_enabled));
 390
 391         return was_enabled;
 392 }
 393
 394 /**
 395  * intel_set_memory_cxsr - Configure CxSR state
 396  * @dev_priv: i915 device
 397  * @enable: Allow vs. disallow CxSR
 398  *
 399  * Allow or disallow the system to enter a special CxSR
 400  * (C-state self refresh) state. What typically happens in CxSR mode
 401  * is that several display FIFOs may get combined into a single larger
 402  * FIFO for a particular plane (so called max FIFO mode) to allow the
 403  * system to defer memory fetches longer, and the memory will enter
 404  * self refresh.
 405  *
 406  * Note that enabling CxSR does not guarantee that the system enter
 407  * this special mode, nor does it guarantee that the system stays
 408  * in that mode once entered. So this just allows/disallows the system
 409  * to autonomously utilize the CxSR mode. Other factors such as core
 410  * C-states will affect when/if the system actually enters/exits the
 411  * CxSR mode.
 412  *
 413  * Note that on VLV/CHV this actually only controls the max FIFO mode,
 414  * and the system is free to enter/exit memory self refresh at any time
 415  * even when the use of CxSR has been disallowed.
 416  *
 417  * While the system is actually in the CxSR/max FIFO mode, some plane
 418  * control registers will not get latched on vblank. Thus in order to
 419  * guarantee the system will respond to changes in the plane registers
 420  * we must always disallow CxSR prior to making changes to those registers.
 421  * Unfortunately the system will re-evaluate the CxSR conditions at
 422  * frame start which happens after vblank start (which is when the plane
 423  * registers would get latched), so we can't proceed with the plane update
 424  * during the same frame where we disallowed CxSR.
 425  *
 426  * Certain platforms also have a deeper HPLL SR mode. Fortunately the
 427  * HPLL SR mode depends on CxSR itself, so we don't have to hand hold
 428  * the hardware w.r.t. HPLL SR when writing to plane registers.
 429  * Disallowing just CxSR is sufficient.
 430  */
 431 bool intel_set_memory_cxsr(struct drm_i915_private *dev_priv, bool enable)
 432 {
 433         bool ret;
 434
 435         mutex_lock(&dev_priv->wm.wm_mutex);
 436         ret = _intel_set_memory_cxsr(dev_priv, enable);
 437         if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
 438                 dev_priv->wm.vlv.cxsr = enable;
 439         else if (IS_G4X(dev_priv))
 440                 dev_priv->wm.g4x.cxsr = enable;
 441         mutex_unlock(&dev_priv->wm.wm_mutex);
 442
 443         return ret;
 444 }
 445
 446 /*
 447  * Latency for FIFO fetches is dependent on several factors:
 448  *   - memory configuration (speed, channels)
 449  *   - chipset
 450  *   - current MCH state
 451  * It can be fairly high in some situations, so here we assume a fairly
 452  * pessimal value.  It's a tradeoff between extra memory fetches (if we
 453  * set this value too high, the FIFO will fetch frequently to stay full)
 454  * and power consumption (set it too low to save power and we might see
 455  * FIFO underruns and display "flicker").
 456  *
 457  * A value of 5us seems to be a good balance; safe for very low end
 458  * platforms but not overly aggressive on lower latency configs.
 459  */
 460 static const int pessimal_latency_ns = 5000;
 461
 462 #define VLV_FIFO_START(dsparb, dsparb2, lo_shift, hi_shift) \
 463         ((((dsparb) >> (lo_shift)) & 0xff) | ((((dsparb2) >> (hi_shift)) & 0x1) << 8))
 464
 465 static void vlv_get_fifo_size(struct intel_crtc_state *crtc_state)
 466 {
 467         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
 468         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
 469         struct vlv_fifo_state *fifo_state = &crtc_state->wm.vlv.fifo_state;
 470         enum pipe pipe = crtc->pipe;
 471         int sprite0_start, sprite1_start;
 472
 473         switch (pipe) {
 474                 u32 dsparb, dsparb2, dsparb3;
 475         case PIPE_A:
 476                 dsparb = I915_READ(DSPARB);
 477                 dsparb2 = I915_READ(DSPARB2);
 478                 sprite0_start = VLV_FIFO_START(dsparb, dsparb2, 0, 0);
 479                 sprite1_start = VLV_FIFO_START(dsparb, dsparb2, 8, 4);
 480                 break;
 481         case PIPE_B:
 482                 dsparb = I915_READ(DSPARB);
 483                 dsparb2 = I915_READ(DSPARB2);
 484                 sprite0_start = VLV_FIFO_START(dsparb, dsparb2, 16, 8);
 485                 sprite1_start = VLV_FIFO_START(dsparb, dsparb2, 24, 12);
 486                 break;
 487         case PIPE_C:
 488                 dsparb2 = I915_READ(DSPARB2);
 489                 dsparb3 = I915_READ(DSPARB3);
 490                 sprite0_start = VLV_FIFO_START(dsparb3, dsparb2, 0, 16);
 491                 sprite1_start = VLV_FIFO_START(dsparb3, dsparb2, 8, 20);
 492                 break;
 493         default:
 494                 MISSING_CASE(pipe);
 495                 return;
 496         }
 497
 498         fifo_state->plane[PLANE_PRIMARY] = sprite0_start;
 499         fifo_state->plane[PLANE_SPRITE0] = sprite1_start - sprite0_start;
 500         fifo_state->plane[PLANE_SPRITE1] = 511 - sprite1_start;
 501         fifo_state->plane[PLANE_CURSOR] = 63;
 502 }
 503
 504 static int i9xx_get_fifo_size(struct drm_i915_private *dev_priv,
 505                               enum i9xx_plane_id i9xx_plane)
 506 {
 507         u32 dsparb = I915_READ(DSPARB);
 508         int size;
 509
 510         size = dsparb & 0x7f;
 511         if (i9xx_plane == PLANE_B)
 512                 size = ((dsparb >> DSPARB_CSTART_SHIFT) & 0x7f) - size;
 513
 514         drm_dbg_kms(&dev_priv->drm, "FIFO size - (0x%08x) %c: %d\n",
 515                     dsparb, plane_name(i9xx_plane), size);
 516
 517         return size;
 518 }
 519
 520 static int i830_get_fifo_size(struct drm_i915_private *dev_priv,
 521                               enum i9xx_plane_id i9xx_plane)
 522 {
 523         u32 dsparb = I915_READ(DSPARB);
 524         int size;
 525
 526         size = dsparb & 0x1ff;
 527         if (i9xx_plane == PLANE_B)
 528                 size = ((dsparb >> DSPARB_BEND_SHIFT) & 0x1ff) - size;
 529         size >>= 1; /* Convert to cachelines */
 530
 531         drm_dbg_kms(&dev_priv->drm, "FIFO size - (0x%08x) %c: %d\n",
 532                     dsparb, plane_name(i9xx_plane), size);
 533
 534         return size;
 535 }
 536
 537 static int i845_get_fifo_size(struct drm_i915_private *dev_priv,
 538                               enum i9xx_plane_id i9xx_plane)
 539 {
 540         u32 dsparb = I915_READ(DSPARB);
 541         int size;
 542
 543         size = dsparb & 0x7f;
 544         size >>= 2; /* Convert to cachelines */
 545
 546         drm_dbg_kms(&dev_priv->drm, "FIFO size - (0x%08x) %c: %d\n",
 547                     dsparb, plane_name(i9xx_plane), size);
 548
 549         return size;
 550 }
 551
 552 /* Pineview has different values for various configs */
 553 static const struct intel_watermark_params pnv_display_wm = {
 554         .fifo_size = PINEVIEW_DISPLAY_FIFO,
 555         .max_wm = PINEVIEW_MAX_WM,
 556         .default_wm = PINEVIEW_DFT_WM,
 557         .guard_size = PINEVIEW_GUARD_WM,
 558         .cacheline_size = PINEVIEW_FIFO_LINE_SIZE,
 559 };
 560
 561 static const struct intel_watermark_params pnv_display_hplloff_wm = {
 562         .fifo_size = PINEVIEW_DISPLAY_FIFO,
 563         .max_wm = PINEVIEW_MAX_WM,
 564         .default_wm = PINEVIEW_DFT_HPLLOFF_WM,
 565         .guard_size = PINEVIEW_GUARD_WM,
 566         .cacheline_size = PINEVIEW_FIFO_LINE_SIZE,
 567 };
 568
 569 static const struct intel_watermark_params pnv_cursor_wm = {
 570         .fifo_size = PINEVIEW_CURSOR_FIFO,
 571         .max_wm = PINEVIEW_CURSOR_MAX_WM,
 572         .default_wm = PINEVIEW_CURSOR_DFT_WM,
 573         .guard_size = PINEVIEW_CURSOR_GUARD_WM,
 574         .cacheline_size = PINEVIEW_FIFO_LINE_SIZE,
 575 };
 576
 577 static const struct intel_watermark_params pnv_cursor_hplloff_wm = {
 578         .fifo_size = PINEVIEW_CURSOR_FIFO,
 579         .max_wm = PINEVIEW_CURSOR_MAX_WM,
 580         .default_wm = PINEVIEW_CURSOR_DFT_WM,
 581         .guard_size = PINEVIEW_CURSOR_GUARD_WM,
 582         .cacheline_size = PINEVIEW_FIFO_LINE_SIZE,
 583 };
 584
 585 static const struct intel_watermark_params i965_cursor_wm_info = {
 586         .fifo_size = I965_CURSOR_FIFO,
 587         .max_wm = I965_CURSOR_MAX_WM,
 588         .default_wm = I965_CURSOR_DFT_WM,
 589         .guard_size = 2,
 590         .cacheline_size = I915_FIFO_LINE_SIZE,
 591 };
 592
 593 static const struct intel_watermark_params i945_wm_info = {
 594         .fifo_size = I945_FIFO_SIZE,
 595         .max_wm = I915_MAX_WM,
 596         .default_wm = 1,
 597         .guard_size = 2,
 598         .cacheline_size = I915_FIFO_LINE_SIZE,
 599 };
 600
 601 static const struct intel_watermark_params i915_wm_info = {
 602         .fifo_size = I915_FIFO_SIZE,
 603         .max_wm = I915_MAX_WM,
 604         .default_wm = 1,
 605         .guard_size = 2,
 606         .cacheline_size = I915_FIFO_LINE_SIZE,
 607 };
 608
 609 static const struct intel_watermark_params i830_a_wm_info = {
 610         .fifo_size = I855GM_FIFO_SIZE,
 611         .max_wm = I915_MAX_WM,
 612         .default_wm = 1,
 613         .guard_size = 2,
 614         .cacheline_size = I830_FIFO_LINE_SIZE,
 615 };
 616
 617 static const struct intel_watermark_params i830_bc_wm_info = {
 618         .fifo_size = I855GM_FIFO_SIZE,
 619         .max_wm = I915_MAX_WM/2,
 620         .default_wm = 1,
 621         .guard_size = 2,
 622         .cacheline_size = I830_FIFO_LINE_SIZE,
 623 };
 624
 625 static const struct intel_watermark_params i845_wm_info = {
 626         .fifo_size = I830_FIFO_SIZE,
 627         .max_wm = I915_MAX_WM,
 628         .default_wm = 1,
 629         .guard_size = 2,
 630         .cacheline_size = I830_FIFO_LINE_SIZE,
 631 };
 632
 633 /**
 634  * intel_wm_method1 - Method 1 / "small buffer" watermark formula
 635  * @pixel_rate: Pipe pixel rate in kHz
 636  * @cpp: Plane bytes per pixel
 637  * @latency: Memory wakeup latency in 0.1us units
 638  *
 639  * Compute the watermark using the method 1 or "small buffer"
 640  * formula. The caller may additonally add extra cachelines
 641  * to account for TLB misses and clock crossings.
 642  *
 643  * This method is concerned with the short term drain rate
 644  * of the FIFO, ie. it does not account for blanking periods
 645  * which would effectively reduce the average drain rate across
 646  * a longer period. The name "small" refers to the fact the
 647  * FIFO is relatively small compared to the amount of data
 648  * fetched.
 649  *
 650  * The FIFO level vs. time graph might look something like:
 651  *
 652  *   |\   |\
 653  *   | \  | \
 654  * __---__---__ (- plane active, _ blanking)
 655  * -> time
 656  *
 657  * or perhaps like this:
 658  *
 659  *   |\|\  |\|\
 660  * __----__----__ (- plane active, _ blanking)
 661  * -> time
 662  *
 663  * Returns:
 664  * The watermark in bytes
 665  */
 666 static unsigned int intel_wm_method1(unsigned int pixel_rate,
 667                                      unsigned int cpp,
 668                                      unsigned int latency)
 669 {
 670         u64 ret;
 671
 672         ret = mul_u32_u32(pixel_rate, cpp * latency);
 673         ret = DIV_ROUND_UP_ULL(ret, 10000);
 674
 675         return ret;
 676 }
 677
 678 /**
 679  * intel_wm_method2 - Method 2 / "large buffer" watermark formula
 680  * @pixel_rate: Pipe pixel rate in kHz
 681  * @htotal: Pipe horizontal total
 682  * @width: Plane width in pixels
 683  * @cpp: Plane bytes per pixel
 684  * @latency: Memory wakeup latency in 0.1us units
 685  *
 686  * Compute the watermark using the method 2 or "large buffer"
 687  * formula. The caller may additonally add extra cachelines
 688  * to account for TLB misses and clock crossings.
 689  *
 690  * This method is concerned with the long term drain rate
 691  * of the FIFO, ie. it does account for blanking periods
 692  * which effectively reduce the average drain rate across
 693  * a longer period. The name "large" refers to the fact the
 694  * FIFO is relatively large compared to the amount of data
 695  * fetched.
 696  *
 697  * The FIFO level vs. time graph might look something like:
 698  *
 699  *    |\___       |\___
 700  *    |    \___   |    \___
 701  *    |        \  |        \
 702  * __ --__--__--__--__--__--__ (- plane active, _ blanking)
 703  * -> time
 704  *
 705  * Returns:
 706  * The watermark in bytes
 707  */
 708 static unsigned int intel_wm_method2(unsigned int pixel_rate,
 709                                      unsigned int htotal,
 710                                      unsigned int width,
 711                                      unsigned int cpp,
 712                                      unsigned int latency)
 713 {
 714         unsigned int ret;
 715
 716         /*
 717          * FIXME remove once all users are computing
 718          * watermarks in the correct place.
 719          */
 720         if (WARN_ON_ONCE(htotal == 0))
 721                 htotal = 1;
 722
 723         ret = (latency * pixel_rate) / (htotal * 10000);
 724         ret = (ret + 1) * width * cpp;
 725
 726         return ret;
 727 }
 728
 729 /**
 730  * intel_calculate_wm - calculate watermark level
 731  * @pixel_rate: pixel clock
 732  * @wm: chip FIFO params
 733  * @fifo_size: size of the FIFO buffer
 734  * @cpp: bytes per pixel
 735  * @latency_ns: memory latency for the platform
 736  *
 737  * Calculate the watermark level (the level at which the display plane will
 738  * start fetching from memory again).  Each chip has a different display
 739  * FIFO size and allocation, so the caller needs to figure that out and pass
 740  * in the correct intel_watermark_params structure.
 741  *
 742  * As the pixel clock runs, the FIFO will be drained at a rate that depends
 743  * on the pixel size.  When it reaches the watermark level, it'll start
 744  * fetching FIFO line sized based chunks from memory until the FIFO fills
 745  * past the watermark point.  If the FIFO drains completely, a FIFO underrun
 746  * will occur, and a display engine hang could result.
 747  */
 748 static unsigned int intel_calculate_wm(int pixel_rate,
 749                                        const struct intel_watermark_params *wm,
 750                                        int fifo_size, int cpp,
 751                                        unsigned int latency_ns)
 752 {
 753         int entries, wm_size;
 754
 755         /*
 756          * Note: we need to make sure we don't overflow for various clock &
 757          * latency values.
 758          * clocks go from a few thousand to several hundred thousand.
 759          * latency is usually a few thousand
 760          */
 761         entries = intel_wm_method1(pixel_rate, cpp,
 762                                    latency_ns / 100);
 763         entries = DIV_ROUND_UP(entries, wm->cacheline_size) +
 764                 wm->guard_size;
 765         DRM_DEBUG_KMS("FIFO entries required for mode: %d\n", entries);
 766
 767         wm_size = fifo_size - entries;
 768         DRM_DEBUG_KMS("FIFO watermark level: %d\n", wm_size);
 769
 770         /* Don't promote wm_size to unsigned... */
 771         if (wm_size > wm->max_wm)
 772                 wm_size = wm->max_wm;
 773         if (wm_size <= 0)
 774                 wm_size = wm->default_wm;
 775
 776         /*
 777          * Bspec seems to indicate that the value shouldn't be lower than
 778          * 'burst size + 1'. Certainly 830 is quite unhappy with low values.
 779          * Lets go for 8 which is the burst size since certain platforms
 780          * already use a hardcoded 8 (which is what the spec says should be
 781          * done).
 782          */
 783         if (wm_size <= 8)
 784                 wm_size = 8;
 785
 786         return wm_size;
 787 }
 788
 789 static bool is_disabling(int old, int new, int threshold)
 790 {
 791         return old >= threshold && new < threshold;
 792 }
 793
 794 static bool is_enabling(int old, int new, int threshold)
 795 {
 796         return old < threshold && new >= threshold;
 797 }
 798
 799 static int intel_wm_num_levels(struct drm_i915_private *dev_priv)
 800 {
 801         return dev_priv->wm.max_level + 1;
 802 }
 803
 804 static bool intel_wm_plane_visible(const struct intel_crtc_state *crtc_state,
 805                                    const struct intel_plane_state *plane_state)
 806 {
 807         struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
 808
 809         /* FIXME check the 'enable' instead */
 810         if (!crtc_state->hw.active)
 811                 return false;
 812
 813         /*
 814          * Treat cursor with fb as always visible since cursor updates
 815          * can happen faster than the vrefresh rate, and the current
 816          * watermark code doesn't handle that correctly. Cursor updates
 817          * which set/clear the fb or change the cursor size are going
 818          * to get throttled by intel_legacy_cursor_update() to work
 819          * around this problem with the watermark code.
 820          */
 821         if (plane->id == PLANE_CURSOR)
 822                 return plane_state->hw.fb != NULL;
 823         else
 824                 return plane_state->uapi.visible;
 825 }
 826
 827 static bool intel_crtc_active(struct intel_crtc *crtc)
 828 {
 829         /* Be paranoid as we can arrive here with only partial
 830          * state retrieved from the hardware during setup.
 831          *
 832          * We can ditch the adjusted_mode.crtc_clock check as soon
 833          * as Haswell has gained clock readout/fastboot support.
 834          *
 835          * We can ditch the crtc->primary->state->fb check as soon as we can
 836          * properly reconstruct framebuffers.
 837          *
 838          * FIXME: The intel_crtc->active here should be switched to
 839          * crtc->state->active once we have proper CRTC states wired up
 840          * for atomic.
 841          */
 842         return crtc->active && crtc->base.primary->state->fb &&
 843                 crtc->config->hw.adjusted_mode.crtc_clock;
 844 }
 845
 846 static struct intel_crtc *single_enabled_crtc(struct drm_i915_private *dev_priv)
 847 {
 848         struct intel_crtc *crtc, *enabled = NULL;
 849
 850         for_each_intel_crtc(&dev_priv->drm, crtc) {
 851                 if (intel_crtc_active(crtc)) {
 852                         if (enabled)
 853                                 return NULL;
 854                         enabled = crtc;
 855                 }
 856         }
 857
 858         return enabled;
 859 }
 860
 861 static void pnv_update_wm(struct intel_crtc *unused_crtc)
 862 {
 863         struct drm_i915_private *dev_priv = to_i915(unused_crtc->base.dev);
 864         struct intel_crtc *crtc;
 865         const struct cxsr_latency *latency;
 866         u32 reg;
 867         unsigned int wm;
 868
 869         latency = intel_get_cxsr_latency(!IS_MOBILE(dev_priv),
 870                                          dev_priv->is_ddr3,
 871                                          dev_priv->fsb_freq,
 872                                          dev_priv->mem_freq);
 873         if (!latency) {
 874                 drm_dbg_kms(&dev_priv->drm,
 875                             "Unknown FSB/MEM found, disable CxSR\n");
 876                 intel_set_memory_cxsr(dev_priv, false);
 877                 return;
 878         }
 879
 880         crtc = single_enabled_crtc(dev_priv);
 881         if (crtc) {
 882                 const struct drm_display_mode *adjusted_mode =
 883                         &crtc->config->hw.adjusted_mode;
 884                 const struct drm_framebuffer *fb =
 885                         crtc->base.primary->state->fb;
 886                 int cpp = fb->format->cpp[0];
 887                 int clock = adjusted_mode->crtc_clock;
 888
 889                 /* Display SR */
 890                 wm = intel_calculate_wm(clock, &pnv_display_wm,
 891                                         pnv_display_wm.fifo_size,
 892                                         cpp, latency->display_sr);
 893                 reg = I915_READ(DSPFW1);
 894                 reg &= ~DSPFW_SR_MASK;
 895                 reg |= FW_WM(wm, SR);
 896                 I915_WRITE(DSPFW1, reg);
 897                 drm_dbg_kms(&dev_priv->drm, "DSPFW1 register is %x\n", reg);
 898
 899                 /* cursor SR */
 900                 wm = intel_calculate_wm(clock, &pnv_cursor_wm,
 901                                         pnv_display_wm.fifo_size,
 902                                         4, latency->cursor_sr);
 903                 reg = I915_READ(DSPFW3);
 904                 reg &= ~DSPFW_CURSOR_SR_MASK;
 905                 reg |= FW_WM(wm, CURSOR_SR);
 906                 I915_WRITE(DSPFW3, reg);
 907
 908                 /* Display HPLL off SR */
 909                 wm = intel_calculate_wm(clock, &pnv_display_hplloff_wm,
 910                                         pnv_display_hplloff_wm.fifo_size,
 911                                         cpp, latency->display_hpll_disable);
 912                 reg = I915_READ(DSPFW3);
 913                 reg &= ~DSPFW_HPLL_SR_MASK;
 914                 reg |= FW_WM(wm, HPLL_SR);
 915                 I915_WRITE(DSPFW3, reg);
 916
 917                 /* cursor HPLL off SR */
 918                 wm = intel_calculate_wm(clock, &pnv_cursor_hplloff_wm,
 919                                         pnv_display_hplloff_wm.fifo_size,
 920                                         4, latency->cursor_hpll_disable);
 921                 reg = I915_READ(DSPFW3);
 922                 reg &= ~DSPFW_HPLL_CURSOR_MASK;
 923                 reg |= FW_WM(wm, HPLL_CURSOR);
 924                 I915_WRITE(DSPFW3, reg);
 925                 drm_dbg_kms(&dev_priv->drm, "DSPFW3 register is %x\n", reg);
 926
 927                 intel_set_memory_cxsr(dev_priv, true);
 928         } else {
 929                 intel_set_memory_cxsr(dev_priv, false);
 930         }
 931 }
 932
 933 /*
 934  * Documentation says:
 935  * "If the line size is small, the TLB fetches can get in the way of the
 936  *  data fetches, causing some lag in the pixel data return which is not
 937  *  accounted for in the above formulas. The following adjustment only
 938  *  needs to be applied if eight whole lines fit in the buffer at once.
 939  *  The WM is adjusted upwards by the difference between the FIFO size
 940  *  and the size of 8 whole lines. This adjustment is always performed
 941  *  in the actual pixel depth regardless of whether FBC is enabled or not."
 942  */
 943 static unsigned int g4x_tlb_miss_wa(int fifo_size, int width, int cpp)
 944 {
 945         int tlb_miss = fifo_size * 64 - width * cpp * 8;
 946
 947         return max(0, tlb_miss);
 948 }
 949
 950 static void g4x_write_wm_values(struct drm_i915_private *dev_priv,
 951                                 const struct g4x_wm_values *wm)
 952 {
 953         enum pipe pipe;
 954
 955         for_each_pipe(dev_priv, pipe)
 956                 trace_g4x_wm(intel_get_crtc_for_pipe(dev_priv, pipe), wm);
 957
 958         I915_WRITE(DSPFW1,
 959                    FW_WM(wm->sr.plane, SR) |
 960                    FW_WM(wm->pipe[PIPE_B].plane[PLANE_CURSOR], CURSORB) |
 961                    FW_WM(wm->pipe[PIPE_B].plane[PLANE_PRIMARY], PLANEB) |
 962                    FW_WM(wm->pipe[PIPE_A].plane[PLANE_PRIMARY], PLANEA));
 963         I915_WRITE(DSPFW2,
 964                    (wm->fbc_en ? DSPFW_FBC_SR_EN : 0) |
 965                    FW_WM(wm->sr.fbc, FBC_SR) |
 966                    FW_WM(wm->hpll.fbc, FBC_HPLL_SR) |
 967                    FW_WM(wm->pipe[PIPE_B].plane[PLANE_SPRITE0], SPRITEB) |
 968                    FW_WM(wm->pipe[PIPE_A].plane[PLANE_CURSOR], CURSORA) |
 969                    FW_WM(wm->pipe[PIPE_A].plane[PLANE_SPRITE0], SPRITEA));
 970         I915_WRITE(DSPFW3,
 971                    (wm->hpll_en ? DSPFW_HPLL_SR_EN : 0) |
 972                    FW_WM(wm->sr.cursor, CURSOR_SR) |
 973                    FW_WM(wm->hpll.cursor, HPLL_CURSOR) |
 974                    FW_WM(wm->hpll.plane, HPLL_SR));
 975
 976         POSTING_READ(DSPFW1);
 977 }
 978
 979 #define FW_WM_VLV(value, plane) \
 980         (((value) << DSPFW_ ## plane ## _SHIFT) & DSPFW_ ## plane ## _MASK_VLV)
 981
 982 static void vlv_write_wm_values(struct drm_i915_private *dev_priv,
 983                                 const struct vlv_wm_values *wm)
 984 {
 985         enum pipe pipe;
 986
 987         for_each_pipe(dev_priv, pipe) {
 988                 trace_vlv_wm(intel_get_crtc_for_pipe(dev_priv, pipe), wm);
 989
 990                 I915_WRITE(VLV_DDL(pipe),
 991                            (wm->ddl[pipe].plane[PLANE_CURSOR] << DDL_CURSOR_SHIFT) |
 992                            (wm->ddl[pipe].plane[PLANE_SPRITE1] << DDL_SPRITE_SHIFT(1)) |
 993                            (wm->ddl[pipe].plane[PLANE_SPRITE0] << DDL_SPRITE_SHIFT(0)) |
 994                            (wm->ddl[pipe].plane[PLANE_PRIMARY] << DDL_PLANE_SHIFT));
 995         }
 996
 997         /*
 998          * Zero the (unused) WM1 watermarks, and also clear all the
 999          * high order bits so that there are no out of bounds values
1000          * present in the registers during the reprogramming.
1001          */
1002         I915_WRITE(DSPHOWM, 0);
1003         I915_WRITE(DSPHOWM1, 0);
1004         I915_WRITE(DSPFW4, 0);
1005         I915_WRITE(DSPFW5, 0);
1006         I915_WRITE(DSPFW6, 0);
1007
1008         I915_WRITE(DSPFW1,
1009                    FW_WM(wm->sr.plane, SR) |
1010                    FW_WM(wm->pipe[PIPE_B].plane[PLANE_CURSOR], CURSORB) |
1011                    FW_WM_VLV(wm->pipe[PIPE_B].plane[PLANE_PRIMARY], PLANEB) |
1012                    FW_WM_VLV(wm->pipe[PIPE_A].plane[PLANE_PRIMARY], PLANEA));
1013         I915_WRITE(DSPFW2,
1014                    FW_WM_VLV(wm->pipe[PIPE_A].plane[PLANE_SPRITE1], SPRITEB) |
1015                    FW_WM(wm->pipe[PIPE_A].plane[PLANE_CURSOR], CURSORA) |
1016                    FW_WM_VLV(wm->pipe[PIPE_A].plane[PLANE_SPRITE0], SPRITEA));
1017         I915_WRITE(DSPFW3,
1018                    FW_WM(wm->sr.cursor, CURSOR_SR));
1019
1020         if (IS_CHERRYVIEW(dev_priv)) {
1021                 I915_WRITE(DSPFW7_CHV,
1022                            FW_WM_VLV(wm->pipe[PIPE_B].plane[PLANE_SPRITE1], SPRITED) |
1023                            FW_WM_VLV(wm->pipe[PIPE_B].plane[PLANE_SPRITE0], SPRITEC));
1024                 I915_WRITE(DSPFW8_CHV,
1025                            FW_WM_VLV(wm->pipe[PIPE_C].plane[PLANE_SPRITE1], SPRITEF) |
1026                            FW_WM_VLV(wm->pipe[PIPE_C].plane[PLANE_SPRITE0], SPRITEE));
1027                 I915_WRITE(DSPFW9_CHV,
1028                            FW_WM_VLV(wm->pipe[PIPE_C].plane[PLANE_PRIMARY], PLANEC) |
1029                            FW_WM(wm->pipe[PIPE_C].plane[PLANE_CURSOR], CURSORC));
1030                 I915_WRITE(DSPHOWM,
1031                            FW_WM(wm->sr.plane >> 9, SR_HI) |
1032                            FW_WM(wm->pipe[PIPE_C].plane[PLANE_SPRITE1] >> 8, SPRITEF_HI) |
1033                            FW_WM(wm->pipe[PIPE_C].plane[PLANE_SPRITE0] >> 8, SPRITEE_HI) |
1034                            FW_WM(wm->pipe[PIPE_C].plane[PLANE_PRIMARY] >> 8, PLANEC_HI) |
1035                            FW_WM(wm->pipe[PIPE_B].plane[PLANE_SPRITE1] >> 8, SPRITED_HI) |
1036                            FW_WM(wm->pipe[PIPE_B].plane[PLANE_SPRITE0] >> 8, SPRITEC_HI) |
1037                            FW_WM(wm->pipe[PIPE_B].plane[PLANE_PRIMARY] >> 8, PLANEB_HI) |
1038                            FW_WM(wm->pipe[PIPE_A].plane[PLANE_SPRITE1] >> 8, SPRITEB_HI) |
1039                            FW_WM(wm->pipe[PIPE_A].plane[PLANE_SPRITE0] >> 8, SPRITEA_HI) |
1040                            FW_WM(wm->pipe[PIPE_A].plane[PLANE_PRIMARY] >> 8, PLANEA_HI));
1041         } else {
1042                 I915_WRITE(DSPFW7,
1043                            FW_WM_VLV(wm->pipe[PIPE_B].plane[PLANE_SPRITE1], SPRITED) |
1044                            FW_WM_VLV(wm->pipe[PIPE_B].plane[PLANE_SPRITE0], SPRITEC));
1045                 I915_WRITE(DSPHOWM,
1046                            FW_WM(wm->sr.plane >> 9, SR_HI) |
1047                            FW_WM(wm->pipe[PIPE_B].plane[PLANE_SPRITE1] >> 8, SPRITED_HI) |
1048                            FW_WM(wm->pipe[PIPE_B].plane[PLANE_SPRITE0] >> 8, SPRITEC_HI) |
1049                            FW_WM(wm->pipe[PIPE_B].plane[PLANE_PRIMARY] >> 8, PLANEB_HI) |
1050                            FW_WM(wm->pipe[PIPE_A].plane[PLANE_SPRITE1] >> 8, SPRITEB_HI) |
1051                            FW_WM(wm->pipe[PIPE_A].plane[PLANE_SPRITE0] >> 8, SPRITEA_HI) |
1052                            FW_WM(wm->pipe[PIPE_A].plane[PLANE_PRIMARY] >> 8, PLANEA_HI));
1053         }
1054
1055         POSTING_READ(DSPFW1);
1056 }
1057
1058 #undef FW_WM_VLV
1059
1060 static void g4x_setup_wm_latency(struct drm_i915_private *dev_priv)
1061 {
1062         /* all latencies in usec */
1063         dev_priv->wm.pri_latency[G4X_WM_LEVEL_NORMAL] = 5;
1064         dev_priv->wm.pri_latency[G4X_WM_LEVEL_SR] = 12;
1065         dev_priv->wm.pri_latency[G4X_WM_LEVEL_HPLL] = 35;
1066
1067         dev_priv->wm.max_level = G4X_WM_LEVEL_HPLL;
1068 }
1069
1070 static int g4x_plane_fifo_size(enum plane_id plane_id, int level)
1071 {
1072         /*
1073          * DSPCNTR[13] supposedly controls whether the
1074          * primary plane can use the FIFO space otherwise
1075          * reserved for the sprite plane. It's not 100% clear
1076          * what the actual FIFO size is, but it looks like we
1077          * can happily set both primary and sprite watermarks
1078          * up to 127 cachelines. So that would seem to mean
1079          * that either DSPCNTR[13] doesn't do anything, or that
1080          * the total FIFO is >= 256 cachelines in size. Either
1081          * way, we don't seem to have to worry about this
1082          * repartitioning as the maximum watermark value the
1083          * register can hold for each plane is lower than the
1084          * minimum FIFO size.
1085          */
1086         switch (plane_id) {
1087         case PLANE_CURSOR:
1088                 return 63;
1089         case PLANE_PRIMARY:
1090                 return level == G4X_WM_LEVEL_NORMAL ? 127 : 511;
1091         case PLANE_SPRITE0:
1092                 return level == G4X_WM_LEVEL_NORMAL ? 127 : 0;
1093         default:
1094                 MISSING_CASE(plane_id);
1095                 return 0;
1096         }
1097 }
1098
1099 static int g4x_fbc_fifo_size(int level)
1100 {
1101         switch (level) {
1102         case G4X_WM_LEVEL_SR:
1103                 return 7;
1104         case G4X_WM_LEVEL_HPLL:
1105                 return 15;
1106         default:
1107                 MISSING_CASE(level);
1108                 return 0;
1109         }
1110 }
1111
1112 static u16 g4x_compute_wm(const struct intel_crtc_state *crtc_state,
1113                           const struct intel_plane_state *plane_state,
1114                           int level)
1115 {
1116         struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
1117         struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
1118         const struct drm_display_mode *adjusted_mode =
1119                 &crtc_state->hw.adjusted_mode;
1120         unsigned int latency = dev_priv->wm.pri_latency[level] * 10;
1121         unsigned int clock, htotal, cpp, width, wm;
1122
1123         if (latency == 0)
1124                 return USHRT_MAX;
1125
1126         if (!intel_wm_plane_visible(crtc_state, plane_state))
1127                 return 0;
1128
1129         cpp = plane_state->hw.fb->format->cpp[0];
1130
1131         /*
1132          * Not 100% sure which way ELK should go here as the
1133          * spec only says CL/CTG should assume 32bpp and BW
1134          * doesn't need to. But as these things followed the
1135          * mobile vs. desktop lines on gen3 as well, let's
1136          * assume ELK doesn't need this.
1137          *
1138          * The spec also fails to list such a restriction for
1139          * the HPLL watermark, which seems a little strange.
1140          * Let's use 32bpp for the HPLL watermark as well.
1141          */
1142         if (IS_GM45(dev_priv) && plane->id == PLANE_PRIMARY &&
1143             level != G4X_WM_LEVEL_NORMAL)
1144                 cpp = max(cpp, 4u);
1145
1146         clock = adjusted_mode->crtc_clock;
1147         htotal = adjusted_mode->crtc_htotal;
1148
1149         width = drm_rect_width(&plane_state->uapi.dst);
1150
1151         if (plane->id == PLANE_CURSOR) {
1152                 wm = intel_wm_method2(clock, htotal, width, cpp, latency);
1153         } else if (plane->id == PLANE_PRIMARY &&
1154                    level == G4X_WM_LEVEL_NORMAL) {
1155                 wm = intel_wm_method1(clock, cpp, latency);
1156         } else {
1157                 unsigned int small, large;
1158
1159                 small = intel_wm_method1(clock, cpp, latency);
1160                 large = intel_wm_method2(clock, htotal, width, cpp, latency);
1161
1162                 wm = min(small, large);
1163         }
1164
1165         wm += g4x_tlb_miss_wa(g4x_plane_fifo_size(plane->id, level),
1166                               width, cpp);
1167
1168         wm = DIV_ROUND_UP(wm, 64) + 2;
1169
1170         return min_t(unsigned int, wm, USHRT_MAX);
1171 }
1172
1173 static bool g4x_raw_plane_wm_set(struct intel_crtc_state *crtc_state,
1174                                  int level, enum plane_id plane_id, u16 value)
1175 {
1176         struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
1177         bool dirty = false;
1178
1179         for (; level < intel_wm_num_levels(dev_priv); level++) {
1180                 struct g4x_pipe_wm *raw = &crtc_state->wm.g4x.raw[level];
1181
1182                 dirty |= raw->plane[plane_id] != value;
1183                 raw->plane[plane_id] = value;
1184         }
1185
1186         return dirty;
1187 }
1188
1189 static bool g4x_raw_fbc_wm_set(struct intel_crtc_state *crtc_state,
1190                                int level, u16 value)
1191 {
1192         struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
1193         bool dirty = false;
1194
1195         /* NORMAL level doesn't have an FBC watermark */
1196         level = max(level, G4X_WM_LEVEL_SR);
1197
1198         for (; level < intel_wm_num_levels(dev_priv); level++) {
1199                 struct g4x_pipe_wm *raw = &crtc_state->wm.g4x.raw[level];
1200
1201                 dirty |= raw->fbc != value;
1202                 raw->fbc = value;
1203         }
1204
1205         return dirty;
1206 }
1207
1208 static u32 ilk_compute_fbc_wm(const struct intel_crtc_state *crtc_state,
1209                               const struct intel_plane_state *plane_state,
1210                               u32 pri_val);
1211
1212 static bool g4x_raw_plane_wm_compute(struct intel_crtc_state *crtc_state,
1213                                      const struct intel_plane_state *plane_state)
1214 {
1215         struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
1216         struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
1217         int num_levels = intel_wm_num_levels(to_i915(plane->base.dev));
1218         enum plane_id plane_id = plane->id;
1219         bool dirty = false;
1220         int level;
1221
1222         if (!intel_wm_plane_visible(crtc_state, plane_state)) {
1223                 dirty |= g4x_raw_plane_wm_set(crtc_state, 0, plane_id, 0);
1224                 if (plane_id == PLANE_PRIMARY)
1225                         dirty |= g4x_raw_fbc_wm_set(crtc_state, 0, 0);
1226                 goto out;
1227         }
1228
1229         for (level = 0; level < num_levels; level++) {
1230                 struct g4x_pipe_wm *raw = &crtc_state->wm.g4x.raw[level];
1231                 int wm, max_wm;
1232
1233                 wm = g4x_compute_wm(crtc_state, plane_state, level);
1234                 max_wm = g4x_plane_fifo_size(plane_id, level);
1235
1236                 if (wm > max_wm)
1237                         break;
1238
1239                 dirty |= raw->plane[plane_id] != wm;
1240                 raw->plane[plane_id] = wm;
1241
1242                 if (plane_id != PLANE_PRIMARY ||
1243                     level == G4X_WM_LEVEL_NORMAL)
1244                         continue;
1245
1246                 wm = ilk_compute_fbc_wm(crtc_state, plane_state,
1247                                         raw->plane[plane_id]);
1248                 max_wm = g4x_fbc_fifo_size(level);
1249
1250                 /*
1251                  * FBC wm is not mandatory as we
1252                  * can always just disable its use.
1253                  */
1254                 if (wm > max_wm)
1255                         wm = USHRT_MAX;
1256
1257                 dirty |= raw->fbc != wm;
1258                 raw->fbc = wm;
1259         }
1260
1261         /* mark watermarks as invalid */
1262         dirty |= g4x_raw_plane_wm_set(crtc_state, level, plane_id, USHRT_MAX);
1263
1264         if (plane_id == PLANE_PRIMARY)
1265                 dirty |= g4x_raw_fbc_wm_set(crtc_state, level, USHRT_MAX);
1266
1267  out:
1268         if (dirty) {
1269                 drm_dbg_kms(&dev_priv->drm,
1270                             "%s watermarks: normal=%d, SR=%d, HPLL=%d\n",
1271                             plane->base.name,
1272                             crtc_state->wm.g4x.raw[G4X_WM_LEVEL_NORMAL].plane[plane_id],
1273                             crtc_state->wm.g4x.raw[G4X_WM_LEVEL_SR].plane[plane_id],
1274                             crtc_state->wm.g4x.raw[G4X_WM_LEVEL_HPLL].plane[plane_id]);
1275
1276                 if (plane_id == PLANE_PRIMARY)
1277                         drm_dbg_kms(&dev_priv->drm,
1278                                     "FBC watermarks: SR=%d, HPLL=%d\n",
1279                                     crtc_state->wm.g4x.raw[G4X_WM_LEVEL_SR].fbc,
1280                                     crtc_state->wm.g4x.raw[G4X_WM_LEVEL_HPLL].fbc);
1281         }
1282
1283         return dirty;
1284 }
1285
1286 static bool g4x_raw_plane_wm_is_valid(const struct intel_crtc_state *crtc_state,
1287                                       enum plane_id plane_id, int level)
1288 {
1289         const struct g4x_pipe_wm *raw = &crtc_state->wm.g4x.raw[level];
1290
1291         return raw->plane[plane_id] <= g4x_plane_fifo_size(plane_id, level);
1292 }
1293
1294 static bool g4x_raw_crtc_wm_is_valid(const struct intel_crtc_state *crtc_state,
1295                                      int level)
1296 {
1297         struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
1298
1299         if (level > dev_priv->wm.max_level)
1300                 return false;
1301
1302         return g4x_raw_plane_wm_is_valid(crtc_state, PLANE_PRIMARY, level) &&
1303                 g4x_raw_plane_wm_is_valid(crtc_state, PLANE_SPRITE0, level) &&
1304                 g4x_raw_plane_wm_is_valid(crtc_state, PLANE_CURSOR, level);
1305 }
1306
1307 /* mark all levels starting from 'level' as invalid */
1308 static void g4x_invalidate_wms(struct intel_crtc *crtc,
1309                                struct g4x_wm_state *wm_state, int level)
1310 {
1311         if (level <= G4X_WM_LEVEL_NORMAL) {
1312                 enum plane_id plane_id;
1313
1314                 for_each_plane_id_on_crtc(crtc, plane_id)
1315                         wm_state->wm.plane[plane_id] = USHRT_MAX;
1316         }
1317
1318         if (level <= G4X_WM_LEVEL_SR) {
1319                 wm_state->cxsr = false;
1320                 wm_state->sr.cursor = USHRT_MAX;
1321                 wm_state->sr.plane = USHRT_MAX;
1322                 wm_state->sr.fbc = USHRT_MAX;
1323         }
1324
1325         if (level <= G4X_WM_LEVEL_HPLL) {
1326                 wm_state->hpll_en = false;
1327                 wm_state->hpll.cursor = USHRT_MAX;
1328                 wm_state->hpll.plane = USHRT_MAX;
1329                 wm_state->hpll.fbc = USHRT_MAX;
1330         }
1331 }
1332
1333 static int g4x_compute_pipe_wm(struct intel_crtc_state *crtc_state)
1334 {
1335         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
1336         struct intel_atomic_state *state =
1337                 to_intel_atomic_state(crtc_state->uapi.state);
1338         struct g4x_wm_state *wm_state = &crtc_state->wm.g4x.optimal;
1339         int num_active_planes = hweight8(crtc_state->active_planes &
1340                                          ~BIT(PLANE_CURSOR));
1341         const struct g4x_pipe_wm *raw;
1342         const struct intel_plane_state *old_plane_state;
1343         const struct intel_plane_state *new_plane_state;
1344         struct intel_plane *plane;
1345         enum plane_id plane_id;
1346         int i, level;
1347         unsigned int dirty = 0;
1348
1349         for_each_oldnew_intel_plane_in_state(state, plane,
1350                                              old_plane_state,
1351                                              new_plane_state, i) {
1352                 if (new_plane_state->hw.crtc != &crtc->base &&
1353                     old_plane_state->hw.crtc != &crtc->base)
1354                         continue;
1355
1356                 if (g4x_raw_plane_wm_compute(crtc_state, new_plane_state))
1357                         dirty |= BIT(plane->id);
1358         }
1359
1360         if (!dirty)
1361                 return 0;
1362
1363         level = G4X_WM_LEVEL_NORMAL;
1364         if (!g4x_raw_crtc_wm_is_valid(crtc_state, level))
1365                 goto out;
1366
1367         raw = &crtc_state->wm.g4x.raw[level];
1368         for_each_plane_id_on_crtc(crtc, plane_id)
1369                 wm_state->wm.plane[plane_id] = raw->plane[plane_id];
1370
1371         level = G4X_WM_LEVEL_SR;
1372
1373         if (!g4x_raw_crtc_wm_is_valid(crtc_state, level))
1374                 goto out;
1375
1376         raw = &crtc_state->wm.g4x.raw[level];
1377         wm_state->sr.plane = raw->plane[PLANE_PRIMARY];
1378         wm_state->sr.cursor = raw->plane[PLANE_CURSOR];
1379         wm_state->sr.fbc = raw->fbc;
1380
1381         wm_state->cxsr = num_active_planes == BIT(PLANE_PRIMARY);
1382
1383         level = G4X_WM_LEVEL_HPLL;
1384
1385         if (!g4x_raw_crtc_wm_is_valid(crtc_state, level))
1386                 goto out;
1387
1388         raw = &crtc_state->wm.g4x.raw[level];
1389         wm_state->hpll.plane = raw->plane[PLANE_PRIMARY];
1390         wm_state->hpll.cursor = raw->plane[PLANE_CURSOR];
1391         wm_state->hpll.fbc = raw->fbc;
1392
1393         wm_state->hpll_en = wm_state->cxsr;
1394
1395         level++;
1396
1397  out:
1398         if (level == G4X_WM_LEVEL_NORMAL)
1399                 return -EINVAL;
1400
1401         /* invalidate the higher levels */
1402         g4x_invalidate_wms(crtc, wm_state, level);
1403
1404         /*
1405          * Determine if the FBC watermark(s) can be used. IF
1406          * this isn't the case we prefer to disable the FBC
1407          ( watermark(s) rather than disable the SR/HPLL
1408          * level(s) entirely.
1409          */
1410         wm_state->fbc_en = level > G4X_WM_LEVEL_NORMAL;
1411
1412         if (level >= G4X_WM_LEVEL_SR &&
1413             wm_state->sr.fbc > g4x_fbc_fifo_size(G4X_WM_LEVEL_SR))
1414                 wm_state->fbc_en = false;
1415         else if (level >= G4X_WM_LEVEL_HPLL &&
1416                  wm_state->hpll.fbc > g4x_fbc_fifo_size(G4X_WM_LEVEL_HPLL))
1417                 wm_state->fbc_en = false;
1418
1419         return 0;
1420 }
1421
1422 static int g4x_compute_intermediate_wm(struct intel_crtc_state *new_crtc_state)
1423 {
1424         struct intel_crtc *crtc = to_intel_crtc(new_crtc_state->uapi.crtc);
1425         struct g4x_wm_state *intermediate = &new_crtc_state->wm.g4x.intermediate;
1426         const struct g4x_wm_state *optimal = &new_crtc_state->wm.g4x.optimal;
1427         struct intel_atomic_state *intel_state =
1428                 to_intel_atomic_state(new_crtc_state->uapi.state);
1429         const struct intel_crtc_state *old_crtc_state =
1430                 intel_atomic_get_old_crtc_state(intel_state, crtc);
1431         const struct g4x_wm_state *active = &old_crtc_state->wm.g4x.optimal;
1432         enum plane_id plane_id;
1433
1434         if (!new_crtc_state->hw.active || drm_atomic_crtc_needs_modeset(&new_crtc_state->uapi)) {
1435                 *intermediate = *optimal;
1436
1437                 intermediate->cxsr = false;
1438                 intermediate->hpll_en = false;
1439                 goto out;
1440         }
1441
1442         intermediate->cxsr = optimal->cxsr && active->cxsr &&
1443                 !new_crtc_state->disable_cxsr;
1444         intermediate->hpll_en = optimal->hpll_en && active->hpll_en &&
1445                 !new_crtc_state->disable_cxsr;
1446         intermediate->fbc_en = optimal->fbc_en && active->fbc_en;
1447
1448         for_each_plane_id_on_crtc(crtc, plane_id) {
1449                 intermediate->wm.plane[plane_id] =
1450                         max(optimal->wm.plane[plane_id],
1451                             active->wm.plane[plane_id]);
1452
1453                 WARN_ON(intermediate->wm.plane[plane_id] >
1454                         g4x_plane_fifo_size(plane_id, G4X_WM_LEVEL_NORMAL));
1455         }
1456
1457         intermediate->sr.plane = max(optimal->sr.plane,
1458                                      active->sr.plane);
1459         intermediate->sr.cursor = max(optimal->sr.cursor,
1460                                       active->sr.cursor);
1461         intermediate->sr.fbc = max(optimal->sr.fbc,
1462                                    active->sr.fbc);
1463
1464         intermediate->hpll.plane = max(optimal->hpll.plane,
1465                                        active->hpll.plane);
1466         intermediate->hpll.cursor = max(optimal->hpll.cursor,
1467                                         active->hpll.cursor);
1468         intermediate->hpll.fbc = max(optimal->hpll.fbc,
1469                                      active->hpll.fbc);
1470
1471         WARN_ON((intermediate->sr.plane >
1472                  g4x_plane_fifo_size(PLANE_PRIMARY, G4X_WM_LEVEL_SR) ||
1473                  intermediate->sr.cursor >
1474                  g4x_plane_fifo_size(PLANE_CURSOR, G4X_WM_LEVEL_SR)) &&
1475                 intermediate->cxsr);
1476         WARN_ON((intermediate->sr.plane >
1477                  g4x_plane_fifo_size(PLANE_PRIMARY, G4X_WM_LEVEL_HPLL) ||
1478                  intermediate->sr.cursor >
1479                  g4x_plane_fifo_size(PLANE_CURSOR, G4X_WM_LEVEL_HPLL)) &&
1480                 intermediate->hpll_en);
1481
1482         WARN_ON(intermediate->sr.fbc > g4x_fbc_fifo_size(1) &&
1483                 intermediate->fbc_en && intermediate->cxsr);
1484         WARN_ON(intermediate->hpll.fbc > g4x_fbc_fifo_size(2) &&
1485                 intermediate->fbc_en && intermediate->hpll_en);
1486
1487 out:
1488         /*
1489          * If our intermediate WM are identical to the final WM, then we can
1490          * omit the post-vblank programming; only update if it's different.
1491          */
1492         if (memcmp(intermediate, optimal, sizeof(*intermediate)) != 0)
1493                 new_crtc_state->wm.need_postvbl_update = true;
1494
1495         return 0;
1496 }
1497
1498 static void g4x_merge_wm(struct drm_i915_private *dev_priv,
1499                          struct g4x_wm_values *wm)
1500 {
1501         struct intel_crtc *crtc;
1502         int num_active_pipes = 0;
1503
1504         wm->cxsr = true;
1505         wm->hpll_en = true;
1506         wm->fbc_en = true;
1507
1508         for_each_intel_crtc(&dev_priv->drm, crtc) {
1509                 const struct g4x_wm_state *wm_state = &crtc->wm.active.g4x;
1510
1511                 if (!crtc->active)
1512                         continue;
1513
1514                 if (!wm_state->cxsr)
1515                         wm->cxsr = false;
1516                 if (!wm_state->hpll_en)
1517                         wm->hpll_en = false;
1518                 if (!wm_state->fbc_en)
1519                         wm->fbc_en = false;
1520
1521                 num_active_pipes++;
1522         }
1523
1524         if (num_active_pipes != 1) {
1525                 wm->cxsr = false;
1526                 wm->hpll_en = false;
1527                 wm->fbc_en = false;
1528         }
1529
1530         for_each_intel_crtc(&dev_priv->drm, crtc) {
1531                 const struct g4x_wm_state *wm_state = &crtc->wm.active.g4x;
1532                 enum pipe pipe = crtc->pipe;
1533
1534                 wm->pipe[pipe] = wm_state->wm;
1535                 if (crtc->active && wm->cxsr)
1536                         wm->sr = wm_state->sr;
1537                 if (crtc->active && wm->hpll_en)
1538                         wm->hpll = wm_state->hpll;
1539         }
1540 }
1541
1542 static void g4x_program_watermarks(struct drm_i915_private *dev_priv)
1543 {
1544         struct g4x_wm_values *old_wm = &dev_priv->wm.g4x;
1545         struct g4x_wm_values new_wm = {};
1546
1547         g4x_merge_wm(dev_priv, &new_wm);
1548
1549         if (memcmp(old_wm, &new_wm, sizeof(new_wm)) == 0)
1550                 return;
1551
1552         if (is_disabling(old_wm->cxsr, new_wm.cxsr, true))
1553                 _intel_set_memory_cxsr(dev_priv, false);
1554
1555         g4x_write_wm_values(dev_priv, &new_wm);
1556
1557         if (is_enabling(old_wm->cxsr, new_wm.cxsr, true))
1558                 _intel_set_memory_cxsr(dev_priv, true);
1559
1560         *old_wm = new_wm;
1561 }
1562
1563 static void g4x_initial_watermarks(struct intel_atomic_state *state,
1564                                    struct intel_crtc *crtc)
1565 {
1566         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1567         const struct intel_crtc_state *crtc_state =
1568                 intel_atomic_get_new_crtc_state(state, crtc);
1569
1570         mutex_lock(&dev_priv->wm.wm_mutex);
1571         crtc->wm.active.g4x = crtc_state->wm.g4x.intermediate;
1572         g4x_program_watermarks(dev_priv);
1573         mutex_unlock(&dev_priv->wm.wm_mutex);
1574 }
1575
1576 static void g4x_optimize_watermarks(struct intel_atomic_state *state,
1577                                     struct intel_crtc *crtc)
1578 {
1579         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1580         const struct intel_crtc_state *crtc_state =
1581                 intel_atomic_get_new_crtc_state(state, crtc);
1582
1583         if (!crtc_state->wm.need_postvbl_update)
1584                 return;
1585
1586         mutex_lock(&dev_priv->wm.wm_mutex);
1587         crtc->wm.active.g4x = crtc_state->wm.g4x.optimal;
1588         g4x_program_watermarks(dev_priv);
1589         mutex_unlock(&dev_priv->wm.wm_mutex);
1590 }
1591
1592 /* latency must be in 0.1us units. */
1593 static unsigned int vlv_wm_method2(unsigned int pixel_rate,
1594                                    unsigned int htotal,
1595                                    unsigned int width,
1596                                    unsigned int cpp,
1597                                    unsigned int latency)
1598 {
1599         unsigned int ret;
1600
1601         ret = intel_wm_method2(pixel_rate, htotal,
1602                                width, cpp, latency);
1603         ret = DIV_ROUND_UP(ret, 64);
1604
1605         return ret;
1606 }
1607
1608 static void vlv_setup_wm_latency(struct drm_i915_private *dev_priv)
1609 {
1610         /* all latencies in usec */
1611         dev_priv->wm.pri_latency[VLV_WM_LEVEL_PM2] = 3;
1612
1613         dev_priv->wm.max_level = VLV_WM_LEVEL_PM2;
1614
1615         if (IS_CHERRYVIEW(dev_priv)) {
1616                 dev_priv->wm.pri_latency[VLV_WM_LEVEL_PM5] = 12;
1617                 dev_priv->wm.pri_latency[VLV_WM_LEVEL_DDR_DVFS] = 33;
1618
1619                 dev_priv->wm.max_level = VLV_WM_LEVEL_DDR_DVFS;
1620         }
1621 }
1622
1623 static u16 vlv_compute_wm_level(const struct intel_crtc_state *crtc_state,
1624                                 const struct intel_plane_state *plane_state,
1625                                 int level)
1626 {
1627         struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
1628         struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
1629         const struct drm_display_mode *adjusted_mode =
1630                 &crtc_state->hw.adjusted_mode;
1631         unsigned int clock, htotal, cpp, width, wm;
1632
1633         if (dev_priv->wm.pri_latency[level] == 0)
1634                 return USHRT_MAX;
1635
1636         if (!intel_wm_plane_visible(crtc_state, plane_state))
1637                 return 0;
1638
1639         cpp = plane_state->hw.fb->format->cpp[0];
1640         clock = adjusted_mode->crtc_clock;
1641         htotal = adjusted_mode->crtc_htotal;
1642         width = crtc_state->pipe_src_w;
1643
1644         if (plane->id == PLANE_CURSOR) {
1645                 /*
1646                  * FIXME the formula gives values that are
1647                  * too big for the cursor FIFO, and hence we
1648                  * would never be able to use cursors. For
1649                  * now just hardcode the watermark.
1650                  */
1651                 wm = 63;
1652         } else {
1653                 wm = vlv_wm_method2(clock, htotal, width, cpp,
1654                                     dev_priv->wm.pri_latency[level] * 10);
1655         }
1656
1657         return min_t(unsigned int, wm, USHRT_MAX);
1658 }
1659
1660 static bool vlv_need_sprite0_fifo_workaround(unsigned int active_planes)
1661 {
1662         return (active_planes & (BIT(PLANE_SPRITE0) |
1663                                  BIT(PLANE_SPRITE1))) == BIT(PLANE_SPRITE1);
1664 }
1665
1666 static int vlv_compute_fifo(struct intel_crtc_state *crtc_state)
1667 {
1668         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
1669         const struct g4x_pipe_wm *raw =
1670                 &crtc_state->wm.vlv.raw[VLV_WM_LEVEL_PM2];
1671         struct vlv_fifo_state *fifo_state = &crtc_state->wm.vlv.fifo_state;
1672         unsigned int active_planes = crtc_state->active_planes & ~BIT(PLANE_CURSOR);
1673         int num_active_planes = hweight8(active_planes);
1674         const int fifo_size = 511;
1675         int fifo_extra, fifo_left = fifo_size;
1676         int sprite0_fifo_extra = 0;
1677         unsigned int total_rate;
1678         enum plane_id plane_id;
1679
1680         /*
1681          * When enabling sprite0 after sprite1 has already been enabled
1682          * we tend to get an underrun unless sprite0 already has some
1683          * FIFO space allcoated. Hence we always allocate at least one
1684          * cacheline for sprite0 whenever sprite1 is enabled.
1685          *
1686          * All other plane enable sequences appear immune to this problem.
1687          */
1688         if (vlv_need_sprite0_fifo_workaround(active_planes))
1689                 sprite0_fifo_extra = 1;
1690
1691         total_rate = raw->plane[PLANE_PRIMARY] +
1692                 raw->plane[PLANE_SPRITE0] +
1693                 raw->plane[PLANE_SPRITE1] +
1694                 sprite0_fifo_extra;
1695
1696         if (total_rate > fifo_size)
1697                 return -EINVAL;
1698
1699         if (total_rate == 0)
1700                 total_rate = 1;
1701
1702         for_each_plane_id_on_crtc(crtc, plane_id) {
1703                 unsigned int rate;
1704
1705                 if ((active_planes & BIT(plane_id)) == 0) {
1706                         fifo_state->plane[plane_id] = 0;
1707                         continue;
1708                 }
1709
1710                 rate = raw->plane[plane_id];
1711                 fifo_state->plane[plane_id] = fifo_size * rate / total_rate;
1712                 fifo_left -= fifo_state->plane[plane_id];
1713         }
1714
1715         fifo_state->plane[PLANE_SPRITE0] += sprite0_fifo_extra;
1716         fifo_left -= sprite0_fifo_extra;
1717
1718         fifo_state->plane[PLANE_CURSOR] = 63;
1719
1720         fifo_extra = DIV_ROUND_UP(fifo_left, num_active_planes ?: 1);
1721
1722         /* spread the remainder evenly */
1723         for_each_plane_id_on_crtc(crtc, plane_id) {
1724                 int plane_extra;
1725
1726                 if (fifo_left == 0)
1727                         break;
1728
1729                 if ((active_planes & BIT(plane_id)) == 0)
1730                         continue;
1731
1732                 plane_extra = min(fifo_extra, fifo_left);
1733                 fifo_state->plane[plane_id] += plane_extra;
1734                 fifo_left -= plane_extra;
1735         }
1736
1737         WARN_ON(active_planes != 0 && fifo_left != 0);
1738
1739         /* give it all to the first plane if none are active */
1740         if (active_planes == 0) {
1741                 WARN_ON(fifo_left != fifo_size);
1742                 fifo_state->plane[PLANE_PRIMARY] = fifo_left;
1743         }
1744
1745         return 0;
1746 }
1747
1748 /* mark all levels starting from 'level' as invalid */
1749 static void vlv_invalidate_wms(struct intel_crtc *crtc,
1750                                struct vlv_wm_state *wm_state, int level)
1751 {
1752         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1753
1754         for (; level < intel_wm_num_levels(dev_priv); level++) {
1755                 enum plane_id plane_id;
1756
1757                 for_each_plane_id_on_crtc(crtc, plane_id)
1758                         wm_state->wm[level].plane[plane_id] = USHRT_MAX;
1759
1760                 wm_state->sr[level].cursor = USHRT_MAX;
1761                 wm_state->sr[level].plane = USHRT_MAX;
1762         }
1763 }
1764
1765 static u16 vlv_invert_wm_value(u16 wm, u16 fifo_size)
1766 {
1767         if (wm > fifo_size)
1768                 return USHRT_MAX;
1769         else
1770                 return fifo_size - wm;
1771 }
1772
1773 /*
1774  * Starting from 'level' set all higher
1775  * levels to 'value' in the "raw" watermarks.
1776  */
1777 static bool vlv_raw_plane_wm_set(struct intel_crtc_state *crtc_state,
1778                                  int level, enum plane_id plane_id, u16 value)
1779 {
1780         struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
1781         int num_levels = intel_wm_num_levels(dev_priv);
1782         bool dirty = false;
1783
1784         for (; level < num_levels; level++) {
1785                 struct g4x_pipe_wm *raw = &crtc_state->wm.vlv.raw[level];
1786
1787                 dirty |= raw->plane[plane_id] != value;
1788                 raw->plane[plane_id] = value;
1789         }
1790
1791         return dirty;
1792 }
1793
1794 static bool vlv_raw_plane_wm_compute(struct intel_crtc_state *crtc_state,
1795                                      const struct intel_plane_state *plane_state)
1796 {
1797         struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
1798         struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
1799         enum plane_id plane_id = plane->id;
1800         int num_levels = intel_wm_num_levels(to_i915(plane->base.dev));
1801         int level;
1802         bool dirty = false;
1803
1804         if (!intel_wm_plane_visible(crtc_state, plane_state)) {
1805                 dirty |= vlv_raw_plane_wm_set(crtc_state, 0, plane_id, 0);
1806                 goto out;
1807         }
1808
1809         for (level = 0; level < num_levels; level++) {
1810                 struct g4x_pipe_wm *raw = &crtc_state->wm.vlv.raw[level];
1811                 int wm = vlv_compute_wm_level(crtc_state, plane_state, level);
1812                 int max_wm = plane_id == PLANE_CURSOR ? 63 : 511;
1813
1814                 if (wm > max_wm)
1815                         break;
1816
1817                 dirty |= raw->plane[plane_id] != wm;
1818                 raw->plane[plane_id] = wm;
1819         }
1820
1821         /* mark all higher levels as invalid */
1822         dirty |= vlv_raw_plane_wm_set(crtc_state, level, plane_id, USHRT_MAX);
1823
1824 out:
1825         if (dirty)
1826                 drm_dbg_kms(&dev_priv->drm,
1827                             "%s watermarks: PM2=%d, PM5=%d, DDR DVFS=%d\n",
1828                             plane->base.name,
1829                             crtc_state->wm.vlv.raw[VLV_WM_LEVEL_PM2].plane[plane_id],
1830                             crtc_state->wm.vlv.raw[VLV_WM_LEVEL_PM5].plane[plane_id],
1831                             crtc_state->wm.vlv.raw[VLV_WM_LEVEL_DDR_DVFS].plane[plane_id]);
1832
1833         return dirty;
1834 }
1835
1836 static bool vlv_raw_plane_wm_is_valid(const struct intel_crtc_state *crtc_state,
1837                                       enum plane_id plane_id, int level)
1838 {
1839         const struct g4x_pipe_wm *raw =
1840                 &crtc_state->wm.vlv.raw[level];
1841         const struct vlv_fifo_state *fifo_state =
1842                 &crtc_state->wm.vlv.fifo_state;
1843
1844         return raw->plane[plane_id] <= fifo_state->plane[plane_id];
1845 }
1846
1847 static bool vlv_raw_crtc_wm_is_valid(const struct intel_crtc_state *crtc_state, int level)
1848 {
1849         return vlv_raw_plane_wm_is_valid(crtc_state, PLANE_PRIMARY, level) &&
1850                 vlv_raw_plane_wm_is_valid(crtc_state, PLANE_SPRITE0, level) &&
1851                 vlv_raw_plane_wm_is_valid(crtc_state, PLANE_SPRITE1, level) &&
1852                 vlv_raw_plane_wm_is_valid(crtc_state, PLANE_CURSOR, level);
1853 }
1854
1855 static int vlv_compute_pipe_wm(struct intel_crtc_state *crtc_state)
1856 {
1857         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
1858         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1859         struct intel_atomic_state *state =
1860                 to_intel_atomic_state(crtc_state->uapi.state);
1861         struct vlv_wm_state *wm_state = &crtc_state->wm.vlv.optimal;
1862         const struct vlv_fifo_state *fifo_state =
1863                 &crtc_state->wm.vlv.fifo_state;
1864         int num_active_planes = hweight8(crtc_state->active_planes &
1865                                          ~BIT(PLANE_CURSOR));
1866         bool needs_modeset = drm_atomic_crtc_needs_modeset(&crtc_state->uapi);
1867         const struct intel_plane_state *old_plane_state;
1868         const struct intel_plane_state *new_plane_state;
1869         struct intel_plane *plane;
1870         enum plane_id plane_id;
1871         int level, ret, i;
1872         unsigned int dirty = 0;
1873
1874         for_each_oldnew_intel_plane_in_state(state, plane,
1875                                              old_plane_state,
1876                                              new_plane_state, i) {
1877                 if (new_plane_state->hw.crtc != &crtc->base &&
1878                     old_plane_state->hw.crtc != &crtc->base)
1879                         continue;
1880
1881                 if (vlv_raw_plane_wm_compute(crtc_state, new_plane_state))
1882                         dirty |= BIT(plane->id);
1883         }
1884
1885         /*
1886          * DSPARB registers may have been reset due to the
1887          * power well being turned off. Make sure we restore
1888          * them to a consistent state even if no primary/sprite
1889          * planes are initially active.
1890          */
1891         if (needs_modeset)
1892                 crtc_state->fifo_changed = true;
1893
1894         if (!dirty)
1895                 return 0;
1896
1897         /* cursor changes don't warrant a FIFO recompute */
1898         if (dirty & ~BIT(PLANE_CURSOR)) {
1899                 const struct intel_crtc_state *old_crtc_state =
1900                         intel_atomic_get_old_crtc_state(state, crtc);
1901                 const struct vlv_fifo_state *old_fifo_state =
1902                         &old_crtc_state->wm.vlv.fifo_state;
1903
1904                 ret = vlv_compute_fifo(crtc_state);
1905                 if (ret)
1906                         return ret;
1907
1908                 if (needs_modeset ||
1909                     memcmp(old_fifo_state, fifo_state,
1910                            sizeof(*fifo_state)) != 0)
1911                         crtc_state->fifo_changed = true;
1912         }
1913
1914         /* initially allow all levels */
1915         wm_state->num_levels = intel_wm_num_levels(dev_priv);
1916         /*
1917          * Note that enabling cxsr with no primary/sprite planes
1918          * enabled can wedge the pipe. Hence we only allow cxsr
1919          * with exactly one enabled primary/sprite plane.
1920          */
1921         wm_state->cxsr = crtc->pipe != PIPE_C && num_active_planes == 1;
1922
1923         for (level = 0; level < wm_state->num_levels; level++) {
1924                 const struct g4x_pipe_wm *raw = &crtc_state->wm.vlv.raw[level];
1925                 const int sr_fifo_size = INTEL_NUM_PIPES(dev_priv) * 512 - 1;
1926
1927                 if (!vlv_raw_crtc_wm_is_valid(crtc_state, level))
1928                         break;
1929
1930                 for_each_plane_id_on_crtc(crtc, plane_id) {
1931                         wm_state->wm[level].plane[plane_id] =
1932                                 vlv_invert_wm_value(raw->plane[plane_id],
1933                                                     fifo_state->plane[plane_id]);
1934                 }
1935
1936                 wm_state->sr[level].plane =
1937                         vlv_invert_wm_value(max3(raw->plane[PLANE_PRIMARY],
1938                                                  raw->plane[PLANE_SPRITE0],
1939                                                  raw->plane[PLANE_SPRITE1]),
1940                                             sr_fifo_size);
1941
1942                 wm_state->sr[level].cursor =
1943                         vlv_invert_wm_value(raw->plane[PLANE_CURSOR],
1944                                             63);
1945         }
1946
1947         if (level == 0)
1948                 return -EINVAL;
1949
1950         /* limit to only levels we can actually handle */
1951         wm_state->num_levels = level;
1952
1953         /* invalidate the higher levels */
1954         vlv_invalidate_wms(crtc, wm_state, level);
1955
1956         return 0;
1957 }
1958
1959 #define VLV_FIFO(plane, value) \
1960         (((value) << DSPARB_ ## plane ## _SHIFT_VLV) & DSPARB_ ## plane ## _MASK_VLV)
1961
1962 static void vlv_atomic_update_fifo(struct intel_atomic_state *state,
1963                                    struct intel_crtc *crtc)
1964 {
1965         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1966         struct intel_uncore *uncore = &dev_priv->uncore;
1967         const struct intel_crtc_state *crtc_state =
1968                 intel_atomic_get_new_crtc_state(state, crtc);
1969         const struct vlv_fifo_state *fifo_state =
1970                 &crtc_state->wm.vlv.fifo_state;
1971         int sprite0_start, sprite1_start, fifo_size;
1972
1973         if (!crtc_state->fifo_changed)
1974                 return;
1975
1976         sprite0_start = fifo_state->plane[PLANE_PRIMARY];
1977         sprite1_start = fifo_state->plane[PLANE_SPRITE0] + sprite0_start;
1978         fifo_size = fifo_state->plane[PLANE_SPRITE1] + sprite1_start;
1979
1980         drm_WARN_ON(&dev_priv->drm, fifo_state->plane[PLANE_CURSOR] != 63);
1981         drm_WARN_ON(&dev_priv->drm, fifo_size != 511);
1982
1983         trace_vlv_fifo_size(crtc, sprite0_start, sprite1_start, fifo_size);
1984
1985         /*
1986          * uncore.lock serves a double purpose here. It allows us to
1987          * use the less expensive I915_{READ,WRITE}_FW() functions, and
1988          * it protects the DSPARB registers from getting clobbered by
1989          * parallel updates from multiple pipes.
1990          *
1991          * intel_pipe_update_start() has already disabled interrupts
1992          * for us, so a plain spin_lock() is sufficient here.
1993          */
1994         spin_lock(&uncore->lock);
1995
1996         switch (crtc->pipe) {
1997                 u32 dsparb, dsparb2, dsparb3;
1998         case PIPE_A:
1999                 dsparb = intel_uncore_read_fw(uncore, DSPARB);
2000                 dsparb2 = intel_uncore_read_fw(uncore, DSPARB2);
2001
2002                 dsparb &= ~(VLV_FIFO(SPRITEA, 0xff) |
2003                             VLV_FIFO(SPRITEB, 0xff));
2004                 dsparb |= (VLV_FIFO(SPRITEA, sprite0_start) |
2005                            VLV_FIFO(SPRITEB, sprite1_start));
2006
2007                 dsparb2 &= ~(VLV_FIFO(SPRITEA_HI, 0x1) |
2008                              VLV_FIFO(SPRITEB_HI, 0x1));
2009                 dsparb2 |= (VLV_FIFO(SPRITEA_HI, sprite0_start >> 8) |
2010                            VLV_FIFO(SPRITEB_HI, sprite1_start >> 8));
2011
2012                 intel_uncore_write_fw(uncore, DSPARB, dsparb);
2013                 intel_uncore_write_fw(uncore, DSPARB2, dsparb2);
2014                 break;
2015         case PIPE_B:
2016                 dsparb = intel_uncore_read_fw(uncore, DSPARB);
2017                 dsparb2 = intel_uncore_read_fw(uncore, DSPARB2);
2018
2019                 dsparb &= ~(VLV_FIFO(SPRITEC, 0xff) |
2020                             VLV_FIFO(SPRITED, 0xff));
2021                 dsparb |= (VLV_FIFO(SPRITEC, sprite0_start) |
2022                            VLV_FIFO(SPRITED, sprite1_start));
2023
2024                 dsparb2 &= ~(VLV_FIFO(SPRITEC_HI, 0xff) |
2025                              VLV_FIFO(SPRITED_HI, 0xff));
2026                 dsparb2 |= (VLV_FIFO(SPRITEC_HI, sprite0_start >> 8) |
2027                            VLV_FIFO(SPRITED_HI, sprite1_start >> 8));
2028
2029                 intel_uncore_write_fw(uncore, DSPARB, dsparb);
2030                 intel_uncore_write_fw(uncore, DSPARB2, dsparb2);
2031                 break;
2032         case PIPE_C:
2033                 dsparb3 = intel_uncore_read_fw(uncore, DSPARB3);
2034                 dsparb2 = intel_uncore_read_fw(uncore, DSPARB2);
2035
2036                 dsparb3 &= ~(VLV_FIFO(SPRITEE, 0xff) |
2037                              VLV_FIFO(SPRITEF, 0xff));
2038                 dsparb3 |= (VLV_FIFO(SPRITEE, sprite0_start) |
2039                             VLV_FIFO(SPRITEF, sprite1_start));
2040
2041                 dsparb2 &= ~(VLV_FIFO(SPRITEE_HI, 0xff) |
2042                              VLV_FIFO(SPRITEF_HI, 0xff));
2043                 dsparb2 |= (VLV_FIFO(SPRITEE_HI, sprite0_start >> 8) |
2044                            VLV_FIFO(SPRITEF_HI, sprite1_start >> 8));
2045
2046                 intel_uncore_write_fw(uncore, DSPARB3, dsparb3);
2047                 intel_uncore_write_fw(uncore, DSPARB2, dsparb2);
2048                 break;
2049         default:
2050                 break;
2051         }
2052
2053         intel_uncore_posting_read_fw(uncore, DSPARB);
2054
2055         spin_unlock(&uncore->lock);
2056 }
2057
2058 #undef VLV_FIFO
2059
2060 static int vlv_compute_intermediate_wm(struct intel_crtc_state *new_crtc_state)
2061 {
2062         struct intel_crtc *crtc = to_intel_crtc(new_crtc_state->uapi.crtc);
2063         struct vlv_wm_state *intermediate = &new_crtc_state->wm.vlv.intermediate;
2064         const struct vlv_wm_state *optimal = &new_crtc_state->wm.vlv.optimal;
2065         struct intel_atomic_state *intel_state =
2066                 to_intel_atomic_state(new_crtc_state->uapi.state);
2067         const struct intel_crtc_state *old_crtc_state =
2068                 intel_atomic_get_old_crtc_state(intel_state, crtc);
2069         const struct vlv_wm_state *active = &old_crtc_state->wm.vlv.optimal;
2070         int level;
2071
2072         if (!new_crtc_state->hw.active || drm_atomic_crtc_needs_modeset(&new_crtc_state->uapi)) {
2073                 *intermediate = *optimal;
2074
2075                 intermediate->cxsr = false;
2076                 goto out;
2077         }
2078
2079         intermediate->num_levels = min(optimal->num_levels, active->num_levels);
2080         intermediate->cxsr = optimal->cxsr && active->cxsr &&
2081                 !new_crtc_state->disable_cxsr;
2082
2083         for (level = 0; level < intermediate->num_levels; level++) {
2084                 enum plane_id plane_id;
2085
2086                 for_each_plane_id_on_crtc(crtc, plane_id) {
2087                         intermediate->wm[level].plane[plane_id] =
2088                                 min(optimal->wm[level].plane[plane_id],
2089                                     active->wm[level].plane[plane_id]);
2090                 }
2091
2092                 intermediate->sr[level].plane = min(optimal->sr[level].plane,
2093                                                     active->sr[level].plane);
2094                 intermediate->sr[level].cursor = min(optimal->sr[level].cursor,
2095                                                      active->sr[level].cursor);
2096         }
2097
2098         vlv_invalidate_wms(crtc, intermediate, level);
2099
2100 out:
2101         /*
2102          * If our intermediate WM are identical to the final WM, then we can
2103          * omit the post-vblank programming; only update if it's different.
2104          */
2105         if (memcmp(intermediate, optimal, sizeof(*intermediate)) != 0)
2106                 new_crtc_state->wm.need_postvbl_update = true;
2107
2108         return 0;
2109 }
2110
2111 static void vlv_merge_wm(struct drm_i915_private *dev_priv,
2112                          struct vlv_wm_values *wm)
2113 {
2114         struct intel_crtc *crtc;
2115         int num_active_pipes = 0;
2116
2117         wm->level = dev_priv->wm.max_level;
2118         wm->cxsr = true;
2119
2120         for_each_intel_crtc(&dev_priv->drm, crtc) {
2121                 const struct vlv_wm_state *wm_state = &crtc->wm.active.vlv;
2122
2123                 if (!crtc->active)
2124                         continue;
2125
2126                 if (!wm_state->cxsr)
2127                         wm->cxsr = false;
2128
2129                 num_active_pipes++;
2130                 wm->level = min_t(int, wm->level, wm_state->num_levels - 1);
2131         }
2132
2133         if (num_active_pipes != 1)
2134                 wm->cxsr = false;
2135
2136         if (num_active_pipes > 1)
2137                 wm->level = VLV_WM_LEVEL_PM2;
2138
2139         for_each_intel_crtc(&dev_priv->drm, crtc) {
2140                 const struct vlv_wm_state *wm_state = &crtc->wm.active.vlv;
2141                 enum pipe pipe = crtc->pipe;
2142
2143                 wm->pipe[pipe] = wm_state->wm[wm->level];
2144                 if (crtc->active && wm->cxsr)
2145                         wm->sr = wm_state->sr[wm->level];
2146
2147                 wm->ddl[pipe].plane[PLANE_PRIMARY] = DDL_PRECISION_HIGH | 2;
2148                 wm->ddl[pipe].plane[PLANE_SPRITE0] = DDL_PRECISION_HIGH | 2;
2149                 wm->ddl[pipe].plane[PLANE_SPRITE1] = DDL_PRECISION_HIGH | 2;
2150                 wm->ddl[pipe].plane[PLANE_CURSOR] = DDL_PRECISION_HIGH | 2;
2151         }
2152 }
2153
2154 static void vlv_program_watermarks(struct drm_i915_private *dev_priv)
2155 {
2156         struct vlv_wm_values *old_wm = &dev_priv->wm.vlv;
2157         struct vlv_wm_values new_wm = {};
2158
2159         vlv_merge_wm(dev_priv, &new_wm);
2160
2161         if (memcmp(old_wm, &new_wm, sizeof(new_wm)) == 0)
2162                 return;
2163
2164         if (is_disabling(old_wm->level, new_wm.level, VLV_WM_LEVEL_DDR_DVFS))
2165                 chv_set_memory_dvfs(dev_priv, false);
2166
2167         if (is_disabling(old_wm->level, new_wm.level, VLV_WM_LEVEL_PM5))
2168                 chv_set_memory_pm5(dev_priv, false);
2169
2170         if (is_disabling(old_wm->cxsr, new_wm.cxsr, true))
2171                 _intel_set_memory_cxsr(dev_priv, false);
2172
2173         vlv_write_wm_values(dev_priv, &new_wm);
2174
2175         if (is_enabling(old_wm->cxsr, new_wm.cxsr, true))
2176                 _intel_set_memory_cxsr(dev_priv, true);
2177
2178         if (is_enabling(old_wm->level, new_wm.level, VLV_WM_LEVEL_PM5))
2179                 chv_set_memory_pm5(dev_priv, true);
2180
2181         if (is_enabling(old_wm->level, new_wm.level, VLV_WM_LEVEL_DDR_DVFS))
2182                 chv_set_memory_dvfs(dev_priv, true);
2183
2184         *old_wm = new_wm;
2185 }
2186
2187 static void vlv_initial_watermarks(struct intel_atomic_state *state,
2188                                    struct intel_crtc *crtc)
2189 {
2190         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
2191         const struct intel_crtc_state *crtc_state =
2192                 intel_atomic_get_new_crtc_state(state, crtc);
2193
2194         mutex_lock(&dev_priv->wm.wm_mutex);
2195         crtc->wm.active.vlv = crtc_state->wm.vlv.intermediate;
2196         vlv_program_watermarks(dev_priv);
2197         mutex_unlock(&dev_priv->wm.wm_mutex);
2198 }
2199
2200 static void vlv_optimize_watermarks(struct intel_atomic_state *state,
2201                                     struct intel_crtc *crtc)
2202 {
2203         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
2204         const struct intel_crtc_state *crtc_state =
2205                 intel_atomic_get_new_crtc_state(state, crtc);
2206
2207         if (!crtc_state->wm.need_postvbl_update)
2208                 return;
2209
2210         mutex_lock(&dev_priv->wm.wm_mutex);
2211         crtc->wm.active.vlv = crtc_state->wm.vlv.optimal;
2212         vlv_program_watermarks(dev_priv);
2213         mutex_unlock(&dev_priv->wm.wm_mutex);
2214 }
2215
2216 static void i965_update_wm(struct intel_crtc *unused_crtc)
2217 {
2218         struct drm_i915_private *dev_priv = to_i915(unused_crtc->base.dev);
2219         struct intel_crtc *crtc;
2220         int srwm = 1;
2221         int cursor_sr = 16;
2222         bool cxsr_enabled;
2223
2224         /* Calc sr entries for one plane configs */
2225         crtc = single_enabled_crtc(dev_priv);
2226         if (crtc) {
2227                 /* self-refresh has much higher latency */
2228                 static const int sr_latency_ns = 12000;
2229                 const struct drm_display_mode *adjusted_mode =
2230                         &crtc->config->hw.adjusted_mode;
2231                 const struct drm_framebuffer *fb =
2232                         crtc->base.primary->state->fb;
2233                 int clock = adjusted_mode->crtc_clock;
2234                 int htotal = adjusted_mode->crtc_htotal;
2235                 int hdisplay = crtc->config->pipe_src_w;
2236                 int cpp = fb->format->cpp[0];
2237                 int entries;
2238
2239                 entries = intel_wm_method2(clock, htotal,
2240                                            hdisplay, cpp, sr_latency_ns / 100);
2241                 entries = DIV_ROUND_UP(entries, I915_FIFO_LINE_SIZE);
2242                 srwm = I965_FIFO_SIZE - entries;
2243                 if (srwm < 0)
2244                         srwm = 1;
2245                 srwm &= 0x1ff;
2246                 drm_dbg_kms(&dev_priv->drm,
2247                             "self-refresh entries: %d, wm: %d\n",
2248                             entries, srwm);
2249
2250                 entries = intel_wm_method2(clock, htotal,
2251                                            crtc->base.cursor->state->crtc_w, 4,
2252                                            sr_latency_ns / 100);
2253                 entries = DIV_ROUND_UP(entries,
2254                                        i965_cursor_wm_info.cacheline_size) +
2255                         i965_cursor_wm_info.guard_size;
2256
2257                 cursor_sr = i965_cursor_wm_info.fifo_size - entries;
2258                 if (cursor_sr > i965_cursor_wm_info.max_wm)
2259                         cursor_sr = i965_cursor_wm_info.max_wm;
2260
2261                 drm_dbg_kms(&dev_priv->drm,
2262                             "self-refresh watermark: display plane %d "
2263                             "cursor %d\n", srwm, cursor_sr);
2264
2265                 cxsr_enabled = true;
2266         } else {
2267                 cxsr_enabled = false;
2268                 /* Turn off self refresh if both pipes are enabled */
2269                 intel_set_memory_cxsr(dev_priv, false);
2270         }
2271
2272         drm_dbg_kms(&dev_priv->drm,
2273                     "Setting FIFO watermarks - A: 8, B: 8, C: 8, SR %d\n",
2274                     srwm);
2275
2276         /* 965 has limitations... */
2277         I915_WRITE(DSPFW1, FW_WM(srwm, SR) |
2278                    FW_WM(8, CURSORB) |
2279                    FW_WM(8, PLANEB) |
2280                    FW_WM(8, PLANEA));
2281         I915_WRITE(DSPFW2, FW_WM(8, CURSORA) |
2282                    FW_WM(8, PLANEC_OLD));
2283         /* update cursor SR watermark */
2284         I915_WRITE(DSPFW3, FW_WM(cursor_sr, CURSOR_SR));
2285
2286         if (cxsr_enabled)
2287                 intel_set_memory_cxsr(dev_priv, true);
2288 }
2289
2290 #undef FW_WM
2291
2292 static void i9xx_update_wm(struct intel_crtc *unused_crtc)
2293 {
2294         struct drm_i915_private *dev_priv = to_i915(unused_crtc->base.dev);
2295         const struct intel_watermark_params *wm_info;
2296         u32 fwater_lo;
2297         u32 fwater_hi;
2298         int cwm, srwm = 1;
2299         int fifo_size;
2300         int planea_wm, planeb_wm;
2301         struct intel_crtc *crtc, *enabled = NULL;
2302
2303         if (IS_I945GM(dev_priv))
2304                 wm_info = &i945_wm_info;
2305         else if (!IS_GEN(dev_priv, 2))
2306                 wm_info = &i915_wm_info;
2307         else
2308                 wm_info = &i830_a_wm_info;
2309
2310         fifo_size = dev_priv->display.get_fifo_size(dev_priv, PLANE_A);
2311         crtc = intel_get_crtc_for_plane(dev_priv, PLANE_A);
2312         if (intel_crtc_active(crtc)) {
2313                 const struct drm_display_mode *adjusted_mode =
2314                         &crtc->config->hw.adjusted_mode;
2315                 const struct drm_framebuffer *fb =
2316                         crtc->base.primary->state->fb;
2317                 int cpp;
2318
2319                 if (IS_GEN(dev_priv, 2))
2320                         cpp = 4;
2321                 else
2322                         cpp = fb->format->cpp[0];
2323
2324                 planea_wm = intel_calculate_wm(adjusted_mode->crtc_clock,
2325                                                wm_info, fifo_size, cpp,
2326                                                pessimal_latency_ns);
2327                 enabled = crtc;
2328         } else {
2329                 planea_wm = fifo_size - wm_info->guard_size;
2330                 if (planea_wm > (long)wm_info->max_wm)
2331                         planea_wm = wm_info->max_wm;
2332         }
2333
2334         if (IS_GEN(dev_priv, 2))
2335                 wm_info = &i830_bc_wm_info;
2336
2337         fifo_size = dev_priv->display.get_fifo_size(dev_priv, PLANE_B);
2338         crtc = intel_get_crtc_for_plane(dev_priv, PLANE_B);
2339         if (intel_crtc_active(crtc)) {
2340                 const struct drm_display_mode *adjusted_mode =
2341                         &crtc->config->hw.adjusted_mode;
2342                 const struct drm_framebuffer *fb =
2343                         crtc->base.primary->state->fb;
2344                 int cpp;
2345
2346                 if (IS_GEN(dev_priv, 2))
2347                         cpp = 4;
2348                 else
2349                         cpp = fb->format->cpp[0];
2350
2351                 planeb_wm = intel_calculate_wm(adjusted_mode->crtc_clock,
2352                                                wm_info, fifo_size, cpp,
2353                                                pessimal_latency_ns);
2354                 if (enabled == NULL)
2355                         enabled = crtc;
2356                 else
2357                         enabled = NULL;
2358         } else {
2359                 planeb_wm = fifo_size - wm_info->guard_size;
2360                 if (planeb_wm > (long)wm_info->max_wm)
2361                         planeb_wm = wm_info->max_wm;
2362         }
2363
2364         drm_dbg_kms(&dev_priv->drm,
2365                     "FIFO watermarks - A: %d, B: %d\n", planea_wm, planeb_wm);
2366
2367         if (IS_I915GM(dev_priv) && enabled) {
2368                 struct drm_i915_gem_object *obj;
2369
2370                 obj = intel_fb_obj(enabled->base.primary->state->fb);
2371
2372                 /* self-refresh seems busted with untiled */
2373                 if (!i915_gem_object_is_tiled(obj))
2374                         enabled = NULL;
2375         }
2376
2377         /*
2378          * Overlay gets an aggressive default since video jitter is bad.
2379          */
2380         cwm = 2;
2381
2382         /* Play safe and disable self-refresh before adjusting watermarks. */
2383         intel_set_memory_cxsr(dev_priv, false);
2384
2385         /* Calc sr entries for one plane configs */
2386         if (HAS_FW_BLC(dev_priv) && enabled) {
2387                 /* self-refresh has much higher latency */
2388                 static const int sr_latency_ns = 6000;
2389                 const struct drm_display_mode *adjusted_mode =
2390                         &enabled->config->hw.adjusted_mode;
2391                 const struct drm_framebuffer *fb =
2392                         enabled->base.primary->state->fb;
2393                 int clock = adjusted_mode->crtc_clock;
2394                 int htotal = adjusted_mode->crtc_htotal;
2395                 int hdisplay = enabled->config->pipe_src_w;
2396                 int cpp;
2397                 int entries;
2398
2399                 if (IS_I915GM(dev_priv) || IS_I945GM(dev_priv))
2400                         cpp = 4;
2401                 else
2402                         cpp = fb->format->cpp[0];
2403
2404                 entries = intel_wm_method2(clock, htotal, hdisplay, cpp,
2405                                            sr_latency_ns / 100);
2406                 entries = DIV_ROUND_UP(entries, wm_info->cacheline_size);
2407                 drm_dbg_kms(&dev_priv->drm,
2408                             "self-refresh entries: %d\n", entries);
2409                 srwm = wm_info->fifo_size - entries;
2410                 if (srwm < 0)
2411                         srwm = 1;
2412
2413                 if (IS_I945G(dev_priv) || IS_I945GM(dev_priv))
2414                         I915_WRITE(FW_BLC_SELF,
2415                                    FW_BLC_SELF_FIFO_MASK | (srwm & 0xff));
2416                 else
2417                         I915_WRITE(FW_BLC_SELF, srwm & 0x3f);
2418         }
2419
2420         drm_dbg_kms(&dev_priv->drm,
2421                     "Setting FIFO watermarks - A: %d, B: %d, C: %d, SR %d\n",
2422                      planea_wm, planeb_wm, cwm, srwm);
2423
2424         fwater_lo = ((planeb_wm & 0x3f) << 16) | (planea_wm & 0x3f);
2425         fwater_hi = (cwm & 0x1f);
2426
2427         /* Set request length to 8 cachelines per fetch */
2428         fwater_lo = fwater_lo | (1 << 24) | (1 << 8);
2429         fwater_hi = fwater_hi | (1 << 8);
2430
2431         I915_WRITE(FW_BLC, fwater_lo);
2432         I915_WRITE(FW_BLC2, fwater_hi);
2433
2434         if (enabled)
2435                 intel_set_memory_cxsr(dev_priv, true);
2436 }
2437
2438 static void i845_update_wm(struct intel_crtc *unused_crtc)
2439 {
2440         struct drm_i915_private *dev_priv = to_i915(unused_crtc->base.dev);
2441         struct intel_crtc *crtc;
2442         const struct drm_display_mode *adjusted_mode;
2443         u32 fwater_lo;
2444         int planea_wm;
2445
2446         crtc = single_enabled_crtc(dev_priv);
2447         if (crtc == NULL)
2448                 return;
2449
2450         adjusted_mode = &crtc->config->hw.adjusted_mode;
2451         planea_wm = intel_calculate_wm(adjusted_mode->crtc_clock,
2452                                        &i845_wm_info,
2453                                        dev_priv->display.get_fifo_size(dev_priv, PLANE_A),
2454                                        4, pessimal_latency_ns);
2455         fwater_lo = I915_READ(FW_BLC) & ~0xfff;
2456         fwater_lo |= (3<<8) | planea_wm;
2457
2458         drm_dbg_kms(&dev_priv->drm,
2459                     "Setting FIFO watermarks - A: %d\n", planea_wm);
2460
2461         I915_WRITE(FW_BLC, fwater_lo);
2462 }
2463
2464 /* latency must be in 0.1us units. */
2465 static unsigned int ilk_wm_method1(unsigned int pixel_rate,
2466                                    unsigned int cpp,
2467                                    unsigned int latency)
2468 {
2469         unsigned int ret;
2470
2471         ret = intel_wm_method1(pixel_rate, cpp, latency);
2472         ret = DIV_ROUND_UP(ret, 64) + 2;
2473
2474         return ret;
2475 }
2476
2477 /* latency must be in 0.1us units. */
2478 static unsigned int ilk_wm_method2(unsigned int pixel_rate,
2479                                    unsigned int htotal,
2480                                    unsigned int width,
2481                                    unsigned int cpp,
2482                                    unsigned int latency)
2483 {
2484         unsigned int ret;
2485
2486         ret = intel_wm_method2(pixel_rate, htotal,
2487                                width, cpp, latency);
2488         ret = DIV_ROUND_UP(ret, 64) + 2;
2489
2490         return ret;
2491 }
2492
2493 static u32 ilk_wm_fbc(u32 pri_val, u32 horiz_pixels, u8 cpp)
2494 {
2495         /*
2496          * Neither of these should be possible since this function shouldn't be
2497          * called if the CRTC is off or the plane is invisible.  But let's be
2498          * extra paranoid to avoid a potential divide-by-zero if we screw up
2499          * elsewhere in the driver.
2500          */
2501         if (WARN_ON(!cpp))
2502                 return 0;
2503         if (WARN_ON(!horiz_pixels))
2504                 return 0;
2505
2506         return DIV_ROUND_UP(pri_val * 64, horiz_pixels * cpp) + 2;
2507 }
2508
2509 struct ilk_wm_maximums {
2510         u16 pri;
2511         u16 spr;
2512         u16 cur;
2513         u16 fbc;
2514 };
2515
2516 /*
2517  * For both WM_PIPE and WM_LP.
2518  * mem_value must be in 0.1us units.
2519  */
2520 static u32 ilk_compute_pri_wm(const struct intel_crtc_state *crtc_state,
2521                               const struct intel_plane_state *plane_state,
2522                               u32 mem_value, bool is_lp)
2523 {
2524         u32 method1, method2;
2525         int cpp;
2526
2527         if (mem_value == 0)
2528                 return U32_MAX;
2529
2530         if (!intel_wm_plane_visible(crtc_state, plane_state))
2531                 return 0;
2532
2533         cpp = plane_state->hw.fb->format->cpp[0];
2534
2535         method1 = ilk_wm_method1(crtc_state->pixel_rate, cpp, mem_value);
2536
2537         if (!is_lp)
2538                 return method1;
2539
2540         method2 = ilk_wm_method2(crtc_state->pixel_rate,
2541                                  crtc_state->hw.adjusted_mode.crtc_htotal,
2542                                  drm_rect_width(&plane_state->uapi.dst),
2543                                  cpp, mem_value);
2544
2545         return min(method1, method2);
2546 }
2547
2548 /*
2549  * For both WM_PIPE and WM_LP.
2550  * mem_value must be in 0.1us units.
2551  */
2552 static u32 ilk_compute_spr_wm(const struct intel_crtc_state *crtc_state,
2553                               const struct intel_plane_state *plane_state,
2554                               u32 mem_value)
2555 {
2556         u32 method1, method2;
2557         int cpp;
2558
2559         if (mem_value == 0)
2560                 return U32_MAX;
2561
2562         if (!intel_wm_plane_visible(crtc_state, plane_state))
2563                 return 0;
2564
2565         cpp = plane_state->hw.fb->format->cpp[0];
2566
2567         method1 = ilk_wm_method1(crtc_state->pixel_rate, cpp, mem_value);
2568         method2 = ilk_wm_method2(crtc_state->pixel_rate,
2569                                  crtc_state->hw.adjusted_mode.crtc_htotal,
2570                                  drm_rect_width(&plane_state->uapi.dst),
2571                                  cpp, mem_value);
2572         return min(method1, method2);
2573 }
2574
2575 /*
2576  * For both WM_PIPE and WM_LP.
2577  * mem_value must be in 0.1us units.
2578  */
2579 static u32 ilk_compute_cur_wm(const struct intel_crtc_state *crtc_state,
2580                               const struct intel_plane_state *plane_state,
2581                               u32 mem_value)
2582 {
2583         int cpp;
2584
2585         if (mem_value == 0)
2586                 return U32_MAX;
2587
2588         if (!intel_wm_plane_visible(crtc_state, plane_state))
2589                 return 0;
2590
2591         cpp = plane_state->hw.fb->format->cpp[0];
2592
2593         return ilk_wm_method2(crtc_state->pixel_rate,
2594                               crtc_state->hw.adjusted_mode.crtc_htotal,
2595                               drm_rect_width(&plane_state->uapi.dst),
2596                               cpp, mem_value);
2597 }
2598
2599 /* Only for WM_LP. */
2600 static u32 ilk_compute_fbc_wm(const struct intel_crtc_state *crtc_state,
2601                               const struct intel_plane_state *plane_state,
2602                               u32 pri_val)
2603 {
2604         int cpp;
2605
2606         if (!intel_wm_plane_visible(crtc_state, plane_state))
2607                 return 0;
2608
2609         cpp = plane_state->hw.fb->format->cpp[0];
2610
2611         return ilk_wm_fbc(pri_val, drm_rect_width(&plane_state->uapi.dst),
2612                           cpp);
2613 }
2614
2615 static unsigned int
2616 ilk_display_fifo_size(const struct drm_i915_private *dev_priv)
2617 {
2618         if (INTEL_GEN(dev_priv) >= 8)
2619                 return 3072;
2620         else if (INTEL_GEN(dev_priv) >= 7)
2621                 return 768;
2622         else
2623                 return 512;
2624 }
2625
2626 static unsigned int
2627 ilk_plane_wm_reg_max(const struct drm_i915_private *dev_priv,
2628                      int level, bool is_sprite)
2629 {
2630         if (INTEL_GEN(dev_priv) >= 8)
2631                 /* BDW primary/sprite plane watermarks */
2632                 return level == 0 ? 255 : 2047;
2633         else if (INTEL_GEN(dev_priv) >= 7)
2634                 /* IVB/HSW primary/sprite plane watermarks */
2635                 return level == 0 ? 127 : 1023;
2636         else if (!is_sprite)
2637                 /* ILK/SNB primary plane watermarks */
2638                 return level == 0 ? 127 : 511;
2639         else
2640                 /* ILK/SNB sprite plane watermarks */
2641                 return level == 0 ? 63 : 255;
2642 }
2643
2644 static unsigned int
2645 ilk_cursor_wm_reg_max(const struct drm_i915_private *dev_priv, int level)
2646 {
2647         if (INTEL_GEN(dev_priv) >= 7)
2648                 return level == 0 ? 63 : 255;
2649         else
2650                 return level == 0 ? 31 : 63;
2651 }
2652
2653 static unsigned int ilk_fbc_wm_reg_max(const struct drm_i915_private *dev_priv)
2654 {
2655         if (INTEL_GEN(dev_priv) >= 8)
2656                 return 31;
2657         else
2658                 return 15;
2659 }
2660
2661 /* Calculate the maximum primary/sprite plane watermark */
2662 static unsigned int ilk_plane_wm_max(const struct drm_i915_private *dev_priv,
2663                                      int level,
2664                                      const struct intel_wm_config *config,
2665                                      enum intel_ddb_partitioning ddb_partitioning,
2666                                      bool is_sprite)
2667 {
2668         unsigned int fifo_size = ilk_display_fifo_size(dev_priv);
2669
2670         /* if sprites aren't enabled, sprites get nothing */
2671         if (is_sprite && !config->sprites_enabled)
2672                 return 0;
2673
2674         /* HSW allows LP1+ watermarks even with multiple pipes */
2675         if (level == 0 || config->num_pipes_active > 1) {
2676                 fifo_size /= INTEL_NUM_PIPES(dev_priv);
2677
2678                 /*
2679                  * For some reason the non self refresh
2680                  * FIFO size is only half of the self
2681                  * refresh FIFO size on ILK/SNB.
2682                  */
2683                 if (INTEL_GEN(dev_priv) <= 6)
2684                         fifo_size /= 2;
2685         }
2686
2687         if (config->sprites_enabled) {
2688                 /* level 0 is always calculated with 1:1 split */
2689                 if (level > 0 && ddb_partitioning == INTEL_DDB_PART_5_6) {
2690                         if (is_sprite)
2691                                 fifo_size *= 5;
2692                         fifo_size /= 6;
2693                 } else {
2694                         fifo_size /= 2;
2695                 }
2696         }
2697
2698         /* clamp to max that the registers can hold */
2699         return min(fifo_size, ilk_plane_wm_reg_max(dev_priv, level, is_sprite));
2700 }
2701
2702 /* Calculate the maximum cursor plane watermark */
2703 static unsigned int ilk_cursor_wm_max(const struct drm_i915_private *dev_priv,
2704                                       int level,
2705                                       const struct intel_wm_config *config)
2706 {
2707         /* HSW LP1+ watermarks w/ multiple pipes */
2708         if (level > 0 && config->num_pipes_active > 1)
2709                 return 64;
2710
2711         /* otherwise just report max that registers can hold */
2712         return ilk_cursor_wm_reg_max(dev_priv, level);
2713 }
2714
2715 static void ilk_compute_wm_maximums(const struct drm_i915_private *dev_priv,
2716                                     int level,
2717                                     const struct intel_wm_config *config,
2718                                     enum intel_ddb_partitioning ddb_partitioning,
2719                                     struct ilk_wm_maximums *max)
2720 {
2721         max->pri = ilk_plane_wm_max(dev_priv, level, config, ddb_partitioning, false);
2722         max->spr = ilk_plane_wm_max(dev_priv, level, config, ddb_partitioning, true);
2723         max->cur = ilk_cursor_wm_max(dev_priv, level, config);
2724         max->fbc = ilk_fbc_wm_reg_max(dev_priv);
2725 }
2726
2727 static void ilk_compute_wm_reg_maximums(const struct drm_i915_private *dev_priv,
2728                                         int level,
2729                                         struct ilk_wm_maximums *max)
2730 {
2731         max->pri = ilk_plane_wm_reg_max(dev_priv, level, false);
2732         max->spr = ilk_plane_wm_reg_max(dev_priv, level, true);
2733         max->cur = ilk_cursor_wm_reg_max(dev_priv, level);
2734         max->fbc = ilk_fbc_wm_reg_max(dev_priv);
2735 }
2736
2737 static bool ilk_validate_wm_level(int level,
2738                                   const struct ilk_wm_maximums *max,
2739                                   struct intel_wm_level *result)
2740 {
2741         bool ret;
2742
2743         /* already determined to be invalid? */
2744         if (!result->enable)
2745                 return false;
2746
2747         result->enable = result->pri_val <= max->pri &&
2748                          result->spr_val <= max->spr &&
2749                          result->cur_val <= max->cur;
2750
2751         ret = result->enable;
2752
2753         /*
2754          * HACK until we can pre-compute everything,
2755          * and thus fail gracefully if LP0 watermarks
2756          * are exceeded...
2757          */
2758         if (level == 0 && !result->enable) {
2759                 if (result->pri_val > max->pri)
2760                         DRM_DEBUG_KMS("Primary WM%d too large %u (max %u)\n",
2761                                       level, result->pri_val, max->pri);
2762                 if (result->spr_val > max->spr)
2763                         DRM_DEBUG_KMS("Sprite WM%d too large %u (max %u)\n",
2764                                       level, result->spr_val, max->spr);
2765                 if (result->cur_val > max->cur)
2766                         DRM_DEBUG_KMS("Cursor WM%d too large %u (max %u)\n",
2767                                       level, result->cur_val, max->cur);
2768
2769                 result->pri_val = min_t(u32, result->pri_val, max->pri);
2770                 result->spr_val = min_t(u32, result->spr_val, max->spr);
2771                 result->cur_val = min_t(u32, result->cur_val, max->cur);
2772                 result->enable = true;
2773         }
2774
2775         return ret;
2776 }
2777
2778 static void ilk_compute_wm_level(const struct drm_i915_private *dev_priv,
2779                                  const struct intel_crtc *intel_crtc,
2780                                  int level,
2781                                  struct intel_crtc_state *crtc_state,
2782                                  const struct intel_plane_state *pristate,
2783                                  const struct intel_plane_state *sprstate,
2784                                  const struct intel_plane_state *curstate,
2785                                  struct intel_wm_level *result)
2786 {
2787         u16 pri_latency = dev_priv->wm.pri_latency[level];
2788         u16 spr_latency = dev_priv->wm.spr_latency[level];
2789         u16 cur_latency = dev_priv->wm.cur_latency[level];
2790
2791         /* WM1+ latency values stored in 0.5us units */
2792         if (level > 0) {
2793                 pri_latency *= 5;
2794                 spr_latency *= 5;
2795                 cur_latency *= 5;
2796         }
2797
2798         if (pristate) {
2799                 result->pri_val = ilk_compute_pri_wm(crtc_state, pristate,
2800                                                      pri_latency, level);
2801                 result->fbc_val = ilk_compute_fbc_wm(crtc_state, pristate, result->pri_val);
2802         }
2803
2804         if (sprstate)
2805                 result->spr_val = ilk_compute_spr_wm(crtc_state, sprstate, spr_latency);
2806
2807         if (curstate)
2808                 result->cur_val = ilk_compute_cur_wm(crtc_state, curstate, cur_latency);
2809
2810         result->enable = true;
2811 }
2812
2813 static void intel_read_wm_latency(struct drm_i915_private *dev_priv,
2814                                   u16 wm[8])
2815 {
2816         struct intel_uncore *uncore = &dev_priv->uncore;
2817
2818         if (INTEL_GEN(dev_priv) >= 9) {
2819                 u32 val;
2820                 int ret, i;
2821                 int level, max_level = ilk_wm_max_level(dev_priv);
2822
2823                 /* read the first set of memory latencies[0:3] */
2824                 val = 0; /* data0 to be programmed to 0 for first set */
2825                 ret = sandybridge_pcode_read(dev_priv,
2826                                              GEN9_PCODE_READ_MEM_LATENCY,
2827                                              &val, NULL);
2828
2829                 if (ret) {
2830                         drm_err(&dev_priv->drm,
2831                                 "SKL Mailbox read error = %d\n", ret);
2832                         return;
2833                 }
2834
2835                 wm[0] = val & GEN9_MEM_LATENCY_LEVEL_MASK;
2836                 wm[1] = (val >> GEN9_MEM_LATENCY_LEVEL_1_5_SHIFT) &
2837                                 GEN9_MEM_LATENCY_LEVEL_MASK;
2838                 wm[2] = (val >> GEN9_MEM_LATENCY_LEVEL_2_6_SHIFT) &
2839                                 GEN9_MEM_LATENCY_LEVEL_MASK;
2840                 wm[3] = (val >> GEN9_MEM_LATENCY_LEVEL_3_7_SHIFT) &
2841                                 GEN9_MEM_LATENCY_LEVEL_MASK;
2842
2843                 /* read the second set of memory latencies[4:7] */
2844                 val = 1; /* data0 to be programmed to 1 for second set */
2845                 ret = sandybridge_pcode_read(dev_priv,
2846                                              GEN9_PCODE_READ_MEM_LATENCY,
2847                                              &val, NULL);
2848                 if (ret) {
2849                         drm_err(&dev_priv->drm,
2850                                 "SKL Mailbox read error = %d\n", ret);
2851                         return;
2852                 }
2853
2854                 wm[4] = val & GEN9_MEM_LATENCY_LEVEL_MASK;
2855                 wm[5] = (val >> GEN9_MEM_LATENCY_LEVEL_1_5_SHIFT) &
2856                                 GEN9_MEM_LATENCY_LEVEL_MASK;
2857                 wm[6] = (val >> GEN9_MEM_LATENCY_LEVEL_2_6_SHIFT) &
2858                                 GEN9_MEM_LATENCY_LEVEL_MASK;
2859                 wm[7] = (val >> GEN9_MEM_LATENCY_LEVEL_3_7_SHIFT) &
2860                                 GEN9_MEM_LATENCY_LEVEL_MASK;
2861
2862                 /*
2863                  * If a level n (n > 1) has a 0us latency, all levels m (m >= n)
2864                  * need to be disabled. We make sure to sanitize the values out
2865                  * of the punit to satisfy this requirement.
2866                  */
2867                 for (level = 1; level <= max_level; level++) {
2868                         if (wm[level] == 0) {
2869                                 for (i = level + 1; i <= max_level; i++)
2870                                         wm[i] = 0;
2871                                 break;
2872                         }
2873                 }
2874
2875                 /*
2876                  * WaWmMemoryReadLatency:skl+,glk
2877                  *
2878                  * punit doesn't take into account the read latency so we need
2879                  * to add 2us to the various latency levels we retrieve from the
2880                  * punit when level 0 response data us 0us.
2881                  */
2882                 if (wm[0] == 0) {
2883                         wm[0] += 2;
2884                         for (level = 1; level <= max_level; level++) {
2885                                 if (wm[level] == 0)
2886                                         break;
2887                                 wm[level] += 2;
2888                         }
2889                 }
2890
2891                 /*
2892                  * WA Level-0 adjustment for 16GB DIMMs: SKL+
2893                  * If we could not get dimm info enable this WA to prevent from
2894                  * any underrun. If not able to get Dimm info assume 16GB dimm
2895                  * to avoid any underrun.
2896                  */
2897                 if (dev_priv->dram_info.is_16gb_dimm)
2898                         wm[0] += 1;
2899
2900         } else if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv)) {
2901                 u64 sskpd = intel_uncore_read64(uncore, MCH_SSKPD);
2902
2903                 wm[0] = (sskpd >> 56) & 0xFF;
2904                 if (wm[0] == 0)
2905                         wm[0] = sskpd & 0xF;
2906                 wm[1] = (sskpd >> 4) & 0xFF;
2907                 wm[2] = (sskpd >> 12) & 0xFF;
2908                 wm[3] = (sskpd >> 20) & 0x1FF;
2909                 wm[4] = (sskpd >> 32) & 0x1FF;
2910         } else if (INTEL_GEN(dev_priv) >= 6) {
2911                 u32 sskpd = intel_uncore_read(uncore, MCH_SSKPD);
2912
2913                 wm[0] = (sskpd >> SSKPD_WM0_SHIFT) & SSKPD_WM_MASK;
2914                 wm[1] = (sskpd >> SSKPD_WM1_SHIFT) & SSKPD_WM_MASK;
2915                 wm[2] = (sskpd >> SSKPD_WM2_SHIFT) & SSKPD_WM_MASK;
2916                 wm[3] = (sskpd >> SSKPD_WM3_SHIFT) & SSKPD_WM_MASK;
2917         } else if (INTEL_GEN(dev_priv) >= 5) {
2918                 u32 mltr = intel_uncore_read(uncore, MLTR_ILK);
2919
2920                 /* ILK primary LP0 latency is 700 ns */
2921                 wm[0] = 7;
2922                 wm[1] = (mltr >> MLTR_WM1_SHIFT) & ILK_SRLT_MASK;
2923                 wm[2] = (mltr >> MLTR_WM2_SHIFT) & ILK_SRLT_MASK;
2924         } else {
2925                 MISSING_CASE(INTEL_DEVID(dev_priv));
2926         }
2927 }
2928
2929 static void intel_fixup_spr_wm_latency(struct drm_i915_private *dev_priv,
2930                                        u16 wm[5])
2931 {
2932         /* ILK sprite LP0 latency is 1300 ns */
2933         if (IS_GEN(dev_priv, 5))
2934                 wm[0] = 13;
2935 }
2936
2937 static void intel_fixup_cur_wm_latency(struct drm_i915_private *dev_priv,
2938                                        u16 wm[5])
2939 {
2940         /* ILK cursor LP0 latency is 1300 ns */
2941         if (IS_GEN(dev_priv, 5))
2942                 wm[0] = 13;
2943 }
2944
2945 int ilk_wm_max_level(const struct drm_i915_private *dev_priv)
2946 {
2947         /* how many WM levels are we expecting */
2948         if (INTEL_GEN(dev_priv) >= 9)
2949                 return 7;
2950         else if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
2951                 return 4;
2952         else if (INTEL_GEN(dev_priv) >= 6)
2953                 return 3;
2954         else
2955                 return 2;
2956 }
2957
2958 static void intel_print_wm_latency(struct drm_i915_private *dev_priv,
2959                                    const char *name,
2960                                    const u16 wm[8])
2961 {
2962         int level, max_level = ilk_wm_max_level(dev_priv);
2963
2964         for (level = 0; level <= max_level; level++) {
2965                 unsigned int latency = wm[level];
2966
2967                 if (latency == 0) {
2968                         drm_dbg_kms(&dev_priv->drm,
2969                                     "%s WM%d latency not provided\n",
2970                                     name, level);
2971                         continue;
2972                 }
2973
2974                 /*
2975                  * - latencies are in us on gen9.
2976                  * - before then, WM1+ latency values are in 0.5us units
2977                  */
2978                 if (INTEL_GEN(dev_priv) >= 9)
2979                         latency *= 10;
2980                 else if (level > 0)
2981                         latency *= 5;
2982
2983                 drm_dbg_kms(&dev_priv->drm,
2984                             "%s WM%d latency %u (%u.%u usec)\n", name, level,
2985                             wm[level], latency / 10, latency % 10);
2986         }
2987 }
2988
2989 static bool ilk_increase_wm_latency(struct drm_i915_private *dev_priv,
2990                                     u16 wm[5], u16 min)
2991 {
2992         int level, max_level = ilk_wm_max_level(dev_priv);
2993
2994         if (wm[0] >= min)
2995                 return false;
2996
2997         wm[0] = max(wm[0], min);
2998         for (level = 1; level <= max_level; level++)
2999                 wm[level] = max_t(u16, wm[level], DIV_ROUND_UP(min, 5));
3000
3001         return true;
3002 }
3003
3004 static void snb_wm_latency_quirk(struct drm_i915_private *dev_priv)
3005 {
3006         bool changed;
3007
3008         /*
3009          * The BIOS provided WM memory latency values are often
3010          * inadequate for high resolution displays. Adjust them.
3011          */
3012         changed = ilk_increase_wm_latency(dev_priv, dev_priv->wm.pri_latency, 12) |
3013                 ilk_increase_wm_latency(dev_priv, dev_priv->wm.spr_latency, 12) |
3014                 ilk_increase_wm_latency(dev_priv, dev_priv->wm.cur_latency, 12);
3015
3016         if (!changed)
3017                 return;
3018
3019         drm_dbg_kms(&dev_priv->drm,
3020                     "WM latency values increased to avoid potential underruns\n");
3021         intel_print_wm_latency(dev_priv, "Primary", dev_priv->wm.pri_latency);
3022         intel_print_wm_latency(dev_priv, "Sprite", dev_priv->wm.spr_latency);
3023         intel_print_wm_latency(dev_priv, "Cursor", dev_priv->wm.cur_latency);
3024 }
3025
3026 static void snb_wm_lp3_irq_quirk(struct drm_i915_private *dev_priv)
3027 {
3028         /*
3029          * On some SNB machines (Thinkpad X220 Tablet at least)
3030          * LP3 usage can cause vblank interrupts to be lost.
3031          * The DEIIR bit will go high but it looks like the CPU
3032          * never gets interrupted.
3033          *
3034          * It's not clear whether other interrupt source could
3035          * be affected or if this is somehow limited to vblank
3036          * interrupts only. To play it safe we disable LP3
3037          * watermarks entirely.
3038          */
3039         if (dev_priv->wm.pri_latency[3] == 0 &&
3040             dev_priv->wm.spr_latency[3] == 0 &&
3041             dev_priv->wm.cur_latency[3] == 0)
3042                 return;
3043
3044         dev_priv->wm.pri_latency[3] = 0;
3045         dev_priv->wm.spr_latency[3] = 0;
3046         dev_priv->wm.cur_latency[3] = 0;
3047
3048         drm_dbg_kms(&dev_priv->drm,
3049                     "LP3 watermarks disabled due to potential for lost interrupts\n");
3050         intel_print_wm_latency(dev_priv, "Primary", dev_priv->wm.pri_latency);
3051         intel_print_wm_latency(dev_priv, "Sprite", dev_priv->wm.spr_latency);
3052         intel_print_wm_latency(dev_priv, "Cursor", dev_priv->wm.cur_latency);
3053 }
3054
3055 static void ilk_setup_wm_latency(struct drm_i915_private *dev_priv)
3056 {
3057         intel_read_wm_latency(dev_priv, dev_priv->wm.pri_latency);
3058
3059         memcpy(dev_priv->wm.spr_latency, dev_priv->wm.pri_latency,
3060                sizeof(dev_priv->wm.pri_latency));
3061         memcpy(dev_priv->wm.cur_latency, dev_priv->wm.pri_latency,
3062                sizeof(dev_priv->wm.pri_latency));
3063
3064         intel_fixup_spr_wm_latency(dev_priv, dev_priv->wm.spr_latency);
3065         intel_fixup_cur_wm_latency(dev_priv, dev_priv->wm.cur_latency);
3066
3067         intel_print_wm_latency(dev_priv, "Primary", dev_priv->wm.pri_latency);
3068         intel_print_wm_latency(dev_priv, "Sprite", dev_priv->wm.spr_latency);
3069         intel_print_wm_latency(dev_priv, "Cursor", dev_priv->wm.cur_latency);
3070
3071         if (IS_GEN(dev_priv, 6)) {
3072                 snb_wm_latency_quirk(dev_priv);
3073                 snb_wm_lp3_irq_quirk(dev_priv);
3074         }
3075 }
3076
3077 static void skl_setup_wm_latency(struct drm_i915_private *dev_priv)
3078 {
3079         intel_read_wm_latency(dev_priv, dev_priv->wm.skl_latency);
3080         intel_print_wm_latency(dev_priv, "Gen9 Plane", dev_priv->wm.skl_latency);
3081 }
3082
3083 static bool ilk_validate_pipe_wm(const struct drm_i915_private *dev_priv,
3084                                  struct intel_pipe_wm *pipe_wm)
3085 {
3086         /* LP0 watermark maximums depend on this pipe alone */
3087         const struct intel_wm_config config = {
3088                 .num_pipes_active = 1,
3089                 .sprites_enabled = pipe_wm->sprites_enabled,
3090                 .sprites_scaled = pipe_wm->sprites_scaled,
3091         };
3092         struct ilk_wm_maximums max;
3093
3094         /* LP0 watermarks always use 1/2 DDB partitioning */
3095         ilk_compute_wm_maximums(dev_priv, 0, &config, INTEL_DDB_PART_1_2, &max);
3096
3097         /* At least LP0 must be valid */
3098         if (!ilk_validate_wm_level(0, &max, &pipe_wm->wm[0])) {
3099                 drm_dbg_kms(&dev_priv->drm, "LP0 watermark invalid\n");
3100                 return false;
3101         }
3102
3103         return true;
3104 }
3105
3106 /* Compute new watermarks for the pipe */
3107 static int ilk_compute_pipe_wm(struct intel_crtc_state *crtc_state)
3108 {
3109         struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
3110         struct intel_crtc *intel_crtc = to_intel_crtc(crtc_state->uapi.crtc);
3111         struct intel_pipe_wm *pipe_wm;
3112         struct intel_plane *plane;
3113         const struct intel_plane_state *plane_state;
3114         const struct intel_plane_state *pristate = NULL;
3115         const struct intel_plane_state *sprstate = NULL;
3116         const struct intel_plane_state *curstate = NULL;
3117         int level, max_level = ilk_wm_max_level(dev_priv), usable_level;
3118         struct ilk_wm_maximums max;
3119
3120         pipe_wm = &crtc_state->wm.ilk.optimal;
3121
3122         intel_atomic_crtc_state_for_each_plane_state(plane, plane_state, crtc_state) {
3123                 if (plane->base.type == DRM_PLANE_TYPE_PRIMARY)
3124                         pristate = plane_state;
3125                 else if (plane->base.type == DRM_PLANE_TYPE_OVERLAY)
3126                         sprstate = plane_state;
3127                 else if (plane->base.type == DRM_PLANE_TYPE_CURSOR)
3128                         curstate = plane_state;
3129         }
3130
3131         pipe_wm->pipe_enabled = crtc_state->hw.active;
3132         if (sprstate) {
3133                 pipe_wm->sprites_enabled = sprstate->uapi.visible;
3134                 pipe_wm->sprites_scaled = sprstate->uapi.visible &&
3135                         (drm_rect_width(&sprstate->uapi.dst) != drm_rect_width(&sprstate->uapi.src) >> 16 ||
3136                          drm_rect_height(&sprstate->uapi.dst) != drm_rect_height(&sprstate->uapi.src) >> 16);
3137         }
3138
3139         usable_level = max_level;
3140
3141         /* ILK/SNB: LP2+ watermarks only w/o sprites */
3142         if (INTEL_GEN(dev_priv) <= 6 && pipe_wm->sprites_enabled)
3143                 usable_level = 1;
3144
3145         /* ILK/SNB/IVB: LP1+ watermarks only w/o scaling */
3146         if (pipe_wm->sprites_scaled)
3147                 usable_level = 0;
3148
3149         memset(&pipe_wm->wm, 0, sizeof(pipe_wm->wm));
3150         ilk_compute_wm_level(dev_priv, intel_crtc, 0, crtc_state,
3151                              pristate, sprstate, curstate, &pipe_wm->wm[0]);
3152
3153         if (!ilk_validate_pipe_wm(dev_priv, pipe_wm))
3154                 return -EINVAL;
3155
3156         ilk_compute_wm_reg_maximums(dev_priv, 1, &max);
3157
3158         for (level = 1; level <= usable_level; level++) {
3159                 struct intel_wm_level *wm = &pipe_wm->wm[level];
3160
3161                 ilk_compute_wm_level(dev_priv, intel_crtc, level, crtc_state,
3162                                      pristate, sprstate, curstate, wm);
3163
3164                 /*
3165                  * Disable any watermark level that exceeds the
3166                  * register maximums since such watermarks are
3167                  * always invalid.
3168                  */
3169                 if (!ilk_validate_wm_level(level, &max, wm)) {
3170                         memset(wm, 0, sizeof(*wm));
3171                         break;
3172                 }
3173         }
3174
3175         return 0;
3176 }
3177
3178 /*
3179  * Build a set of 'intermediate' watermark values that satisfy both the old
3180  * state and the new state.  These can be programmed to the hardware
3181  * immediately.
3182  */
3183 static int ilk_compute_intermediate_wm(struct intel_crtc_state *newstate)
3184 {
3185         struct intel_crtc *intel_crtc = to_intel_crtc(newstate->uapi.crtc);
3186         struct drm_i915_private *dev_priv = to_i915(intel_crtc->base.dev);
3187         struct intel_pipe_wm *a = &newstate->wm.ilk.intermediate;
3188         struct intel_atomic_state *intel_state =
3189                 to_intel_atomic_state(newstate->uapi.state);
3190         const struct intel_crtc_state *oldstate =
3191                 intel_atomic_get_old_crtc_state(intel_state, intel_crtc);
3192         const struct intel_pipe_wm *b = &oldstate->wm.ilk.optimal;
3193         int level, max_level = ilk_wm_max_level(dev_priv);
3194
3195         /*
3196          * Start with the final, target watermarks, then combine with the
3197          * currently active watermarks to get values that are safe both before
3198          * and after the vblank.
3199          */
3200         *a = newstate->wm.ilk.optimal;
3201         if (!newstate->hw.active || drm_atomic_crtc_needs_modeset(&newstate->uapi) ||
3202             intel_state->skip_intermediate_wm)
3203                 return 0;
3204
3205         a->pipe_enabled |= b->pipe_enabled;
3206         a->sprites_enabled |= b->sprites_enabled;
3207         a->sprites_scaled |= b->sprites_scaled;
3208
3209         for (level = 0; level <= max_level; level++) {
3210                 struct intel_wm_level *a_wm = &a->wm[level];
3211                 const struct intel_wm_level *b_wm = &b->wm[level];
3212
3213                 a_wm->enable &= b_wm->enable;
3214                 a_wm->pri_val = max(a_wm->pri_val, b_wm->pri_val);
3215                 a_wm->spr_val = max(a_wm->spr_val, b_wm->spr_val);
3216                 a_wm->cur_val = max(a_wm->cur_val, b_wm->cur_val);
3217                 a_wm->fbc_val = max(a_wm->fbc_val, b_wm->fbc_val);
3218         }
3219
3220         /*
3221          * We need to make sure that these merged watermark values are
3222          * actually a valid configuration themselves.  If they're not,
3223          * there's no safe way to transition from the old state to
3224          * the new state, so we need to fail the atomic transaction.
3225          */
3226         if (!ilk_validate_pipe_wm(dev_priv, a))
3227                 return -EINVAL;
3228
3229         /*
3230          * If our intermediate WM are identical to the final WM, then we can
3231          * omit the post-vblank programming; only update if it's different.
3232          */
3233         if (memcmp(a, &newstate->wm.ilk.optimal, sizeof(*a)) != 0)
3234                 newstate->wm.need_postvbl_update = true;
3235
3236         return 0;
3237 }
3238
3239 /*
3240  * Merge the watermarks from all active pipes for a specific level.
3241  */
3242 static void ilk_merge_wm_level(struct drm_i915_private *dev_priv,
3243                                int level,
3244                                struct intel_wm_level *ret_wm)
3245 {
3246         const struct intel_crtc *intel_crtc;
3247
3248         ret_wm->enable = true;
3249
3250         for_each_intel_crtc(&dev_priv->drm, intel_crtc) {
3251                 const struct intel_pipe_wm *active = &intel_crtc->wm.active.ilk;
3252                 const struct intel_wm_level *wm = &active->wm[level];
3253
3254                 if (!active->pipe_enabled)
3255                         continue;
3256
3257                 /*
3258                  * The watermark values may have been used in the past,
3259                  * so we must maintain them in the registers for some
3260                  * time even if the level is now disabled.
3261                  */
3262                 if (!wm->enable)
3263                         ret_wm->enable = false;
3264
3265                 ret_wm->pri_val = max(ret_wm->pri_val, wm->pri_val);
3266                 ret_wm->spr_val = max(ret_wm->spr_val, wm->spr_val);
3267                 ret_wm->cur_val = max(ret_wm->cur_val, wm->cur_val);
3268                 ret_wm->fbc_val = max(ret_wm->fbc_val, wm->fbc_val);
3269         }
3270 }
3271
3272 /*
3273  * Merge all low power watermarks for all active pipes.
3274  */
3275 static void ilk_wm_merge(struct drm_i915_private *dev_priv,
3276                          const struct intel_wm_config *config,
3277                          const struct ilk_wm_maximums *max,
3278                          struct intel_pipe_wm *merged)
3279 {
3280         int level, max_level = ilk_wm_max_level(dev_priv);
3281         int last_enabled_level = max_level;
3282
3283         /* ILK/SNB/IVB: LP1+ watermarks only w/ single pipe */
3284         if ((INTEL_GEN(dev_priv) <= 6 || IS_IVYBRIDGE(dev_priv)) &&
3285             config->num_pipes_active > 1)
3286                 last_enabled_level = 0;
3287
3288         /* ILK: FBC WM must be disabled always */
3289         merged->fbc_wm_enabled = INTEL_GEN(dev_priv) >= 6;
3290
3291         /* merge each WM1+ level */
3292         for (level = 1; level <= max_level; level++) {
3293                 struct intel_wm_level *wm = &merged->wm[level];
3294
3295                 ilk_merge_wm_level(dev_priv, level, wm);
3296
3297                 if (level > last_enabled_level)
3298                         wm->enable = false;
3299                 else if (!ilk_validate_wm_level(level, max, wm))
3300                         /* make sure all following levels get disabled */
3301                         last_enabled_level = level - 1;
3302
3303                 /*
3304                  * The spec says it is preferred to disable
3305                  * FBC WMs instead of disabling a WM level.
3306                  */
3307                 if (wm->fbc_val > max->fbc) {
3308                         if (wm->enable)
3309                                 merged->fbc_wm_enabled = false;
3310                         wm->fbc_val = 0;
3311                 }
3312         }
3313
3314         /* ILK: LP2+ must be disabled when FBC WM is disabled but FBC enabled */
3315         /*
3316          * FIXME this is racy. FBC might get enabled later.
3317          * What we should check here is whether FBC can be
3318          * enabled sometime later.
3319          */
3320         if (IS_GEN(dev_priv, 5) && !merged->fbc_wm_enabled &&
3321             intel_fbc_is_active(dev_priv)) {
3322                 for (level = 2; level <= max_level; level++) {
3323                         struct intel_wm_level *wm = &merged->wm[level];
3324
3325                         wm->enable = false;
3326                 }
3327         }
3328 }
3329
3330 static int ilk_wm_lp_to_level(int wm_lp, const struct intel_pipe_wm *pipe_wm)
3331 {
3332         /* LP1,LP2,LP3 levels are either 1,2,3 or 1,3,4 */
3333         return wm_lp + (wm_lp >= 2 && pipe_wm->wm[4].enable);
3334 }
3335
3336 /* The value we need to program into the WM_LPx latency field */
3337 static unsigned int ilk_wm_lp_latency(struct drm_i915_private *dev_priv,
3338                                       int level)
3339 {
3340         if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
3341                 return 2 * level;
3342         else
3343                 return dev_priv->wm.pri_latency[level];
3344 }
3345
3346 static void ilk_compute_wm_results(struct drm_i915_private *dev_priv,
3347                                    const struct intel_pipe_wm *merged,
3348                                    enum intel_ddb_partitioning partitioning,
3349                                    struct ilk_wm_values *results)
3350 {
3351         struct intel_crtc *intel_crtc;
3352         int level, wm_lp;
3353
3354         results->enable_fbc_wm = merged->fbc_wm_enabled;
3355         results->partitioning = partitioning;
3356
3357         /* LP1+ register values */
3358         for (wm_lp = 1; wm_lp <= 3; wm_lp++) {
3359                 const struct intel_wm_level *r;
3360
3361                 level = ilk_wm_lp_to_level(wm_lp, merged);
3362
3363                 r = &merged->wm[level];
3364
3365                 /*
3366                  * Maintain the watermark values even if the level is
3367                  * disabled. Doing otherwise could cause underruns.
3368                  */
3369                 results->wm_lp[wm_lp - 1] =
3370                         (ilk_wm_lp_latency(dev_priv, level) << WM1_LP_LATENCY_SHIFT) |
3371                         (r->pri_val << WM1_LP_SR_SHIFT) |
3372                         r->cur_val;
3373
3374                 if (r->enable)
3375                         results->wm_lp[wm_lp - 1] |= WM1_LP_SR_EN;
3376
3377                 if (INTEL_GEN(dev_priv) >= 8)
3378                         results->wm_lp[wm_lp - 1] |=
3379                                 r->fbc_val << WM1_LP_FBC_SHIFT_BDW;
3380                 else
3381                         results->wm_lp[wm_lp - 1] |=
3382                                 r->fbc_val << WM1_LP_FBC_SHIFT;
3383
3384                 /*
3385                  * Always set WM1S_LP_EN when spr_val != 0, even if the
3386                  * level is disabled. Doing otherwise could cause underruns.
3387                  */
3388                 if (INTEL_GEN(dev_priv) <= 6 && r->spr_val) {
3389                         drm_WARN_ON(&dev_priv->drm, wm_lp != 1);
3390                         results->wm_lp_spr[wm_lp - 1] = WM1S_LP_EN | r->spr_val;
3391                 } else
3392                         results->wm_lp_spr[wm_lp - 1] = r->spr_val;
3393         }
3394
3395         /* LP0 register values */
3396         for_each_intel_crtc(&dev_priv->drm, intel_crtc) {
3397                 enum pipe pipe = intel_crtc->pipe;
3398                 const struct intel_pipe_wm *pipe_wm = &intel_crtc->wm.active.ilk;
3399                 const struct intel_wm_level *r = &pipe_wm->wm[0];
3400
3401                 if (drm_WARN_ON(&dev_priv->drm, !r->enable))
3402                         continue;
3403
3404                 results->wm_pipe[pipe] =
3405                         (r->pri_val << WM0_PIPE_PLANE_SHIFT) |
3406                         (r->spr_val << WM0_PIPE_SPRITE_SHIFT) |
3407                         r->cur_val;
3408         }
3409 }
3410
3411 /* Find the result with the highest level enabled. Check for enable_fbc_wm in
3412  * case both are at the same level. Prefer r1 in case they're the same. */
3413 static struct intel_pipe_wm *
3414 ilk_find_best_result(struct drm_i915_private *dev_priv,
3415                      struct intel_pipe_wm *r1,
3416                      struct intel_pipe_wm *r2)
3417 {
3418         int level, max_level = ilk_wm_max_level(dev_priv);
3419         int level1 = 0, level2 = 0;
3420
3421         for (level = 1; level <= max_level; level++) {
3422                 if (r1->wm[level].enable)
3423                         level1 = level;
3424                 if (r2->wm[level].enable)
3425                         level2 = level;
3426         }
3427
3428         if (level1 == level2) {
3429                 if (r2->fbc_wm_enabled && !r1->fbc_wm_enabled)
3430                         return r2;
3431                 else
3432                         return r1;
3433         } else if (level1 > level2) {
3434                 return r1;
3435         } else {
3436                 return r2;
3437         }
3438 }
3439
3440 /* dirty bits used to track which watermarks need changes */
3441 #define WM_DIRTY_PIPE(pipe) (1 << (pipe))
3442 #define WM_DIRTY_LP(wm_lp) (1 << (15 + (wm_lp)))
3443 #define WM_DIRTY_LP_ALL (WM_DIRTY_LP(1) | WM_DIRTY_LP(2) | WM_DIRTY_LP(3))
3444 #define WM_DIRTY_FBC (1 << 24)
3445 #define WM_DIRTY_DDB (1 << 25)
3446
3447 static unsigned int ilk_compute_wm_dirty(struct drm_i915_private *dev_priv,
3448                                          const struct ilk_wm_values *old,
3449                                          const struct ilk_wm_values *new)
3450 {
3451         unsigned int dirty = 0;
3452         enum pipe pipe;
3453         int wm_lp;
3454
3455         for_each_pipe(dev_priv, pipe) {
3456                 if (old->wm_pipe[pipe] != new->wm_pipe[pipe]) {
3457                         dirty |= WM_DIRTY_PIPE(pipe);
3458                         /* Must disable LP1+ watermarks too */
3459                         dirty |= WM_DIRTY_LP_ALL;
3460                 }
3461         }
3462
3463         if (old->enable_fbc_wm != new->enable_fbc_wm) {
3464                 dirty |= WM_DIRTY_FBC;
3465                 /* Must disable LP1+ watermarks too */
3466                 dirty |= WM_DIRTY_LP_ALL;
3467         }
3468
3469         if (old->partitioning != new->partitioning) {
3470                 dirty |= WM_DIRTY_DDB;
3471                 /* Must disable LP1+ watermarks too */
3472                 dirty |= WM_DIRTY_LP_ALL;
3473         }
3474
3475         /* LP1+ watermarks already deemed dirty, no need to continue */
3476         if (dirty & WM_DIRTY_LP_ALL)
3477                 return dirty;
3478
3479         /* Find the lowest numbered LP1+ watermark in need of an update... */
3480         for (wm_lp = 1; wm_lp <= 3; wm_lp++) {
3481                 if (old->wm_lp[wm_lp - 1] != new->wm_lp[wm_lp - 1] ||
3482                     old->wm_lp_spr[wm_lp - 1] != new->wm_lp_spr[wm_lp - 1])
3483                         break;
3484         }
3485
3486         /* ...and mark it and all higher numbered LP1+ watermarks as dirty */
3487         for (; wm_lp <= 3; wm_lp++)
3488                 dirty |= WM_DIRTY_LP(wm_lp);
3489
3490         return dirty;
3491 }
3492
3493 static bool _ilk_disable_lp_wm(struct drm_i915_private *dev_priv,
3494                                unsigned int dirty)
3495 {
3496         struct ilk_wm_values *previous = &dev_priv->wm.hw;
3497         bool changed = false;
3498
3499         if (dirty & WM_DIRTY_LP(3) && previous->wm_lp[2] & WM1_LP_SR_EN) {
3500                 previous->wm_lp[2] &= ~WM1_LP_SR_EN;
3501                 I915_WRITE(WM3_LP_ILK, previous->wm_lp[2]);
3502                 changed = true;
3503         }
3504         if (dirty & WM_DIRTY_LP(2) && previous->wm_lp[1] & WM1_LP_SR_EN) {
3505                 previous->wm_lp[1] &= ~WM1_LP_SR_EN;
3506                 I915_WRITE(WM2_LP_ILK, previous->wm_lp[1]);
3507                 changed = true;
3508         }
3509         if (dirty & WM_DIRTY_LP(1) && previous->wm_lp[0] & WM1_LP_SR_EN) {
3510                 previous->wm_lp[0] &= ~WM1_LP_SR_EN;
3511                 I915_WRITE(WM1_LP_ILK, previous->wm_lp[0]);
3512                 changed = true;
3513         }
3514
3515         /*
3516          * Don't touch WM1S_LP_EN here.
3517          * Doing so could cause underruns.
3518          */
3519
3520         return changed;
3521 }
3522
3523 /*
3524  * The spec says we shouldn't write when we don't need, because every write
3525  * causes WMs to be re-evaluated, expending some power.
3526  */
3527 static void ilk_write_wm_values(struct drm_i915_private *dev_priv,
3528                                 struct ilk_wm_values *results)
3529 {
3530         struct ilk_wm_values *previous = &dev_priv->wm.hw;
3531         unsigned int dirty;
3532         u32 val;
3533
3534         dirty = ilk_compute_wm_dirty(dev_priv, previous, results);
3535         if (!dirty)
3536                 return;
3537
3538         _ilk_disable_lp_wm(dev_priv, dirty);
3539
3540         if (dirty & WM_DIRTY_PIPE(PIPE_A))
3541                 I915_WRITE(WM0_PIPEA_ILK, results->wm_pipe[0]);
3542         if (dirty & WM_DIRTY_PIPE(PIPE_B))
3543                 I915_WRITE(WM0_PIPEB_ILK, results->wm_pipe[1]);
3544         if (dirty & WM_DIRTY_PIPE(PIPE_C))
3545                 I915_WRITE(WM0_PIPEC_IVB, results->wm_pipe[2]);
3546
3547         if (dirty & WM_DIRTY_DDB) {
3548                 if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv)) {
3549                         val = I915_READ(WM_MISC);
3550                         if (results->partitioning == INTEL_DDB_PART_1_2)
3551                                 val &= ~WM_MISC_DATA_PARTITION_5_6;
3552                         else
3553                                 val |= WM_MISC_DATA_PARTITION_5_6;
3554                         I915_WRITE(WM_MISC, val);
3555                 } else {
3556                         val = I915_READ(DISP_ARB_CTL2);
3557                         if (results->partitioning == INTEL_DDB_PART_1_2)
3558                                 val &= ~DISP_DATA_PARTITION_5_6;
3559                         else
3560                                 val |= DISP_DATA_PARTITION_5_6;
3561                         I915_WRITE(DISP_ARB_CTL2, val);
3562                 }
3563         }
3564
3565         if (dirty & WM_DIRTY_FBC) {
3566                 val = I915_READ(DISP_ARB_CTL);
3567                 if (results->enable_fbc_wm)
3568                         val &= ~DISP_FBC_WM_DIS;
3569                 else
3570                         val |= DISP_FBC_WM_DIS;
3571                 I915_WRITE(DISP_ARB_CTL, val);
3572         }
3573
3574         if (dirty & WM_DIRTY_LP(1) &&
3575             previous->wm_lp_spr[0] != results->wm_lp_spr[0])
3576                 I915_WRITE(WM1S_LP_ILK, results->wm_lp_spr[0]);
3577
3578         if (INTEL_GEN(dev_priv) >= 7) {
3579                 if (dirty & WM_DIRTY_LP(2) && previous->wm_lp_spr[1] != results->wm_lp_spr[1])
3580                         I915_WRITE(WM2S_LP_IVB, results->wm_lp_spr[1]);
3581                 if (dirty & WM_DIRTY_LP(3) && previous->wm_lp_spr[2] != results->wm_lp_spr[2])
3582                         I915_WRITE(WM3S_LP_IVB, results->wm_lp_spr[2]);
3583         }
3584
3585         if (dirty & WM_DIRTY_LP(1) && previous->wm_lp[0] != results->wm_lp[0])
3586                 I915_WRITE(WM1_LP_ILK, results->wm_lp[0]);
3587         if (dirty & WM_DIRTY_LP(2) && previous->wm_lp[1] != results->wm_lp[1])
3588                 I915_WRITE(WM2_LP_ILK, results->wm_lp[1]);
3589         if (dirty & WM_DIRTY_LP(3) && previous->wm_lp[2] != results->wm_lp[2])
3590                 I915_WRITE(WM3_LP_ILK, results->wm_lp[2]);
3591
3592         dev_priv->wm.hw = *results;
3593 }
3594
3595 bool ilk_disable_lp_wm(struct drm_i915_private *dev_priv)
3596 {
3597         return _ilk_disable_lp_wm(dev_priv, WM_DIRTY_LP_ALL);
3598 }
3599
3600 u8 intel_enabled_dbuf_slices_mask(struct drm_i915_private *dev_priv)
3601 {
3602         int i;
3603         int max_slices = INTEL_INFO(dev_priv)->num_supported_dbuf_slices;
3604         u8 enabled_slices_mask = 0;
3605
3606         for (i = 0; i < max_slices; i++) {
3607                 if (I915_READ(DBUF_CTL_S(i)) & DBUF_POWER_STATE)
3608                         enabled_slices_mask |= BIT(i);
3609         }
3610
3611         return enabled_slices_mask;
3612 }
3613
3614 /*
3615  * FIXME: We still don't have the proper code detect if we need to apply the WA,
3616  * so assume we'll always need it in order to avoid underruns.
3617  */
3618 static bool skl_needs_memory_bw_wa(struct drm_i915_private *dev_priv)
3619 {
3620         return IS_GEN9_BC(dev_priv) || IS_BROXTON(dev_priv);
3621 }
3622
3623 static bool
3624 intel_has_sagv(struct drm_i915_private *dev_priv)
3625 {
3626         /* HACK! */
3627         if (IS_GEN(dev_priv, 12))
3628                 return false;
3629
3630         return (IS_GEN9_BC(dev_priv) || INTEL_GEN(dev_priv) >= 10) &&
3631                 dev_priv->sagv_status != I915_SAGV_NOT_CONTROLLED;
3632 }
3633
3634 static void
3635 skl_setup_sagv_block_time(struct drm_i915_private *dev_priv)
3636 {
3637         if (INTEL_GEN(dev_priv) >= 12) {
3638                 u32 val = 0;
3639                 int ret;
3640
3641                 ret = sandybridge_pcode_read(dev_priv,
3642                                              GEN12_PCODE_READ_SAGV_BLOCK_TIME_US,
3643                                              &val, NULL);
3644                 if (!ret) {
3645                         dev_priv->sagv_block_time_us = val;
3646                         return;
3647                 }
3648
3649                 drm_dbg(&dev_priv->drm, "Couldn't read SAGV block time!\n");
3650         } else if (IS_GEN(dev_priv, 11)) {
3651                 dev_priv->sagv_block_time_us = 10;
3652                 return;
3653         } else if (IS_GEN(dev_priv, 10)) {
3654                 dev_priv->sagv_block_time_us = 20;
3655                 return;
3656         } else if (IS_GEN(dev_priv, 9)) {
3657                 dev_priv->sagv_block_time_us = 30;
3658                 return;
3659         } else {
3660                 MISSING_CASE(INTEL_GEN(dev_priv));
3661         }
3662
3663         /* Default to an unusable block time */
3664         dev_priv->sagv_block_time_us = -1;
3665 }
3666
3667 /*
3668  * SAGV dynamically adjusts the system agent voltage and clock frequencies
3669  * depending on power and performance requirements. The display engine access
3670  * to system memory is blocked during the adjustment time. Because of the
3671  * blocking time, having this enabled can cause full system hangs and/or pipe
3672  * underruns if we don't meet all of the following requirements:
3673  *
3674  *  - <= 1 pipe enabled
3675  *  - All planes can enable watermarks for latencies >= SAGV engine block time
3676  *  - We're not using an interlaced display configuration
3677  */
3678 int
3679 intel_enable_sagv(struct drm_i915_private *dev_priv)
3680 {
3681         int ret;
3682
3683         if (!intel_has_sagv(dev_priv))
3684                 return 0;
3685
3686         if (dev_priv->sagv_status == I915_SAGV_ENABLED)
3687                 return 0;
3688
3689         drm_dbg_kms(&dev_priv->drm, "Enabling SAGV\n");
3690         ret = sandybridge_pcode_write(dev_priv, GEN9_PCODE_SAGV_CONTROL,
3691                                       GEN9_SAGV_ENABLE);
3692
3693         /* We don't need to wait for SAGV when enabling */
3694
3695         /*
3696          * Some skl systems, pre-release machines in particular,
3697          * don't actually have SAGV.
3698          */
3699         if (IS_SKYLAKE(dev_priv) && ret == -ENXIO) {
3700                 drm_dbg(&dev_priv->drm, "No SAGV found on system, ignoring\n");
3701                 dev_priv->sagv_status = I915_SAGV_NOT_CONTROLLED;
3702                 return 0;
3703         } else if (ret < 0) {
3704                 drm_err(&dev_priv->drm, "Failed to enable SAGV\n");
3705                 return ret;
3706         }
3707
3708         dev_priv->sagv_status = I915_SAGV_ENABLED;
3709         return 0;
3710 }
3711
3712 int
3713 intel_disable_sagv(struct drm_i915_private *dev_priv)
3714 {
3715         int ret;
3716
3717         if (!intel_has_sagv(dev_priv))
3718                 return 0;
3719
3720         if (dev_priv->sagv_status == I915_SAGV_DISABLED)
3721                 return 0;
3722
3723         drm_dbg_kms(&dev_priv->drm, "Disabling SAGV\n");
3724         /* bspec says to keep retrying for at least 1 ms */
3725         ret = skl_pcode_request(dev_priv, GEN9_PCODE_SAGV_CONTROL,
3726                                 GEN9_SAGV_DISABLE,
3727                                 GEN9_SAGV_IS_DISABLED, GEN9_SAGV_IS_DISABLED,
3728                                 1);
3729         /*
3730          * Some skl systems, pre-release machines in particular,
3731          * don't actually have SAGV.
3732          */
3733         if (IS_SKYLAKE(dev_priv) && ret == -ENXIO) {
3734                 drm_dbg(&dev_priv->drm, "No SAGV found on system, ignoring\n");
3735                 dev_priv->sagv_status = I915_SAGV_NOT_CONTROLLED;
3736                 return 0;
3737         } else if (ret < 0) {
3738                 drm_err(&dev_priv->drm, "Failed to disable SAGV (%d)\n", ret);
3739                 return ret;
3740         }
3741
3742         dev_priv->sagv_status = I915_SAGV_DISABLED;
3743         return 0;
3744 }
3745
3746 bool intel_can_enable_sagv(struct intel_atomic_state *state)
3747 {
3748         struct drm_device *dev = state->base.dev;
3749         struct drm_i915_private *dev_priv = to_i915(dev);
3750         struct intel_crtc *crtc;
3751         struct intel_plane *plane;
3752         struct intel_crtc_state *crtc_state;
3753         enum pipe pipe;
3754         int level, latency;
3755
3756         if (!intel_has_sagv(dev_priv))
3757                 return false;
3758
3759         /*
3760          * If there are no active CRTCs, no additional checks need be performed
3761          */
3762         if (hweight8(state->active_pipes) == 0)
3763                 return true;
3764
3765         /*
3766          * SKL+ workaround: bspec recommends we disable SAGV when we have
3767          * more then one pipe enabled
3768          */
3769         if (hweight8(state->active_pipes) > 1)
3770                 return false;
3771
3772         /* Since we're now guaranteed to only have one active CRTC... */
3773         pipe = ffs(state->active_pipes) - 1;
3774         crtc = intel_get_crtc_for_pipe(dev_priv, pipe);
3775         crtc_state = to_intel_crtc_state(crtc->base.state);
3776
3777         if (crtc_state->hw.adjusted_mode.flags & DRM_MODE_FLAG_INTERLACE)
3778                 return false;
3779
3780         for_each_intel_plane_on_crtc(dev, crtc, plane) {
3781                 struct skl_plane_wm *wm =
3782                         &crtc_state->wm.skl.optimal.planes[plane->id];
3783
3784                 /* Skip this plane if it's not enabled */
3785                 if (!wm->wm[0].plane_en)
3786                         continue;
3787
3788                 /* Find the highest enabled wm level for this plane */
3789                 for (level = ilk_wm_max_level(dev_priv);
3790                      !wm->wm[level].plane_en; --level)
3791                      { }
3792
3793                 latency = dev_priv->wm.skl_latency[level];
3794
3795                 if (skl_needs_memory_bw_wa(dev_priv) &&
3796                     plane->base.state->fb->modifier ==
3797                     I915_FORMAT_MOD_X_TILED)
3798                         latency += 15;
3799
3800                 /*
3801                  * If any of the planes on this pipe don't enable wm levels that
3802                  * incur memory latencies higher than sagv_block_time_us we
3803                  * can't enable SAGV.
3804                  */
3805                 if (latency < dev_priv->sagv_block_time_us)
3806                         return false;
3807         }
3808
3809         return true;
3810 }
3811
3812 /*
3813  * Calculate initial DBuf slice offset, based on slice size
3814  * and mask(i.e if slice size is 1024 and second slice is enabled
3815  * offset would be 1024)
3816  */
3817 static unsigned int
3818 icl_get_first_dbuf_slice_offset(u32 dbuf_slice_mask,
3819                                 u32 slice_size,
3820                                 u32 ddb_size)
3821 {
3822         unsigned int offset = 0;
3823
3824         if (!dbuf_slice_mask)
3825                 return 0;
3826
3827         offset = (ffs(dbuf_slice_mask) - 1) * slice_size;
3828
3829         WARN_ON(offset >= ddb_size);
3830         return offset;
3831 }
3832
3833 static u16 intel_get_ddb_size(struct drm_i915_private *dev_priv)
3834 {
3835         u16 ddb_size = INTEL_INFO(dev_priv)->ddb_size;
3836
3837         drm_WARN_ON(&dev_priv->drm, ddb_size == 0);
3838
3839         if (INTEL_GEN(dev_priv) < 11)
3840                 return ddb_size - 4; /* 4 blocks for bypass path allocation */
3841
3842         return ddb_size;
3843 }
3844
3845 static u8 skl_compute_dbuf_slices(const struct intel_crtc_state *crtc_state,
3846                                   u32 active_pipes);
3847
3848 static void
3849 skl_ddb_get_pipe_allocation_limits(struct drm_i915_private *dev_priv,
3850                                    const struct intel_crtc_state *crtc_state,
3851                                    const u64 total_data_rate,
3852                                    struct skl_ddb_entry *alloc, /* out */
3853                                    int *num_active /* out */)
3854 {
3855         struct drm_atomic_state *state = crtc_state->uapi.state;
3856         struct intel_atomic_state *intel_state = to_intel_atomic_state(state);
3857         struct drm_crtc *for_crtc = crtc_state->uapi.crtc;
3858         const struct intel_crtc *crtc;
3859         u32 pipe_width = 0, total_width_in_range = 0, width_before_pipe_in_range = 0;
3860         enum pipe for_pipe = to_intel_crtc(for_crtc)->pipe;
3861         u16 ddb_size;
3862         u32 ddb_range_size;
3863         u32 i;
3864         u32 dbuf_slice_mask;
3865         u32 active_pipes;
3866         u32 offset;
3867         u32 slice_size;
3868         u32 total_slice_mask;
3869         u32 start, end;
3870
3871         if (drm_WARN_ON(&dev_priv->drm, !state) || !crtc_state->hw.active) {
3872                 alloc->start = 0;
3873                 alloc->end = 0;
3874                 *num_active = hweight8(dev_priv->active_pipes);
3875                 return;
3876         }
3877
3878         if (intel_state->active_pipe_changes)
3879                 active_pipes = intel_state->active_pipes;
3880         else
3881                 active_pipes = dev_priv->active_pipes;
3882
3883         *num_active = hweight8(active_pipes);
3884
3885         ddb_size = intel_get_ddb_size(dev_priv);
3886
3887         slice_size = ddb_size / INTEL_INFO(dev_priv)->num_supported_dbuf_slices;
3888
3889         /*
3890          * If the state doesn't change the active CRTC's or there is no
3891          * modeset request, then there's no need to recalculate;
3892          * the existing pipe allocation limits should remain unchanged.
3893          * Note that we're safe from racing commits since any racing commit
3894          * that changes the active CRTC list or do modeset would need to
3895          * grab _all_ crtc locks, including the one we currently hold.
3896          */
3897         if (!intel_state->active_pipe_changes && !intel_state->modeset) {
3898                 /*
3899                  * alloc may be cleared by clear_intel_crtc_state,
3900                  * copy from old state to be sure
3901                  */
3902                 *alloc = to_intel_crtc_state(for_crtc->state)->wm.skl.ddb;
3903                 return;
3904         }
3905
3906         /*
3907          * Get allowed DBuf slices for correspondent pipe and platform.
3908          */
3909         dbuf_slice_mask = skl_compute_dbuf_slices(crtc_state, active_pipes);
3910
3911         DRM_DEBUG_KMS("DBuf slice mask %x pipe %c active pipes %x\n",
3912                       dbuf_slice_mask,
3913                       pipe_name(for_pipe), active_pipes);
3914
3915         /*
3916          * Figure out at which DBuf slice we start, i.e if we start at Dbuf S2
3917          * and slice size is 1024, the offset would be 1024
3918          */
3919         offset = icl_get_first_dbuf_slice_offset(dbuf_slice_mask,
3920                                                  slice_size, ddb_size);
3921
3922         /*
3923          * Figure out total size of allowed DBuf slices, which is basically
3924          * a number of allowed slices for that pipe multiplied by slice size.
3925          * Inside of this
3926          * range ddb entries are still allocated in proportion to display width.
3927          */
3928         ddb_range_size = hweight8(dbuf_slice_mask) * slice_size;
3929
3930         /*
3931          * Watermark/ddb requirement highly depends upon width of the
3932          * framebuffer, So instead of allocating DDB equally among pipes
3933          * distribute DDB based on resolution/width of the display.
3934          */
3935         total_slice_mask = dbuf_slice_mask;
3936         for_each_new_intel_crtc_in_state(intel_state, crtc, crtc_state, i) {
3937                 const struct drm_display_mode *adjusted_mode =
3938                         &crtc_state->hw.adjusted_mode;
3939                 enum pipe pipe = crtc->pipe;
3940                 int hdisplay, vdisplay;
3941                 u32 pipe_dbuf_slice_mask;
3942
3943                 if (!crtc_state->hw.active)
3944                         continue;
3945
3946                 pipe_dbuf_slice_mask = skl_compute_dbuf_slices(crtc_state,
3947                                                                active_pipes);
3948
3949                 /*
3950                  * According to BSpec pipe can share one dbuf slice with another
3951                  * pipes or pipe can use multiple dbufs, in both cases we
3952                  * account for other pipes only if they have exactly same mask.
3953                  * However we need to account how many slices we should enable
3954                  * in total.
3955                  */
3956                 total_slice_mask |= pipe_dbuf_slice_mask;
3957
3958                 /*
3959                  * Do not account pipes using other slice sets
3960                  * luckily as of current BSpec slice sets do not partially
3961                  * intersect(pipes share either same one slice or same slice set
3962                  * i.e no partial intersection), so it is enough to check for
3963                  * equality for now.
3964                  */
3965                 if (dbuf_slice_mask != pipe_dbuf_slice_mask)
3966                         continue;
3967
3968                 drm_mode_get_hv_timing(adjusted_mode, &hdisplay, &vdisplay);
3969
3970                 total_width_in_range += hdisplay;
3971
3972                 if (pipe < for_pipe)
3973                         width_before_pipe_in_range += hdisplay;
3974                 else if (pipe == for_pipe)
3975                         pipe_width = hdisplay;
3976         }
3977
3978         /*
3979          * FIXME: For now we always enable slice S1 as per
3980          * the Bspec display initialization sequence.
3981          */
3982         intel_state->enabled_dbuf_slices_mask = total_slice_mask | BIT(DBUF_S1);
3983
3984         start = ddb_range_size * width_before_pipe_in_range / total_width_in_range;
3985         end = ddb_range_size *
3986                 (width_before_pipe_in_range + pipe_width) / total_width_in_range;
3987
3988         alloc->start = offset + start;
3989         alloc->end = offset + end;
3990
3991         DRM_DEBUG_KMS("Pipe %d ddb %d-%d\n", for_pipe,
3992                       alloc->start, alloc->end);
3993         DRM_DEBUG_KMS("Enabled ddb slices mask %x num supported %d\n",
3994                       intel_state->enabled_dbuf_slices_mask,
3995                       INTEL_INFO(dev_priv)->num_supported_dbuf_slices);
3996 }
3997
3998 static int skl_compute_wm_params(const struct intel_crtc_state *crtc_state,
3999                                  int width, const struct drm_format_info *format,
4000                                  u64 modifier, unsigned int rotation,
4001                                  u32 plane_pixel_rate, struct skl_wm_params *wp,
4002                                  int color_plane);
4003 static void skl_compute_plane_wm(const struct intel_crtc_state *crtc_state,
4004                                  int level,
4005                                  const struct skl_wm_params *wp,
4006                                  const struct skl_wm_level *result_prev,
4007                                  struct skl_wm_level *result /* out */);
4008
4009 static unsigned int
4010 skl_cursor_allocation(const struct intel_crtc_state *crtc_state,
4011                       int num_active)
4012 {
4013         struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
4014         int level, max_level = ilk_wm_max_level(dev_priv);
4015         struct skl_wm_level wm = {};
4016         int ret, min_ddb_alloc = 0;
4017         struct skl_wm_params wp;
4018
4019         ret = skl_compute_wm_params(crtc_state, 256,
4020                                     drm_format_info(DRM_FORMAT_ARGB8888),
4021                                     DRM_FORMAT_MOD_LINEAR,
4022                                     DRM_MODE_ROTATE_0,
4023                                     crtc_state->pixel_rate, &wp, 0);
4024         drm_WARN_ON(&dev_priv->drm, ret);
4025
4026         for (level = 0; level <= max_level; level++) {
4027                 skl_compute_plane_wm(crtc_state, level, &wp, &wm, &wm);
4028                 if (wm.min_ddb_alloc == U16_MAX)
4029                         break;
4030
4031                 min_ddb_alloc = wm.min_ddb_alloc;
4032         }
4033
4034         return max(num_active == 1 ? 32 : 8, min_ddb_alloc);
4035 }
4036
4037 static void skl_ddb_entry_init_from_hw(struct drm_i915_private *dev_priv,
4038                                        struct skl_ddb_entry *entry, u32 reg)
4039 {
4040
4041         entry->start = reg & DDB_ENTRY_MASK;
4042         entry->end = (reg >> DDB_ENTRY_END_SHIFT) & DDB_ENTRY_MASK;
4043
4044         if (entry->end)
4045                 entry->end += 1;
4046 }
4047
4048 static void
4049 skl_ddb_get_hw_plane_state(struct drm_i915_private *dev_priv,
4050                            const enum pipe pipe,
4051                            const enum plane_id plane_id,
4052                            struct skl_ddb_entry *ddb_y,
4053                            struct skl_ddb_entry *ddb_uv)
4054 {
4055         u32 val, val2;
4056         u32 fourcc = 0;
4057
4058         /* Cursor doesn't support NV12/planar, so no extra calculation needed */
4059         if (plane_id == PLANE_CURSOR) {
4060                 val = I915_READ(CUR_BUF_CFG(pipe));
4061                 skl_ddb_entry_init_from_hw(dev_priv, ddb_y, val);
4062                 return;
4063         }
4064
4065         val = I915_READ(PLANE_CTL(pipe, plane_id));
4066
4067         /* No DDB allocated for disabled planes */
4068         if (val & PLANE_CTL_ENABLE)
4069                 fourcc = skl_format_to_fourcc(val & PLANE_CTL_FORMAT_MASK,
4070                                               val & PLANE_CTL_ORDER_RGBX,
4071                                               val & PLANE_CTL_ALPHA_MASK);
4072
4073         if (INTEL_GEN(dev_priv) >= 11) {
4074                 val = I915_READ(PLANE_BUF_CFG(pipe, plane_id));
4075                 skl_ddb_entry_init_from_hw(dev_priv, ddb_y, val);
4076         } else {
4077                 val = I915_READ(PLANE_BUF_CFG(pipe, plane_id));
4078                 val2 = I915_READ(PLANE_NV12_BUF_CFG(pipe, plane_id));
4079
4080                 if (fourcc &&
4081                     drm_format_info_is_yuv_semiplanar(drm_format_info(fourcc)))
4082                         swap(val, val2);
4083
4084                 skl_ddb_entry_init_from_hw(dev_priv, ddb_y, val);
4085                 skl_ddb_entry_init_from_hw(dev_priv, ddb_uv, val2);
4086         }
4087 }
4088
4089 void skl_pipe_ddb_get_hw_state(struct intel_crtc *crtc,
4090                                struct skl_ddb_entry *ddb_y,
4091                                struct skl_ddb_entry *ddb_uv)
4092 {
4093         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
4094         enum intel_display_power_domain power_domain;
4095         enum pipe pipe = crtc->pipe;
4096         intel_wakeref_t wakeref;
4097         enum plane_id plane_id;
4098
4099         power_domain = POWER_DOMAIN_PIPE(pipe);
4100         wakeref = intel_display_power_get_if_enabled(dev_priv, power_domain);
4101         if (!wakeref)
4102                 return;
4103
4104         for_each_plane_id_on_crtc(crtc, plane_id)
4105                 skl_ddb_get_hw_plane_state(dev_priv, pipe,
4106                                            plane_id,
4107                                            &ddb_y[plane_id],
4108                                            &ddb_uv[plane_id]);
4109
4110         intel_display_power_put(dev_priv, power_domain, wakeref);
4111 }
4112
4113 void skl_ddb_get_hw_state(struct drm_i915_private *dev_priv)
4114 {
4115         dev_priv->enabled_dbuf_slices_mask =
4116                                 intel_enabled_dbuf_slices_mask(dev_priv);
4117 }
4118
4119 /*
4120  * Determines the downscale amount of a plane for the purposes of watermark calculations.
4121  * The bspec defines downscale amount as:
4122  *
4123  * """
4124  * Horizontal down scale amount = maximum[1, Horizontal source size /
4125  *                                           Horizontal destination size]
4126  * Vertical down scale amount = maximum[1, Vertical source size /
4127  *                                         Vertical destination size]
4128  * Total down scale amount = Horizontal down scale amount *
4129  *                           Vertical down scale amount
4130  * """
4131  *
4132  * Return value is provided in 16.16 fixed point form to retain fractional part.
4133  * Caller should take care of dividing & rounding off the value.
4134  */
4135 static uint_fixed_16_16_t
4136 skl_plane_downscale_amount(const struct intel_crtc_state *crtc_state,
4137                            const struct intel_plane_state *plane_state)
4138 {
4139         u32 src_w, src_h, dst_w, dst_h;
4140         uint_fixed_16_16_t fp_w_ratio, fp_h_ratio;
4141         uint_fixed_16_16_t downscale_h, downscale_w;
4142
4143         if (WARN_ON(!intel_wm_plane_visible(crtc_state, plane_state)))
4144                 return u32_to_fixed16(0);
4145
4146         /*
4147          * Src coordinates are already rotated by 270 degrees for
4148          * the 90/270 degree plane rotation cases (to match the
4149          * GTT mapping), hence no need to account for rotation here.
4150          *
4151          * n.b., src is 16.16 fixed point, dst is whole integer.
4152          */
4153         src_w = drm_rect_width(&plane_state->uapi.src) >> 16;
4154         src_h = drm_rect_height(&plane_state->uapi.src) >> 16;
4155         dst_w = drm_rect_width(&plane_state->uapi.dst);
4156         dst_h = drm_rect_height(&plane_state->uapi.dst);
4157
4158         fp_w_ratio = div_fixed16(src_w, dst_w);
4159         fp_h_ratio = div_fixed16(src_h, dst_h);
4160         downscale_w = max_fixed16(fp_w_ratio, u32_to_fixed16(1));
4161         downscale_h = max_fixed16(fp_h_ratio, u32_to_fixed16(1));
4162
4163         return mul_fixed16(downscale_w, downscale_h);
4164 }
4165
4166 struct dbuf_slice_conf_entry {
4167         u8 active_pipes;
4168         u8 dbuf_mask[I915_MAX_PIPES];
4169 };
4170
4171 /*
4172  * Table taken from Bspec 12716
4173  * Pipes do have some preferred DBuf slice affinity,
4174  * plus there are some hardcoded requirements on how
4175  * those should be distributed for multipipe scenarios.
4176  * For more DBuf slices algorithm can get even more messy
4177  * and less readable, so decided to use a table almost
4178  * as is from BSpec itself - that way it is at least easier
4179  * to compare, change and check.
4180  */
4181 static const struct dbuf_slice_conf_entry icl_allowed_dbufs[] =
4182 /* Autogenerated with igt/tools/intel_dbuf_map tool: */
4183 {
4184         {
4185                 .active_pipes = BIT(PIPE_A),
4186                 .dbuf_mask = {
4187                         [PIPE_A] = BIT(DBUF_S1)
4188                 }
4189         },
4190         {
4191                 .active_pipes = BIT(PIPE_B),
4192                 .dbuf_mask = {
4193                         [PIPE_B] = BIT(DBUF_S1)
4194                 }
4195         },
4196         {
4197                 .active_pipes = BIT(PIPE_A) | BIT(PIPE_B),
4198                 .dbuf_mask = {
4199                         [PIPE_A] = BIT(DBUF_S1),
4200                         [PIPE_B] = BIT(DBUF_S2)
4201                 }
4202         },
4203         {
4204                 .active_pipes = BIT(PIPE_C),
4205                 .dbuf_mask = {
4206                         [PIPE_C] = BIT(DBUF_S2)
4207                 }
4208         },
4209         {
4210                 .active_pipes = BIT(PIPE_A) | BIT(PIPE_C),
4211                 .dbuf_mask = {
4212                         [PIPE_A] = BIT(DBUF_S1),
4213                         [PIPE_C] = BIT(DBUF_S2)
4214                 }
4215         },
4216         {
4217                 .active_pipes = BIT(PIPE_B) | BIT(PIPE_C),
4218                 .dbuf_mask = {
4219                         [PIPE_B] = BIT(DBUF_S1),
4220                         [PIPE_C] = BIT(DBUF_S2)
4221                 }
4222         },
4223         {
4224                 .active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_C),
4225                 .dbuf_mask = {
4226                         [PIPE_A] = BIT(DBUF_S1),
4227                         [PIPE_B] = BIT(DBUF_S1),
4228                         [PIPE_C] = BIT(DBUF_S2)
4229                 }
4230         },
4231 };
4232
4233 /*
4234  * Table taken from Bspec 49255
4235  * Pipes do have some preferred DBuf slice affinity,
4236  * plus there are some hardcoded requirements on how
4237  * those should be distributed for multipipe scenarios.
4238  * For more DBuf slices algorithm can get even more messy
4239  * and less readable, so decided to use a table almost
4240  * as is from BSpec itself - that way it is at least easier
4241  * to compare, change and check.
4242  */
4243 static const struct dbuf_slice_conf_entry tgl_allowed_dbufs[] =
4244 /* Autogenerated with igt/tools/intel_dbuf_map tool: */
4245 {
4246         {
4247                 .active_pipes = BIT(PIPE_A),
4248                 .dbuf_mask = {
4249                         [PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2)
4250                 }
4251         },
4252         {
4253                 .active_pipes = BIT(PIPE_B),
4254                 .dbuf_mask = {
4255                         [PIPE_B] = BIT(DBUF_S1) | BIT(DBUF_S2)
4256                 }
4257         },
4258         {
4259                 .active_pipes = BIT(PIPE_A) | BIT(PIPE_B),
4260                 .dbuf_mask = {
4261                         [PIPE_A] = BIT(DBUF_S2),
4262                         [PIPE_B] = BIT(DBUF_S1)
4263                 }
4264         },
4265         {
4266                 .active_pipes = BIT(PIPE_C),
4267                 .dbuf_mask = {
4268                         [PIPE_C] = BIT(DBUF_S2) | BIT(DBUF_S1)
4269                 }
4270         },
4271         {
4272                 .active_pipes = BIT(PIPE_A) | BIT(PIPE_C),
4273                 .dbuf_mask = {
4274                         [PIPE_A] = BIT(DBUF_S1),
4275                         [PIPE_C] = BIT(DBUF_S2)
4276                 }
4277         },
4278         {
4279                 .active_pipes = BIT(PIPE_B) | BIT(PIPE_C),
4280                 .dbuf_mask = {
4281                         [PIPE_B] = BIT(DBUF_S1),
4282                         [PIPE_C] = BIT(DBUF_S2)
4283                 }
4284         },
4285         {
4286                 .active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_C),
4287                 .dbuf_mask = {
4288                         [PIPE_A] = BIT(DBUF_S1),
4289                         [PIPE_B] = BIT(DBUF_S1),
4290                         [PIPE_C] = BIT(DBUF_S2)
4291                 }
4292         },
4293         {
4294                 .active_pipes = BIT(PIPE_D),
4295                 .dbuf_mask = {
4296                         [PIPE_D] = BIT(DBUF_S2) | BIT(DBUF_S1)
4297                 }
4298         },
4299         {
4300                 .active_pipes = BIT(PIPE_A) | BIT(PIPE_D),
4301                 .dbuf_mask = {
4302                         [PIPE_A] = BIT(DBUF_S1),
4303                         [PIPE_D] = BIT(DBUF_S2)
4304                 }
4305         },
4306         {
4307                 .active_pipes = BIT(PIPE_B) | BIT(PIPE_D),
4308                 .dbuf_mask = {
4309                         [PIPE_B] = BIT(DBUF_S1),
4310                         [PIPE_D] = BIT(DBUF_S2)
4311                 }
4312         },
4313         {
4314                 .active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_D),
4315                 .dbuf_mask = {
4316                         [PIPE_A] = BIT(DBUF_S1),
4317                         [PIPE_B] = BIT(DBUF_S1),
4318                         [PIPE_D] = BIT(DBUF_S2)
4319                 }
4320         },
4321         {
4322                 .active_pipes = BIT(PIPE_C) | BIT(PIPE_D),
4323                 .dbuf_mask = {
4324                         [PIPE_C] = BIT(DBUF_S1),
4325                         [PIPE_D] = BIT(DBUF_S2)
4326                 }
4327         },
4328         {
4329                 .active_pipes = BIT(PIPE_A) | BIT(PIPE_C) | BIT(PIPE_D),
4330                 .dbuf_mask = {
4331                         [PIPE_A] = BIT(DBUF_S1),
4332                         [PIPE_C] = BIT(DBUF_S2),
4333                         [PIPE_D] = BIT(DBUF_S2)
4334                 }
4335         },
4336         {
4337                 .active_pipes = BIT(PIPE_B) | BIT(PIPE_C) | BIT(PIPE_D),
4338                 .dbuf_mask = {
4339                         [PIPE_B] = BIT(DBUF_S1),
4340                         [PIPE_C] = BIT(DBUF_S2),
4341                         [PIPE_D] = BIT(DBUF_S2)
4342                 }
4343         },
4344         {
4345                 .active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_C) | BIT(PIPE_D),
4346                 .dbuf_mask = {
4347                         [PIPE_A] = BIT(DBUF_S1),
4348                         [PIPE_B] = BIT(DBUF_S1),
4349                         [PIPE_C] = BIT(DBUF_S2),
4350                         [PIPE_D] = BIT(DBUF_S2)
4351                 }
4352         },
4353 };
4354
4355 static u8 compute_dbuf_slices(enum pipe pipe,
4356                               u32 active_pipes,
4357                               const struct dbuf_slice_conf_entry *dbuf_slices,
4358                               int size)
4359 {
4360         int i;
4361
4362         for (i = 0; i < size; i++) {
4363                 if (dbuf_slices[i].active_pipes == active_pipes)
4364                         return dbuf_slices[i].dbuf_mask[pipe];
4365         }
4366         return 0;
4367 }
4368
4369 /*
4370  * This function finds an entry with same enabled pipe configuration and
4371  * returns correspondent DBuf slice mask as stated in BSpec for particular
4372  * platform.
4373  */
4374 static u32 icl_compute_dbuf_slices(enum pipe pipe,
4375                                    u32 active_pipes)
4376 {
4377         /*
4378          * FIXME: For ICL this is still a bit unclear as prev BSpec revision
4379          * required calculating "pipe ratio" in order to determine
4380          * if one or two slices can be used for single pipe configurations
4381          * as additional constraint to the existing table.
4382          * However based on recent info, it should be not "pipe ratio"
4383          * but rather ratio between pixel_rate and cdclk with additional
4384          * constants, so for now we are using only table until this is
4385          * clarified. Also this is the reason why crtc_state param is
4386          * still here - we will need it once those additional constraints
4387          * pop up.
4388          */
4389         return compute_dbuf_slices(pipe, active_pipes,
4390                                    icl_allowed_dbufs,
4391                                    ARRAY_SIZE(icl_allowed_dbufs));
4392 }
4393
4394 static u32 tgl_compute_dbuf_slices(enum pipe pipe,
4395                                    u32 active_pipes)
4396 {
4397         return compute_dbuf_slices(pipe, active_pipes,
4398                                    tgl_allowed_dbufs,
4399                                    ARRAY_SIZE(tgl_allowed_dbufs));
4400 }
4401
4402 static u8 skl_compute_dbuf_slices(const struct intel_crtc_state *crtc_state,
4403                                   u32 active_pipes)
4404 {
4405         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
4406         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
4407         enum pipe pipe = crtc->pipe;
4408
4409         if (IS_GEN(dev_priv, 12))
4410                 return tgl_compute_dbuf_slices(pipe,
4411                                                active_pipes);
4412         else if (IS_GEN(dev_priv, 11))
4413                 return icl_compute_dbuf_slices(pipe,
4414                                                active_pipes);
4415         /*
4416          * For anything else just return one slice yet.
4417          * Should be extended for other platforms.
4418          */
4419         return BIT(DBUF_S1);
4420 }
4421
4422 static u64
4423 skl_plane_relative_data_rate(const struct intel_crtc_state *crtc_state,
4424                              const struct intel_plane_state *plane_state,
4425                              int color_plane)
4426 {
4427         struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
4428         const struct drm_framebuffer *fb = plane_state->hw.fb;
4429         u32 data_rate;
4430         u32 width = 0, height = 0;
4431         uint_fixed_16_16_t down_scale_amount;
4432         u64 rate;
4433
4434         if (!plane_state->uapi.visible)
4435                 return 0;
4436
4437         if (plane->id == PLANE_CURSOR)
4438                 return 0;
4439
4440         if (color_plane == 1 &&
4441             !intel_format_info_is_yuv_semiplanar(fb->format, fb->modifier))
4442                 return 0;
4443
4444         /*
4445          * Src coordinates are already rotated by 270 degrees for
4446          * the 90/270 degree plane rotation cases (to match the
4447          * GTT mapping), hence no need to account for rotation here.
4448          */
4449         width = drm_rect_width(&plane_state->uapi.src) >> 16;
4450         height = drm_rect_height(&plane_state->uapi.src) >> 16;
4451
4452         /* UV plane does 1/2 pixel sub-sampling */
4453         if (color_plane == 1) {
4454                 width /= 2;
4455                 height /= 2;
4456         }
4457
4458         data_rate = width * height;
4459
4460         down_scale_amount = skl_plane_downscale_amount(crtc_state, plane_state);
4461
4462         rate = mul_round_up_u32_fixed16(data_rate, down_scale_amount);
4463
4464         rate *= fb->format->cpp[color_plane];
4465         return rate;
4466 }
4467
4468 static u64
4469 skl_get_total_relative_data_rate(struct intel_crtc_state *crtc_state,
4470                                  u64 *plane_data_rate,
4471                                  u64 *uv_plane_data_rate)
4472 {
4473         struct drm_atomic_state *state = crtc_state->uapi.state;
4474         struct intel_plane *plane;
4475         const struct intel_plane_state *plane_state;
4476         u64 total_data_rate = 0;
4477
4478         if (WARN_ON(!state))
4479                 return 0;
4480
4481         /* Calculate and cache data rate for each plane */
4482         intel_atomic_crtc_state_for_each_plane_state(plane, plane_state, crtc_state) {
4483                 enum plane_id plane_id = plane->id;
4484                 u64 rate;
4485
4486                 /* packed/y */
4487                 rate = skl_plane_relative_data_rate(crtc_state, plane_state, 0);
4488                 plane_data_rate[plane_id] = rate;
4489                 total_data_rate += rate;
4490
4491                 /* uv-plane */
4492                 rate = skl_plane_relative_data_rate(crtc_state, plane_state, 1);
4493                 uv_plane_data_rate[plane_id] = rate;
4494                 total_data_rate += rate;
4495         }
4496
4497         return total_data_rate;
4498 }
4499
4500 static u64
4501 icl_get_total_relative_data_rate(struct intel_crtc_state *crtc_state,
4502                                  u64 *plane_data_rate)
4503 {
4504         struct intel_plane *plane;
4505         const struct intel_plane_state *plane_state;
4506         u64 total_data_rate = 0;
4507
4508         if (WARN_ON(!crtc_state->uapi.state))
4509                 return 0;
4510
4511         /* Calculate and cache data rate for each plane */
4512         intel_atomic_crtc_state_for_each_plane_state(plane, plane_state, crtc_state) {
4513                 enum plane_id plane_id = plane->id;
4514                 u64 rate;
4515
4516                 if (!plane_state->planar_linked_plane) {
4517                         rate = skl_plane_relative_data_rate(crtc_state, plane_state, 0);
4518                         plane_data_rate[plane_id] = rate;
4519                         total_data_rate += rate;
4520                 } else {
4521                         enum plane_id y_plane_id;
4522
4523                         /*
4524                          * The slave plane might not iterate in
4525                          * intel_atomic_crtc_state_for_each_plane_state(),
4526                          * and needs the master plane state which may be
4527                          * NULL if we try get_new_plane_state(), so we
4528                          * always calculate from the master.
4529                          */
4530                         if (plane_state->planar_slave)
4531                                 continue;
4532
4533                         /* Y plane rate is calculated on the slave */
4534                         rate = skl_plane_relative_data_rate(crtc_state, plane_state, 0);
4535                         y_plane_id = plane_state->planar_linked_plane->id;
4536                         plane_data_rate[y_plane_id] = rate;
4537                         total_data_rate += rate;
4538
4539                         rate = skl_plane_relative_data_rate(crtc_state, plane_state, 1);
4540                         plane_data_rate[plane_id] = rate;
4541                         total_data_rate += rate;
4542                 }
4543         }
4544
4545         return total_data_rate;
4546 }
4547
4548 static int
4549 skl_allocate_pipe_ddb(struct intel_crtc_state *crtc_state)
4550 {
4551         struct drm_atomic_state *state = crtc_state->uapi.state;
4552         struct drm_crtc *crtc = crtc_state->uapi.crtc;
4553         struct drm_i915_private *dev_priv = to_i915(crtc->dev);
4554         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
4555         struct skl_ddb_entry *alloc = &crtc_state->wm.skl.ddb;
4556         u16 alloc_size, start = 0;
4557         u16 total[I915_MAX_PLANES] = {};
4558         u16 uv_total[I915_MAX_PLANES] = {};
4559         u64 total_data_rate;
4560         enum plane_id plane_id;
4561         int num_active;
4562         u64 plane_data_rate[I915_MAX_PLANES] = {};
4563         u64 uv_plane_data_rate[I915_MAX_PLANES] = {};
4564         u32 blocks;
4565         int level;
4566
4567         /* Clear the partitioning for disabled planes. */
4568         memset(crtc_state->wm.skl.plane_ddb_y, 0, sizeof(crtc_state->wm.skl.plane_ddb_y));
4569         memset(crtc_state->wm.skl.plane_ddb_uv, 0, sizeof(crtc_state->wm.skl.plane_ddb_uv));
4570
4571         if (drm_WARN_ON(&dev_priv->drm, !state))
4572                 return 0;
4573
4574         if (!crtc_state->hw.active) {
4575                 alloc->start = alloc->end = 0;
4576                 return 0;
4577         }
4578
4579         if (INTEL_GEN(dev_priv) >= 11)
4580                 total_data_rate =
4581                         icl_get_total_relative_data_rate(crtc_state,
4582                                                          plane_data_rate);
4583         else
4584                 total_data_rate =
4585                         skl_get_total_relative_data_rate(crtc_state,
4586                                                          plane_data_rate,
4587                                                          uv_plane_data_rate);
4588
4589
4590         skl_ddb_get_pipe_allocation_limits(dev_priv, crtc_state, total_data_rate,
4591                                            alloc, &num_active);
4592         alloc_size = skl_ddb_entry_size(alloc);
4593         if (alloc_size == 0)
4594                 return 0;
4595
4596         /* Allocate fixed number of blocks for cursor. */
4597         total[PLANE_CURSOR] = skl_cursor_allocation(crtc_state, num_active);
4598         alloc_size -= total[PLANE_CURSOR];
4599         crtc_state->wm.skl.plane_ddb_y[PLANE_CURSOR].start =
4600                 alloc->end - total[PLANE_CURSOR];
4601         crtc_state->wm.skl.plane_ddb_y[PLANE_CURSOR].end = alloc->end;
4602
4603         if (total_data_rate == 0)
4604                 return 0;
4605
4606         /*
4607          * Find the highest watermark level for which we can satisfy the block
4608          * requirement of active planes.
4609          */
4610         for (level = ilk_wm_max_level(dev_priv); level >= 0; level--) {
4611                 blocks = 0;
4612                 for_each_plane_id_on_crtc(intel_crtc, plane_id) {
4613                         const struct skl_plane_wm *wm =
4614                                 &crtc_state->wm.skl.optimal.planes[plane_id];
4615
4616                         if (plane_id == PLANE_CURSOR) {
4617                                 if (wm->wm[level].min_ddb_alloc > total[PLANE_CURSOR]) {
4618                                         drm_WARN_ON(&dev_priv->drm,
4619                                                     wm->wm[level].min_ddb_alloc != U16_MAX);
4620                                         blocks = U32_MAX;
4621                                         break;
4622                                 }
4623                                 continue;
4624                         }
4625
4626                         blocks += wm->wm[level].min_ddb_alloc;
4627                         blocks += wm->uv_wm[level].min_ddb_alloc;
4628                 }
4629
4630                 if (blocks <= alloc_size) {
4631                         alloc_size -= blocks;
4632                         break;
4633                 }
4634         }
4635
4636         if (level < 0) {
4637                 drm_dbg_kms(&dev_priv->drm,
4638                             "Requested display configuration exceeds system DDB limitations");
4639                 drm_dbg_kms(&dev_priv->drm, "minimum required %d/%d\n",
4640                             blocks, alloc_size);
4641                 return -EINVAL;
4642         }
4643
4644         /*
4645          * Grant each plane the blocks it requires at the highest achievable
4646          * watermark level, plus an extra share of the leftover blocks
4647          * proportional to its relative data rate.
4648          */
4649         for_each_plane_id_on_crtc(intel_crtc, plane_id) {
4650                 const struct skl_plane_wm *wm =
4651                         &crtc_state->wm.skl.optimal.planes[plane_id];
4652                 u64 rate;
4653                 u16 extra;
4654
4655                 if (plane_id == PLANE_CURSOR)
4656                         continue;
4657
4658                 /*
4659                  * We've accounted for all active planes; remaining planes are
4660                  * all disabled.
4661                  */
4662                 if (total_data_rate == 0)
4663                         break;
4664
4665                 rate = plane_data_rate[plane_id];
4666                 extra = min_t(u16, alloc_size,
4667                               DIV64_U64_ROUND_UP(alloc_size * rate,
4668                                                  total_data_rate));
4669                 total[plane_id] = wm->wm[level].min_ddb_alloc + extra;
4670                 alloc_size -= extra;
4671                 total_data_rate -= rate;
4672
4673                 if (total_data_rate == 0)
4674                         break;
4675
4676                 rate = uv_plane_data_rate[plane_id];
4677                 extra = min_t(u16, alloc_size,
4678                               DIV64_U64_ROUND_UP(alloc_size * rate,
4679                                                  total_data_rate));
4680                 uv_total[plane_id] = wm->uv_wm[level].min_ddb_alloc + extra;
4681                 alloc_size -= extra;
4682                 total_data_rate -= rate;
4683         }
4684         drm_WARN_ON(&dev_priv->drm, alloc_size != 0 || total_data_rate != 0);
4685
4686         /* Set the actual DDB start/end points for each plane */
4687         start = alloc->start;
4688         for_each_plane_id_on_crtc(intel_crtc, plane_id) {
4689                 struct skl_ddb_entry *plane_alloc =
4690                         &crtc_state->wm.skl.plane_ddb_y[plane_id];
4691                 struct skl_ddb_entry *uv_plane_alloc =
4692                         &crtc_state->wm.skl.plane_ddb_uv[plane_id];
4693
4694                 if (plane_id == PLANE_CURSOR)
4695                         continue;
4696
4697                 /* Gen11+ uses a separate plane for UV watermarks */
4698                 drm_WARN_ON(&dev_priv->drm,
4699                             INTEL_GEN(dev_priv) >= 11 && uv_total[plane_id]);
4700
4701                 /* Leave disabled planes at (0,0) */
4702                 if (total[plane_id]) {
4703                         plane_alloc->start = start;
4704                         start += total[plane_id];
4705                         plane_alloc->end = start;
4706                 }
4707
4708                 if (uv_total[plane_id]) {
4709                         uv_plane_alloc->start = start;
4710                         start += uv_total[plane_id];
4711                         uv_plane_alloc->end = start;
4712                 }
4713         }
4714
4715         /*
4716          * When we calculated watermark values we didn't know how high
4717          * of a level we'd actually be able to hit, so we just marked
4718          * all levels as "enabled."  Go back now and disable the ones
4719          * that aren't actually possible.
4720          */
4721         for (level++; level <= ilk_wm_max_level(dev_priv); level++) {
4722                 for_each_plane_id_on_crtc(intel_crtc, plane_id) {
4723                         struct skl_plane_wm *wm =
4724                                 &crtc_state->wm.skl.optimal.planes[plane_id];
4725
4726                         /*
4727                          * We only disable the watermarks for each plane if
4728                          * they exceed the ddb allocation of said plane. This
4729                          * is done so that we don't end up touching cursor
4730                          * watermarks needlessly when some other plane reduces
4731                          * our max possible watermark level.
4732                          *
4733                          * Bspec has this to say about the PLANE_WM enable bit:
4734                          * "All the watermarks at this level for all enabled
4735                          *  planes must be enabled before the level will be used."
4736                          * So this is actually safe to do.
4737                          */
4738                         if (wm->wm[level].min_ddb_alloc > total[plane_id] ||
4739                             wm->uv_wm[level].min_ddb_alloc > uv_total[plane_id])
4740                                 memset(&wm->wm[level], 0, sizeof(wm->wm[level]));
4741
4742                         /*
4743                          * Wa_1408961008:icl, ehl
4744                          * Underruns with WM1+ disabled
4745                          */
4746                         if (IS_GEN(dev_priv, 11) &&
4747                             level == 1 && wm->wm[0].plane_en) {
4748                                 wm->wm[level].plane_res_b = wm->wm[0].plane_res_b;
4749                                 wm->wm[level].plane_res_l = wm->wm[0].plane_res_l;
4750                                 wm->wm[level].ignore_lines = wm->wm[0].ignore_lines;
4751                         }
4752                 }
4753         }
4754
4755         /*
4756          * Go back and disable the transition watermark if it turns out we
4757          * don't have enough DDB blocks for it.
4758          */
4759         for_each_plane_id_on_crtc(intel_crtc, plane_id) {
4760                 struct skl_plane_wm *wm =
4761                         &crtc_state->wm.skl.optimal.planes[plane_id];
4762
4763                 if (wm->trans_wm.plane_res_b >= total[plane_id])
4764                         memset(&wm->trans_wm, 0, sizeof(wm->trans_wm));
4765         }
4766
4767         return 0;
4768 }
4769
4770 /*
4771  * The max latency should be 257 (max the punit can code is 255 and we add 2us
4772  * for the read latency) and cpp should always be <= 8, so that
4773  * should allow pixel_rate up to ~2 GHz which seems sufficient since max
4774  * 2xcdclk is 1350 MHz and the pixel rate should never exceed that.
4775 */
4776 static uint_fixed_16_16_t
4777 skl_wm_method1(const struct drm_i915_private *dev_priv, u32 pixel_rate,
4778                u8 cpp, u32 latency, u32 dbuf_block_size)
4779 {
4780         u32 wm_intermediate_val;
4781         uint_fixed_16_16_t ret;
4782
4783         if (latency == 0)
4784                 return FP_16_16_MAX;
4785
4786         wm_intermediate_val = latency * pixel_rate * cpp;
4787         ret = div_fixed16(wm_intermediate_val, 1000 * dbuf_block_size);
4788
4789         if (INTEL_GEN(dev_priv) >= 10)
4790                 ret = add_fixed16_u32(ret, 1);
4791
4792         return ret;
4793 }
4794
4795 static uint_fixed_16_16_t
4796 skl_wm_method2(u32 pixel_rate, u32 pipe_htotal, u32 latency,
4797                uint_fixed_16_16_t plane_blocks_per_line)
4798 {
4799         u32 wm_intermediate_val;
4800         uint_fixed_16_16_t ret;
4801
4802         if (latency == 0)
4803                 return FP_16_16_MAX;
4804
4805         wm_intermediate_val = latency * pixel_rate;
4806         wm_intermediate_val = DIV_ROUND_UP(wm_intermediate_val,
4807                                            pipe_htotal * 1000);
4808         ret = mul_u32_fixed16(wm_intermediate_val, plane_blocks_per_line);
4809         return ret;
4810 }
4811
4812 static uint_fixed_16_16_t
4813 intel_get_linetime_us(const struct intel_crtc_state *crtc_state)
4814 {
4815         u32 pixel_rate;
4816         u32 crtc_htotal;
4817         uint_fixed_16_16_t linetime_us;
4818
4819         if (!crtc_state->hw.active)
4820                 return u32_to_fixed16(0);
4821
4822         pixel_rate = crtc_state->pixel_rate;
4823
4824         if (WARN_ON(pixel_rate == 0))
4825                 return u32_to_fixed16(0);
4826
4827         crtc_htotal = crtc_state->hw.adjusted_mode.crtc_htotal;
4828         linetime_us = div_fixed16(crtc_htotal * 1000, pixel_rate);
4829
4830         return linetime_us;
4831 }
4832
4833 static u32
4834 skl_adjusted_plane_pixel_rate(const struct intel_crtc_state *crtc_state,
4835                               const struct intel_plane_state *plane_state)
4836 {
4837         u64 adjusted_pixel_rate;
4838         uint_fixed_16_16_t downscale_amount;
4839
4840         /* Shouldn't reach here on disabled planes... */
4841         if (WARN_ON(!intel_wm_plane_visible(crtc_state, plane_state)))
4842                 return 0;
4843
4844         /*
4845          * Adjusted plane pixel rate is just the pipe's adjusted pixel rate
4846          * with additional adjustments for plane-specific scaling.
4847          */
4848         adjusted_pixel_rate = crtc_state->pixel_rate;
4849         downscale_amount = skl_plane_downscale_amount(crtc_state, plane_state);
4850
4851         return mul_round_up_u32_fixed16(adjusted_pixel_rate,
4852                                             downscale_amount);
4853 }
4854
4855 static int
4856 skl_compute_wm_params(const struct intel_crtc_state *crtc_state,
4857                       int width, const struct drm_format_info *format,
4858                       u64 modifier, unsigned int rotation,
4859                       u32 plane_pixel_rate, struct skl_wm_params *wp,
4860                       int color_plane)
4861 {
4862         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
4863         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
4864         u32 interm_pbpl;
4865
4866         /* only planar format has two planes */
4867         if (color_plane == 1 &&
4868             !intel_format_info_is_yuv_semiplanar(format, modifier)) {
4869                 drm_dbg_kms(&dev_priv->drm,
4870                             "Non planar format have single plane\n");
4871                 return -EINVAL;
4872         }
4873
4874         wp->y_tiled = modifier == I915_FORMAT_MOD_Y_TILED ||
4875                       modifier == I915_FORMAT_MOD_Yf_TILED ||
4876                       modifier == I915_FORMAT_MOD_Y_TILED_CCS ||
4877                       modifier == I915_FORMAT_MOD_Yf_TILED_CCS;
4878         wp->x_tiled = modifier == I915_FORMAT_MOD_X_TILED;
4879         wp->rc_surface = modifier == I915_FORMAT_MOD_Y_TILED_CCS ||
4880                          modifier == I915_FORMAT_MOD_Yf_TILED_CCS;
4881         wp->is_planar = intel_format_info_is_yuv_semiplanar(format, modifier);
4882
4883         wp->width = width;
4884         if (color_plane == 1 && wp->is_planar)
4885                 wp->width /= 2;
4886
4887         wp->cpp = format->cpp[color_plane];
4888         wp->plane_pixel_rate = plane_pixel_rate;
4889
4890         if (INTEL_GEN(dev_priv) >= 11 &&
4891             modifier == I915_FORMAT_MOD_Yf_TILED  && wp->cpp == 1)
4892                 wp->dbuf_block_size = 256;
4893         else
4894                 wp->dbuf_block_size = 512;
4895
4896         if (drm_rotation_90_or_270(rotation)) {
4897                 switch (wp->cpp) {
4898                 case 1:
4899                         wp->y_min_scanlines = 16;
4900                         break;
4901                 case 2:
4902                         wp->y_min_scanlines = 8;
4903                         break;
4904                 case 4:
4905                         wp->y_min_scanlines = 4;
4906                         break;
4907                 default:
4908                         MISSING_CASE(wp->cpp);
4909                         return -EINVAL;
4910                 }
4911         } else {
4912                 wp->y_min_scanlines = 4;
4913         }
4914
4915         if (skl_needs_memory_bw_wa(dev_priv))
4916                 wp->y_min_scanlines *= 2;
4917
4918         wp->plane_bytes_per_line = wp->width * wp->cpp;
4919         if (wp->y_tiled) {
4920                 interm_pbpl = DIV_ROUND_UP(wp->plane_bytes_per_line *
4921                                            wp->y_min_scanlines,
4922                                            wp->dbuf_block_size);
4923
4924                 if (INTEL_GEN(dev_priv) >= 10)
4925                         interm_pbpl++;
4926
4927                 wp->plane_blocks_per_line = div_fixed16(interm_pbpl,
4928                                                         wp->y_min_scanlines);
4929         } else if (wp->x_tiled && IS_GEN(dev_priv, 9)) {
4930                 interm_pbpl = DIV_ROUND_UP(wp->plane_bytes_per_line,
4931                                            wp->dbuf_block_size);
4932                 wp->plane_blocks_per_line = u32_to_fixed16(interm_pbpl);
4933         } else {
4934                 interm_pbpl = DIV_ROUND_UP(wp->plane_bytes_per_line,
4935                                            wp->dbuf_block_size) + 1;
4936                 wp->plane_blocks_per_line = u32_to_fixed16(interm_pbpl);
4937         }
4938
4939         wp->y_tile_minimum = mul_u32_fixed16(wp->y_min_scanlines,
4940                                              wp->plane_blocks_per_line);
4941
4942         wp->linetime_us = fixed16_to_u32_round_up(
4943                                         intel_get_linetime_us(crtc_state));
4944
4945         return 0;
4946 }
4947
4948 static int
4949 skl_compute_plane_wm_params(const struct intel_crtc_state *crtc_state,
4950                             const struct intel_plane_state *plane_state,
4951                             struct skl_wm_params *wp, int color_plane)
4952 {
4953         const struct drm_framebuffer *fb = plane_state->hw.fb;
4954         int width;
4955
4956         /*
4957          * Src coordinates are already rotated by 270 degrees for
4958          * the 90/270 degree plane rotation cases (to match the
4959          * GTT mapping), hence no need to account for rotation here.
4960          */
4961         width = drm_rect_width(&plane_state->uapi.src) >> 16;
4962
4963         return skl_compute_wm_params(crtc_state, width,
4964                                      fb->format, fb->modifier,
4965                                      plane_state->hw.rotation,
4966                                      skl_adjusted_plane_pixel_rate(crtc_state, plane_state),
4967                                      wp, color_plane);
4968 }
4969
4970 static bool skl_wm_has_lines(struct drm_i915_private *dev_priv, int level)
4971 {
4972         if (INTEL_GEN(dev_priv) >= 10 || IS_GEMINILAKE(dev_priv))
4973                 return true;
4974
4975         /* The number of lines are ignored for the level 0 watermark. */
4976         return level > 0;
4977 }
4978
4979 static void skl_compute_plane_wm(const struct intel_crtc_state *crtc_state,
4980                                  int level,
4981                                  const struct skl_wm_params *wp,
4982                                  const struct skl_wm_level *result_prev,
4983                                  struct skl_wm_level *result /* out */)
4984 {
4985         struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
4986         u32 latency = dev_priv->wm.skl_latency[level];
4987         uint_fixed_16_16_t method1, method2;
4988         uint_fixed_16_16_t selected_result;
4989         u32 res_blocks, res_lines, min_ddb_alloc = 0;
4990
4991         if (latency == 0) {
4992                 /* reject it */
4993                 result->min_ddb_alloc = U16_MAX;
4994                 return;
4995         }
4996
4997         /*
4998          * WaIncreaseLatencyIPCEnabled: kbl,cfl
4999          * Display WA #1141: kbl,cfl
5000          */
5001         if ((IS_KABYLAKE(dev_priv) || IS_COFFEELAKE(dev_priv)) ||
5002             dev_priv->ipc_enabled)
5003                 latency += 4;
5004
5005         if (skl_needs_memory_bw_wa(dev_priv) && wp->x_tiled)
5006                 latency += 15;
5007
5008         method1 = skl_wm_method1(dev_priv, wp->plane_pixel_rate,
5009                                  wp->cpp, latency, wp->dbuf_block_size);
5010         method2 = skl_wm_method2(wp->plane_pixel_rate,
5011                                  crtc_state->hw.adjusted_mode.crtc_htotal,
5012                                  latency,
5013                                  wp->plane_blocks_per_line);
5014
5015         if (wp->y_tiled) {
5016                 selected_result = max_fixed16(method2, wp->y_tile_minimum);
5017         } else {
5018                 if ((wp->cpp * crtc_state->hw.adjusted_mode.crtc_htotal /
5019                      wp->dbuf_block_size < 1) &&
5020                      (wp->plane_bytes_per_line / wp->dbuf_block_size < 1)) {
5021                         selected_result = method2;
5022                 } else if (latency >= wp->linetime_us) {
5023                         if (IS_GEN(dev_priv, 9) &&
5024                             !IS_GEMINILAKE(dev_priv))
5025                                 selected_result = min_fixed16(method1, method2);
5026                         else
5027                                 selected_result = method2;
5028                 } else {
5029                         selected_result = method1;
5030                 }
5031         }
5032
5033         res_blocks = fixed16_to_u32_round_up(selected_result) + 1;
5034         res_lines = div_round_up_fixed16(selected_result,
5035                                          wp->plane_blocks_per_line);
5036
5037         if (IS_GEN9_BC(dev_priv) || IS_BROXTON(dev_priv)) {
5038                 /* Display WA #1125: skl,bxt,kbl */
5039                 if (level == 0 && wp->rc_surface)
5040                         res_blocks +=
5041                                 fixed16_to_u32_round_up(wp->y_tile_minimum);
5042
5043                 /* Display WA #1126: skl,bxt,kbl */
5044                 if (level >= 1 && level <= 7) {
5045                         if (wp->y_tiled) {
5046                                 res_blocks +=
5047                                     fixed16_to_u32_round_up(wp->y_tile_minimum);
5048                                 res_lines += wp->y_min_scanlines;
5049                         } else {
5050                                 res_blocks++;
5051                         }
5052
5053                         /*
5054                          * Make sure result blocks for higher latency levels are
5055                          * atleast as high as level below the current level.
5056                          * Assumption in DDB algorithm optimization for special
5057                          * cases. Also covers Display WA #1125 for RC.
5058                          */
5059                         if (result_prev->plane_res_b > res_blocks)
5060                                 res_blocks = result_prev->plane_res_b;
5061                 }
5062         }
5063
5064         if (INTEL_GEN(dev_priv) >= 11) {
5065                 if (wp->y_tiled) {
5066                         int extra_lines;
5067
5068                         if (res_lines % wp->y_min_scanlines == 0)
5069                                 extra_lines = wp->y_min_scanlines;
5070                         else
5071                                 extra_lines = wp->y_min_scanlines * 2 -
5072                                         res_lines % wp->y_min_scanlines;
5073
5074                         min_ddb_alloc = mul_round_up_u32_fixed16(res_lines + extra_lines,
5075                                                                  wp->plane_blocks_per_line);
5076                 } else {
5077                         min_ddb_alloc = res_blocks +
5078                                 DIV_ROUND_UP(res_blocks, 10);
5079                 }
5080         }
5081
5082         if (!skl_wm_has_lines(dev_priv, level))
5083                 res_lines = 0;
5084
5085         if (res_lines > 31) {
5086                 /* reject it */
5087                 result->min_ddb_alloc = U16_MAX;
5088                 return;
5089         }
5090
5091         /*
5092          * If res_lines is valid, assume we can use this watermark level
5093          * for now.  We'll come back and disable it after we calculate the
5094          * DDB allocation if it turns out we don't actually have enough
5095          * blocks to satisfy it.
5096          */
5097         result->plane_res_b = res_blocks;
5098         result->plane_res_l = res_lines;
5099         /* Bspec says: value >= plane ddb allocation -> invalid, hence the +1 here */
5100         result->min_ddb_alloc = max(min_ddb_alloc, res_blocks) + 1;
5101         result->plane_en = true;
5102 }
5103
5104 static void
5105 skl_compute_wm_levels(const struct intel_crtc_state *crtc_state,
5106                       const struct skl_wm_params *wm_params,
5107                       struct skl_wm_level *levels)
5108 {
5109         struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
5110         int level, max_level = ilk_wm_max_level(dev_priv);
5111         struct skl_wm_level *result_prev = &levels[0];
5112
5113         for (level = 0; level <= max_level; level++) {
5114                 struct skl_wm_level *result = &levels[level];
5115
5116                 skl_compute_plane_wm(crtc_state, level, wm_params,
5117                                      result_prev, result);
5118
5119                 result_prev = result;
5120         }
5121 }
5122
5123 static void skl_compute_transition_wm(const struct intel_crtc_state *crtc_state,
5124                                       const struct skl_wm_params *wp,
5125                                       struct skl_plane_wm *wm)
5126 {
5127         struct drm_device *dev = crtc_state->uapi.crtc->dev;
5128         const struct drm_i915_private *dev_priv = to_i915(dev);
5129         u16 trans_min, trans_y_tile_min;
5130         const u16 trans_amount = 10; /* This is configurable amount */
5131         u16 wm0_sel_res_b, trans_offset_b, res_blocks;
5132
5133         /* Transition WM are not recommended by HW team for GEN9 */
5134         if (INTEL_GEN(dev_priv) <= 9)
5135                 return;
5136
5137         /* Transition WM don't make any sense if ipc is disabled */
5138         if (!dev_priv->ipc_enabled)
5139                 return;
5140
5141         trans_min = 14;
5142         if (INTEL_GEN(dev_priv) >= 11)
5143                 trans_min = 4;
5144
5145         trans_offset_b = trans_min + trans_amount;
5146
5147         /*
5148          * The spec asks for Selected Result Blocks for wm0 (the real value),
5149          * not Result Blocks (the integer value). Pay attention to the capital
5150          * letters. The value wm_l0->plane_res_b is actually Result Blocks, but
5151          * since Result Blocks is the ceiling of Selected Result Blocks plus 1,
5152          * and since we later will have to get the ceiling of the sum in the
5153          * transition watermarks calculation, we can just pretend Selected
5154          * Result Blocks is Result Blocks minus 1 and it should work for the
5155          * current platforms.
5156          */
5157         wm0_sel_res_b = wm->wm[0].plane_res_b - 1;
5158
5159         if (wp->y_tiled) {
5160                 trans_y_tile_min =
5161                         (u16)mul_round_up_u32_fixed16(2, wp->y_tile_minimum);
5162                 res_blocks = max(wm0_sel_res_b, trans_y_tile_min) +
5163                                 trans_offset_b;
5164         } else {
5165                 res_blocks = wm0_sel_res_b + trans_offset_b;
5166
5167                 /* WA BUG:1938466 add one block for non y-tile planes */
5168                 if (IS_CNL_REVID(dev_priv, CNL_REVID_A0, CNL_REVID_A0))
5169                         res_blocks += 1;
5170
5171         }
5172
5173         /*
5174          * Just assume we can enable the transition watermark.  After
5175          * computing the DDB we'll come back and disable it if that
5176          * assumption turns out to be false.
5177          */
5178         wm->trans_wm.plane_res_b = res_blocks + 1;
5179         wm->trans_wm.plane_en = true;
5180 }
5181
5182 static int skl_build_plane_wm_single(struct intel_crtc_state *crtc_state,
5183                                      const struct intel_plane_state *plane_state,
5184                                      enum plane_id plane_id, int color_plane)
5185 {
5186         struct skl_plane_wm *wm = &crtc_state->wm.skl.optimal.planes[plane_id];
5187         struct skl_wm_params wm_params;
5188         int ret;
5189
5190         ret = skl_compute_plane_wm_params(crtc_state, plane_state,
5191                                           &wm_params, color_plane);
5192         if (ret)
5193                 return ret;
5194
5195         skl_compute_wm_levels(crtc_state, &wm_params, wm->wm);
5196         skl_compute_transition_wm(crtc_state, &wm_params, wm);
5197
5198         return 0;
5199 }
5200
5201 static int skl_build_plane_wm_uv(struct intel_crtc_state *crtc_state,
5202                                  const struct intel_plane_state *plane_state,
5203                                  enum plane_id plane_id)
5204 {
5205         struct skl_plane_wm *wm = &crtc_state->wm.skl.optimal.planes[plane_id];
5206         struct skl_wm_params wm_params;
5207         int ret;
5208
5209         wm->is_planar = true;
5210
5211         /* uv plane watermarks must also be validated for NV12/Planar */
5212         ret = skl_compute_plane_wm_params(crtc_state, plane_state,
5213                                           &wm_params, 1);
5214         if (ret)
5215                 return ret;
5216
5217         skl_compute_wm_levels(crtc_state, &wm_params, wm->uv_wm);
5218
5219         return 0;
5220 }
5221
5222 static int skl_build_plane_wm(struct intel_crtc_state *crtc_state,
5223                               const struct intel_plane_state *plane_state)
5224 {
5225         struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
5226         const struct drm_framebuffer *fb = plane_state->hw.fb;
5227         enum plane_id plane_id = plane->id;
5228         int ret;
5229
5230         if (!intel_wm_plane_visible(crtc_state, plane_state))
5231                 return 0;
5232
5233         ret = skl_build_plane_wm_single(crtc_state, plane_state,
5234                                         plane_id, 0);
5235         if (ret)
5236                 return ret;
5237
5238         if (fb->format->is_yuv && fb->format->num_planes > 1) {
5239                 ret = skl_build_plane_wm_uv(crtc_state, plane_state,
5240                                             plane_id);
5241                 if (ret)
5242                         return ret;
5243         }
5244
5245         return 0;
5246 }
5247
5248 static int icl_build_plane_wm(struct intel_crtc_state *crtc_state,
5249                               const struct intel_plane_state *plane_state)
5250 {
5251         enum plane_id plane_id = to_intel_plane(plane_state->uapi.plane)->id;
5252         int ret;
5253
5254         /* Watermarks calculated in master */
5255         if (plane_state->planar_slave)
5256                 return 0;
5257
5258         if (plane_state->planar_linked_plane) {
5259                 const struct drm_framebuffer *fb = plane_state->hw.fb;
5260                 enum plane_id y_plane_id = plane_state->planar_linked_plane->id;
5261
5262                 WARN_ON(!intel_wm_plane_visible(crtc_state, plane_state));
5263                 WARN_ON(!fb->format->is_yuv ||
5264                         fb->format->num_planes == 1);
5265
5266                 ret = skl_build_plane_wm_single(crtc_state, plane_state,
5267                                                 y_plane_id, 0);
5268                 if (ret)
5269                         return ret;
5270
5271                 ret = skl_build_plane_wm_single(crtc_state, plane_state,
5272                                                 plane_id, 1);
5273                 if (ret)
5274                         return ret;
5275         } else if (intel_wm_plane_visible(crtc_state, plane_state)) {
5276                 ret = skl_build_plane_wm_single(crtc_state, plane_state,
5277                                                 plane_id, 0);
5278                 if (ret)
5279                         return ret;
5280         }
5281
5282         return 0;
5283 }
5284
5285 static int skl_build_pipe_wm(struct intel_crtc_state *crtc_state)
5286 {
5287         struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
5288         struct skl_pipe_wm *pipe_wm = &crtc_state->wm.skl.optimal;
5289         struct intel_plane *plane;
5290         const struct intel_plane_state *plane_state;
5291         int ret;
5292
5293         /*
5294          * We'll only calculate watermarks for planes that are actually
5295          * enabled, so make sure all other planes are set as disabled.
5296          */
5297         memset(pipe_wm->planes, 0, sizeof(pipe_wm->planes));
5298
5299         intel_atomic_crtc_state_for_each_plane_state(plane, plane_state,
5300                                                      crtc_state) {
5301
5302                 if (INTEL_GEN(dev_priv) >= 11)
5303                         ret = icl_build_plane_wm(crtc_state, plane_state);
5304                 else
5305                         ret = skl_build_plane_wm(crtc_state, plane_state);
5306                 if (ret)
5307                         return ret;
5308         }
5309
5310         return 0;
5311 }
5312
5313 static void skl_ddb_entry_write(struct drm_i915_private *dev_priv,
5314                                 i915_reg_t reg,
5315                                 const struct skl_ddb_entry *entry)
5316 {
5317         if (entry->end)
5318                 intel_de_write_fw(dev_priv, reg,
5319                                   (entry->end - 1) << 16 | entry->start);
5320         else
5321                 intel_de_write_fw(dev_priv, reg, 0);
5322 }
5323
5324 static void skl_write_wm_level(struct drm_i915_private *dev_priv,
5325                                i915_reg_t reg,
5326                                const struct skl_wm_level *level)
5327 {
5328         u32 val = 0;
5329
5330         if (level->plane_en)
5331                 val |= PLANE_WM_EN;
5332         if (level->ignore_lines)
5333                 val |= PLANE_WM_IGNORE_LINES;
5334         val |= level->plane_res_b;
5335         val |= level->plane_res_l << PLANE_WM_LINES_SHIFT;
5336
5337         intel_de_write_fw(dev_priv, reg, val);
5338 }
5339
5340 void skl_write_plane_wm(struct intel_plane *plane,
5341                         const struct intel_crtc_state *crtc_state)
5342 {
5343         struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
5344         int level, max_level = ilk_wm_max_level(dev_priv);
5345         enum plane_id plane_id = plane->id;
5346         enum pipe pipe = plane->pipe;
5347         const struct skl_plane_wm *wm =
5348                 &crtc_state->wm.skl.optimal.planes[plane_id];
5349         const struct skl_ddb_entry *ddb_y =
5350                 &crtc_state->wm.skl.plane_ddb_y[plane_id];
5351         const struct skl_ddb_entry *ddb_uv =
5352                 &crtc_state->wm.skl.plane_ddb_uv[plane_id];
5353
5354         for (level = 0; level <= max_level; level++) {
5355                 skl_write_wm_level(dev_priv, PLANE_WM(pipe, plane_id, level),
5356                                    &wm->wm[level]);
5357         }
5358         skl_write_wm_level(dev_priv, PLANE_WM_TRANS(pipe, plane_id),
5359                            &wm->trans_wm);
5360
5361         if (INTEL_GEN(dev_priv) >= 11) {
5362                 skl_ddb_entry_write(dev_priv,
5363                                     PLANE_BUF_CFG(pipe, plane_id), ddb_y);
5364                 return;
5365         }
5366
5367         if (wm->is_planar)
5368                 swap(ddb_y, ddb_uv);
5369
5370         skl_ddb_entry_write(dev_priv,
5371                             PLANE_BUF_CFG(pipe, plane_id), ddb_y);
5372         skl_ddb_entry_write(dev_priv,
5373                             PLANE_NV12_BUF_CFG(pipe, plane_id), ddb_uv);
5374 }
5375
5376 void skl_write_cursor_wm(struct intel_plane *plane,
5377                          const struct intel_crtc_state *crtc_state)
5378 {
5379         struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
5380         int level, max_level = ilk_wm_max_level(dev_priv);
5381         enum plane_id plane_id = plane->id;
5382         enum pipe pipe = plane->pipe;
5383         const struct skl_plane_wm *wm =
5384                 &crtc_state->wm.skl.optimal.planes[plane_id];
5385         const struct skl_ddb_entry *ddb =
5386                 &crtc_state->wm.skl.plane_ddb_y[plane_id];
5387
5388         for (level = 0; level <= max_level; level++) {
5389                 skl_write_wm_level(dev_priv, CUR_WM(pipe, level),
5390                                    &wm->wm[level]);
5391         }
5392         skl_write_wm_level(dev_priv, CUR_WM_TRANS(pipe), &wm->trans_wm);
5393
5394         skl_ddb_entry_write(dev_priv, CUR_BUF_CFG(pipe), ddb);
5395 }
5396
5397 bool skl_wm_level_equals(const struct skl_wm_level *l1,
5398                          const struct skl_wm_level *l2)
5399 {
5400         return l1->plane_en == l2->plane_en &&
5401                 l1->ignore_lines == l2->ignore_lines &&
5402                 l1->plane_res_l == l2->plane_res_l &&
5403                 l1->plane_res_b == l2->plane_res_b;
5404 }
5405
5406 static bool skl_plane_wm_equals(struct drm_i915_private *dev_priv,
5407                                 const struct skl_plane_wm *wm1,
5408                                 const struct skl_plane_wm *wm2)
5409 {
5410         int level, max_level = ilk_wm_max_level(dev_priv);
5411
5412         for (level = 0; level <= max_level; level++) {
5413                 if (!skl_wm_level_equals(&wm1->wm[level], &wm2->wm[level]) ||
5414                     !skl_wm_level_equals(&wm1->uv_wm[level], &wm2->uv_wm[level]))
5415                         return false;
5416         }
5417
5418         return skl_wm_level_equals(&wm1->trans_wm, &wm2->trans_wm);
5419 }
5420
5421 static inline bool skl_ddb_entries_overlap(const struct skl_ddb_entry *a,
5422                                            const struct skl_ddb_entry *b)
5423 {
5424         return a->start < b->end && b->start < a->end;
5425 }
5426
5427 bool skl_ddb_allocation_overlaps(const struct skl_ddb_entry *ddb,
5428                                  const struct skl_ddb_entry *entries,
5429                                  int num_entries, int ignore_idx)
5430 {
5431         int i;
5432
5433         for (i = 0; i < num_entries; i++) {
5434                 if (i != ignore_idx &&
5435                     skl_ddb_entries_overlap(ddb, &entries[i]))
5436                         return true;
5437         }
5438
5439         return false;
5440 }
5441
5442 static int
5443 skl_ddb_add_affected_planes(const struct intel_crtc_state *old_crtc_state,
5444                             struct intel_crtc_state *new_crtc_state)
5445 {
5446         struct intel_atomic_state *state = to_intel_atomic_state(new_crtc_state->uapi.state);
5447         struct intel_crtc *crtc = to_intel_crtc(new_crtc_state->uapi.crtc);
5448         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
5449         struct intel_plane *plane;
5450
5451         for_each_intel_plane_on_crtc(&dev_priv->drm, crtc, plane) {
5452                 struct intel_plane_state *plane_state;
5453                 enum plane_id plane_id = plane->id;
5454
5455                 if (skl_ddb_entry_equal(&old_crtc_state->wm.skl.plane_ddb_y[plane_id],
5456                                         &new_crtc_state->wm.skl.plane_ddb_y[plane_id]) &&
5457                     skl_ddb_entry_equal(&old_crtc_state->wm.skl.plane_ddb_uv[plane_id],
5458                                         &new_crtc_state->wm.skl.plane_ddb_uv[plane_id]))
5459                         continue;
5460
5461                 plane_state = intel_atomic_get_plane_state(state, plane);
5462                 if (IS_ERR(plane_state))
5463                         return PTR_ERR(plane_state);
5464
5465                 new_crtc_state->update_planes |= BIT(plane_id);
5466         }
5467
5468         return 0;
5469 }
5470
5471 static int
5472 skl_compute_ddb(struct intel_atomic_state *state)
5473 {
5474         struct drm_i915_private *dev_priv = to_i915(state->base.dev);
5475         struct intel_crtc_state *old_crtc_state;
5476         struct intel_crtc_state *new_crtc_state;
5477         struct intel_crtc *crtc;
5478         int ret, i;
5479
5480         state->enabled_dbuf_slices_mask = dev_priv->enabled_dbuf_slices_mask;
5481
5482         for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state,
5483                                             new_crtc_state, i) {
5484                 ret = skl_allocate_pipe_ddb(new_crtc_state);
5485                 if (ret)
5486                         return ret;
5487
5488                 ret = skl_ddb_add_affected_planes(old_crtc_state,
5489                                                   new_crtc_state);
5490                 if (ret)
5491                         return ret;
5492         }
5493
5494         return 0;
5495 }
5496
5497 static char enast(bool enable)
5498 {
5499         return enable ? '*' : ' ';
5500 }
5501
5502 static void
5503 skl_print_wm_changes(struct intel_atomic_state *state)
5504 {
5505         struct drm_i915_private *dev_priv = to_i915(state->base.dev);
5506         const struct intel_crtc_state *old_crtc_state;
5507         const struct intel_crtc_state *new_crtc_state;
5508         struct intel_plane *plane;
5509         struct intel_crtc *crtc;
5510         int i;
5511
5512         if (!drm_debug_enabled(DRM_UT_KMS))
5513                 return;
5514
5515         for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state,
5516                                             new_crtc_state, i) {
5517                 const struct skl_pipe_wm *old_pipe_wm, *new_pipe_wm;
5518
5519                 old_pipe_wm = &old_crtc_state->wm.skl.optimal;
5520                 new_pipe_wm = &new_crtc_state->wm.skl.optimal;
5521
5522                 for_each_intel_plane_on_crtc(&dev_priv->drm, crtc, plane) {
5523                         enum plane_id plane_id = plane->id;
5524                         const struct skl_ddb_entry *old, *new;
5525
5526                         old = &old_crtc_state->wm.skl.plane_ddb_y[plane_id];
5527                         new = &new_crtc_state->wm.skl.plane_ddb_y[plane_id];
5528
5529                         if (skl_ddb_entry_equal(old, new))
5530                                 continue;
5531
5532                         drm_dbg_kms(&dev_priv->drm,
5533                                     "[PLANE:%d:%s] ddb (%4d - %4d) -> (%4d - %4d), size %4d -> %4d\n",
5534                                     plane->base.base.id, plane->base.name,
5535                                     old->start, old->end, new->start, new->end,
5536                                     skl_ddb_entry_size(old), skl_ddb_entry_size(new));
5537                 }
5538
5539                 for_each_intel_plane_on_crtc(&dev_priv->drm, crtc, plane) {
5540                         enum plane_id plane_id = plane->id;
5541                         const struct skl_plane_wm *old_wm, *new_wm;
5542
5543                         old_wm = &old_pipe_wm->planes[plane_id];
5544                         new_wm = &new_pipe_wm->planes[plane_id];
5545
5546                         if (skl_plane_wm_equals(dev_priv, old_wm, new_wm))
5547                                 continue;
5548
5549                         drm_dbg_kms(&dev_priv->drm,
5550                                     "[PLANE:%d:%s]   level %cwm0,%cwm1,%cwm2,%cwm3,%cwm4,%cwm5,%cwm6,%cwm7,%ctwm"
5551                                     " -> %cwm0,%cwm1,%cwm2,%cwm3,%cwm4,%cwm5,%cwm6,%cwm7,%ctwm\n",
5552                                     plane->base.base.id, plane->base.name,
5553                                     enast(old_wm->wm[0].plane_en), enast(old_wm->wm[1].plane_en),
5554                                     enast(old_wm->wm[2].plane_en), enast(old_wm->wm[3].plane_en),
5555                                     enast(old_wm->wm[4].plane_en), enast(old_wm->wm[5].plane_en),
5556                                     enast(old_wm->wm[6].plane_en), enast(old_wm->wm[7].plane_en),
5557                                     enast(old_wm->trans_wm.plane_en),
5558                                     enast(new_wm->wm[0].plane_en), enast(new_wm->wm[1].plane_en),
5559                                     enast(new_wm->wm[2].plane_en), enast(new_wm->wm[3].plane_en),
5560                                     enast(new_wm->wm[4].plane_en), enast(new_wm->wm[5].plane_en),
5561                                     enast(new_wm->wm[6].plane_en), enast(new_wm->wm[7].plane_en),
5562                                     enast(new_wm->trans_wm.plane_en));
5563
5564                         drm_dbg_kms(&dev_priv->drm,
5565                                     "[PLANE:%d:%s]   lines %c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d"
5566                                       " -> %c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d\n",
5567                                     plane->base.base.id, plane->base.name,
5568                                     enast(old_wm->wm[0].ignore_lines), old_wm->wm[0].plane_res_l,
5569                                     enast(old_wm->wm[1].ignore_lines), old_wm->wm[1].plane_res_l,
5570                                     enast(old_wm->wm[2].ignore_lines), old_wm->wm[2].plane_res_l,
5571                                     enast(old_wm->wm[3].ignore_lines), old_wm->wm[3].plane_res_l,
5572                                     enast(old_wm->wm[4].ignore_lines), old_wm->wm[4].plane_res_l,
5573                                     enast(old_wm->wm[5].ignore_lines), old_wm->wm[5].plane_res_l,
5574                                     enast(old_wm->wm[6].ignore_lines), old_wm->wm[6].plane_res_l,
5575                                     enast(old_wm->wm[7].ignore_lines), old_wm->wm[7].plane_res_l,
5576                                     enast(old_wm->trans_wm.ignore_lines), old_wm->trans_wm.plane_res_l,
5577
5578                                     enast(new_wm->wm[0].ignore_lines), new_wm->wm[0].plane_res_l,
5579                                     enast(new_wm->wm[1].ignore_lines), new_wm->wm[1].plane_res_l,
5580                                     enast(new_wm->wm[2].ignore_lines), new_wm->wm[2].plane_res_l,
5581                                     enast(new_wm->wm[3].ignore_lines), new_wm->wm[3].plane_res_l,
5582                                     enast(new_wm->wm[4].ignore_lines), new_wm->wm[4].plane_res_l,
5583                                     enast(new_wm->wm[5].ignore_lines), new_wm->wm[5].plane_res_l,
5584                                     enast(new_wm->wm[6].ignore_lines), new_wm->wm[6].plane_res_l,
5585                                     enast(new_wm->wm[7].ignore_lines), new_wm->wm[7].plane_res_l,
5586                                     enast(new_wm->trans_wm.ignore_lines), new_wm->trans_wm.plane_res_l);
5587
5588                         drm_dbg_kms(&dev_priv->drm,
5589                                     "[PLANE:%d:%s]  blocks %4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d"
5590                                     " -> %4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d\n",
5591                                     plane->base.base.id, plane->base.name,
5592                                     old_wm->wm[0].plane_res_b, old_wm->wm[1].plane_res_b,
5593                                     old_wm->wm[2].plane_res_b, old_wm->wm[3].plane_res_b,
5594                                     old_wm->wm[4].plane_res_b, old_wm->wm[5].plane_res_b,
5595                                     old_wm->wm[6].plane_res_b, old_wm->wm[7].plane_res_b,
5596                                     old_wm->trans_wm.plane_res_b,
5597                                     new_wm->wm[0].plane_res_b, new_wm->wm[1].plane_res_b,
5598                                     new_wm->wm[2].plane_res_b, new_wm->wm[3].plane_res_b,
5599                                     new_wm->wm[4].plane_res_b, new_wm->wm[5].plane_res_b,
5600                                     new_wm->wm[6].plane_res_b, new_wm->wm[7].plane_res_b,
5601                                     new_wm->trans_wm.plane_res_b);
5602
5603                         drm_dbg_kms(&dev_priv->drm,
5604                                     "[PLANE:%d:%s] min_ddb %4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d"
5605                                     " -> %4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d\n",
5606                                     plane->base.base.id, plane->base.name,
5607                                     old_wm->wm[0].min_ddb_alloc, old_wm->wm[1].min_ddb_alloc,
5608                                     old_wm->wm[2].min_ddb_alloc, old_wm->wm[3].min_ddb_alloc,
5609                                     old_wm->wm[4].min_ddb_alloc, old_wm->wm[5].min_ddb_alloc,
5610                                     old_wm->wm[6].min_ddb_alloc, old_wm->wm[7].min_ddb_alloc,
5611                                     old_wm->trans_wm.min_ddb_alloc,
5612                                     new_wm->wm[0].min_ddb_alloc, new_wm->wm[1].min_ddb_alloc,
5613                                     new_wm->wm[2].min_ddb_alloc, new_wm->wm[3].min_ddb_alloc,
5614                                     new_wm->wm[4].min_ddb_alloc, new_wm->wm[5].min_ddb_alloc,
5615                                     new_wm->wm[6].min_ddb_alloc, new_wm->wm[7].min_ddb_alloc,
5616                                     new_wm->trans_wm.min_ddb_alloc);
5617                 }
5618         }
5619 }
5620
5621 static int intel_add_all_pipes(struct intel_atomic_state *state)
5622 {
5623         struct drm_i915_private *dev_priv = to_i915(state->base.dev);
5624         struct intel_crtc *crtc;
5625
5626         for_each_intel_crtc(&dev_priv->drm, crtc) {
5627                 struct intel_crtc_state *crtc_state;
5628
5629                 crtc_state = intel_atomic_get_crtc_state(&state->base, crtc);
5630                 if (IS_ERR(crtc_state))
5631                         return PTR_ERR(crtc_state);
5632         }
5633
5634         return 0;
5635 }
5636
5637 static int
5638 skl_ddb_add_affected_pipes(struct intel_atomic_state *state)
5639 {
5640         struct drm_i915_private *dev_priv = to_i915(state->base.dev);
5641         int ret;
5642
5643         /*
5644          * If this is our first atomic update following hardware readout,
5645          * we can't trust the DDB that the BIOS programmed for us.  Let's
5646          * pretend that all pipes switched active status so that we'll
5647          * ensure a full DDB recompute.
5648          */
5649         if (dev_priv->wm.distrust_bios_wm) {
5650                 ret = drm_modeset_lock(&dev_priv->drm.mode_config.connection_mutex,
5651                                        state->base.acquire_ctx);
5652                 if (ret)
5653                         return ret;
5654
5655                 state->active_pipe_changes = INTEL_INFO(dev_priv)->pipe_mask;
5656
5657                 /*
5658                  * We usually only initialize state->active_pipes if we
5659                  * we're doing a modeset; make sure this field is always
5660                  * initialized during the sanitization process that happens
5661                  * on the first commit too.
5662                  */
5663                 if (!state->modeset)
5664                         state->active_pipes = dev_priv->active_pipes;
5665         }
5666
5667         /*
5668          * If the modeset changes which CRTC's are active, we need to
5669          * recompute the DDB allocation for *all* active pipes, even
5670          * those that weren't otherwise being modified in any way by this
5671          * atomic commit.  Due to the shrinking of the per-pipe allocations
5672          * when new active CRTC's are added, it's possible for a pipe that
5673          * we were already using and aren't changing at all here to suddenly
5674          * become invalid if its DDB needs exceeds its new allocation.
5675          *
5676          * Note that if we wind up doing a full DDB recompute, we can't let
5677          * any other display updates race with this transaction, so we need
5678          * to grab the lock on *all* CRTC's.
5679          */
5680         if (state->active_pipe_changes || state->modeset) {
5681                 ret = intel_add_all_pipes(state);
5682                 if (ret)
5683                         return ret;
5684         }
5685
5686         return 0;
5687 }
5688
5689 /*
5690  * To make sure the cursor watermark registers are always consistent
5691  * with our computed state the following scenario needs special
5692  * treatment:
5693  *
5694  * 1. enable cursor
5695  * 2. move cursor entirely offscreen
5696  * 3. disable cursor
5697  *
5698  * Step 2. does call .disable_plane() but does not zero the watermarks
5699  * (since we consider an offscreen cursor still active for the purposes
5700  * of watermarks). Step 3. would not normally call .disable_plane()
5701  * because the actual plane visibility isn't changing, and we don't
5702  * deallocate the cursor ddb until the pipe gets disabled. So we must
5703  * force step 3. to call .disable_plane() to update the watermark
5704  * registers properly.
5705  *
5706  * Other planes do not suffer from this issues as their watermarks are
5707  * calculated based on the actual plane visibility. The only time this
5708  * can trigger for the other planes is during the initial readout as the
5709  * default value of the watermarks registers is not zero.
5710  */
5711 static int skl_wm_add_affected_planes(struct intel_atomic_state *state,
5712                                       struct intel_crtc *crtc)
5713 {
5714         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
5715         const struct intel_crtc_state *old_crtc_state =
5716                 intel_atomic_get_old_crtc_state(state, crtc);
5717         struct intel_crtc_state *new_crtc_state =
5718                 intel_atomic_get_new_crtc_state(state, crtc);
5719         struct intel_plane *plane;
5720
5721         for_each_intel_plane_on_crtc(&dev_priv->drm, crtc, plane) {
5722                 struct intel_plane_state *plane_state;
5723                 enum plane_id plane_id = plane->id;
5724
5725                 /*
5726                  * Force a full wm update for every plane on modeset.
5727                  * Required because the reset value of the wm registers
5728                  * is non-zero, whereas we want all disabled planes to
5729                  * have zero watermarks. So if we turn off the relevant
5730                  * power well the hardware state will go out of sync
5731                  * with the software state.
5732                  */
5733                 if (!drm_atomic_crtc_needs_modeset(&new_crtc_state->uapi) &&
5734                     skl_plane_wm_equals(dev_priv,
5735                                         &old_crtc_state->wm.skl.optimal.planes[plane_id],
5736                                         &new_crtc_state->wm.skl.optimal.planes[plane_id]))
5737                         continue;
5738
5739                 plane_state = intel_atomic_get_plane_state(state, plane);
5740                 if (IS_ERR(plane_state))
5741                         return PTR_ERR(plane_state);
5742
5743                 new_crtc_state->update_planes |= BIT(plane_id);
5744         }
5745
5746         return 0;
5747 }
5748
5749 static int
5750 skl_compute_wm(struct intel_atomic_state *state)
5751 {
5752         struct intel_crtc *crtc;
5753         struct intel_crtc_state *new_crtc_state;
5754         struct intel_crtc_state *old_crtc_state;
5755         int ret, i;
5756
5757         ret = skl_ddb_add_affected_pipes(state);
5758         if (ret)
5759                 return ret;
5760
5761         /*
5762          * Calculate WM's for all pipes that are part of this transaction.
5763          * Note that skl_ddb_add_affected_pipes may have added more CRTC's that
5764          * weren't otherwise being modified if pipe allocations had to change.
5765          */
5766         for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state,
5767                                             new_crtc_state, i) {
5768                 ret = skl_build_pipe_wm(new_crtc_state);
5769                 if (ret)
5770                         return ret;
5771
5772                 ret = skl_wm_add_affected_planes(state, crtc);
5773                 if (ret)
5774                         return ret;
5775         }
5776
5777         ret = skl_compute_ddb(state);
5778         if (ret)
5779                 return ret;
5780
5781         skl_print_wm_changes(state);
5782
5783         return 0;
5784 }
5785
5786 static void ilk_compute_wm_config(struct drm_i915_private *dev_priv,
5787                                   struct intel_wm_config *config)
5788 {
5789         struct intel_crtc *crtc;
5790
5791         /* Compute the currently _active_ config */
5792         for_each_intel_crtc(&dev_priv->drm, crtc) {
5793                 const struct intel_pipe_wm *wm = &crtc->wm.active.ilk;
5794
5795                 if (!wm->pipe_enabled)
5796                         continue;
5797
5798                 config->sprites_enabled |= wm->sprites_enabled;
5799                 config->sprites_scaled |= wm->sprites_scaled;
5800                 config->num_pipes_active++;
5801         }
5802 }
5803
5804 static void ilk_program_watermarks(struct drm_i915_private *dev_priv)
5805 {
5806         struct intel_pipe_wm lp_wm_1_2 = {}, lp_wm_5_6 = {}, *best_lp_wm;
5807         struct ilk_wm_maximums max;
5808         struct intel_wm_config config = {};
5809         struct ilk_wm_values results = {};
5810         enum intel_ddb_partitioning partitioning;
5811
5812         ilk_compute_wm_config(dev_priv, &config);
5813
5814         ilk_compute_wm_maximums(dev_priv, 1, &config, INTEL_DDB_PART_1_2, &max);
5815         ilk_wm_merge(dev_priv, &config, &max, &lp_wm_1_2);
5816
5817         /* 5/6 split only in single pipe config on IVB+ */
5818         if (INTEL_GEN(dev_priv) >= 7 &&
5819             config.num_pipes_active == 1 && config.sprites_enabled) {
5820                 ilk_compute_wm_maximums(dev_priv, 1, &config, INTEL_DDB_PART_5_6, &max);
5821                 ilk_wm_merge(dev_priv, &config, &max, &lp_wm_5_6);
5822
5823                 best_lp_wm = ilk_find_best_result(dev_priv, &lp_wm_1_2, &lp_wm_5_6);
5824         } else {
5825                 best_lp_wm = &lp_wm_1_2;
5826         }
5827
5828         partitioning = (best_lp_wm == &lp_wm_1_2) ?
5829                        INTEL_DDB_PART_1_2 : INTEL_DDB_PART_5_6;
5830
5831         ilk_compute_wm_results(dev_priv, best_lp_wm, partitioning, &results);
5832
5833         ilk_write_wm_values(dev_priv, &results);
5834 }
5835
5836 static void ilk_initial_watermarks(struct intel_atomic_state *state,
5837                                    struct intel_crtc *crtc)
5838 {
5839         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
5840         const struct intel_crtc_state *crtc_state =
5841                 intel_atomic_get_new_crtc_state(state, crtc);
5842
5843         mutex_lock(&dev_priv->wm.wm_mutex);
5844         crtc->wm.active.ilk = crtc_state->wm.ilk.intermediate;
5845         ilk_program_watermarks(dev_priv);
5846         mutex_unlock(&dev_priv->wm.wm_mutex);
5847 }
5848
5849 static void ilk_optimize_watermarks(struct intel_atomic_state *state,
5850                                     struct intel_crtc *crtc)
5851 {
5852         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
5853         const struct intel_crtc_state *crtc_state =
5854                 intel_atomic_get_new_crtc_state(state, crtc);
5855
5856         if (!crtc_state->wm.need_postvbl_update)
5857                 return;
5858
5859         mutex_lock(&dev_priv->wm.wm_mutex);
5860         crtc->wm.active.ilk = crtc_state->wm.ilk.optimal;
5861         ilk_program_watermarks(dev_priv);
5862         mutex_unlock(&dev_priv->wm.wm_mutex);
5863 }
5864
5865 static inline void skl_wm_level_from_reg_val(u32 val,
5866                                              struct skl_wm_level *level)
5867 {
5868         level->plane_en = val & PLANE_WM_EN;
5869         level->ignore_lines = val & PLANE_WM_IGNORE_LINES;
5870         level->plane_res_b = val & PLANE_WM_BLOCKS_MASK;
5871         level->plane_res_l = (val >> PLANE_WM_LINES_SHIFT) &
5872                 PLANE_WM_LINES_MASK;
5873 }
5874
5875 void skl_pipe_wm_get_hw_state(struct intel_crtc *crtc,
5876                               struct skl_pipe_wm *out)
5877 {
5878         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
5879         enum pipe pipe = crtc->pipe;
5880         int level, max_level;
5881         enum plane_id plane_id;
5882         u32 val;
5883
5884         max_level = ilk_wm_max_level(dev_priv);
5885
5886         for_each_plane_id_on_crtc(crtc, plane_id) {
5887                 struct skl_plane_wm *wm = &out->planes[plane_id];
5888
5889                 for (level = 0; level <= max_level; level++) {
5890                         if (plane_id != PLANE_CURSOR)
5891                                 val = I915_READ(PLANE_WM(pipe, plane_id, level));
5892                         else
5893                                 val = I915_READ(CUR_WM(pipe, level));
5894
5895                         skl_wm_level_from_reg_val(val, &wm->wm[level]);
5896                 }
5897
5898                 if (plane_id != PLANE_CURSOR)
5899                         val = I915_READ(PLANE_WM_TRANS(pipe, plane_id));
5900                 else
5901                         val = I915_READ(CUR_WM_TRANS(pipe));
5902
5903                 skl_wm_level_from_reg_val(val, &wm->trans_wm);
5904         }
5905
5906         if (!crtc->active)
5907                 return;
5908 }
5909
5910 void skl_wm_get_hw_state(struct drm_i915_private *dev_priv)
5911 {
5912         struct intel_crtc *crtc;
5913         struct intel_crtc_state *crtc_state;
5914
5915         skl_ddb_get_hw_state(dev_priv);
5916         for_each_intel_crtc(&dev_priv->drm, crtc) {
5917                 crtc_state = to_intel_crtc_state(crtc->base.state);
5918
5919                 skl_pipe_wm_get_hw_state(crtc, &crtc_state->wm.skl.optimal);
5920         }
5921
5922         if (dev_priv->active_pipes) {
5923                 /* Fully recompute DDB on first atomic commit */
5924                 dev_priv->wm.distrust_bios_wm = true;
5925         }
5926 }
5927
5928 static void ilk_pipe_wm_get_hw_state(struct intel_crtc *crtc)
5929 {
5930         struct drm_device *dev = crtc->base.dev;
5931         struct drm_i915_private *dev_priv = to_i915(dev);
5932         struct ilk_wm_values *hw = &dev_priv->wm.hw;
5933         struct intel_crtc_state *crtc_state = to_intel_crtc_state(crtc->base.state);
5934         struct intel_pipe_wm *active = &crtc_state->wm.ilk.optimal;
5935         enum pipe pipe = crtc->pipe;
5936         static const i915_reg_t wm0_pipe_reg[] = {
5937                 [PIPE_A] = WM0_PIPEA_ILK,
5938                 [PIPE_B] = WM0_PIPEB_ILK,
5939                 [PIPE_C] = WM0_PIPEC_IVB,
5940         };
5941
5942         hw->wm_pipe[pipe] = I915_READ(wm0_pipe_reg[pipe]);
5943
5944         memset(active, 0, sizeof(*active));
5945
5946         active->pipe_enabled = crtc->active;
5947
5948         if (active->pipe_enabled) {
5949                 u32 tmp = hw->wm_pipe[pipe];
5950
5951                 /*
5952                  * For active pipes LP0 watermark is marked as
5953                  * enabled, and LP1+ watermaks as disabled since
5954                  * we can't really reverse compute them in case
5955                  * multiple pipes are active.
5956                  */
5957                 active->wm[0].enable = true;
5958                 active->wm[0].pri_val = (tmp & WM0_PIPE_PLANE_MASK) >> WM0_PIPE_PLANE_SHIFT;
5959                 active->wm[0].spr_val = (tmp & WM0_PIPE_SPRITE_MASK) >> WM0_PIPE_SPRITE_SHIFT;
5960                 active->wm[0].cur_val = tmp & WM0_PIPE_CURSOR_MASK;
5961         } else {
5962                 int level, max_level = ilk_wm_max_level(dev_priv);
5963
5964                 /*
5965                  * For inactive pipes, all watermark levels
5966                  * should be marked as enabled but zeroed,
5967                  * which is what we'd compute them to.
5968                  */
5969                 for (level = 0; level <= max_level; level++)
5970                         active->wm[level].enable = true;
5971         }
5972
5973         crtc->wm.active.ilk = *active;
5974 }
5975
5976 #define _FW_WM(value, plane) \
5977         (((value) & DSPFW_ ## plane ## _MASK) >> DSPFW_ ## plane ## _SHIFT)
5978 #define _FW_WM_VLV(value, plane) \
5979         (((value) & DSPFW_ ## plane ## _MASK_VLV) >> DSPFW_ ## plane ## _SHIFT)
5980
5981 static void g4x_read_wm_values(struct drm_i915_private *dev_priv,
5982                                struct g4x_wm_values *wm)
5983 {
5984         u32 tmp;
5985
5986         tmp = I915_READ(DSPFW1);
5987         wm->sr.plane = _FW_WM(tmp, SR);
5988         wm->pipe[PIPE_B].plane[PLANE_CURSOR] = _FW_WM(tmp, CURSORB);
5989         wm->pipe[PIPE_B].plane[PLANE_PRIMARY] = _FW_WM(tmp, PLANEB);
5990         wm->pipe[PIPE_A].plane[PLANE_PRIMARY] = _FW_WM(tmp, PLANEA);
5991
5992         tmp = I915_READ(DSPFW2);
5993         wm->fbc_en = tmp & DSPFW_FBC_SR_EN;
5994         wm->sr.fbc = _FW_WM(tmp, FBC_SR);
5995         wm->hpll.fbc = _FW_WM(tmp, FBC_HPLL_SR);
5996         wm->pipe[PIPE_B].plane[PLANE_SPRITE0] = _FW_WM(tmp, SPRITEB);
5997         wm->pipe[PIPE_A].plane[PLANE_CURSOR] = _FW_WM(tmp, CURSORA);
5998         wm->pipe[PIPE_A].plane[PLANE_SPRITE0] = _FW_WM(tmp, SPRITEA);
5999
6000         tmp = I915_READ(DSPFW3);
6001         wm->hpll_en = tmp & DSPFW_HPLL_SR_EN;
6002         wm->sr.cursor = _FW_WM(tmp, CURSOR_SR);
6003         wm->hpll.cursor = _FW_WM(tmp, HPLL_CURSOR);
6004         wm->hpll.plane = _FW_WM(tmp, HPLL_SR);
6005 }
6006
6007 static void vlv_read_wm_values(struct drm_i915_private *dev_priv,
6008                                struct vlv_wm_values *wm)
6009 {
6010         enum pipe pipe;
6011         u32 tmp;
6012
6013         for_each_pipe(dev_priv, pipe) {
6014                 tmp = I915_READ(VLV_DDL(pipe));
6015
6016                 wm->ddl[pipe].plane[PLANE_PRIMARY] =
6017                         (tmp >> DDL_PLANE_SHIFT) & (DDL_PRECISION_HIGH | DRAIN_LATENCY_MASK);
6018                 wm->ddl[pipe].plane[PLANE_CURSOR] =
6019                         (tmp >> DDL_CURSOR_SHIFT) & (DDL_PRECISION_HIGH | DRAIN_LATENCY_MASK);
6020                 wm->ddl[pipe].plane[PLANE_SPRITE0] =
6021                         (tmp >> DDL_SPRITE_SHIFT(0)) & (DDL_PRECISION_HIGH | DRAIN_LATENCY_MASK);
6022                 wm->ddl[pipe].plane[PLANE_SPRITE1] =
6023                         (tmp >> DDL_SPRITE_SHIFT(1)) & (DDL_PRECISION_HIGH | DRAIN_LATENCY_MASK);
6024         }
6025
6026         tmp = I915_READ(DSPFW1);
6027         wm->sr.plane = _FW_WM(tmp, SR);
6028         wm->pipe[PIPE_B].plane[PLANE_CURSOR] = _FW_WM(tmp, CURSORB);
6029         wm->pipe[PIPE_B].plane[PLANE_PRIMARY] = _FW_WM_VLV(tmp, PLANEB);
6030         wm->pipe[PIPE_A].plane[PLANE_PRIMARY] = _FW_WM_VLV(tmp, PLANEA);
6031
6032         tmp = I915_READ(DSPFW2);
6033         wm->pipe[PIPE_A].plane[PLANE_SPRITE1] = _FW_WM_VLV(tmp, SPRITEB);
6034         wm->pipe[PIPE_A].plane[PLANE_CURSOR] = _FW_WM(tmp, CURSORA);
6035         wm->pipe[PIPE_A].plane[PLANE_SPRITE0] = _FW_WM_VLV(tmp, SPRITEA);
6036
6037         tmp = I915_READ(DSPFW3);
6038         wm->sr.cursor = _FW_WM(tmp, CURSOR_SR);
6039
6040         if (IS_CHERRYVIEW(dev_priv)) {
6041                 tmp = I915_READ(DSPFW7_CHV);
6042                 wm->pipe[PIPE_B].plane[PLANE_SPRITE1] = _FW_WM_VLV(tmp, SPRITED);
6043                 wm->pipe[PIPE_B].plane[PLANE_SPRITE0] = _FW_WM_VLV(tmp, SPRITEC);
6044
6045                 tmp = I915_READ(DSPFW8_CHV);
6046                 wm->pipe[PIPE_C].plane[PLANE_SPRITE1] = _FW_WM_VLV(tmp, SPRITEF);
6047                 wm->pipe[PIPE_C].plane[PLANE_SPRITE0] = _FW_WM_VLV(tmp, SPRITEE);
6048
6049                 tmp = I915_READ(DSPFW9_CHV);
6050                 wm->pipe[PIPE_C].plane[PLANE_PRIMARY] = _FW_WM_VLV(tmp, PLANEC);
6051                 wm->pipe[PIPE_C].plane[PLANE_CURSOR] = _FW_WM(tmp, CURSORC);
6052
6053                 tmp = I915_READ(DSPHOWM);
6054                 wm->sr.plane |= _FW_WM(tmp, SR_HI) << 9;
6055                 wm->pipe[PIPE_C].plane[PLANE_SPRITE1] |= _FW_WM(tmp, SPRITEF_HI) << 8;
6056                 wm->pipe[PIPE_C].plane[PLANE_SPRITE0] |= _FW_WM(tmp, SPRITEE_HI) << 8;
6057                 wm->pipe[PIPE_C].plane[PLANE_PRIMARY] |= _FW_WM(tmp, PLANEC_HI) << 8;
6058                 wm->pipe[PIPE_B].plane[PLANE_SPRITE1] |= _FW_WM(tmp, SPRITED_HI) << 8;
6059                 wm->pipe[PIPE_B].plane[PLANE_SPRITE0] |= _FW_WM(tmp, SPRITEC_HI) << 8;
6060                 wm->pipe[PIPE_B].plane[PLANE_PRIMARY] |= _FW_WM(tmp, PLANEB_HI) << 8;
6061                 wm->pipe[PIPE_A].plane[PLANE_SPRITE1] |= _FW_WM(tmp, SPRITEB_HI) << 8;
6062                 wm->pipe[PIPE_A].plane[PLANE_SPRITE0] |= _FW_WM(tmp, SPRITEA_HI) << 8;
6063                 wm->pipe[PIPE_A].plane[PLANE_PRIMARY] |= _FW_WM(tmp, PLANEA_HI) << 8;
6064         } else {
6065                 tmp = I915_READ(DSPFW7);
6066                 wm->pipe[PIPE_B].plane[PLANE_SPRITE1] = _FW_WM_VLV(tmp, SPRITED);
6067                 wm->pipe[PIPE_B].plane[PLANE_SPRITE0] = _FW_WM_VLV(tmp, SPRITEC);
6068
6069                 tmp = I915_READ(DSPHOWM);
6070                 wm->sr.plane |= _FW_WM(tmp, SR_HI) << 9;
6071                 wm->pipe[PIPE_B].plane[PLANE_SPRITE1] |= _FW_WM(tmp, SPRITED_HI) << 8;
6072                 wm->pipe[PIPE_B].plane[PLANE_SPRITE0] |= _FW_WM(tmp, SPRITEC_HI) << 8;
6073                 wm->pipe[PIPE_B].plane[PLANE_PRIMARY] |= _FW_WM(tmp, PLANEB_HI) << 8;
6074                 wm->pipe[PIPE_A].plane[PLANE_SPRITE1] |= _FW_WM(tmp, SPRITEB_HI) << 8;
6075                 wm->pipe[PIPE_A].plane[PLANE_SPRITE0] |= _FW_WM(tmp, SPRITEA_HI) << 8;
6076                 wm->pipe[PIPE_A].plane[PLANE_PRIMARY] |= _FW_WM(tmp, PLANEA_HI) << 8;
6077         }
6078 }
6079
6080 #undef _FW_WM
6081 #undef _FW_WM_VLV
6082
6083 void g4x_wm_get_hw_state(struct drm_i915_private *dev_priv)
6084 {
6085         struct g4x_wm_values *wm = &dev_priv->wm.g4x;
6086         struct intel_crtc *crtc;
6087
6088         g4x_read_wm_values(dev_priv, wm);
6089
6090         wm->cxsr = I915_READ(FW_BLC_SELF) & FW_BLC_SELF_EN;
6091
6092         for_each_intel_crtc(&dev_priv->drm, crtc) {
6093                 struct intel_crtc_state *crtc_state =
6094                         to_intel_crtc_state(crtc->base.state);
6095                 struct g4x_wm_state *active = &crtc->wm.active.g4x;
6096                 struct g4x_pipe_wm *raw;
6097                 enum pipe pipe = crtc->pipe;
6098                 enum plane_id plane_id;
6099                 int level, max_level;
6100
6101                 active->cxsr = wm->cxsr;
6102                 active->hpll_en = wm->hpll_en;
6103                 active->fbc_en = wm->fbc_en;
6104
6105                 active->sr = wm->sr;
6106                 active->hpll = wm->hpll;
6107
6108                 for_each_plane_id_on_crtc(crtc, plane_id) {
6109                         active->wm.plane[plane_id] =
6110                                 wm->pipe[pipe].plane[plane_id];
6111                 }
6112
6113                 if (wm->cxsr && wm->hpll_en)
6114                         max_level = G4X_WM_LEVEL_HPLL;
6115                 else if (wm->cxsr)
6116                         max_level = G4X_WM_LEVEL_SR;
6117                 else
6118                         max_level = G4X_WM_LEVEL_NORMAL;
6119
6120                 level = G4X_WM_LEVEL_NORMAL;
6121                 raw = &crtc_state->wm.g4x.raw[level];
6122                 for_each_plane_id_on_crtc(crtc, plane_id)
6123                         raw->plane[plane_id] = active->wm.plane[plane_id];
6124
6125                 if (++level > max_level)
6126                         goto out;
6127
6128                 raw = &crtc_state->wm.g4x.raw[level];
6129                 raw->plane[PLANE_PRIMARY] = active->sr.plane;
6130                 raw->plane[PLANE_CURSOR] = active->sr.cursor;
6131                 raw->plane[PLANE_SPRITE0] = 0;
6132                 raw->fbc = active->sr.fbc;
6133
6134                 if (++level > max_level)
6135                         goto out;
6136
6137                 raw = &crtc_state->wm.g4x.raw[level];
6138                 raw->plane[PLANE_PRIMARY] = active->hpll.plane;
6139                 raw->plane[PLANE_CURSOR] = active->hpll.cursor;
6140                 raw->plane[PLANE_SPRITE0] = 0;
6141                 raw->fbc = active->hpll.fbc;
6142
6143         out:
6144                 for_each_plane_id_on_crtc(crtc, plane_id)
6145                         g4x_raw_plane_wm_set(crtc_state, level,
6146                                              plane_id, USHRT_MAX);
6147                 g4x_raw_fbc_wm_set(crtc_state, level, USHRT_MAX);
6148
6149                 crtc_state->wm.g4x.optimal = *active;
6150                 crtc_state->wm.g4x.intermediate = *active;
6151
6152                 drm_dbg_kms(&dev_priv->drm,
6153                             "Initial watermarks: pipe %c, plane=%d, cursor=%d, sprite=%d\n",
6154                             pipe_name(pipe),
6155                             wm->pipe[pipe].plane[PLANE_PRIMARY],
6156                             wm->pipe[pipe].plane[PLANE_CURSOR],
6157                             wm->pipe[pipe].plane[PLANE_SPRITE0]);
6158         }
6159
6160         drm_dbg_kms(&dev_priv->drm,
6161                     "Initial SR watermarks: plane=%d, cursor=%d fbc=%d\n",
6162                     wm->sr.plane, wm->sr.cursor, wm->sr.fbc);
6163         drm_dbg_kms(&dev_priv->drm,
6164                     "Initial HPLL watermarks: plane=%d, SR cursor=%d fbc=%d\n",
6165                     wm->hpll.plane, wm->hpll.cursor, wm->hpll.fbc);
6166         drm_dbg_kms(&dev_priv->drm, "Initial SR=%s HPLL=%s FBC=%s\n",
6167                     yesno(wm->cxsr), yesno(wm->hpll_en), yesno(wm->fbc_en));
6168 }
6169
6170 void g4x_wm_sanitize(struct drm_i915_private *dev_priv)
6171 {
6172         struct intel_plane *plane;
6173         struct intel_crtc *crtc;
6174
6175         mutex_lock(&dev_priv->wm.wm_mutex);
6176
6177         for_each_intel_plane(&dev_priv->drm, plane) {
6178                 struct intel_crtc *crtc =
6179                         intel_get_crtc_for_pipe(dev_priv, plane->pipe);
6180                 struct intel_crtc_state *crtc_state =
6181                         to_intel_crtc_state(crtc->base.state);
6182                 struct intel_plane_state *plane_state =
6183                         to_intel_plane_state(plane->base.state);
6184                 struct g4x_wm_state *wm_state = &crtc_state->wm.g4x.optimal;
6185                 enum plane_id plane_id = plane->id;
6186                 int level;
6187
6188                 if (plane_state->uapi.visible)
6189                         continue;
6190
6191                 for (level = 0; level < 3; level++) {
6192                         struct g4x_pipe_wm *raw =
6193                                 &crtc_state->wm.g4x.raw[level];
6194
6195                         raw->plane[plane_id] = 0;
6196                         wm_state->wm.plane[plane_id] = 0;
6197                 }
6198
6199                 if (plane_id == PLANE_PRIMARY) {
6200                         for (level = 0; level < 3; level++) {
6201                                 struct g4x_pipe_wm *raw =
6202                                         &crtc_state->wm.g4x.raw[level];
6203                                 raw->fbc = 0;
6204                         }
6205
6206                         wm_state->sr.fbc = 0;
6207                         wm_state->hpll.fbc = 0;
6208                         wm_state->fbc_en = false;
6209                 }
6210         }
6211
6212         for_each_intel_crtc(&dev_priv->drm, crtc) {
6213                 struct intel_crtc_state *crtc_state =
6214                         to_intel_crtc_state(crtc->base.state);
6215
6216                 crtc_state->wm.g4x.intermediate =
6217                         crtc_state->wm.g4x.optimal;
6218                 crtc->wm.active.g4x = crtc_state->wm.g4x.optimal;
6219         }
6220
6221         g4x_program_watermarks(dev_priv);
6222
6223         mutex_unlock(&dev_priv->wm.wm_mutex);
6224 }
6225
6226 void vlv_wm_get_hw_state(struct drm_i915_private *dev_priv)
6227 {
6228         struct vlv_wm_values *wm = &dev_priv->wm.vlv;
6229         struct intel_crtc *crtc;
6230         u32 val;
6231
6232         vlv_read_wm_values(dev_priv, wm);
6233
6234         wm->cxsr = I915_READ(FW_BLC_SELF_VLV) & FW_CSPWRDWNEN;
6235         wm->level = VLV_WM_LEVEL_PM2;
6236
6237         if (IS_CHERRYVIEW(dev_priv)) {
6238                 vlv_punit_get(dev_priv);
6239
6240                 val = vlv_punit_read(dev_priv, PUNIT_REG_DSPSSPM);
6241                 if (val & DSP_MAXFIFO_PM5_ENABLE)
6242                         wm->level = VLV_WM_LEVEL_PM5;
6243
6244                 /*
6245                  * If DDR DVFS is disabled in the BIOS, Punit
6246                  * will never ack the request. So if that happens
6247                  * assume we don't have to enable/disable DDR DVFS
6248                  * dynamically. To test that just set the REQ_ACK
6249                  * bit to poke the Punit, but don't change the
6250                  * HIGH/LOW bits so that we don't actually change
6251                  * the current state.
6252                  */
6253                 val = vlv_punit_read(dev_priv, PUNIT_REG_DDR_SETUP2);
6254                 val |= FORCE_DDR_FREQ_REQ_ACK;
6255                 vlv_punit_write(dev_priv, PUNIT_REG_DDR_SETUP2, val);
6256
6257                 if (wait_for((vlv_punit_read(dev_priv, PUNIT_REG_DDR_SETUP2) &
6258                               FORCE_DDR_FREQ_REQ_ACK) == 0, 3)) {
6259                         drm_dbg_kms(&dev_priv->drm,
6260                                     "Punit not acking DDR DVFS request, "
6261                                     "assuming DDR DVFS is disabled\n");
6262                         dev_priv->wm.max_level = VLV_WM_LEVEL_PM5;
6263                 } else {
6264                         val = vlv_punit_read(dev_priv, PUNIT_REG_DDR_SETUP2);
6265                         if ((val & FORCE_DDR_HIGH_FREQ) == 0)
6266                                 wm->level = VLV_WM_LEVEL_DDR_DVFS;
6267                 }
6268
6269                 vlv_punit_put(dev_priv);
6270         }
6271
6272         for_each_intel_crtc(&dev_priv->drm, crtc) {
6273                 struct intel_crtc_state *crtc_state =
6274                         to_intel_crtc_state(crtc->base.state);
6275                 struct vlv_wm_state *active = &crtc->wm.active.vlv;
6276                 const struct vlv_fifo_state *fifo_state =
6277                         &crtc_state->wm.vlv.fifo_state;
6278                 enum pipe pipe = crtc->pipe;
6279                 enum plane_id plane_id;
6280                 int level;
6281
6282                 vlv_get_fifo_size(crtc_state);
6283
6284                 active->num_levels = wm->level + 1;
6285                 active->cxsr = wm->cxsr;
6286
6287                 for (level = 0; level < active->num_levels; level++) {
6288                         struct g4x_pipe_wm *raw =
6289                                 &crtc_state->wm.vlv.raw[level];
6290
6291                         active->sr[level].plane = wm->sr.plane;
6292                         active->sr[level].cursor = wm->sr.cursor;
6293
6294                         for_each_plane_id_on_crtc(crtc, plane_id) {
6295                                 active->wm[level].plane[plane_id] =
6296                                         wm->pipe[pipe].plane[plane_id];
6297
6298                                 raw->plane[plane_id] =
6299                                         vlv_invert_wm_value(active->wm[level].plane[plane_id],
6300                                                             fifo_state->plane[plane_id]);
6301                         }
6302                 }
6303
6304                 for_each_plane_id_on_crtc(crtc, plane_id)
6305                         vlv_raw_plane_wm_set(crtc_state, level,
6306                                              plane_id, USHRT_MAX);
6307                 vlv_invalidate_wms(crtc, active, level);
6308
6309                 crtc_state->wm.vlv.optimal = *active;
6310                 crtc_state->wm.vlv.intermediate = *active;
6311
6312                 drm_dbg_kms(&dev_priv->drm,
6313                             "Initial watermarks: pipe %c, plane=%d, cursor=%d, sprite0=%d, sprite1=%d\n",
6314                             pipe_name(pipe),
6315                             wm->pipe[pipe].plane[PLANE_PRIMARY],
6316                             wm->pipe[pipe].plane[PLANE_CURSOR],
6317                             wm->pipe[pipe].plane[PLANE_SPRITE0],
6318                             wm->pipe[pipe].plane[PLANE_SPRITE1]);
6319         }
6320
6321         drm_dbg_kms(&dev_priv->drm,
6322                     "Initial watermarks: SR plane=%d, SR cursor=%d level=%d cxsr=%d\n",
6323                     wm->sr.plane, wm->sr.cursor, wm->level, wm->cxsr);
6324 }
6325
6326 void vlv_wm_sanitize(struct drm_i915_private *dev_priv)
6327 {
6328         struct intel_plane *plane;
6329         struct intel_crtc *crtc;
6330
6331         mutex_lock(&dev_priv->wm.wm_mutex);
6332
6333         for_each_intel_plane(&dev_priv->drm, plane) {
6334                 struct intel_crtc *crtc =
6335                         intel_get_crtc_for_pipe(dev_priv, plane->pipe);
6336                 struct intel_crtc_state *crtc_state =
6337                         to_intel_crtc_state(crtc->base.state);
6338                 struct intel_plane_state *plane_state =
6339                         to_intel_plane_state(plane->base.state);
6340                 struct vlv_wm_state *wm_state = &crtc_state->wm.vlv.optimal;
6341                 const struct vlv_fifo_state *fifo_state =
6342                         &crtc_state->wm.vlv.fifo_state;
6343                 enum plane_id plane_id = plane->id;
6344                 int level;
6345
6346                 if (plane_state->uapi.visible)
6347                         continue;
6348
6349                 for (level = 0; level < wm_state->num_levels; level++) {
6350                         struct g4x_pipe_wm *raw =
6351                                 &crtc_state->wm.vlv.raw[level];
6352
6353                         raw->plane[plane_id] = 0;
6354
6355                         wm_state->wm[level].plane[plane_id] =
6356                                 vlv_invert_wm_value(raw->plane[plane_id],
6357                                                     fifo_state->plane[plane_id]);
6358                 }
6359         }
6360
6361         for_each_intel_crtc(&dev_priv->drm, crtc) {
6362                 struct intel_crtc_state *crtc_state =
6363                         to_intel_crtc_state(crtc->base.state);
6364
6365                 crtc_state->wm.vlv.intermediate =
6366                         crtc_state->wm.vlv.optimal;
6367                 crtc->wm.active.vlv = crtc_state->wm.vlv.optimal;
6368         }
6369
6370         vlv_program_watermarks(dev_priv);
6371
6372         mutex_unlock(&dev_priv->wm.wm_mutex);
6373 }
6374
6375 /*
6376  * FIXME should probably kill this and improve
6377  * the real watermark readout/sanitation instead
6378  */
6379 static void ilk_init_lp_watermarks(struct drm_i915_private *dev_priv)
6380 {
6381         I915_WRITE(WM3_LP_ILK, I915_READ(WM3_LP_ILK) & ~WM1_LP_SR_EN);
6382         I915_WRITE(WM2_LP_ILK, I915_READ(WM2_LP_ILK) & ~WM1_LP_SR_EN);
6383         I915_WRITE(WM1_LP_ILK, I915_READ(WM1_LP_ILK) & ~WM1_LP_SR_EN);
6384
6385         /*
6386          * Don't touch WM1S_LP_EN here.
6387          * Doing so could cause underruns.
6388          */
6389 }
6390
6391 void ilk_wm_get_hw_state(struct drm_i915_private *dev_priv)
6392 {
6393         struct ilk_wm_values *hw = &dev_priv->wm.hw;
6394         struct intel_crtc *crtc;
6395
6396         ilk_init_lp_watermarks(dev_priv);
6397
6398         for_each_intel_crtc(&dev_priv->drm, crtc)
6399                 ilk_pipe_wm_get_hw_state(crtc);
6400
6401         hw->wm_lp[0] = I915_READ(WM1_LP_ILK);
6402         hw->wm_lp[1] = I915_READ(WM2_LP_ILK);
6403         hw->wm_lp[2] = I915_READ(WM3_LP_ILK);
6404
6405         hw->wm_lp_spr[0] = I915_READ(WM1S_LP_ILK);
6406         if (INTEL_GEN(dev_priv) >= 7) {
6407                 hw->wm_lp_spr[1] = I915_READ(WM2S_LP_IVB);
6408                 hw->wm_lp_spr[2] = I915_READ(WM3S_LP_IVB);
6409         }
6410
6411         if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
6412                 hw->partitioning = (I915_READ(WM_MISC) & WM_MISC_DATA_PARTITION_5_6) ?
6413                         INTEL_DDB_PART_5_6 : INTEL_DDB_PART_1_2;
6414         else if (IS_IVYBRIDGE(dev_priv))
6415                 hw->partitioning = (I915_READ(DISP_ARB_CTL2) & DISP_DATA_PARTITION_5_6) ?
6416                         INTEL_DDB_PART_5_6 : INTEL_DDB_PART_1_2;
6417
6418         hw->enable_fbc_wm =
6419                 !(I915_READ(DISP_ARB_CTL) & DISP_FBC_WM_DIS);
6420 }
6421
6422 /**
6423  * intel_update_watermarks - update FIFO watermark values based on current modes
6424  * @crtc: the #intel_crtc on which to compute the WM
6425  *
6426  * Calculate watermark values for the various WM regs based on current mode
6427  * and plane configuration.
6428  *
6429  * There are several cases to deal with here:
6430  *   - normal (i.e. non-self-refresh)
6431  *   - self-refresh (SR) mode
6432  *   - lines are large relative to FIFO size (buffer can hold up to 2)
6433  *   - lines are small relative to FIFO size (buffer can hold more than 2
6434  *     lines), so need to account for TLB latency
6435  *
6436  *   The normal calculation is:
6437  *     watermark = dotclock * bytes per pixel * latency
6438  *   where latency is platform & configuration dependent (we assume pessimal
6439  *   values here).
6440  *
6441  *   The SR calculation is:
6442  *     watermark = (trunc(latency/line time)+1) * surface width *
6443  *       bytes per pixel
6444  *   where
6445  *     line time = htotal / dotclock
6446  *     surface width = hdisplay for normal plane and 64 for cursor
6447  *   and latency is assumed to be high, as above.
6448  *
6449  * The final value programmed to the register should always be rounded up,
6450  * and include an extra 2 entries to account for clock crossings.
6451  *
6452  * We don't use the sprite, so we can ignore that.  And on Crestline we have
6453  * to set the non-SR watermarks to 8.
6454  */
6455 void intel_update_watermarks(struct intel_crtc *crtc)
6456 {
6457         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
6458
6459         if (dev_priv->display.update_wm)
6460                 dev_priv->display.update_wm(crtc);
6461 }
6462
6463 void intel_enable_ipc(struct drm_i915_private *dev_priv)
6464 {
6465         u32 val;
6466
6467         if (!HAS_IPC(dev_priv))
6468                 return;
6469
6470         val = I915_READ(DISP_ARB_CTL2);
6471
6472         if (dev_priv->ipc_enabled)
6473                 val |= DISP_IPC_ENABLE;
6474         else
6475                 val &= ~DISP_IPC_ENABLE;
6476
6477         I915_WRITE(DISP_ARB_CTL2, val);
6478 }
6479
6480 static bool intel_can_enable_ipc(struct drm_i915_private *dev_priv)
6481 {
6482         /* Display WA #0477 WaDisableIPC: skl */
6483         if (IS_SKYLAKE(dev_priv))
6484                 return false;
6485
6486         /* Display WA #1141: SKL:all KBL:all CFL */
6487         if (IS_KABYLAKE(dev_priv) || IS_COFFEELAKE(dev_priv))
6488                 return dev_priv->dram_info.symmetric_memory;
6489
6490         return true;
6491 }
6492
6493 void intel_init_ipc(struct drm_i915_private *dev_priv)
6494 {
6495         if (!HAS_IPC(dev_priv))
6496                 return;
6497
6498         dev_priv->ipc_enabled = intel_can_enable_ipc(dev_priv);
6499
6500         intel_enable_ipc(dev_priv);
6501 }
6502
6503 static void ibx_init_clock_gating(struct drm_i915_private *dev_priv)
6504 {
6505         /*
6506          * On Ibex Peak and Cougar Point, we need to disable clock
6507          * gating for the panel power sequencer or it will fail to
6508          * start up when no ports are active.
6509          */
6510         I915_WRITE(SOUTH_DSPCLK_GATE_D, PCH_DPLSUNIT_CLOCK_GATE_DISABLE);
6511 }
6512
6513 static void g4x_disable_trickle_feed(struct drm_i915_private *dev_priv)
6514 {
6515         enum pipe pipe;
6516
6517         for_each_pipe(dev_priv, pipe) {
6518                 I915_WRITE(DSPCNTR(pipe),
6519                            I915_READ(DSPCNTR(pipe)) |
6520                            DISPPLANE_TRICKLE_FEED_DISABLE);
6521
6522                 I915_WRITE(DSPSURF(pipe), I915_READ(DSPSURF(pipe)));
6523                 POSTING_READ(DSPSURF(pipe));
6524         }
6525 }
6526
6527 static void ilk_init_clock_gating(struct drm_i915_private *dev_priv)
6528 {
6529         u32 dspclk_gate = ILK_VRHUNIT_CLOCK_GATE_DISABLE;
6530
6531         /*
6532          * Required for FBC
6533          * WaFbcDisableDpfcClockGating:ilk
6534          */
6535         dspclk_gate |= ILK_DPFCRUNIT_CLOCK_GATE_DISABLE |
6536                    ILK_DPFCUNIT_CLOCK_GATE_DISABLE |
6537                    ILK_DPFDUNIT_CLOCK_GATE_ENABLE;
6538
6539         I915_WRITE(PCH_3DCGDIS0,
6540                    MARIUNIT_CLOCK_GATE_DISABLE |
6541                    SVSMUNIT_CLOCK_GATE_DISABLE);
6542         I915_WRITE(PCH_3DCGDIS1,
6543                    VFMUNIT_CLOCK_GATE_DISABLE);
6544
6545         /*
6546          * According to the spec the following bits should be set in
6547          * order to enable memory self-refresh
6548          * The bit 22/21 of 0x42004
6549          * The bit 5 of 0x42020
6550          * The bit 15 of 0x45000
6551          */
6552         I915_WRITE(ILK_DISPLAY_CHICKEN2,
6553                    (I915_READ(ILK_DISPLAY_CHICKEN2) |
6554                     ILK_DPARB_GATE | ILK_VSDPFD_FULL));
6555         dspclk_gate |= ILK_DPARBUNIT_CLOCK_GATE_ENABLE;
6556         I915_WRITE(DISP_ARB_CTL,
6557                    (I915_READ(DISP_ARB_CTL) |
6558                     DISP_FBC_WM_DIS));
6559
6560         /*
6561          * Based on the document from hardware guys the following bits
6562          * should be set unconditionally in order to enable FBC.
6563          * The bit 22 of 0x42000
6564          * The bit 22 of 0x42004
6565          * The bit 7,8,9 of 0x42020.
6566          */
6567         if (IS_IRONLAKE_M(dev_priv)) {
6568                 /* WaFbcAsynchFlipDisableFbcQueue:ilk */
6569                 I915_WRITE(ILK_DISPLAY_CHICKEN1,
6570                            I915_READ(ILK_DISPLAY_CHICKEN1) |
6571                            ILK_FBCQ_DIS);
6572                 I915_WRITE(ILK_DISPLAY_CHICKEN2,
6573                            I915_READ(ILK_DISPLAY_CHICKEN2) |
6574                            ILK_DPARB_GATE);
6575         }
6576
6577         I915_WRITE(ILK_DSPCLK_GATE_D, dspclk_gate);
6578
6579         I915_WRITE(ILK_DISPLAY_CHICKEN2,
6580                    I915_READ(ILK_DISPLAY_CHICKEN2) |
6581                    ILK_ELPIN_409_SELECT);
6582         I915_WRITE(_3D_CHICKEN2,
6583                    _3D_CHICKEN2_WM_READ_PIPELINED << 16 |
6584                    _3D_CHICKEN2_WM_READ_PIPELINED);
6585
6586         /* WaDisableRenderCachePipelinedFlush:ilk */
6587         I915_WRITE(CACHE_MODE_0,
6588                    _MASKED_BIT_ENABLE(CM0_PIPELINED_RENDER_FLUSH_DISABLE));
6589
6590         /* WaDisable_RenderCache_OperationalFlush:ilk */
6591         I915_WRITE(CACHE_MODE_0, _MASKED_BIT_DISABLE(RC_OP_FLUSH_ENABLE));
6592
6593         g4x_disable_trickle_feed(dev_priv);
6594
6595         ibx_init_clock_gating(dev_priv);
6596 }
6597
6598 static void cpt_init_clock_gating(struct drm_i915_private *dev_priv)
6599 {
6600         enum pipe pipe;
6601         u32 val;
6602
6603         /*
6604          * On Ibex Peak and Cougar Point, we need to disable clock
6605          * gating for the panel power sequencer or it will fail to
6606          * start up when no ports are active.
6607          */
6608         I915_WRITE(SOUTH_DSPCLK_GATE_D, PCH_DPLSUNIT_CLOCK_GATE_DISABLE |
6609                    PCH_DPLUNIT_CLOCK_GATE_DISABLE |
6610                    PCH_CPUNIT_CLOCK_GATE_DISABLE);
6611         I915_WRITE(SOUTH_CHICKEN2, I915_READ(SOUTH_CHICKEN2) |
6612                    DPLS_EDP_PPS_FIX_DIS);
6613         /* The below fixes the weird display corruption, a few pixels shifted
6614          * downward, on (only) LVDS of some HP laptops with IVY.
6615          */
6616         for_each_pipe(dev_priv, pipe) {
6617                 val = I915_READ(TRANS_CHICKEN2(pipe));
6618                 val |= TRANS_CHICKEN2_TIMING_OVERRIDE;
6619                 val &= ~TRANS_CHICKEN2_FDI_POLARITY_REVERSED;
6620                 if (dev_priv->vbt.fdi_rx_polarity_inverted)
6621                         val |= TRANS_CHICKEN2_FDI_POLARITY_REVERSED;
6622                 val &= ~TRANS_CHICKEN2_DISABLE_DEEP_COLOR_COUNTER;
6623                 val &= ~TRANS_CHICKEN2_DISABLE_DEEP_COLOR_MODESWITCH;
6624                 I915_WRITE(TRANS_CHICKEN2(pipe), val);
6625         }
6626         /* WADP0ClockGatingDisable */
6627         for_each_pipe(dev_priv, pipe) {
6628                 I915_WRITE(TRANS_CHICKEN1(pipe),
6629                            TRANS_CHICKEN1_DP0UNIT_GC_DISABLE);
6630         }
6631 }
6632
6633 static void gen6_check_mch_setup(struct drm_i915_private *dev_priv)
6634 {
6635         u32 tmp;
6636
6637         tmp = I915_READ(MCH_SSKPD);
6638         if ((tmp & MCH_SSKPD_WM0_MASK) != MCH_SSKPD_WM0_VAL)
6639                 drm_dbg_kms(&dev_priv->drm,
6640                             "Wrong MCH_SSKPD value: 0x%08x This can cause underruns.\n",
6641                             tmp);
6642 }
6643
6644 static void gen6_init_clock_gating(struct drm_i915_private *dev_priv)
6645 {
6646         u32 dspclk_gate = ILK_VRHUNIT_CLOCK_GATE_DISABLE;
6647
6648         I915_WRITE(ILK_DSPCLK_GATE_D, dspclk_gate);
6649
6650         I915_WRITE(ILK_DISPLAY_CHICKEN2,
6651                    I915_READ(ILK_DISPLAY_CHICKEN2) |
6652                    ILK_ELPIN_409_SELECT);
6653
6654         /* WaDisableHiZPlanesWhenMSAAEnabled:snb */
6655         I915_WRITE(_3D_CHICKEN,
6656                    _MASKED_BIT_ENABLE(_3D_CHICKEN_HIZ_PLANE_DISABLE_MSAA_4X_SNB));
6657
6658         /* WaDisable_RenderCache_OperationalFlush:snb */
6659         I915_WRITE(CACHE_MODE_0, _MASKED_BIT_DISABLE(RC_OP_FLUSH_ENABLE));
6660
6661         /*
6662          * BSpec recoomends 8x4 when MSAA is used,
6663          * however in practice 16x4 seems fastest.
6664          *
6665          * Note that PS/WM thread counts depend on the WIZ hashing
6666          * disable bit, which we don't touch here, but it's good
6667          * to keep in mind (see 3DSTATE_PS and 3DSTATE_WM).
6668          */
6669         I915_WRITE(GEN6_GT_MODE,
6670                    _MASKED_FIELD(GEN6_WIZ_HASHING_MASK, GEN6_WIZ_HASHING_16x4));
6671
6672         I915_WRITE(CACHE_MODE_0,
6673                    _MASKED_BIT_DISABLE(CM0_STC_EVICT_DISABLE_LRA_SNB));
6674
6675         I915_WRITE(GEN6_UCGCTL1,
6676                    I915_READ(GEN6_UCGCTL1) |
6677                    GEN6_BLBUNIT_CLOCK_GATE_DISABLE |
6678                    GEN6_CSUNIT_CLOCK_GATE_DISABLE);
6679
6680         /* According to the BSpec vol1g, bit 12 (RCPBUNIT) clock
6681          * gating disable must be set.  Failure to set it results in
6682          * flickering pixels due to Z write ordering failures after
6683          * some amount of runtime in the Mesa "fire" demo, and Unigine
6684          * Sanctuary and Tropics, and apparently anything else with
6685          * alpha test or pixel discard.
6686          *
6687          * According to the spec, bit 11 (RCCUNIT) must also be set,
6688          * but we didn't debug actual testcases to find it out.
6689          *
6690          * WaDisableRCCUnitClockGating:snb
6691          * WaDisableRCPBUnitClockGating:snb
6692          */
6693         I915_WRITE(GEN6_UCGCTL2,
6694                    GEN6_RCPBUNIT_CLOCK_GATE_DISABLE |
6695                    GEN6_RCCUNIT_CLOCK_GATE_DISABLE);
6696
6697         /* WaStripsFansDisableFastClipPerformanceFix:snb */
6698         I915_WRITE(_3D_CHICKEN3,
6699                    _MASKED_BIT_ENABLE(_3D_CHICKEN3_SF_DISABLE_FASTCLIP_CULL));
6700
6701         /*
6702          * Bspec says:
6703          * "This bit must be set if 3DSTATE_CLIP clip mode is set to normal and
6704          * 3DSTATE_SF number of SF output attributes is more than 16."
6705          */
6706         I915_WRITE(_3D_CHICKEN3,
6707                    _MASKED_BIT_ENABLE(_3D_CHICKEN3_SF_DISABLE_PIPELINED_ATTR_FETCH));
6708
6709         /*
6710          * According to the spec the following bits should be
6711          * set in order to enable memory self-refresh and fbc:
6712          * The bit21 and bit22 of 0x42000
6713          * The bit21 and bit22 of 0x42004
6714          * The bit5 and bit7 of 0x42020
6715          * The bit14 of 0x70180
6716          * The bit14 of 0x71180
6717          *
6718          * WaFbcAsynchFlipDisableFbcQueue:snb
6719          */
6720         I915_WRITE(ILK_DISPLAY_CHICKEN1,
6721                    I915_READ(ILK_DISPLAY_CHICKEN1) |
6722                    ILK_FBCQ_DIS | ILK_PABSTRETCH_DIS);
6723         I915_WRITE(ILK_DISPLAY_CHICKEN2,
6724                    I915_READ(ILK_DISPLAY_CHICKEN2) |
6725                    ILK_DPARB_GATE | ILK_VSDPFD_FULL);
6726         I915_WRITE(ILK_DSPCLK_GATE_D,
6727                    I915_READ(ILK_DSPCLK_GATE_D) |
6728                    ILK_DPARBUNIT_CLOCK_GATE_ENABLE  |
6729                    ILK_DPFDUNIT_CLOCK_GATE_ENABLE);
6730
6731         g4x_disable_trickle_feed(dev_priv);
6732
6733         cpt_init_clock_gating(dev_priv);
6734
6735         gen6_check_mch_setup(dev_priv);
6736 }
6737
6738 static void gen7_setup_fixed_func_scheduler(struct drm_i915_private *dev_priv)
6739 {
6740         u32 reg = I915_READ(GEN7_FF_THREAD_MODE);
6741
6742         /*
6743          * WaVSThreadDispatchOverride:ivb,vlv
6744          *
6745          * This actually overrides the dispatch
6746          * mode for all thread types.
6747          */
6748         reg &= ~GEN7_FF_SCHED_MASK;
6749         reg |= GEN7_FF_TS_SCHED_HW;
6750         reg |= GEN7_FF_VS_SCHED_HW;
6751         reg |= GEN7_FF_DS_SCHED_HW;
6752
6753         I915_WRITE(GEN7_FF_THREAD_MODE, reg);
6754 }
6755
6756 static void lpt_init_clock_gating(struct drm_i915_private *dev_priv)
6757 {
6758         /*
6759          * TODO: this bit should only be enabled when really needed, then
6760          * disabled when not needed anymore in order to save power.
6761          */
6762         if (HAS_PCH_LPT_LP(dev_priv))
6763                 I915_WRITE(SOUTH_DSPCLK_GATE_D,
6764                            I915_READ(SOUTH_DSPCLK_GATE_D) |
6765                            PCH_LP_PARTITION_LEVEL_DISABLE);
6766
6767         /* WADPOClockGatingDisable:hsw */
6768         I915_WRITE(TRANS_CHICKEN1(PIPE_A),
6769                    I915_READ(TRANS_CHICKEN1(PIPE_A)) |
6770                    TRANS_CHICKEN1_DP0UNIT_GC_DISABLE);
6771 }
6772
6773 static void lpt_suspend_hw(struct drm_i915_private *dev_priv)
6774 {
6775         if (HAS_PCH_LPT_LP(dev_priv)) {
6776                 u32 val = I915_READ(SOUTH_DSPCLK_GATE_D);
6777
6778                 val &= ~PCH_LP_PARTITION_LEVEL_DISABLE;
6779                 I915_WRITE(SOUTH_DSPCLK_GATE_D, val);
6780         }
6781 }
6782
6783 static void gen8_set_l3sqc_credits(struct drm_i915_private *dev_priv,
6784                                    int general_prio_credits,
6785                                    int high_prio_credits)
6786 {
6787         u32 misccpctl;
6788         u32 val;
6789
6790         /* WaTempDisableDOPClkGating:bdw */
6791         misccpctl = I915_READ(GEN7_MISCCPCTL);
6792         I915_WRITE(GEN7_MISCCPCTL, misccpctl & ~GEN7_DOP_CLOCK_GATE_ENABLE);
6793
6794         val = I915_READ(GEN8_L3SQCREG1);
6795         val &= ~L3_PRIO_CREDITS_MASK;
6796         val |= L3_GENERAL_PRIO_CREDITS(general_prio_credits);
6797         val |= L3_HIGH_PRIO_CREDITS(high_prio_credits);
6798         I915_WRITE(GEN8_L3SQCREG1, val);
6799
6800         /*
6801          * Wait at least 100 clocks before re-enabling clock gating.
6802          * See the definition of L3SQCREG1 in BSpec.
6803          */
6804         POSTING_READ(GEN8_L3SQCREG1);
6805         udelay(1);
6806         I915_WRITE(GEN7_MISCCPCTL, misccpctl);
6807 }
6808
6809 static void icl_init_clock_gating(struct drm_i915_private *dev_priv)
6810 {
6811         /* This is not an Wa. Enable to reduce Sampler power */
6812         I915_WRITE(GEN10_DFR_RATIO_EN_AND_CHICKEN,
6813                    I915_READ(GEN10_DFR_RATIO_EN_AND_CHICKEN) & ~DFR_DISABLE);
6814
6815         /* WaEnable32PlaneMode:icl */
6816         I915_WRITE(GEN9_CSFE_CHICKEN1_RCS,
6817                    _MASKED_BIT_ENABLE(GEN11_ENABLE_32_PLANE_MODE));
6818
6819         /*
6820          * Wa_1408615072:icl,ehl  (vsunit)
6821          * Wa_1407596294:icl,ehl  (hsunit)
6822          */
6823         intel_uncore_rmw(&dev_priv->uncore, UNSLICE_UNIT_LEVEL_CLKGATE,
6824                          0, VSUNIT_CLKGATE_DIS | HSUNIT_CLKGATE_DIS);
6825
6826         /* Wa_1407352427:icl,ehl */
6827         intel_uncore_rmw(&dev_priv->uncore, UNSLICE_UNIT_LEVEL_CLKGATE2,
6828                          0, PSDUNIT_CLKGATE_DIS);
6829
6830         /*Wa_14010594013:icl, ehl */
6831         intel_uncore_rmw(&dev_priv->uncore, GEN8_CHICKEN_DCPR_1,
6832                          0, CNL_DELAY_PMRSP);
6833 }
6834
6835 static void tgl_init_clock_gating(struct drm_i915_private *dev_priv)
6836 {
6837         u32 vd_pg_enable = 0;
6838         unsigned int i;
6839
6840         /* Wa_1408615072:tgl */
6841         intel_uncore_rmw(&dev_priv->uncore, UNSLICE_UNIT_LEVEL_CLKGATE2,
6842                          0, VSUNIT_CLKGATE_DIS_TGL);
6843
6844         /* This is not a WA. Enable VD HCP & MFX_ENC powergate */
6845         for (i = 0; i < I915_MAX_VCS; i++) {
6846                 if (HAS_ENGINE(dev_priv, _VCS(i)))
6847                         vd_pg_enable |= VDN_HCP_POWERGATE_ENABLE(i) |
6848                                         VDN_MFX_POWERGATE_ENABLE(i);
6849         }
6850
6851         I915_WRITE(POWERGATE_ENABLE,
6852                    I915_READ(POWERGATE_ENABLE) | vd_pg_enable);
6853
6854         /* Wa_1409825376:tgl (pre-prod)*/
6855         if (IS_TGL_REVID(dev_priv, TGL_REVID_A0, TGL_REVID_A0))
6856                 I915_WRITE(GEN9_CLKGATE_DIS_3, I915_READ(GEN9_CLKGATE_DIS_3) |
6857                            TGL_VRH_GATING_DIS);
6858 }
6859
6860 static void cnp_init_clock_gating(struct drm_i915_private *dev_priv)
6861 {
6862         if (!HAS_PCH_CNP(dev_priv))
6863                 return;
6864
6865         /* Display WA #1181 WaSouthDisplayDisablePWMCGEGating: cnp */
6866         I915_WRITE(SOUTH_DSPCLK_GATE_D, I915_READ(SOUTH_DSPCLK_GATE_D) |
6867                    CNP_PWM_CGE_GATING_DISABLE);
6868 }
6869
6870 static void cnl_init_clock_gating(struct drm_i915_private *dev_priv)
6871 {
6872         u32 val;
6873         cnp_init_clock_gating(dev_priv);
6874
6875         /* This is not an Wa. Enable for better image quality */
6876         I915_WRITE(_3D_CHICKEN3,
6877                    _MASKED_BIT_ENABLE(_3D_CHICKEN3_AA_LINE_QUALITY_FIX_ENABLE));
6878
6879         /* WaEnableChickenDCPR:cnl */
6880         I915_WRITE(GEN8_CHICKEN_DCPR_1,
6881                    I915_READ(GEN8_CHICKEN_DCPR_1) | MASK_WAKEMEM);
6882
6883         /* WaFbcWakeMemOn:cnl */
6884         I915_WRITE(DISP_ARB_CTL, I915_READ(DISP_ARB_CTL) |
6885                    DISP_FBC_MEMORY_WAKE);
6886
6887         val = I915_READ(SLICE_UNIT_LEVEL_CLKGATE);
6888         /* ReadHitWriteOnlyDisable:cnl */
6889         val |= RCCUNIT_CLKGATE_DIS;
6890         /* WaSarbUnitClockGatingDisable:cnl (pre-prod) */
6891         if (IS_CNL_REVID(dev_priv, CNL_REVID_A0, CNL_REVID_B0))
6892                 val |= SARBUNIT_CLKGATE_DIS;
6893         I915_WRITE(SLICE_UNIT_LEVEL_CLKGATE, val);
6894
6895         /* Wa_2201832410:cnl */
6896         val = I915_READ(SUBSLICE_UNIT_LEVEL_CLKGATE);
6897         val |= GWUNIT_CLKGATE_DIS;
6898         I915_WRITE(SUBSLICE_UNIT_LEVEL_CLKGATE, val);
6899
6900         /* WaDisableVFclkgate:cnl */
6901         /* WaVFUnitClockGatingDisable:cnl */
6902         val = I915_READ(UNSLICE_UNIT_LEVEL_CLKGATE);
6903         val |= VFUNIT_CLKGATE_DIS;
6904         I915_WRITE(UNSLICE_UNIT_LEVEL_CLKGATE, val);
6905 }
6906
6907 static void cfl_init_clock_gating(struct drm_i915_private *dev_priv)
6908 {
6909         cnp_init_clock_gating(dev_priv);
6910         gen9_init_clock_gating(dev_priv);
6911
6912         /* WaFbcNukeOnHostModify:cfl */
6913         I915_WRITE(ILK_DPFC_CHICKEN, I915_READ(ILK_DPFC_CHICKEN) |
6914                    ILK_DPFC_NUKE_ON_ANY_MODIFICATION);
6915 }
6916
6917 static void kbl_init_clock_gating(struct drm_i915_private *dev_priv)
6918 {
6919         gen9_init_clock_gating(dev_priv);
6920
6921         /* WaDisableSDEUnitClockGating:kbl */
6922         if (IS_KBL_REVID(dev_priv, 0, KBL_REVID_B0))
6923                 I915_WRITE(GEN8_UCGCTL6, I915_READ(GEN8_UCGCTL6) |
6924                            GEN8_SDEUNIT_CLOCK_GATE_DISABLE);
6925
6926         /* WaDisableGamClockGating:kbl */
6927         if (IS_KBL_REVID(dev_priv, 0, KBL_REVID_B0))
6928                 I915_WRITE(GEN6_UCGCTL1, I915_READ(GEN6_UCGCTL1) |
6929                            GEN6_GAMUNIT_CLOCK_GATE_DISABLE);
6930
6931         /* WaFbcNukeOnHostModify:kbl */
6932         I915_WRITE(ILK_DPFC_CHICKEN, I915_READ(ILK_DPFC_CHICKEN) |
6933                    ILK_DPFC_NUKE_ON_ANY_MODIFICATION);
6934 }
6935
6936 static void skl_init_clock_gating(struct drm_i915_private *dev_priv)
6937 {
6938         gen9_init_clock_gating(dev_priv);
6939
6940         /* WAC6entrylatency:skl */
6941         I915_WRITE(FBC_LLC_READ_CTRL, I915_READ(FBC_LLC_READ_CTRL) |
6942                    FBC_LLC_FULLY_OPEN);
6943
6944         /* WaFbcNukeOnHostModify:skl */
6945         I915_WRITE(ILK_DPFC_CHICKEN, I915_READ(ILK_DPFC_CHICKEN) |
6946                    ILK_DPFC_NUKE_ON_ANY_MODIFICATION);
6947 }
6948
6949 static void bdw_init_clock_gating(struct drm_i915_private *dev_priv)
6950 {
6951         enum pipe pipe;
6952
6953         /* WaSwitchSolVfFArbitrationPriority:bdw */
6954         I915_WRITE(GAM_ECOCHK, I915_READ(GAM_ECOCHK) | HSW_ECOCHK_ARB_PRIO_SOL);
6955
6956         /* WaPsrDPAMaskVBlankInSRD:bdw */
6957         I915_WRITE(CHICKEN_PAR1_1,
6958                    I915_READ(CHICKEN_PAR1_1) | DPA_MASK_VBLANK_SRD);
6959
6960         /* WaPsrDPRSUnmaskVBlankInSRD:bdw */
6961         for_each_pipe(dev_priv, pipe) {
6962                 I915_WRITE(CHICKEN_PIPESL_1(pipe),
6963                            I915_READ(CHICKEN_PIPESL_1(pipe)) |
6964                            BDW_DPRS_MASK_VBLANK_SRD);
6965         }
6966
6967         /* WaVSRefCountFullforceMissDisable:bdw */
6968         /* WaDSRefCountFullforceMissDisable:bdw */
6969         I915_WRITE(GEN7_FF_THREAD_MODE,
6970                    I915_READ(GEN7_FF_THREAD_MODE) &
6971                    ~(GEN8_FF_DS_REF_CNT_FFME | GEN7_FF_VS_REF_CNT_FFME));
6972
6973         I915_WRITE(GEN6_RC_SLEEP_PSMI_CONTROL,
6974                    _MASKED_BIT_ENABLE(GEN8_RC_SEMA_IDLE_MSG_DISABLE));
6975
6976         /* WaDisableSDEUnitClockGating:bdw */
6977         I915_WRITE(GEN8_UCGCTL6, I915_READ(GEN8_UCGCTL6) |
6978                    GEN8_SDEUNIT_CLOCK_GATE_DISABLE);
6979
6980         /* WaProgramL3SqcReg1Default:bdw */
6981         gen8_set_l3sqc_credits(dev_priv, 30, 2);
6982
6983         /* WaKVMNotificationOnConfigChange:bdw */
6984         I915_WRITE(CHICKEN_PAR2_1, I915_READ(CHICKEN_PAR2_1)
6985                    | KVM_CONFIG_CHANGE_NOTIFICATION_SELECT);
6986
6987         lpt_init_clock_gating(dev_priv);
6988
6989         /* WaDisableDopClockGating:bdw
6990          *
6991          * Also see the CHICKEN2 write in bdw_init_workarounds() to disable DOP
6992          * clock gating.
6993          */
6994         I915_WRITE(GEN6_UCGCTL1,
6995                    I915_READ(GEN6_UCGCTL1) | GEN6_EU_TCUNIT_CLOCK_GATE_DISABLE);
6996 }
6997
6998 static void hsw_init_clock_gating(struct drm_i915_private *dev_priv)
6999 {
7000         /* L3 caching of data atomics doesn't work -- disable it. */
7001         I915_WRITE(HSW_SCRATCH1, HSW_SCRATCH1_L3_DATA_ATOMICS_DISABLE);
7002         I915_WRITE(HSW_ROW_CHICKEN3,
7003                    _MASKED_BIT_ENABLE(HSW_ROW_CHICKEN3_L3_GLOBAL_ATOMICS_DISABLE));
7004
7005         /* This is required by WaCatErrorRejectionIssue:hsw */
7006         I915_WRITE(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG,
7007                         I915_READ(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG) |
7008                         GEN7_SQ_CHICKEN_MBCUNIT_SQINTMOB);
7009
7010         /* WaVSRefCountFullforceMissDisable:hsw */
7011         I915_WRITE(GEN7_FF_THREAD_MODE,
7012                    I915_READ(GEN7_FF_THREAD_MODE) & ~GEN7_FF_VS_REF_CNT_FFME);
7013
7014         /* WaDisable_RenderCache_OperationalFlush:hsw */
7015         I915_WRITE(CACHE_MODE_0_GEN7, _MASKED_BIT_DISABLE(RC_OP_FLUSH_ENABLE));
7016
7017         /* enable HiZ Raw Stall Optimization */
7018         I915_WRITE(CACHE_MODE_0_GEN7,
7019                    _MASKED_BIT_DISABLE(HIZ_RAW_STALL_OPT_DISABLE));
7020
7021         /* WaDisable4x2SubspanOptimization:hsw */
7022         I915_WRITE(CACHE_MODE_1,
7023                    _MASKED_BIT_ENABLE(PIXEL_SUBSPAN_COLLECT_OPT_DISABLE));
7024
7025         /*
7026          * BSpec recommends 8x4 when MSAA is used,
7027          * however in practice 16x4 seems fastest.
7028          *
7029          * Note that PS/WM thread counts depend on the WIZ hashing
7030          * disable bit, which we don't touch here, but it's good
7031          * to keep in mind (see 3DSTATE_PS and 3DSTATE_WM).
7032          */
7033         I915_WRITE(GEN7_GT_MODE,
7034                    _MASKED_FIELD(GEN6_WIZ_HASHING_MASK, GEN6_WIZ_HASHING_16x4));
7035
7036         /* WaSampleCChickenBitEnable:hsw */
7037         I915_WRITE(HALF_SLICE_CHICKEN3,
7038                    _MASKED_BIT_ENABLE(HSW_SAMPLE_C_PERFORMANCE));
7039
7040         /* WaSwitchSolVfFArbitrationPriority:hsw */
7041         I915_WRITE(GAM_ECOCHK, I915_READ(GAM_ECOCHK) | HSW_ECOCHK_ARB_PRIO_SOL);
7042
7043         lpt_init_clock_gating(dev_priv);
7044 }
7045
7046 static void ivb_init_clock_gating(struct drm_i915_private *dev_priv)
7047 {
7048         u32 snpcr;
7049
7050         I915_WRITE(ILK_DSPCLK_GATE_D, ILK_VRHUNIT_CLOCK_GATE_DISABLE);
7051
7052         /* WaDisableEarlyCull:ivb */
7053         I915_WRITE(_3D_CHICKEN3,
7054                    _MASKED_BIT_ENABLE(_3D_CHICKEN_SF_DISABLE_OBJEND_CULL));
7055
7056         /* WaDisableBackToBackFlipFix:ivb */
7057         I915_WRITE(IVB_CHICKEN3,
7058                    CHICKEN3_DGMG_REQ_OUT_FIX_DISABLE |
7059                    CHICKEN3_DGMG_DONE_FIX_DISABLE);
7060
7061         /* WaDisablePSDDualDispatchEnable:ivb */
7062         if (IS_IVB_GT1(dev_priv))
7063                 I915_WRITE(GEN7_HALF_SLICE_CHICKEN1,
7064                            _MASKED_BIT_ENABLE(GEN7_PSD_SINGLE_PORT_DISPATCH_ENABLE));
7065
7066         /* WaDisable_RenderCache_OperationalFlush:ivb */
7067         I915_WRITE(CACHE_MODE_0_GEN7, _MASKED_BIT_DISABLE(RC_OP_FLUSH_ENABLE));
7068
7069         /* Apply the WaDisableRHWOOptimizationForRenderHang:ivb workaround. */
7070         I915_WRITE(GEN7_COMMON_SLICE_CHICKEN1,
7071                    GEN7_CSC1_RHWO_OPT_DISABLE_IN_RCC);
7072
7073         /* WaApplyL3ControlAndL3ChickenMode:ivb */
7074         I915_WRITE(GEN7_L3CNTLREG1,
7075                         GEN7_WA_FOR_GEN7_L3_CONTROL);
7076         I915_WRITE(GEN7_L3_CHICKEN_MODE_REGISTER,
7077                    GEN7_WA_L3_CHICKEN_MODE);
7078         if (IS_IVB_GT1(dev_priv))
7079                 I915_WRITE(GEN7_ROW_CHICKEN2,
7080                            _MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
7081         else {
7082                 /* must write both registers */
7083                 I915_WRITE(GEN7_ROW_CHICKEN2,
7084                            _MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
7085                 I915_WRITE(GEN7_ROW_CHICKEN2_GT2,
7086                            _MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
7087         }
7088
7089         /* WaForceL3Serialization:ivb */
7090         I915_WRITE(GEN7_L3SQCREG4, I915_READ(GEN7_L3SQCREG4) &
7091                    ~L3SQ_URB_READ_CAM_MATCH_DISABLE);
7092
7093         /*
7094          * According to the spec, bit 13 (RCZUNIT) must be set on IVB.
7095          * This implements the WaDisableRCZUnitClockGating:ivb workaround.
7096          */
7097         I915_WRITE(GEN6_UCGCTL2,
7098                    GEN6_RCZUNIT_CLOCK_GATE_DISABLE);
7099
7100         /* This is required by WaCatErrorRejectionIssue:ivb */
7101         I915_WRITE(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG,
7102                         I915_READ(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG) |
7103                         GEN7_SQ_CHICKEN_MBCUNIT_SQINTMOB);
7104
7105         g4x_disable_trickle_feed(dev_priv);
7106
7107         gen7_setup_fixed_func_scheduler(dev_priv);
7108
7109         if (0) { /* causes HiZ corruption on ivb:gt1 */
7110                 /* enable HiZ Raw Stall Optimization */
7111                 I915_WRITE(CACHE_MODE_0_GEN7,
7112                            _MASKED_BIT_DISABLE(HIZ_RAW_STALL_OPT_DISABLE));
7113         }
7114
7115         /* WaDisable4x2SubspanOptimization:ivb */
7116         I915_WRITE(CACHE_MODE_1,
7117                    _MASKED_BIT_ENABLE(PIXEL_SUBSPAN_COLLECT_OPT_DISABLE));
7118
7119         /*
7120          * BSpec recommends 8x4 when MSAA is used,
7121          * however in practice 16x4 seems fastest.
7122          *
7123          * Note that PS/WM thread counts depend on the WIZ hashing
7124          * disable bit, which we don't touch here, but it's good
7125          * to keep in mind (see 3DSTATE_PS and 3DSTATE_WM).
7126          */
7127         I915_WRITE(GEN7_GT_MODE,
7128                    _MASKED_FIELD(GEN6_WIZ_HASHING_MASK, GEN6_WIZ_HASHING_16x4));
7129
7130         snpcr = I915_READ(GEN6_MBCUNIT_SNPCR);
7131         snpcr &= ~GEN6_MBC_SNPCR_MASK;
7132         snpcr |= GEN6_MBC_SNPCR_MED;
7133         I915_WRITE(GEN6_MBCUNIT_SNPCR, snpcr);
7134
7135         if (!HAS_PCH_NOP(dev_priv))
7136                 cpt_init_clock_gating(dev_priv);
7137
7138         gen6_check_mch_setup(dev_priv);
7139 }
7140
7141 static void vlv_init_clock_gating(struct drm_i915_private *dev_priv)
7142 {
7143         /* WaDisableEarlyCull:vlv */
7144         I915_WRITE(_3D_CHICKEN3,
7145                    _MASKED_BIT_ENABLE(_3D_CHICKEN_SF_DISABLE_OBJEND_CULL));
7146
7147         /* WaDisableBackToBackFlipFix:vlv */
7148         I915_WRITE(IVB_CHICKEN3,
7149                    CHICKEN3_DGMG_REQ_OUT_FIX_DISABLE |
7150                    CHICKEN3_DGMG_DONE_FIX_DISABLE);
7151
7152         /* WaPsdDispatchEnable:vlv */
7153         /* WaDisablePSDDualDispatchEnable:vlv */
7154         I915_WRITE(GEN7_HALF_SLICE_CHICKEN1,
7155                    _MASKED_BIT_ENABLE(GEN7_MAX_PS_THREAD_DEP |
7156                                       GEN7_PSD_SINGLE_PORT_DISPATCH_ENABLE));
7157
7158         /* WaDisable_RenderCache_OperationalFlush:vlv */
7159         I915_WRITE(CACHE_MODE_0_GEN7, _MASKED_BIT_DISABLE(RC_OP_FLUSH_ENABLE));
7160
7161         /* WaForceL3Serialization:vlv */
7162         I915_WRITE(GEN7_L3SQCREG4, I915_READ(GEN7_L3SQCREG4) &
7163                    ~L3SQ_URB_READ_CAM_MATCH_DISABLE);
7164
7165         /* WaDisableDopClockGating:vlv */
7166         I915_WRITE(GEN7_ROW_CHICKEN2,
7167                    _MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
7168
7169         /* This is required by WaCatErrorRejectionIssue:vlv */
7170         I915_WRITE(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG,
7171                    I915_READ(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG) |
7172                    GEN7_SQ_CHICKEN_MBCUNIT_SQINTMOB);
7173
7174         gen7_setup_fixed_func_scheduler(dev_priv);
7175
7176         /*
7177          * According to the spec, bit 13 (RCZUNIT) must be set on IVB.
7178          * This implements the WaDisableRCZUnitClockGating:vlv workaround.
7179          */
7180         I915_WRITE(GEN6_UCGCTL2,
7181                    GEN6_RCZUNIT_CLOCK_GATE_DISABLE);
7182
7183         /* WaDisableL3Bank2xClockGate:vlv
7184          * Disabling L3 clock gating- MMIO 940c[25] = 1
7185          * Set bit 25, to disable L3_BANK_2x_CLK_GATING */
7186         I915_WRITE(GEN7_UCGCTL4,
7187                    I915_READ(GEN7_UCGCTL4) | GEN7_L3BANK2X_CLOCK_GATE_DISABLE);
7188
7189         /*
7190          * BSpec says this must be set, even though
7191          * WaDisable4x2SubspanOptimization isn't listed for VLV.
7192          */
7193         I915_WRITE(CACHE_MODE_1,
7194                    _MASKED_BIT_ENABLE(PIXEL_SUBSPAN_COLLECT_OPT_DISABLE));
7195
7196         /*
7197          * BSpec recommends 8x4 when MSAA is used,
7198          * however in practice 16x4 seems fastest.
7199          *
7200          * Note that PS/WM thread counts depend on the WIZ hashing
7201          * disable bit, which we don't touch here, but it's good
7202          * to keep in mind (see 3DSTATE_PS and 3DSTATE_WM).
7203          */
7204         I915_WRITE(GEN7_GT_MODE,
7205                    _MASKED_FIELD(GEN6_WIZ_HASHING_MASK, GEN6_WIZ_HASHING_16x4));
7206
7207         /*
7208          * WaIncreaseL3CreditsForVLVB0:vlv
7209          * This is the hardware default actually.
7210          */
7211         I915_WRITE(GEN7_L3SQCREG1, VLV_B0_WA_L3SQCREG1_VALUE);
7212
7213         /*
7214          * WaDisableVLVClockGating_VBIIssue:vlv
7215          * Disable clock gating on th GCFG unit to prevent a delay
7216          * in the reporting of vblank events.
7217          */
7218         I915_WRITE(VLV_GUNIT_CLOCK_GATE, GCFG_DIS);
7219 }
7220
7221 static void chv_init_clock_gating(struct drm_i915_private *dev_priv)
7222 {
7223         /* WaVSRefCountFullforceMissDisable:chv */
7224         /* WaDSRefCountFullforceMissDisable:chv */
7225         I915_WRITE(GEN7_FF_THREAD_MODE,
7226                    I915_READ(GEN7_FF_THREAD_MODE) &
7227                    ~(GEN8_FF_DS_REF_CNT_FFME | GEN7_FF_VS_REF_CNT_FFME));
7228
7229         /* WaDisableSemaphoreAndSyncFlipWait:chv */
7230         I915_WRITE(GEN6_RC_SLEEP_PSMI_CONTROL,
7231                    _MASKED_BIT_ENABLE(GEN8_RC_SEMA_IDLE_MSG_DISABLE));
7232
7233         /* WaDisableCSUnitClockGating:chv */
7234         I915_WRITE(GEN6_UCGCTL1, I915_READ(GEN6_UCGCTL1) |
7235                    GEN6_CSUNIT_CLOCK_GATE_DISABLE);
7236
7237         /* WaDisableSDEUnitClockGating:chv */
7238         I915_WRITE(GEN8_UCGCTL6, I915_READ(GEN8_UCGCTL6) |
7239                    GEN8_SDEUNIT_CLOCK_GATE_DISABLE);
7240
7241         /*
7242          * WaProgramL3SqcReg1Default:chv
7243          * See gfxspecs/Related Documents/Performance Guide/
7244          * LSQC Setting Recommendations.
7245          */
7246         gen8_set_l3sqc_credits(dev_priv, 38, 2);
7247 }
7248
7249 static void g4x_init_clock_gating(struct drm_i915_private *dev_priv)
7250 {
7251         u32 dspclk_gate;
7252
7253         I915_WRITE(RENCLK_GATE_D1, 0);
7254         I915_WRITE(RENCLK_GATE_D2, VF_UNIT_CLOCK_GATE_DISABLE |
7255                    GS_UNIT_CLOCK_GATE_DISABLE |
7256                    CL_UNIT_CLOCK_GATE_DISABLE);
7257         I915_WRITE(RAMCLK_GATE_D, 0);
7258         dspclk_gate = VRHUNIT_CLOCK_GATE_DISABLE |
7259                 OVRUNIT_CLOCK_GATE_DISABLE |
7260                 OVCUNIT_CLOCK_GATE_DISABLE;
7261         if (IS_GM45(dev_priv))
7262                 dspclk_gate |= DSSUNIT_CLOCK_GATE_DISABLE;
7263         I915_WRITE(DSPCLK_GATE_D, dspclk_gate);
7264
7265         /* WaDisableRenderCachePipelinedFlush */
7266         I915_WRITE(CACHE_MODE_0,
7267                    _MASKED_BIT_ENABLE(CM0_PIPELINED_RENDER_FLUSH_DISABLE));
7268
7269         /* WaDisable_RenderCache_OperationalFlush:g4x */
7270         I915_WRITE(CACHE_MODE_0, _MASKED_BIT_DISABLE(RC_OP_FLUSH_ENABLE));
7271
7272         g4x_disable_trickle_feed(dev_priv);
7273 }
7274
7275 static void i965gm_init_clock_gating(struct drm_i915_private *dev_priv)
7276 {
7277         struct intel_uncore *uncore = &dev_priv->uncore;
7278
7279         intel_uncore_write(uncore, RENCLK_GATE_D1, I965_RCC_CLOCK_GATE_DISABLE);
7280         intel_uncore_write(uncore, RENCLK_GATE_D2, 0);
7281         intel_uncore_write(uncore, DSPCLK_GATE_D, 0);
7282         intel_uncore_write(uncore, RAMCLK_GATE_D, 0);
7283         intel_uncore_write16(uncore, DEUC, 0);
7284         intel_uncore_write(uncore,
7285                            MI_ARB_STATE,
7286                            _MASKED_BIT_ENABLE(MI_ARB_DISPLAY_TRICKLE_FEED_DISABLE));
7287
7288         /* WaDisable_RenderCache_OperationalFlush:gen4 */
7289         intel_uncore_write(uncore,
7290                            CACHE_MODE_0,
7291                            _MASKED_BIT_DISABLE(RC_OP_FLUSH_ENABLE));
7292 }
7293
7294 static void i965g_init_clock_gating(struct drm_i915_private *dev_priv)
7295 {
7296         I915_WRITE(RENCLK_GATE_D1, I965_RCZ_CLOCK_GATE_DISABLE |
7297                    I965_RCC_CLOCK_GATE_DISABLE |
7298                    I965_RCPB_CLOCK_GATE_DISABLE |
7299                    I965_ISC_CLOCK_GATE_DISABLE |
7300                    I965_FBC_CLOCK_GATE_DISABLE);
7301         I915_WRITE(RENCLK_GATE_D2, 0);
7302         I915_WRITE(MI_ARB_STATE,
7303                    _MASKED_BIT_ENABLE(MI_ARB_DISPLAY_TRICKLE_FEED_DISABLE));
7304
7305         /* WaDisable_RenderCache_OperationalFlush:gen4 */
7306         I915_WRITE(CACHE_MODE_0, _MASKED_BIT_DISABLE(RC_OP_FLUSH_ENABLE));
7307 }
7308
7309 static void gen3_init_clock_gating(struct drm_i915_private *dev_priv)
7310 {
7311         u32 dstate = I915_READ(D_STATE);
7312
7313         dstate |= DSTATE_PLL_D3_OFF | DSTATE_GFX_CLOCK_GATING |
7314                 DSTATE_DOT_CLOCK_GATING;
7315         I915_WRITE(D_STATE, dstate);
7316
7317         if (IS_PINEVIEW(dev_priv))
7318                 I915_WRITE(ECOSKPD, _MASKED_BIT_ENABLE(ECO_GATING_CX_ONLY));
7319
7320         /* IIR "flip pending" means done if this bit is set */
7321         I915_WRITE(ECOSKPD, _MASKED_BIT_DISABLE(ECO_FLIP_DONE));
7322
7323         /* interrupts should cause a wake up from C3 */
7324         I915_WRITE(INSTPM, _MASKED_BIT_ENABLE(INSTPM_AGPBUSY_INT_EN));
7325
7326         /* On GEN3 we really need to make sure the ARB C3 LP bit is set */
7327         I915_WRITE(MI_ARB_STATE, _MASKED_BIT_ENABLE(MI_ARB_C3_LP_WRITE_ENABLE));
7328
7329         I915_WRITE(MI_ARB_STATE,
7330                    _MASKED_BIT_ENABLE(MI_ARB_DISPLAY_TRICKLE_FEED_DISABLE));
7331 }
7332
7333 static void i85x_init_clock_gating(struct drm_i915_private *dev_priv)
7334 {
7335         I915_WRITE(RENCLK_GATE_D1, SV_CLOCK_GATE_DISABLE);
7336
7337         /* interrupts should cause a wake up from C3 */
7338         I915_WRITE(MI_STATE, _MASKED_BIT_ENABLE(MI_AGPBUSY_INT_EN) |
7339                    _MASKED_BIT_DISABLE(MI_AGPBUSY_830_MODE));
7340
7341         I915_WRITE(MEM_MODE,
7342                    _MASKED_BIT_ENABLE(MEM_DISPLAY_TRICKLE_FEED_DISABLE));
7343 }
7344
7345 static void i830_init_clock_gating(struct drm_i915_private *dev_priv)
7346 {
7347         I915_WRITE(MEM_MODE,
7348                    _MASKED_BIT_ENABLE(MEM_DISPLAY_A_TRICKLE_FEED_DISABLE) |
7349                    _MASKED_BIT_ENABLE(MEM_DISPLAY_B_TRICKLE_FEED_DISABLE));
7350 }
7351
7352 void intel_init_clock_gating(struct drm_i915_private *dev_priv)
7353 {
7354         dev_priv->display.init_clock_gating(dev_priv);
7355 }
7356
7357 void intel_suspend_hw(struct drm_i915_private *dev_priv)
7358 {
7359         if (HAS_PCH_LPT(dev_priv))
7360                 lpt_suspend_hw(dev_priv);
7361 }
7362
7363 static void nop_init_clock_gating(struct drm_i915_private *dev_priv)
7364 {
7365         drm_dbg_kms(&dev_priv->drm,
7366                     "No clock gating settings or workarounds applied.\n");
7367 }
7368
7369 /**
7370  * intel_init_clock_gating_hooks - setup the clock gating hooks
7371  * @dev_priv: device private
7372  *
7373  * Setup the hooks that configure which clocks of a given platform can be
7374  * gated and also apply various GT and display specific workarounds for these
7375  * platforms. Note that some GT specific workarounds are applied separately
7376  * when GPU contexts or batchbuffers start their execution.
7377  */
7378 void intel_init_clock_gating_hooks(struct drm_i915_private *dev_priv)
7379 {
7380         if (IS_GEN(dev_priv, 12))
7381                 dev_priv->display.init_clock_gating = tgl_init_clock_gating;
7382         else if (IS_GEN(dev_priv, 11))
7383                 dev_priv->display.init_clock_gating = icl_init_clock_gating;
7384         else if (IS_CANNONLAKE(dev_priv))
7385                 dev_priv->display.init_clock_gating = cnl_init_clock_gating;
7386         else if (IS_COFFEELAKE(dev_priv))
7387                 dev_priv->display.init_clock_gating = cfl_init_clock_gating;
7388         else if (IS_SKYLAKE(dev_priv))
7389                 dev_priv->display.init_clock_gating = skl_init_clock_gating;
7390         else if (IS_KABYLAKE(dev_priv))
7391                 dev_priv->display.init_clock_gating = kbl_init_clock_gating;
7392         else if (IS_BROXTON(dev_priv))
7393                 dev_priv->display.init_clock_gating = bxt_init_clock_gating;
7394         else if (IS_GEMINILAKE(dev_priv))
7395                 dev_priv->display.init_clock_gating = glk_init_clock_gating;
7396         else if (IS_BROADWELL(dev_priv))
7397                 dev_priv->display.init_clock_gating = bdw_init_clock_gating;
7398         else if (IS_CHERRYVIEW(dev_priv))
7399                 dev_priv->display.init_clock_gating = chv_init_clock_gating;
7400         else if (IS_HASWELL(dev_priv))
7401                 dev_priv->display.init_clock_gating = hsw_init_clock_gating;
7402         else if (IS_IVYBRIDGE(dev_priv))
7403                 dev_priv->display.init_clock_gating = ivb_init_clock_gating;
7404         else if (IS_VALLEYVIEW(dev_priv))
7405                 dev_priv->display.init_clock_gating = vlv_init_clock_gating;
7406         else if (IS_GEN(dev_priv, 6))
7407                 dev_priv->display.init_clock_gating = gen6_init_clock_gating;
7408         else if (IS_GEN(dev_priv, 5))
7409                 dev_priv->display.init_clock_gating = ilk_init_clock_gating;
7410         else if (IS_G4X(dev_priv))
7411                 dev_priv->display.init_clock_gating = g4x_init_clock_gating;
7412         else if (IS_I965GM(dev_priv))
7413                 dev_priv->display.init_clock_gating = i965gm_init_clock_gating;
7414         else if (IS_I965G(dev_priv))
7415                 dev_priv->display.init_clock_gating = i965g_init_clock_gating;
7416         else if (IS_GEN(dev_priv, 3))
7417                 dev_priv->display.init_clock_gating = gen3_init_clock_gating;
7418         else if (IS_I85X(dev_priv) || IS_I865G(dev_priv))
7419                 dev_priv->display.init_clock_gating = i85x_init_clock_gating;
7420         else if (IS_GEN(dev_priv, 2))
7421                 dev_priv->display.init_clock_gating = i830_init_clock_gating;
7422         else {
7423                 MISSING_CASE(INTEL_DEVID(dev_priv));
7424                 dev_priv->display.init_clock_gating = nop_init_clock_gating;
7425         }
7426 }
7427
7428 /* Set up chip specific power management-related functions */
7429 void intel_init_pm(struct drm_i915_private *dev_priv)
7430 {
7431         /* For cxsr */
7432         if (IS_PINEVIEW(dev_priv))
7433                 pnv_get_mem_freq(dev_priv);
7434         else if (IS_GEN(dev_priv, 5))
7435                 ilk_get_mem_freq(dev_priv);
7436
7437         if (intel_has_sagv(dev_priv))
7438                 skl_setup_sagv_block_time(dev_priv);
7439
7440         /* For FIFO watermark updates */
7441         if (INTEL_GEN(dev_priv) >= 9) {
7442                 skl_setup_wm_latency(dev_priv);
7443                 dev_priv->display.compute_global_watermarks = skl_compute_wm;
7444         } else if (HAS_PCH_SPLIT(dev_priv)) {
7445                 ilk_setup_wm_latency(dev_priv);
7446
7447                 if ((IS_GEN(dev_priv, 5) && dev_priv->wm.pri_latency[1] &&
7448                      dev_priv->wm.spr_latency[1] && dev_priv->wm.cur_latency[1]) ||
7449                     (!IS_GEN(dev_priv, 5) && dev_priv->wm.pri_latency[0] &&
7450                      dev_priv->wm.spr_latency[0] && dev_priv->wm.cur_latency[0])) {
7451                         dev_priv->display.compute_pipe_wm = ilk_compute_pipe_wm;
7452                         dev_priv->display.compute_intermediate_wm =
7453                                 ilk_compute_intermediate_wm;
7454                         dev_priv->display.initial_watermarks =
7455                                 ilk_initial_watermarks;
7456                         dev_priv->display.optimize_watermarks =
7457                                 ilk_optimize_watermarks;
7458                 } else {
7459                         drm_dbg_kms(&dev_priv->drm,
7460                                     "Failed to read display plane latency. "
7461                                     "Disable CxSR\n");
7462                 }
7463         } else if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) {
7464                 vlv_setup_wm_latency(dev_priv);
7465                 dev_priv->display.compute_pipe_wm = vlv_compute_pipe_wm;
7466                 dev_priv->display.compute_intermediate_wm = vlv_compute_intermediate_wm;
7467                 dev_priv->display.initial_watermarks = vlv_initial_watermarks;
7468                 dev_priv->display.optimize_watermarks = vlv_optimize_watermarks;
7469                 dev_priv->display.atomic_update_watermarks = vlv_atomic_update_fifo;
7470         } else if (IS_G4X(dev_priv)) {
7471                 g4x_setup_wm_latency(dev_priv);
7472                 dev_priv->display.compute_pipe_wm = g4x_compute_pipe_wm;
7473                 dev_priv->display.compute_intermediate_wm = g4x_compute_intermediate_wm;
7474                 dev_priv->display.initial_watermarks = g4x_initial_watermarks;
7475                 dev_priv->display.optimize_watermarks = g4x_optimize_watermarks;
7476         } else if (IS_PINEVIEW(dev_priv)) {
7477                 if (!intel_get_cxsr_latency(!IS_MOBILE(dev_priv),
7478                                             dev_priv->is_ddr3,
7479                                             dev_priv->fsb_freq,
7480                                             dev_priv->mem_freq)) {
7481                         drm_info(&dev_priv->drm,
7482                                  "failed to find known CxSR latency "
7483                                  "(found ddr%s fsb freq %d, mem freq %d), "
7484                                  "disabling CxSR\n",
7485                                  (dev_priv->is_ddr3 == 1) ? "3" : "2",
7486                                  dev_priv->fsb_freq, dev_priv->mem_freq);
7487                         /* Disable CxSR and never update its watermark again */
7488                         intel_set_memory_cxsr(dev_priv, false);
7489                         dev_priv->display.update_wm = NULL;
7490                 } else
7491                         dev_priv->display.update_wm = pnv_update_wm;
7492         } else if (IS_GEN(dev_priv, 4)) {
7493                 dev_priv->display.update_wm = i965_update_wm;
7494         } else if (IS_GEN(dev_priv, 3)) {
7495                 dev_priv->display.update_wm = i9xx_update_wm;
7496                 dev_priv->display.get_fifo_size = i9xx_get_fifo_size;
7497         } else if (IS_GEN(dev_priv, 2)) {
7498                 if (INTEL_NUM_PIPES(dev_priv) == 1) {
7499                         dev_priv->display.update_wm = i845_update_wm;
7500                         dev_priv->display.get_fifo_size = i845_get_fifo_size;
7501                 } else {
7502                         dev_priv->display.update_wm = i9xx_update_wm;
7503                         dev_priv->display.get_fifo_size = i830_get_fifo_size;
7504                 }
7505         } else {
7506                 drm_err(&dev_priv->drm,
7507                         "unexpected fall-through in %s\n", __func__);
7508         }
7509 }
7510
7511 void intel_pm_setup(struct drm_i915_private *dev_priv)
7512 {
7513         dev_priv->runtime_pm.suspended = false;
7514         atomic_set(&dev_priv->runtime_pm.wakeref_count, 0);
7515 }