drivers/gpu/drm/i915/intel_display.c

   1 /*
   2  * Copyright © 2006-2007 Intel Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
  21  * DEALINGS IN THE SOFTWARE.
  22  *
  23  * Authors:
  24  *      Eric Anholt <eric@anholt.net>
  25  */
  26
  27 #include <linux/module.h>
  28 #include <linux/input.h>
  29 #include <linux/i2c.h>
  30 #include <linux/kernel.h>
  31 #include <linux/slab.h>
  32 #include <linux/vgaarb.h>
  33 #include <drm/drm_edid.h>
  34 #include <drm/drmP.h>
  35 #include "intel_drv.h"
  36 #include "intel_frontbuffer.h"
  37 #include <drm/i915_drm.h>
  38 #include "i915_drv.h"
  39 #include "i915_gem_clflush.h"
  40 #include "intel_dsi.h"
  41 #include "i915_trace.h"
  42 #include <drm/drm_atomic.h>
  43 #include <drm/drm_atomic_helper.h>
  44 #include <drm/drm_dp_helper.h>
  45 #include <drm/drm_crtc_helper.h>
  46 #include <drm/drm_plane_helper.h>
  47 #include <drm/drm_rect.h>
  48 #include <drm/drm_atomic_uapi.h>
  49 #include <linux/dma_remapping.h>
  50 #include <linux/reservation.h>
  51
  52 /* Primary plane formats for gen <= 3 */
  53 static const uint32_t i8xx_primary_formats[] = {
  54         DRM_FORMAT_C8,
  55         DRM_FORMAT_RGB565,
  56         DRM_FORMAT_XRGB1555,
  57         DRM_FORMAT_XRGB8888,
  58 };
  59
  60 /* Primary plane formats for gen >= 4 */
  61 static const uint32_t i965_primary_formats[] = {
  62         DRM_FORMAT_C8,
  63         DRM_FORMAT_RGB565,
  64         DRM_FORMAT_XRGB8888,
  65         DRM_FORMAT_XBGR8888,
  66         DRM_FORMAT_XRGB2101010,
  67         DRM_FORMAT_XBGR2101010,
  68 };
  69
  70 static const uint64_t i9xx_format_modifiers[] = {
  71         I915_FORMAT_MOD_X_TILED,
  72         DRM_FORMAT_MOD_LINEAR,
  73         DRM_FORMAT_MOD_INVALID
  74 };
  75
  76 /* Cursor formats */
  77 static const uint32_t intel_cursor_formats[] = {
  78         DRM_FORMAT_ARGB8888,
  79 };
  80
  81 static const uint64_t cursor_format_modifiers[] = {
  82         DRM_FORMAT_MOD_LINEAR,
  83         DRM_FORMAT_MOD_INVALID
  84 };
  85
  86 static void i9xx_crtc_clock_get(struct intel_crtc *crtc,
  87                                 struct intel_crtc_state *pipe_config);
  88 static void ironlake_pch_clock_get(struct intel_crtc *crtc,
  89                                    struct intel_crtc_state *pipe_config);
  90
  91 static int intel_framebuffer_init(struct intel_framebuffer *ifb,
  92                                   struct drm_i915_gem_object *obj,
  93                                   struct drm_mode_fb_cmd2 *mode_cmd);
  94 static void intel_set_pipe_timings(const struct intel_crtc_state *crtc_state);
  95 static void intel_set_pipe_src_size(const struct intel_crtc_state *crtc_state);
  96 static void intel_cpu_transcoder_set_m_n(const struct intel_crtc_state *crtc_state,
  97                                          const struct intel_link_m_n *m_n,
  98                                          const struct intel_link_m_n *m2_n2);
  99 static void i9xx_set_pipeconf(const struct intel_crtc_state *crtc_state);
 100 static void ironlake_set_pipeconf(const struct intel_crtc_state *crtc_state);
 101 static void haswell_set_pipeconf(const struct intel_crtc_state *crtc_state);
 102 static void haswell_set_pipemisc(const struct intel_crtc_state *crtc_state);
 103 static void vlv_prepare_pll(struct intel_crtc *crtc,
 104                             const struct intel_crtc_state *pipe_config);
 105 static void chv_prepare_pll(struct intel_crtc *crtc,
 106                             const struct intel_crtc_state *pipe_config);
 107 static void intel_begin_crtc_commit(struct drm_crtc *, struct drm_crtc_state *);
 108 static void intel_finish_crtc_commit(struct drm_crtc *, struct drm_crtc_state *);
 109 static void intel_crtc_init_scalers(struct intel_crtc *crtc,
 110                                     struct intel_crtc_state *crtc_state);
 111 static void skylake_pfit_enable(const struct intel_crtc_state *crtc_state);
 112 static void ironlake_pfit_disable(const struct intel_crtc_state *old_crtc_state);
 113 static void ironlake_pfit_enable(const struct intel_crtc_state *crtc_state);
 114 static void intel_modeset_setup_hw_state(struct drm_device *dev,
 115                                          struct drm_modeset_acquire_ctx *ctx);
 116 static void intel_pre_disable_primary_noatomic(struct drm_crtc *crtc);
 117
 118 struct intel_limit {
 119         struct {
 120                 int min, max;
 121         } dot, vco, n, m, m1, m2, p, p1;
 122
 123         struct {
 124                 int dot_limit;
 125                 int p2_slow, p2_fast;
 126         } p2;
 127 };
 128
 129 /* returns HPLL frequency in kHz */
 130 int vlv_get_hpll_vco(struct drm_i915_private *dev_priv)
 131 {
 132         int hpll_freq, vco_freq[] = { 800, 1600, 2000, 2400 };
 133
 134         /* Obtain SKU information */
 135         mutex_lock(&dev_priv->sb_lock);
 136         hpll_freq = vlv_cck_read(dev_priv, CCK_FUSE_REG) &
 137                 CCK_FUSE_HPLL_FREQ_MASK;
 138         mutex_unlock(&dev_priv->sb_lock);
 139
 140         return vco_freq[hpll_freq] * 1000;
 141 }
 142
 143 int vlv_get_cck_clock(struct drm_i915_private *dev_priv,
 144                       const char *name, u32 reg, int ref_freq)
 145 {
 146         u32 val;
 147         int divider;
 148
 149         mutex_lock(&dev_priv->sb_lock);
 150         val = vlv_cck_read(dev_priv, reg);
 151         mutex_unlock(&dev_priv->sb_lock);
 152
 153         divider = val & CCK_FREQUENCY_VALUES;
 154
 155         WARN((val & CCK_FREQUENCY_STATUS) !=
 156              (divider << CCK_FREQUENCY_STATUS_SHIFT),
 157              "%s change in progress\n", name);
 158
 159         return DIV_ROUND_CLOSEST(ref_freq << 1, divider + 1);
 160 }
 161
 162 int vlv_get_cck_clock_hpll(struct drm_i915_private *dev_priv,
 163                            const char *name, u32 reg)
 164 {
 165         if (dev_priv->hpll_freq == 0)
 166                 dev_priv->hpll_freq = vlv_get_hpll_vco(dev_priv);
 167
 168         return vlv_get_cck_clock(dev_priv, name, reg,
 169                                  dev_priv->hpll_freq);
 170 }
 171
 172 static void intel_update_czclk(struct drm_i915_private *dev_priv)
 173 {
 174         if (!(IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)))
 175                 return;
 176
 177         dev_priv->czclk_freq = vlv_get_cck_clock_hpll(dev_priv, "czclk",
 178                                                       CCK_CZ_CLOCK_CONTROL);
 179
 180         DRM_DEBUG_DRIVER("CZ clock rate: %d kHz\n", dev_priv->czclk_freq);
 181 }
 182
 183 static inline u32 /* units of 100MHz */
 184 intel_fdi_link_freq(struct drm_i915_private *dev_priv,
 185                     const struct intel_crtc_state *pipe_config)
 186 {
 187         if (HAS_DDI(dev_priv))
 188                 return pipe_config->port_clock; /* SPLL */
 189         else
 190                 return dev_priv->fdi_pll_freq;
 191 }
 192
 193 static const struct intel_limit intel_limits_i8xx_dac = {
 194         .dot = { .min = 25000, .max = 350000 },
 195         .vco = { .min = 908000, .max = 1512000 },
 196         .n = { .min = 2, .max = 16 },
 197         .m = { .min = 96, .max = 140 },
 198         .m1 = { .min = 18, .max = 26 },
 199         .m2 = { .min = 6, .max = 16 },
 200         .p = { .min = 4, .max = 128 },
 201         .p1 = { .min = 2, .max = 33 },
 202         .p2 = { .dot_limit = 165000,
 203                 .p2_slow = 4, .p2_fast = 2 },
 204 };
 205
 206 static const struct intel_limit intel_limits_i8xx_dvo = {
 207         .dot = { .min = 25000, .max = 350000 },
 208         .vco = { .min = 908000, .max = 1512000 },
 209         .n = { .min = 2, .max = 16 },
 210         .m = { .min = 96, .max = 140 },
 211         .m1 = { .min = 18, .max = 26 },
 212         .m2 = { .min = 6, .max = 16 },
 213         .p = { .min = 4, .max = 128 },
 214         .p1 = { .min = 2, .max = 33 },
 215         .p2 = { .dot_limit = 165000,
 216                 .p2_slow = 4, .p2_fast = 4 },
 217 };
 218
 219 static const struct intel_limit intel_limits_i8xx_lvds = {
 220         .dot = { .min = 25000, .max = 350000 },
 221         .vco = { .min = 908000, .max = 1512000 },
 222         .n = { .min = 2, .max = 16 },
 223         .m = { .min = 96, .max = 140 },
 224         .m1 = { .min = 18, .max = 26 },
 225         .m2 = { .min = 6, .max = 16 },
 226         .p = { .min = 4, .max = 128 },
 227         .p1 = { .min = 1, .max = 6 },
 228         .p2 = { .dot_limit = 165000,
 229                 .p2_slow = 14, .p2_fast = 7 },
 230 };
 231
 232 static const struct intel_limit intel_limits_i9xx_sdvo = {
 233         .dot = { .min = 20000, .max = 400000 },
 234         .vco = { .min = 1400000, .max = 2800000 },
 235         .n = { .min = 1, .max = 6 },
 236         .m = { .min = 70, .max = 120 },
 237         .m1 = { .min = 8, .max = 18 },
 238         .m2 = { .min = 3, .max = 7 },
 239         .p = { .min = 5, .max = 80 },
 240         .p1 = { .min = 1, .max = 8 },
 241         .p2 = { .dot_limit = 200000,
 242                 .p2_slow = 10, .p2_fast = 5 },
 243 };
 244
 245 static const struct intel_limit intel_limits_i9xx_lvds = {
 246         .dot = { .min = 20000, .max = 400000 },
 247         .vco = { .min = 1400000, .max = 2800000 },
 248         .n = { .min = 1, .max = 6 },
 249         .m = { .min = 70, .max = 120 },
 250         .m1 = { .min = 8, .max = 18 },
 251         .m2 = { .min = 3, .max = 7 },
 252         .p = { .min = 7, .max = 98 },
 253         .p1 = { .min = 1, .max = 8 },
 254         .p2 = { .dot_limit = 112000,
 255                 .p2_slow = 14, .p2_fast = 7 },
 256 };
 257
 258
 259 static const struct intel_limit intel_limits_g4x_sdvo = {
 260         .dot = { .min = 25000, .max = 270000 },
 261         .vco = { .min = 1750000, .max = 3500000},
 262         .n = { .min = 1, .max = 4 },
 263         .m = { .min = 104, .max = 138 },
 264         .m1 = { .min = 17, .max = 23 },
 265         .m2 = { .min = 5, .max = 11 },
 266         .p = { .min = 10, .max = 30 },
 267         .p1 = { .min = 1, .max = 3},
 268         .p2 = { .dot_limit = 270000,
 269                 .p2_slow = 10,
 270                 .p2_fast = 10
 271         },
 272 };
 273
 274 static const struct intel_limit intel_limits_g4x_hdmi = {
 275         .dot = { .min = 22000, .max = 400000 },
 276         .vco = { .min = 1750000, .max = 3500000},
 277         .n = { .min = 1, .max = 4 },
 278         .m = { .min = 104, .max = 138 },
 279         .m1 = { .min = 16, .max = 23 },
 280         .m2 = { .min = 5, .max = 11 },
 281         .p = { .min = 5, .max = 80 },
 282         .p1 = { .min = 1, .max = 8},
 283         .p2 = { .dot_limit = 165000,
 284                 .p2_slow = 10, .p2_fast = 5 },
 285 };
 286
 287 static const struct intel_limit intel_limits_g4x_single_channel_lvds = {
 288         .dot = { .min = 20000, .max = 115000 },
 289         .vco = { .min = 1750000, .max = 3500000 },
 290         .n = { .min = 1, .max = 3 },
 291         .m = { .min = 104, .max = 138 },
 292         .m1 = { .min = 17, .max = 23 },
 293         .m2 = { .min = 5, .max = 11 },
 294         .p = { .min = 28, .max = 112 },
 295         .p1 = { .min = 2, .max = 8 },
 296         .p2 = { .dot_limit = 0,
 297                 .p2_slow = 14, .p2_fast = 14
 298         },
 299 };
 300
 301 static const struct intel_limit intel_limits_g4x_dual_channel_lvds = {
 302         .dot = { .min = 80000, .max = 224000 },
 303         .vco = { .min = 1750000, .max = 3500000 },
 304         .n = { .min = 1, .max = 3 },
 305         .m = { .min = 104, .max = 138 },
 306         .m1 = { .min = 17, .max = 23 },
 307         .m2 = { .min = 5, .max = 11 },
 308         .p = { .min = 14, .max = 42 },
 309         .p1 = { .min = 2, .max = 6 },
 310         .p2 = { .dot_limit = 0,
 311                 .p2_slow = 7, .p2_fast = 7
 312         },
 313 };
 314
 315 static const struct intel_limit intel_limits_pineview_sdvo = {
 316         .dot = { .min = 20000, .max = 400000},
 317         .vco = { .min = 1700000, .max = 3500000 },
 318         /* Pineview's Ncounter is a ring counter */
 319         .n = { .min = 3, .max = 6 },
 320         .m = { .min = 2, .max = 256 },
 321         /* Pineview only has one combined m divider, which we treat as m2. */
 322         .m1 = { .min = 0, .max = 0 },
 323         .m2 = { .min = 0, .max = 254 },
 324         .p = { .min = 5, .max = 80 },
 325         .p1 = { .min = 1, .max = 8 },
 326         .p2 = { .dot_limit = 200000,
 327                 .p2_slow = 10, .p2_fast = 5 },
 328 };
 329
 330 static const struct intel_limit intel_limits_pineview_lvds = {
 331         .dot = { .min = 20000, .max = 400000 },
 332         .vco = { .min = 1700000, .max = 3500000 },
 333         .n = { .min = 3, .max = 6 },
 334         .m = { .min = 2, .max = 256 },
 335         .m1 = { .min = 0, .max = 0 },
 336         .m2 = { .min = 0, .max = 254 },
 337         .p = { .min = 7, .max = 112 },
 338         .p1 = { .min = 1, .max = 8 },
 339         .p2 = { .dot_limit = 112000,
 340                 .p2_slow = 14, .p2_fast = 14 },
 341 };
 342
 343 /* Ironlake / Sandybridge
 344  *
 345  * We calculate clock using (register_value + 2) for N/M1/M2, so here
 346  * the range value for them is (actual_value - 2).
 347  */
 348 static const struct intel_limit intel_limits_ironlake_dac = {
 349         .dot = { .min = 25000, .max = 350000 },
 350         .vco = { .min = 1760000, .max = 3510000 },
 351         .n = { .min = 1, .max = 5 },
 352         .m = { .min = 79, .max = 127 },
 353         .m1 = { .min = 12, .max = 22 },
 354         .m2 = { .min = 5, .max = 9 },
 355         .p = { .min = 5, .max = 80 },
 356         .p1 = { .min = 1, .max = 8 },
 357         .p2 = { .dot_limit = 225000,
 358                 .p2_slow = 10, .p2_fast = 5 },
 359 };
 360
 361 static const struct intel_limit intel_limits_ironlake_single_lvds = {
 362         .dot = { .min = 25000, .max = 350000 },
 363         .vco = { .min = 1760000, .max = 3510000 },
 364         .n = { .min = 1, .max = 3 },
 365         .m = { .min = 79, .max = 118 },
 366         .m1 = { .min = 12, .max = 22 },
 367         .m2 = { .min = 5, .max = 9 },
 368         .p = { .min = 28, .max = 112 },
 369         .p1 = { .min = 2, .max = 8 },
 370         .p2 = { .dot_limit = 225000,
 371                 .p2_slow = 14, .p2_fast = 14 },
 372 };
 373
 374 static const struct intel_limit intel_limits_ironlake_dual_lvds = {
 375         .dot = { .min = 25000, .max = 350000 },
 376         .vco = { .min = 1760000, .max = 3510000 },
 377         .n = { .min = 1, .max = 3 },
 378         .m = { .min = 79, .max = 127 },
 379         .m1 = { .min = 12, .max = 22 },
 380         .m2 = { .min = 5, .max = 9 },
 381         .p = { .min = 14, .max = 56 },
 382         .p1 = { .min = 2, .max = 8 },
 383         .p2 = { .dot_limit = 225000,
 384                 .p2_slow = 7, .p2_fast = 7 },
 385 };
 386
 387 /* LVDS 100mhz refclk limits. */
 388 static const struct intel_limit intel_limits_ironlake_single_lvds_100m = {
 389         .dot = { .min = 25000, .max = 350000 },
 390         .vco = { .min = 1760000, .max = 3510000 },
 391         .n = { .min = 1, .max = 2 },
 392         .m = { .min = 79, .max = 126 },
 393         .m1 = { .min = 12, .max = 22 },
 394         .m2 = { .min = 5, .max = 9 },
 395         .p = { .min = 28, .max = 112 },
 396         .p1 = { .min = 2, .max = 8 },
 397         .p2 = { .dot_limit = 225000,
 398                 .p2_slow = 14, .p2_fast = 14 },
 399 };
 400
 401 static const struct intel_limit intel_limits_ironlake_dual_lvds_100m = {
 402         .dot = { .min = 25000, .max = 350000 },
 403         .vco = { .min = 1760000, .max = 3510000 },
 404         .n = { .min = 1, .max = 3 },
 405         .m = { .min = 79, .max = 126 },
 406         .m1 = { .min = 12, .max = 22 },
 407         .m2 = { .min = 5, .max = 9 },
 408         .p = { .min = 14, .max = 42 },
 409         .p1 = { .min = 2, .max = 6 },
 410         .p2 = { .dot_limit = 225000,
 411                 .p2_slow = 7, .p2_fast = 7 },
 412 };
 413
 414 static const struct intel_limit intel_limits_vlv = {
 415          /*
 416           * These are the data rate limits (measured in fast clocks)
 417           * since those are the strictest limits we have. The fast
 418           * clock and actual rate limits are more relaxed, so checking
 419           * them would make no difference.
 420           */
 421         .dot = { .min = 25000 * 5, .max = 270000 * 5 },
 422         .vco = { .min = 4000000, .max = 6000000 },
 423         .n = { .min = 1, .max = 7 },
 424         .m1 = { .min = 2, .max = 3 },
 425         .m2 = { .min = 11, .max = 156 },
 426         .p1 = { .min = 2, .max = 3 },
 427         .p2 = { .p2_slow = 2, .p2_fast = 20 }, /* slow=min, fast=max */
 428 };
 429
 430 static const struct intel_limit intel_limits_chv = {
 431         /*
 432          * These are the data rate limits (measured in fast clocks)
 433          * since those are the strictest limits we have.  The fast
 434          * clock and actual rate limits are more relaxed, so checking
 435          * them would make no difference.
 436          */
 437         .dot = { .min = 25000 * 5, .max = 540000 * 5},
 438         .vco = { .min = 4800000, .max = 6480000 },
 439         .n = { .min = 1, .max = 1 },
 440         .m1 = { .min = 2, .max = 2 },
 441         .m2 = { .min = 24 << 22, .max = 175 << 22 },
 442         .p1 = { .min = 2, .max = 4 },
 443         .p2 = { .p2_slow = 1, .p2_fast = 14 },
 444 };
 445
 446 static const struct intel_limit intel_limits_bxt = {
 447         /* FIXME: find real dot limits */
 448         .dot = { .min = 0, .max = INT_MAX },
 449         .vco = { .min = 4800000, .max = 6700000 },
 450         .n = { .min = 1, .max = 1 },
 451         .m1 = { .min = 2, .max = 2 },
 452         /* FIXME: find real m2 limits */
 453         .m2 = { .min = 2 << 22, .max = 255 << 22 },
 454         .p1 = { .min = 2, .max = 4 },
 455         .p2 = { .p2_slow = 1, .p2_fast = 20 },
 456 };
 457
 458 static void
 459 skl_wa_clkgate(struct drm_i915_private *dev_priv, int pipe, bool enable)
 460 {
 461         if (enable)
 462                 I915_WRITE(CLKGATE_DIS_PSL(pipe),
 463                            DUPS1_GATING_DIS | DUPS2_GATING_DIS);
 464         else
 465                 I915_WRITE(CLKGATE_DIS_PSL(pipe),
 466                            I915_READ(CLKGATE_DIS_PSL(pipe)) &
 467                            ~(DUPS1_GATING_DIS | DUPS2_GATING_DIS));
 468 }
 469
 470 static bool
 471 needs_modeset(const struct drm_crtc_state *state)
 472 {
 473         return drm_atomic_crtc_needs_modeset(state);
 474 }
 475
 476 /*
 477  * Platform specific helpers to calculate the port PLL loopback- (clock.m),
 478  * and post-divider (clock.p) values, pre- (clock.vco) and post-divided fast
 479  * (clock.dot) clock rates. This fast dot clock is fed to the port's IO logic.
 480  * The helpers' return value is the rate of the clock that is fed to the
 481  * display engine's pipe which can be the above fast dot clock rate or a
 482  * divided-down version of it.
 483  */
 484 /* m1 is reserved as 0 in Pineview, n is a ring counter */
 485 static int pnv_calc_dpll_params(int refclk, struct dpll *clock)
 486 {
 487         clock->m = clock->m2 + 2;
 488         clock->p = clock->p1 * clock->p2;
 489         if (WARN_ON(clock->n == 0 || clock->p == 0))
 490                 return 0;
 491         clock->vco = DIV_ROUND_CLOSEST(refclk * clock->m, clock->n);
 492         clock->dot = DIV_ROUND_CLOSEST(clock->vco, clock->p);
 493
 494         return clock->dot;
 495 }
 496
 497 static uint32_t i9xx_dpll_compute_m(struct dpll *dpll)
 498 {
 499         return 5 * (dpll->m1 + 2) + (dpll->m2 + 2);
 500 }
 501
 502 static int i9xx_calc_dpll_params(int refclk, struct dpll *clock)
 503 {
 504         clock->m = i9xx_dpll_compute_m(clock);
 505         clock->p = clock->p1 * clock->p2;
 506         if (WARN_ON(clock->n + 2 == 0 || clock->p == 0))
 507                 return 0;
 508         clock->vco = DIV_ROUND_CLOSEST(refclk * clock->m, clock->n + 2);
 509         clock->dot = DIV_ROUND_CLOSEST(clock->vco, clock->p);
 510
 511         return clock->dot;
 512 }
 513
 514 static int vlv_calc_dpll_params(int refclk, struct dpll *clock)
 515 {
 516         clock->m = clock->m1 * clock->m2;
 517         clock->p = clock->p1 * clock->p2;
 518         if (WARN_ON(clock->n == 0 || clock->p == 0))
 519                 return 0;
 520         clock->vco = DIV_ROUND_CLOSEST(refclk * clock->m, clock->n);
 521         clock->dot = DIV_ROUND_CLOSEST(clock->vco, clock->p);
 522
 523         return clock->dot / 5;
 524 }
 525
 526 int chv_calc_dpll_params(int refclk, struct dpll *clock)
 527 {
 528         clock->m = clock->m1 * clock->m2;
 529         clock->p = clock->p1 * clock->p2;
 530         if (WARN_ON(clock->n == 0 || clock->p == 0))
 531                 return 0;
 532         clock->vco = DIV_ROUND_CLOSEST_ULL((uint64_t)refclk * clock->m,
 533                         clock->n << 22);
 534         clock->dot = DIV_ROUND_CLOSEST(clock->vco, clock->p);
 535
 536         return clock->dot / 5;
 537 }
 538
 539 #define INTELPllInvalid(s)   do { /* DRM_DEBUG(s); */ return false; } while (0)
 540
 541 /*
 542  * Returns whether the given set of divisors are valid for a given refclk with
 543  * the given connectors.
 544  */
 545 static bool intel_PLL_is_valid(struct drm_i915_private *dev_priv,
 546                                const struct intel_limit *limit,
 547                                const struct dpll *clock)
 548 {
 549         if (clock->n   < limit->n.min   || limit->n.max   < clock->n)
 550                 INTELPllInvalid("n out of range\n");
 551         if (clock->p1  < limit->p1.min  || limit->p1.max  < clock->p1)
 552                 INTELPllInvalid("p1 out of range\n");
 553         if (clock->m2  < limit->m2.min  || limit->m2.max  < clock->m2)
 554                 INTELPllInvalid("m2 out of range\n");
 555         if (clock->m1  < limit->m1.min  || limit->m1.max  < clock->m1)
 556                 INTELPllInvalid("m1 out of range\n");
 557
 558         if (!IS_PINEVIEW(dev_priv) && !IS_VALLEYVIEW(dev_priv) &&
 559             !IS_CHERRYVIEW(dev_priv) && !IS_GEN9_LP(dev_priv))
 560                 if (clock->m1 <= clock->m2)
 561                         INTELPllInvalid("m1 <= m2\n");
 562
 563         if (!IS_VALLEYVIEW(dev_priv) && !IS_CHERRYVIEW(dev_priv) &&
 564             !IS_GEN9_LP(dev_priv)) {
 565                 if (clock->p < limit->p.min || limit->p.max < clock->p)
 566                         INTELPllInvalid("p out of range\n");
 567                 if (clock->m < limit->m.min || limit->m.max < clock->m)
 568                         INTELPllInvalid("m out of range\n");
 569         }
 570
 571         if (clock->vco < limit->vco.min || limit->vco.max < clock->vco)
 572                 INTELPllInvalid("vco out of range\n");
 573         /* XXX: We may need to be checking "Dot clock" depending on the multiplier,
 574          * connector, etc., rather than just a single range.
 575          */
 576         if (clock->dot < limit->dot.min || limit->dot.max < clock->dot)
 577                 INTELPllInvalid("dot out of range\n");
 578
 579         return true;
 580 }
 581
 582 static int
 583 i9xx_select_p2_div(const struct intel_limit *limit,
 584                    const struct intel_crtc_state *crtc_state,
 585                    int target)
 586 {
 587         struct drm_device *dev = crtc_state->base.crtc->dev;
 588
 589         if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_LVDS)) {
 590                 /*
 591                  * For LVDS just rely on its current settings for dual-channel.
 592                  * We haven't figured out how to reliably set up different
 593                  * single/dual channel state, if we even can.
 594                  */
 595                 if (intel_is_dual_link_lvds(dev))
 596                         return limit->p2.p2_fast;
 597                 else
 598                         return limit->p2.p2_slow;
 599         } else {
 600                 if (target < limit->p2.dot_limit)
 601                         return limit->p2.p2_slow;
 602                 else
 603                         return limit->p2.p2_fast;
 604         }
 605 }
 606
 607 /*
 608  * Returns a set of divisors for the desired target clock with the given
 609  * refclk, or FALSE.  The returned values represent the clock equation:
 610  * reflck * (5 * (m1 + 2) + (m2 + 2)) / (n + 2) / p1 / p2.
 611  *
 612  * Target and reference clocks are specified in kHz.
 613  *
 614  * If match_clock is provided, then best_clock P divider must match the P
 615  * divider from @match_clock used for LVDS downclocking.
 616  */
 617 static bool
 618 i9xx_find_best_dpll(const struct intel_limit *limit,
 619                     struct intel_crtc_state *crtc_state,
 620                     int target, int refclk, struct dpll *match_clock,
 621                     struct dpll *best_clock)
 622 {
 623         struct drm_device *dev = crtc_state->base.crtc->dev;
 624         struct dpll clock;
 625         int err = target;
 626
 627         memset(best_clock, 0, sizeof(*best_clock));
 628
 629         clock.p2 = i9xx_select_p2_div(limit, crtc_state, target);
 630
 631         for (clock.m1 = limit->m1.min; clock.m1 <= limit->m1.max;
 632              clock.m1++) {
 633                 for (clock.m2 = limit->m2.min;
 634                      clock.m2 <= limit->m2.max; clock.m2++) {
 635                         if (clock.m2 >= clock.m1)
 636                                 break;
 637                         for (clock.n = limit->n.min;
 638                              clock.n <= limit->n.max; clock.n++) {
 639                                 for (clock.p1 = limit->p1.min;
 640                                         clock.p1 <= limit->p1.max; clock.p1++) {
 641                                         int this_err;
 642
 643                                         i9xx_calc_dpll_params(refclk, &clock);
 644                                         if (!intel_PLL_is_valid(to_i915(dev),
 645                                                                 limit,
 646                                                                 &clock))
 647                                                 continue;
 648                                         if (match_clock &&
 649                                             clock.p != match_clock->p)
 650                                                 continue;
 651
 652                                         this_err = abs(clock.dot - target);
 653                                         if (this_err < err) {
 654                                                 *best_clock = clock;
 655                                                 err = this_err;
 656                                         }
 657                                 }
 658                         }
 659                 }
 660         }
 661
 662         return (err != target);
 663 }
 664
 665 /*
 666  * Returns a set of divisors for the desired target clock with the given
 667  * refclk, or FALSE.  The returned values represent the clock equation:
 668  * reflck * (5 * (m1 + 2) + (m2 + 2)) / (n + 2) / p1 / p2.
 669  *
 670  * Target and reference clocks are specified in kHz.
 671  *
 672  * If match_clock is provided, then best_clock P divider must match the P
 673  * divider from @match_clock used for LVDS downclocking.
 674  */
 675 static bool
 676 pnv_find_best_dpll(const struct intel_limit *limit,
 677                    struct intel_crtc_state *crtc_state,
 678                    int target, int refclk, struct dpll *match_clock,
 679                    struct dpll *best_clock)
 680 {
 681         struct drm_device *dev = crtc_state->base.crtc->dev;
 682         struct dpll clock;
 683         int err = target;
 684
 685         memset(best_clock, 0, sizeof(*best_clock));
 686
 687         clock.p2 = i9xx_select_p2_div(limit, crtc_state, target);
 688
 689         for (clock.m1 = limit->m1.min; clock.m1 <= limit->m1.max;
 690              clock.m1++) {
 691                 for (clock.m2 = limit->m2.min;
 692                      clock.m2 <= limit->m2.max; clock.m2++) {
 693                         for (clock.n = limit->n.min;
 694                              clock.n <= limit->n.max; clock.n++) {
 695                                 for (clock.p1 = limit->p1.min;
 696                                         clock.p1 <= limit->p1.max; clock.p1++) {
 697                                         int this_err;
 698
 699                                         pnv_calc_dpll_params(refclk, &clock);
 700                                         if (!intel_PLL_is_valid(to_i915(dev),
 701                                                                 limit,
 702                                                                 &clock))
 703                                                 continue;
 704                                         if (match_clock &&
 705                                             clock.p != match_clock->p)
 706                                                 continue;
 707
 708                                         this_err = abs(clock.dot - target);
 709                                         if (this_err < err) {
 710                                                 *best_clock = clock;
 711                                                 err = this_err;
 712                                         }
 713                                 }
 714                         }
 715                 }
 716         }
 717
 718         return (err != target);
 719 }
 720
 721 /*
 722  * Returns a set of divisors for the desired target clock with the given
 723  * refclk, or FALSE.  The returned values represent the clock equation:
 724  * reflck * (5 * (m1 + 2) + (m2 + 2)) / (n + 2) / p1 / p2.
 725  *
 726  * Target and reference clocks are specified in kHz.
 727  *
 728  * If match_clock is provided, then best_clock P divider must match the P
 729  * divider from @match_clock used for LVDS downclocking.
 730  */
 731 static bool
 732 g4x_find_best_dpll(const struct intel_limit *limit,
 733                    struct intel_crtc_state *crtc_state,
 734                    int target, int refclk, struct dpll *match_clock,
 735                    struct dpll *best_clock)
 736 {
 737         struct drm_device *dev = crtc_state->base.crtc->dev;
 738         struct dpll clock;
 739         int max_n;
 740         bool found = false;
 741         /* approximately equals target * 0.00585 */
 742         int err_most = (target >> 8) + (target >> 9);
 743
 744         memset(best_clock, 0, sizeof(*best_clock));
 745
 746         clock.p2 = i9xx_select_p2_div(limit, crtc_state, target);
 747
 748         max_n = limit->n.max;
 749         /* based on hardware requirement, prefer smaller n to precision */
 750         for (clock.n = limit->n.min; clock.n <= max_n; clock.n++) {
 751                 /* based on hardware requirement, prefere larger m1,m2 */
 752                 for (clock.m1 = limit->m1.max;
 753                      clock.m1 >= limit->m1.min; clock.m1--) {
 754                         for (clock.m2 = limit->m2.max;
 755                              clock.m2 >= limit->m2.min; clock.m2--) {
 756                                 for (clock.p1 = limit->p1.max;
 757                                      clock.p1 >= limit->p1.min; clock.p1--) {
 758                                         int this_err;
 759
 760                                         i9xx_calc_dpll_params(refclk, &clock);
 761                                         if (!intel_PLL_is_valid(to_i915(dev),
 762                                                                 limit,
 763                                                                 &clock))
 764                                                 continue;
 765
 766                                         this_err = abs(clock.dot - target);
 767                                         if (this_err < err_most) {
 768                                                 *best_clock = clock;
 769                                                 err_most = this_err;
 770                                                 max_n = clock.n;
 771                                                 found = true;
 772                                         }
 773                                 }
 774                         }
 775                 }
 776         }
 777         return found;
 778 }
 779
 780 /*
 781  * Check if the calculated PLL configuration is more optimal compared to the
 782  * best configuration and error found so far. Return the calculated error.
 783  */
 784 static bool vlv_PLL_is_optimal(struct drm_device *dev, int target_freq,
 785                                const struct dpll *calculated_clock,
 786                                const struct dpll *best_clock,
 787                                unsigned int best_error_ppm,
 788                                unsigned int *error_ppm)
 789 {
 790         /*
 791          * For CHV ignore the error and consider only the P value.
 792          * Prefer a bigger P value based on HW requirements.
 793          */
 794         if (IS_CHERRYVIEW(to_i915(dev))) {
 795                 *error_ppm = 0;
 796
 797                 return calculated_clock->p > best_clock->p;
 798         }
 799
 800         if (WARN_ON_ONCE(!target_freq))
 801                 return false;
 802
 803         *error_ppm = div_u64(1000000ULL *
 804                                 abs(target_freq - calculated_clock->dot),
 805                              target_freq);
 806         /*
 807          * Prefer a better P value over a better (smaller) error if the error
 808          * is small. Ensure this preference for future configurations too by
 809          * setting the error to 0.
 810          */
 811         if (*error_ppm < 100 && calculated_clock->p > best_clock->p) {
 812                 *error_ppm = 0;
 813
 814                 return true;
 815         }
 816
 817         return *error_ppm + 10 < best_error_ppm;
 818 }
 819
 820 /*
 821  * Returns a set of divisors for the desired target clock with the given
 822  * refclk, or FALSE.  The returned values represent the clock equation:
 823  * reflck * (5 * (m1 + 2) + (m2 + 2)) / (n + 2) / p1 / p2.
 824  */
 825 static bool
 826 vlv_find_best_dpll(const struct intel_limit *limit,
 827                    struct intel_crtc_state *crtc_state,
 828                    int target, int refclk, struct dpll *match_clock,
 829                    struct dpll *best_clock)
 830 {
 831         struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
 832         struct drm_device *dev = crtc->base.dev;
 833         struct dpll clock;
 834         unsigned int bestppm = 1000000;
 835         /* min update 19.2 MHz */
 836         int max_n = min(limit->n.max, refclk / 19200);
 837         bool found = false;
 838
 839         target *= 5; /* fast clock */
 840
 841         memset(best_clock, 0, sizeof(*best_clock));
 842
 843         /* based on hardware requirement, prefer smaller n to precision */
 844         for (clock.n = limit->n.min; clock.n <= max_n; clock.n++) {
 845                 for (clock.p1 = limit->p1.max; clock.p1 >= limit->p1.min; clock.p1--) {
 846                         for (clock.p2 = limit->p2.p2_fast; clock.p2 >= limit->p2.p2_slow;
 847                              clock.p2 -= clock.p2 > 10 ? 2 : 1) {
 848                                 clock.p = clock.p1 * clock.p2;
 849                                 /* based on hardware requirement, prefer bigger m1,m2 values */
 850                                 for (clock.m1 = limit->m1.min; clock.m1 <= limit->m1.max; clock.m1++) {
 851                                         unsigned int ppm;
 852
 853                                         clock.m2 = DIV_ROUND_CLOSEST(target * clock.p * clock.n,
 854                                                                      refclk * clock.m1);
 855
 856                                         vlv_calc_dpll_params(refclk, &clock);
 857
 858                                         if (!intel_PLL_is_valid(to_i915(dev),
 859                                                                 limit,
 860                                                                 &clock))
 861                                                 continue;
 862
 863                                         if (!vlv_PLL_is_optimal(dev, target,
 864                                                                 &clock,
 865                                                                 best_clock,
 866                                                                 bestppm, &ppm))
 867                                                 continue;
 868
 869                                         *best_clock = clock;
 870                                         bestppm = ppm;
 871                                         found = true;
 872                                 }
 873                         }
 874                 }
 875         }
 876
 877         return found;
 878 }
 879
 880 /*
 881  * Returns a set of divisors for the desired target clock with the given
 882  * refclk, or FALSE.  The returned values represent the clock equation:
 883  * reflck * (5 * (m1 + 2) + (m2 + 2)) / (n + 2) / p1 / p2.
 884  */
 885 static bool
 886 chv_find_best_dpll(const struct intel_limit *limit,
 887                    struct intel_crtc_state *crtc_state,
 888                    int target, int refclk, struct dpll *match_clock,
 889                    struct dpll *best_clock)
 890 {
 891         struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
 892         struct drm_device *dev = crtc->base.dev;
 893         unsigned int best_error_ppm;
 894         struct dpll clock;
 895         uint64_t m2;
 896         int found = false;
 897
 898         memset(best_clock, 0, sizeof(*best_clock));
 899         best_error_ppm = 1000000;
 900
 901         /*
 902          * Based on hardware doc, the n always set to 1, and m1 always
 903          * set to 2.  If requires to support 200Mhz refclk, we need to
 904          * revisit this because n may not 1 anymore.
 905          */
 906         clock.n = 1, clock.m1 = 2;
 907         target *= 5;    /* fast clock */
 908
 909         for (clock.p1 = limit->p1.max; clock.p1 >= limit->p1.min; clock.p1--) {
 910                 for (clock.p2 = limit->p2.p2_fast;
 911                                 clock.p2 >= limit->p2.p2_slow;
 912                                 clock.p2 -= clock.p2 > 10 ? 2 : 1) {
 913                         unsigned int error_ppm;
 914
 915                         clock.p = clock.p1 * clock.p2;
 916
 917                         m2 = DIV_ROUND_CLOSEST_ULL(((uint64_t)target * clock.p *
 918                                         clock.n) << 22, refclk * clock.m1);
 919
 920                         if (m2 > INT_MAX/clock.m1)
 921                                 continue;
 922
 923                         clock.m2 = m2;
 924
 925                         chv_calc_dpll_params(refclk, &clock);
 926
 927                         if (!intel_PLL_is_valid(to_i915(dev), limit, &clock))
 928                                 continue;
 929
 930                         if (!vlv_PLL_is_optimal(dev, target, &clock, best_clock,
 931                                                 best_error_ppm, &error_ppm))
 932                                 continue;
 933
 934                         *best_clock = clock;
 935                         best_error_ppm = error_ppm;
 936                         found = true;
 937                 }
 938         }
 939
 940         return found;
 941 }
 942
 943 bool bxt_find_best_dpll(struct intel_crtc_state *crtc_state, int target_clock,
 944                         struct dpll *best_clock)
 945 {
 946         int refclk = 100000;
 947         const struct intel_limit *limit = &intel_limits_bxt;
 948
 949         return chv_find_best_dpll(limit, crtc_state,
 950                                   target_clock, refclk, NULL, best_clock);
 951 }
 952
 953 bool intel_crtc_active(struct intel_crtc *crtc)
 954 {
 955         /* Be paranoid as we can arrive here with only partial
 956          * state retrieved from the hardware during setup.
 957          *
 958          * We can ditch the adjusted_mode.crtc_clock check as soon
 959          * as Haswell has gained clock readout/fastboot support.
 960          *
 961          * We can ditch the crtc->primary->state->fb check as soon as we can
 962          * properly reconstruct framebuffers.
 963          *
 964          * FIXME: The intel_crtc->active here should be switched to
 965          * crtc->state->active once we have proper CRTC states wired up
 966          * for atomic.
 967          */
 968         return crtc->active && crtc->base.primary->state->fb &&
 969                 crtc->config->base.adjusted_mode.crtc_clock;
 970 }
 971
 972 enum transcoder intel_pipe_to_cpu_transcoder(struct drm_i915_private *dev_priv,
 973                                              enum pipe pipe)
 974 {
 975         struct intel_crtc *crtc = intel_get_crtc_for_pipe(dev_priv, pipe);
 976
 977         return crtc->config->cpu_transcoder;
 978 }
 979
 980 static bool pipe_scanline_is_moving(struct drm_i915_private *dev_priv,
 981                                     enum pipe pipe)
 982 {
 983         i915_reg_t reg = PIPEDSL(pipe);
 984         u32 line1, line2;
 985         u32 line_mask;
 986
 987         if (IS_GEN(dev_priv, 2))
 988                 line_mask = DSL_LINEMASK_GEN2;
 989         else
 990                 line_mask = DSL_LINEMASK_GEN3;
 991
 992         line1 = I915_READ(reg) & line_mask;
 993         msleep(5);
 994         line2 = I915_READ(reg) & line_mask;
 995
 996         return line1 != line2;
 997 }
 998
 999 static void wait_for_pipe_scanline_moving(struct intel_crtc *crtc, bool state)
1000 {
1001         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1002         enum pipe pipe = crtc->pipe;
1003
1004         /* Wait for the display line to settle/start moving */
1005         if (wait_for(pipe_scanline_is_moving(dev_priv, pipe) == state, 100))
1006                 DRM_ERROR("pipe %c scanline %s wait timed out\n",
1007                           pipe_name(pipe), onoff(state));
1008 }
1009
1010 static void intel_wait_for_pipe_scanline_stopped(struct intel_crtc *crtc)
1011 {
1012         wait_for_pipe_scanline_moving(crtc, false);
1013 }
1014
1015 static void intel_wait_for_pipe_scanline_moving(struct intel_crtc *crtc)
1016 {
1017         wait_for_pipe_scanline_moving(crtc, true);
1018 }
1019
1020 static void
1021 intel_wait_for_pipe_off(const struct intel_crtc_state *old_crtc_state)
1022 {
1023         struct intel_crtc *crtc = to_intel_crtc(old_crtc_state->base.crtc);
1024         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1025
1026         if (INTEL_GEN(dev_priv) >= 4) {
1027                 enum transcoder cpu_transcoder = old_crtc_state->cpu_transcoder;
1028                 i915_reg_t reg = PIPECONF(cpu_transcoder);
1029
1030                 /* Wait for the Pipe State to go off */
1031                 if (intel_wait_for_register(dev_priv,
1032                                             reg, I965_PIPECONF_ACTIVE, 0,
1033                                             100))
1034                         WARN(1, "pipe_off wait timed out\n");
1035         } else {
1036                 intel_wait_for_pipe_scanline_stopped(crtc);
1037         }
1038 }
1039
1040 /* Only for pre-ILK configs */
1041 void assert_pll(struct drm_i915_private *dev_priv,
1042                 enum pipe pipe, bool state)
1043 {
1044         u32 val;
1045         bool cur_state;
1046
1047         val = I915_READ(DPLL(pipe));
1048         cur_state = !!(val & DPLL_VCO_ENABLE);
1049         I915_STATE_WARN(cur_state != state,
1050              "PLL state assertion failure (expected %s, current %s)\n",
1051                         onoff(state), onoff(cur_state));
1052 }
1053
1054 /* XXX: the dsi pll is shared between MIPI DSI ports */
1055 void assert_dsi_pll(struct drm_i915_private *dev_priv, bool state)
1056 {
1057         u32 val;
1058         bool cur_state;
1059
1060         mutex_lock(&dev_priv->sb_lock);
1061         val = vlv_cck_read(dev_priv, CCK_REG_DSI_PLL_CONTROL);
1062         mutex_unlock(&dev_priv->sb_lock);
1063
1064         cur_state = val & DSI_PLL_VCO_EN;
1065         I915_STATE_WARN(cur_state != state,
1066              "DSI PLL state assertion failure (expected %s, current %s)\n",
1067                         onoff(state), onoff(cur_state));
1068 }
1069
1070 static void assert_fdi_tx(struct drm_i915_private *dev_priv,
1071                           enum pipe pipe, bool state)
1072 {
1073         bool cur_state;
1074         enum transcoder cpu_transcoder = intel_pipe_to_cpu_transcoder(dev_priv,
1075                                                                       pipe);
1076
1077         if (HAS_DDI(dev_priv)) {
1078                 /* DDI does not have a specific FDI_TX register */
1079                 u32 val = I915_READ(TRANS_DDI_FUNC_CTL(cpu_transcoder));
1080                 cur_state = !!(val & TRANS_DDI_FUNC_ENABLE);
1081         } else {
1082                 u32 val = I915_READ(FDI_TX_CTL(pipe));
1083                 cur_state = !!(val & FDI_TX_ENABLE);
1084         }
1085         I915_STATE_WARN(cur_state != state,
1086              "FDI TX state assertion failure (expected %s, current %s)\n",
1087                         onoff(state), onoff(cur_state));
1088 }
1089 #define assert_fdi_tx_enabled(d, p) assert_fdi_tx(d, p, true)
1090 #define assert_fdi_tx_disabled(d, p) assert_fdi_tx(d, p, false)
1091
1092 static void assert_fdi_rx(struct drm_i915_private *dev_priv,
1093                           enum pipe pipe, bool state)
1094 {
1095         u32 val;
1096         bool cur_state;
1097
1098         val = I915_READ(FDI_RX_CTL(pipe));
1099         cur_state = !!(val & FDI_RX_ENABLE);
1100         I915_STATE_WARN(cur_state != state,
1101              "FDI RX state assertion failure (expected %s, current %s)\n",
1102                         onoff(state), onoff(cur_state));
1103 }
1104 #define assert_fdi_rx_enabled(d, p) assert_fdi_rx(d, p, true)
1105 #define assert_fdi_rx_disabled(d, p) assert_fdi_rx(d, p, false)
1106
1107 static void assert_fdi_tx_pll_enabled(struct drm_i915_private *dev_priv,
1108                                       enum pipe pipe)
1109 {
1110         u32 val;
1111
1112         /* ILK FDI PLL is always enabled */
1113         if (IS_GEN(dev_priv, 5))
1114                 return;
1115
1116         /* On Haswell, DDI ports are responsible for the FDI PLL setup */
1117         if (HAS_DDI(dev_priv))
1118                 return;
1119
1120         val = I915_READ(FDI_TX_CTL(pipe));
1121         I915_STATE_WARN(!(val & FDI_TX_PLL_ENABLE), "FDI TX PLL assertion failure, should be active but is disabled\n");
1122 }
1123
1124 void assert_fdi_rx_pll(struct drm_i915_private *dev_priv,
1125                        enum pipe pipe, bool state)
1126 {
1127         u32 val;
1128         bool cur_state;
1129
1130         val = I915_READ(FDI_RX_CTL(pipe));
1131         cur_state = !!(val & FDI_RX_PLL_ENABLE);
1132         I915_STATE_WARN(cur_state != state,
1133              "FDI RX PLL assertion failure (expected %s, current %s)\n",
1134                         onoff(state), onoff(cur_state));
1135 }
1136
1137 void assert_panel_unlocked(struct drm_i915_private *dev_priv, enum pipe pipe)
1138 {
1139         i915_reg_t pp_reg;
1140         u32 val;
1141         enum pipe panel_pipe = INVALID_PIPE;
1142         bool locked = true;
1143
1144         if (WARN_ON(HAS_DDI(dev_priv)))
1145                 return;
1146
1147         if (HAS_PCH_SPLIT(dev_priv)) {
1148                 u32 port_sel;
1149
1150                 pp_reg = PP_CONTROL(0);
1151                 port_sel = I915_READ(PP_ON_DELAYS(0)) & PANEL_PORT_SELECT_MASK;
1152
1153                 switch (port_sel) {
1154                 case PANEL_PORT_SELECT_LVDS:
1155                         intel_lvds_port_enabled(dev_priv, PCH_LVDS, &panel_pipe);
1156                         break;
1157                 case PANEL_PORT_SELECT_DPA:
1158                         intel_dp_port_enabled(dev_priv, DP_A, PORT_A, &panel_pipe);
1159                         break;
1160                 case PANEL_PORT_SELECT_DPC:
1161                         intel_dp_port_enabled(dev_priv, PCH_DP_C, PORT_C, &panel_pipe);
1162                         break;
1163                 case PANEL_PORT_SELECT_DPD:
1164                         intel_dp_port_enabled(dev_priv, PCH_DP_D, PORT_D, &panel_pipe);
1165                         break;
1166                 default:
1167                         MISSING_CASE(port_sel);
1168                         break;
1169                 }
1170         } else if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) {
1171                 /* presumably write lock depends on pipe, not port select */
1172                 pp_reg = PP_CONTROL(pipe);
1173                 panel_pipe = pipe;
1174         } else {
1175                 u32 port_sel;
1176
1177                 pp_reg = PP_CONTROL(0);
1178                 port_sel = I915_READ(PP_ON_DELAYS(0)) & PANEL_PORT_SELECT_MASK;
1179
1180                 WARN_ON(port_sel != PANEL_PORT_SELECT_LVDS);
1181                 intel_lvds_port_enabled(dev_priv, LVDS, &panel_pipe);
1182         }
1183
1184         val = I915_READ(pp_reg);
1185         if (!(val & PANEL_POWER_ON) ||
1186             ((val & PANEL_UNLOCK_MASK) == PANEL_UNLOCK_REGS))
1187                 locked = false;
1188
1189         I915_STATE_WARN(panel_pipe == pipe && locked,
1190              "panel assertion failure, pipe %c regs locked\n",
1191              pipe_name(pipe));
1192 }
1193
1194 void assert_pipe(struct drm_i915_private *dev_priv,
1195                  enum pipe pipe, bool state)
1196 {
1197         bool cur_state;
1198         enum transcoder cpu_transcoder = intel_pipe_to_cpu_transcoder(dev_priv,
1199                                                                       pipe);
1200         enum intel_display_power_domain power_domain;
1201
1202         /* we keep both pipes enabled on 830 */
1203         if (IS_I830(dev_priv))
1204                 state = true;
1205
1206         power_domain = POWER_DOMAIN_TRANSCODER(cpu_transcoder);
1207         if (intel_display_power_get_if_enabled(dev_priv, power_domain)) {
1208                 u32 val = I915_READ(PIPECONF(cpu_transcoder));
1209                 cur_state = !!(val & PIPECONF_ENABLE);
1210
1211                 intel_display_power_put(dev_priv, power_domain);
1212         } else {
1213                 cur_state = false;
1214         }
1215
1216         I915_STATE_WARN(cur_state != state,
1217              "pipe %c assertion failure (expected %s, current %s)\n",
1218                         pipe_name(pipe), onoff(state), onoff(cur_state));
1219 }
1220
1221 static void assert_plane(struct intel_plane *plane, bool state)
1222 {
1223         enum pipe pipe;
1224         bool cur_state;
1225
1226         cur_state = plane->get_hw_state(plane, &pipe);
1227
1228         I915_STATE_WARN(cur_state != state,
1229                         "%s assertion failure (expected %s, current %s)\n",
1230                         plane->base.name, onoff(state), onoff(cur_state));
1231 }
1232
1233 #define assert_plane_enabled(p) assert_plane(p, true)
1234 #define assert_plane_disabled(p) assert_plane(p, false)
1235
1236 static void assert_planes_disabled(struct intel_crtc *crtc)
1237 {
1238         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1239         struct intel_plane *plane;
1240
1241         for_each_intel_plane_on_crtc(&dev_priv->drm, crtc, plane)
1242                 assert_plane_disabled(plane);
1243 }
1244
1245 static void assert_vblank_disabled(struct drm_crtc *crtc)
1246 {
1247         if (I915_STATE_WARN_ON(drm_crtc_vblank_get(crtc) == 0))
1248                 drm_crtc_vblank_put(crtc);
1249 }
1250
1251 void assert_pch_transcoder_disabled(struct drm_i915_private *dev_priv,
1252                                     enum pipe pipe)
1253 {
1254         u32 val;
1255         bool enabled;
1256
1257         val = I915_READ(PCH_TRANSCONF(pipe));
1258         enabled = !!(val & TRANS_ENABLE);
1259         I915_STATE_WARN(enabled,
1260              "transcoder assertion failed, should be off on pipe %c but is still active\n",
1261              pipe_name(pipe));
1262 }
1263
1264 static void assert_pch_dp_disabled(struct drm_i915_private *dev_priv,
1265                                    enum pipe pipe, enum port port,
1266                                    i915_reg_t dp_reg)
1267 {
1268         enum pipe port_pipe;
1269         bool state;
1270
1271         state = intel_dp_port_enabled(dev_priv, dp_reg, port, &port_pipe);
1272
1273         I915_STATE_WARN(state && port_pipe == pipe,
1274                         "PCH DP %c enabled on transcoder %c, should be disabled\n",
1275                         port_name(port), pipe_name(pipe));
1276
1277         I915_STATE_WARN(HAS_PCH_IBX(dev_priv) && !state && port_pipe == PIPE_B,
1278                         "IBX PCH DP %c still using transcoder B\n",
1279                         port_name(port));
1280 }
1281
1282 static void assert_pch_hdmi_disabled(struct drm_i915_private *dev_priv,
1283                                      enum pipe pipe, enum port port,
1284                                      i915_reg_t hdmi_reg)
1285 {
1286         enum pipe port_pipe;
1287         bool state;
1288
1289         state = intel_sdvo_port_enabled(dev_priv, hdmi_reg, &port_pipe);
1290
1291         I915_STATE_WARN(state && port_pipe == pipe,
1292                         "PCH HDMI %c enabled on transcoder %c, should be disabled\n",
1293                         port_name(port), pipe_name(pipe));
1294
1295         I915_STATE_WARN(HAS_PCH_IBX(dev_priv) && !state && port_pipe == PIPE_B,
1296                         "IBX PCH HDMI %c still using transcoder B\n",
1297                         port_name(port));
1298 }
1299
1300 static void assert_pch_ports_disabled(struct drm_i915_private *dev_priv,
1301                                       enum pipe pipe)
1302 {
1303         enum pipe port_pipe;
1304
1305         assert_pch_dp_disabled(dev_priv, pipe, PORT_B, PCH_DP_B);
1306         assert_pch_dp_disabled(dev_priv, pipe, PORT_C, PCH_DP_C);
1307         assert_pch_dp_disabled(dev_priv, pipe, PORT_D, PCH_DP_D);
1308
1309         I915_STATE_WARN(intel_crt_port_enabled(dev_priv, PCH_ADPA, &port_pipe) &&
1310                         port_pipe == pipe,
1311                         "PCH VGA enabled on transcoder %c, should be disabled\n",
1312                         pipe_name(pipe));
1313
1314         I915_STATE_WARN(intel_lvds_port_enabled(dev_priv, PCH_LVDS, &port_pipe) &&
1315                         port_pipe == pipe,
1316                         "PCH LVDS enabled on transcoder %c, should be disabled\n",
1317                         pipe_name(pipe));
1318
1319         /* PCH SDVOB multiplex with HDMIB */
1320         assert_pch_hdmi_disabled(dev_priv, pipe, PORT_B, PCH_HDMIB);
1321         assert_pch_hdmi_disabled(dev_priv, pipe, PORT_C, PCH_HDMIC);
1322         assert_pch_hdmi_disabled(dev_priv, pipe, PORT_D, PCH_HDMID);
1323 }
1324
1325 static void _vlv_enable_pll(struct intel_crtc *crtc,
1326                             const struct intel_crtc_state *pipe_config)
1327 {
1328         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1329         enum pipe pipe = crtc->pipe;
1330
1331         I915_WRITE(DPLL(pipe), pipe_config->dpll_hw_state.dpll);
1332         POSTING_READ(DPLL(pipe));
1333         udelay(150);
1334
1335         if (intel_wait_for_register(dev_priv,
1336                                     DPLL(pipe),
1337                                     DPLL_LOCK_VLV,
1338                                     DPLL_LOCK_VLV,
1339                                     1))
1340                 DRM_ERROR("DPLL %d failed to lock\n", pipe);
1341 }
1342
1343 static void vlv_enable_pll(struct intel_crtc *crtc,
1344                            const struct intel_crtc_state *pipe_config)
1345 {
1346         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1347         enum pipe pipe = crtc->pipe;
1348
1349         assert_pipe_disabled(dev_priv, pipe);
1350
1351         /* PLL is protected by panel, make sure we can write it */
1352         assert_panel_unlocked(dev_priv, pipe);
1353
1354         if (pipe_config->dpll_hw_state.dpll & DPLL_VCO_ENABLE)
1355                 _vlv_enable_pll(crtc, pipe_config);
1356
1357         I915_WRITE(DPLL_MD(pipe), pipe_config->dpll_hw_state.dpll_md);
1358         POSTING_READ(DPLL_MD(pipe));
1359 }
1360
1361
1362 static void _chv_enable_pll(struct intel_crtc *crtc,
1363                             const struct intel_crtc_state *pipe_config)
1364 {
1365         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1366         enum pipe pipe = crtc->pipe;
1367         enum dpio_channel port = vlv_pipe_to_channel(pipe);
1368         u32 tmp;
1369
1370         mutex_lock(&dev_priv->sb_lock);
1371
1372         /* Enable back the 10bit clock to display controller */
1373         tmp = vlv_dpio_read(dev_priv, pipe, CHV_CMN_DW14(port));
1374         tmp |= DPIO_DCLKP_EN;
1375         vlv_dpio_write(dev_priv, pipe, CHV_CMN_DW14(port), tmp);
1376
1377         mutex_unlock(&dev_priv->sb_lock);
1378
1379         /*
1380          * Need to wait > 100ns between dclkp clock enable bit and PLL enable.
1381          */
1382         udelay(1);
1383
1384         /* Enable PLL */
1385         I915_WRITE(DPLL(pipe), pipe_config->dpll_hw_state.dpll);
1386
1387         /* Check PLL is locked */
1388         if (intel_wait_for_register(dev_priv,
1389                                     DPLL(pipe), DPLL_LOCK_VLV, DPLL_LOCK_VLV,
1390                                     1))
1391                 DRM_ERROR("PLL %d failed to lock\n", pipe);
1392 }
1393
1394 static void chv_enable_pll(struct intel_crtc *crtc,
1395                            const struct intel_crtc_state *pipe_config)
1396 {
1397         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1398         enum pipe pipe = crtc->pipe;
1399
1400         assert_pipe_disabled(dev_priv, pipe);
1401
1402         /* PLL is protected by panel, make sure we can write it */
1403         assert_panel_unlocked(dev_priv, pipe);
1404
1405         if (pipe_config->dpll_hw_state.dpll & DPLL_VCO_ENABLE)
1406                 _chv_enable_pll(crtc, pipe_config);
1407
1408         if (pipe != PIPE_A) {
1409                 /*
1410                  * WaPixelRepeatModeFixForC0:chv
1411                  *
1412                  * DPLLCMD is AWOL. Use chicken bits to propagate
1413                  * the value from DPLLBMD to either pipe B or C.
1414                  */
1415                 I915_WRITE(CBR4_VLV, CBR_DPLLBMD_PIPE(pipe));
1416                 I915_WRITE(DPLL_MD(PIPE_B), pipe_config->dpll_hw_state.dpll_md);
1417                 I915_WRITE(CBR4_VLV, 0);
1418                 dev_priv->chv_dpll_md[pipe] = pipe_config->dpll_hw_state.dpll_md;
1419
1420                 /*
1421                  * DPLLB VGA mode also seems to cause problems.
1422                  * We should always have it disabled.
1423                  */
1424                 WARN_ON((I915_READ(DPLL(PIPE_B)) & DPLL_VGA_MODE_DIS) == 0);
1425         } else {
1426                 I915_WRITE(DPLL_MD(pipe), pipe_config->dpll_hw_state.dpll_md);
1427                 POSTING_READ(DPLL_MD(pipe));
1428         }
1429 }
1430
1431 static int intel_num_dvo_pipes(struct drm_i915_private *dev_priv)
1432 {
1433         struct intel_crtc *crtc;
1434         int count = 0;
1435
1436         for_each_intel_crtc(&dev_priv->drm, crtc) {
1437                 count += crtc->base.state->active &&
1438                         intel_crtc_has_type(crtc->config, INTEL_OUTPUT_DVO);
1439         }
1440
1441         return count;
1442 }
1443
1444 static void i9xx_enable_pll(struct intel_crtc *crtc,
1445                             const struct intel_crtc_state *crtc_state)
1446 {
1447         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1448         i915_reg_t reg = DPLL(crtc->pipe);
1449         u32 dpll = crtc_state->dpll_hw_state.dpll;
1450         int i;
1451
1452         assert_pipe_disabled(dev_priv, crtc->pipe);
1453
1454         /* PLL is protected by panel, make sure we can write it */
1455         if (IS_MOBILE(dev_priv) && !IS_I830(dev_priv))
1456                 assert_panel_unlocked(dev_priv, crtc->pipe);
1457
1458         /* Enable DVO 2x clock on both PLLs if necessary */
1459         if (IS_I830(dev_priv) && intel_num_dvo_pipes(dev_priv) > 0) {
1460                 /*
1461                  * It appears to be important that we don't enable this
1462                  * for the current pipe before otherwise configuring the
1463                  * PLL. No idea how this should be handled if multiple
1464                  * DVO outputs are enabled simultaneosly.
1465                  */
1466                 dpll |= DPLL_DVO_2X_MODE;
1467                 I915_WRITE(DPLL(!crtc->pipe),
1468                            I915_READ(DPLL(!crtc->pipe)) | DPLL_DVO_2X_MODE);
1469         }
1470
1471         /*
1472          * Apparently we need to have VGA mode enabled prior to changing
1473          * the P1/P2 dividers. Otherwise the DPLL will keep using the old
1474          * dividers, even though the register value does change.
1475          */
1476         I915_WRITE(reg, 0);
1477
1478         I915_WRITE(reg, dpll);
1479
1480         /* Wait for the clocks to stabilize. */
1481         POSTING_READ(reg);
1482         udelay(150);
1483
1484         if (INTEL_GEN(dev_priv) >= 4) {
1485                 I915_WRITE(DPLL_MD(crtc->pipe),
1486                            crtc_state->dpll_hw_state.dpll_md);
1487         } else {
1488                 /* The pixel multiplier can only be updated once the
1489                  * DPLL is enabled and the clocks are stable.
1490                  *
1491                  * So write it again.
1492                  */
1493                 I915_WRITE(reg, dpll);
1494         }
1495
1496         /* We do this three times for luck */
1497         for (i = 0; i < 3; i++) {
1498                 I915_WRITE(reg, dpll);
1499                 POSTING_READ(reg);
1500                 udelay(150); /* wait for warmup */
1501         }
1502 }
1503
1504 static void i9xx_disable_pll(const struct intel_crtc_state *crtc_state)
1505 {
1506         struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
1507         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1508         enum pipe pipe = crtc->pipe;
1509
1510         /* Disable DVO 2x clock on both PLLs if necessary */
1511         if (IS_I830(dev_priv) &&
1512             intel_crtc_has_type(crtc_state, INTEL_OUTPUT_DVO) &&
1513             !intel_num_dvo_pipes(dev_priv)) {
1514                 I915_WRITE(DPLL(PIPE_B),
1515                            I915_READ(DPLL(PIPE_B)) & ~DPLL_DVO_2X_MODE);
1516                 I915_WRITE(DPLL(PIPE_A),
1517                            I915_READ(DPLL(PIPE_A)) & ~DPLL_DVO_2X_MODE);
1518         }
1519
1520         /* Don't disable pipe or pipe PLLs if needed */
1521         if (IS_I830(dev_priv))
1522                 return;
1523
1524         /* Make sure the pipe isn't still relying on us */
1525         assert_pipe_disabled(dev_priv, pipe);
1526
1527         I915_WRITE(DPLL(pipe), DPLL_VGA_MODE_DIS);
1528         POSTING_READ(DPLL(pipe));
1529 }
1530
1531 static void vlv_disable_pll(struct drm_i915_private *dev_priv, enum pipe pipe)
1532 {
1533         u32 val;
1534
1535         /* Make sure the pipe isn't still relying on us */
1536         assert_pipe_disabled(dev_priv, pipe);
1537
1538         val = DPLL_INTEGRATED_REF_CLK_VLV |
1539                 DPLL_REF_CLK_ENABLE_VLV | DPLL_VGA_MODE_DIS;
1540         if (pipe != PIPE_A)
1541                 val |= DPLL_INTEGRATED_CRI_CLK_VLV;
1542
1543         I915_WRITE(DPLL(pipe), val);
1544         POSTING_READ(DPLL(pipe));
1545 }
1546
1547 static void chv_disable_pll(struct drm_i915_private *dev_priv, enum pipe pipe)
1548 {
1549         enum dpio_channel port = vlv_pipe_to_channel(pipe);
1550         u32 val;
1551
1552         /* Make sure the pipe isn't still relying on us */
1553         assert_pipe_disabled(dev_priv, pipe);
1554
1555         val = DPLL_SSC_REF_CLK_CHV |
1556                 DPLL_REF_CLK_ENABLE_VLV | DPLL_VGA_MODE_DIS;
1557         if (pipe != PIPE_A)
1558                 val |= DPLL_INTEGRATED_CRI_CLK_VLV;
1559
1560         I915_WRITE(DPLL(pipe), val);
1561         POSTING_READ(DPLL(pipe));
1562
1563         mutex_lock(&dev_priv->sb_lock);
1564
1565         /* Disable 10bit clock to display controller */
1566         val = vlv_dpio_read(dev_priv, pipe, CHV_CMN_DW14(port));
1567         val &= ~DPIO_DCLKP_EN;
1568         vlv_dpio_write(dev_priv, pipe, CHV_CMN_DW14(port), val);
1569
1570         mutex_unlock(&dev_priv->sb_lock);
1571 }
1572
1573 void vlv_wait_port_ready(struct drm_i915_private *dev_priv,
1574                          struct intel_digital_port *dport,
1575                          unsigned int expected_mask)
1576 {
1577         u32 port_mask;
1578         i915_reg_t dpll_reg;
1579
1580         switch (dport->base.port) {
1581         case PORT_B:
1582                 port_mask = DPLL_PORTB_READY_MASK;
1583                 dpll_reg = DPLL(0);
1584                 break;
1585         case PORT_C:
1586                 port_mask = DPLL_PORTC_READY_MASK;
1587                 dpll_reg = DPLL(0);
1588                 expected_mask <<= 4;
1589                 break;
1590         case PORT_D:
1591                 port_mask = DPLL_PORTD_READY_MASK;
1592                 dpll_reg = DPIO_PHY_STATUS;
1593                 break;
1594         default:
1595                 BUG();
1596         }
1597
1598         if (intel_wait_for_register(dev_priv,
1599                                     dpll_reg, port_mask, expected_mask,
1600                                     1000))
1601                 WARN(1, "timed out waiting for port %c ready: got 0x%x, expected 0x%x\n",
1602                      port_name(dport->base.port),
1603                      I915_READ(dpll_reg) & port_mask, expected_mask);
1604 }
1605
1606 static void ironlake_enable_pch_transcoder(const struct intel_crtc_state *crtc_state)
1607 {
1608         struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
1609         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1610         enum pipe pipe = crtc->pipe;
1611         i915_reg_t reg;
1612         uint32_t val, pipeconf_val;
1613
1614         /* Make sure PCH DPLL is enabled */
1615         assert_shared_dpll_enabled(dev_priv, crtc_state->shared_dpll);
1616
1617         /* FDI must be feeding us bits for PCH ports */
1618         assert_fdi_tx_enabled(dev_priv, pipe);
1619         assert_fdi_rx_enabled(dev_priv, pipe);
1620
1621         if (HAS_PCH_CPT(dev_priv)) {
1622                 /* Workaround: Set the timing override bit before enabling the
1623                  * pch transcoder. */
1624                 reg = TRANS_CHICKEN2(pipe);
1625                 val = I915_READ(reg);
1626                 val |= TRANS_CHICKEN2_TIMING_OVERRIDE;
1627                 I915_WRITE(reg, val);
1628         }
1629
1630         reg = PCH_TRANSCONF(pipe);
1631         val = I915_READ(reg);
1632         pipeconf_val = I915_READ(PIPECONF(pipe));
1633
1634         if (HAS_PCH_IBX(dev_priv)) {
1635                 /*
1636                  * Make the BPC in transcoder be consistent with
1637                  * that in pipeconf reg. For HDMI we must use 8bpc
1638                  * here for both 8bpc and 12bpc.
1639                  */
1640                 val &= ~PIPECONF_BPC_MASK;
1641                 if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_HDMI))
1642                         val |= PIPECONF_8BPC;
1643                 else
1644                         val |= pipeconf_val & PIPECONF_BPC_MASK;
1645         }
1646
1647         val &= ~TRANS_INTERLACE_MASK;
1648         if ((pipeconf_val & PIPECONF_INTERLACE_MASK) == PIPECONF_INTERLACED_ILK)
1649                 if (HAS_PCH_IBX(dev_priv) &&
1650                     intel_crtc_has_type(crtc_state, INTEL_OUTPUT_SDVO))
1651                         val |= TRANS_LEGACY_INTERLACED_ILK;
1652                 else
1653                         val |= TRANS_INTERLACED;
1654         else
1655                 val |= TRANS_PROGRESSIVE;
1656
1657         I915_WRITE(reg, val | TRANS_ENABLE);
1658         if (intel_wait_for_register(dev_priv,
1659                                     reg, TRANS_STATE_ENABLE, TRANS_STATE_ENABLE,
1660                                     100))
1661                 DRM_ERROR("failed to enable transcoder %c\n", pipe_name(pipe));
1662 }
1663
1664 static void lpt_enable_pch_transcoder(struct drm_i915_private *dev_priv,
1665                                       enum transcoder cpu_transcoder)
1666 {
1667         u32 val, pipeconf_val;
1668
1669         /* FDI must be feeding us bits for PCH ports */
1670         assert_fdi_tx_enabled(dev_priv, (enum pipe) cpu_transcoder);
1671         assert_fdi_rx_enabled(dev_priv, PIPE_A);
1672
1673         /* Workaround: set timing override bit. */
1674         val = I915_READ(TRANS_CHICKEN2(PIPE_A));
1675         val |= TRANS_CHICKEN2_TIMING_OVERRIDE;
1676         I915_WRITE(TRANS_CHICKEN2(PIPE_A), val);
1677
1678         val = TRANS_ENABLE;
1679         pipeconf_val = I915_READ(PIPECONF(cpu_transcoder));
1680
1681         if ((pipeconf_val & PIPECONF_INTERLACE_MASK_HSW) ==
1682             PIPECONF_INTERLACED_ILK)
1683                 val |= TRANS_INTERLACED;
1684         else
1685                 val |= TRANS_PROGRESSIVE;
1686
1687         I915_WRITE(LPT_TRANSCONF, val);
1688         if (intel_wait_for_register(dev_priv,
1689                                     LPT_TRANSCONF,
1690                                     TRANS_STATE_ENABLE,
1691                                     TRANS_STATE_ENABLE,
1692                                     100))
1693                 DRM_ERROR("Failed to enable PCH transcoder\n");
1694 }
1695
1696 static void ironlake_disable_pch_transcoder(struct drm_i915_private *dev_priv,
1697                                             enum pipe pipe)
1698 {
1699         i915_reg_t reg;
1700         uint32_t val;
1701
1702         /* FDI relies on the transcoder */
1703         assert_fdi_tx_disabled(dev_priv, pipe);
1704         assert_fdi_rx_disabled(dev_priv, pipe);
1705
1706         /* Ports must be off as well */
1707         assert_pch_ports_disabled(dev_priv, pipe);
1708
1709         reg = PCH_TRANSCONF(pipe);
1710         val = I915_READ(reg);
1711         val &= ~TRANS_ENABLE;
1712         I915_WRITE(reg, val);
1713         /* wait for PCH transcoder off, transcoder state */
1714         if (intel_wait_for_register(dev_priv,
1715                                     reg, TRANS_STATE_ENABLE, 0,
1716                                     50))
1717                 DRM_ERROR("failed to disable transcoder %c\n", pipe_name(pipe));
1718
1719         if (HAS_PCH_CPT(dev_priv)) {
1720                 /* Workaround: Clear the timing override chicken bit again. */
1721                 reg = TRANS_CHICKEN2(pipe);
1722                 val = I915_READ(reg);
1723                 val &= ~TRANS_CHICKEN2_TIMING_OVERRIDE;
1724                 I915_WRITE(reg, val);
1725         }
1726 }
1727
1728 void lpt_disable_pch_transcoder(struct drm_i915_private *dev_priv)
1729 {
1730         u32 val;
1731
1732         val = I915_READ(LPT_TRANSCONF);
1733         val &= ~TRANS_ENABLE;
1734         I915_WRITE(LPT_TRANSCONF, val);
1735         /* wait for PCH transcoder off, transcoder state */
1736         if (intel_wait_for_register(dev_priv,
1737                                     LPT_TRANSCONF, TRANS_STATE_ENABLE, 0,
1738                                     50))
1739                 DRM_ERROR("Failed to disable PCH transcoder\n");
1740
1741         /* Workaround: clear timing override bit. */
1742         val = I915_READ(TRANS_CHICKEN2(PIPE_A));
1743         val &= ~TRANS_CHICKEN2_TIMING_OVERRIDE;
1744         I915_WRITE(TRANS_CHICKEN2(PIPE_A), val);
1745 }
1746
1747 enum pipe intel_crtc_pch_transcoder(struct intel_crtc *crtc)
1748 {
1749         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1750
1751         if (HAS_PCH_LPT(dev_priv))
1752                 return PIPE_A;
1753         else
1754                 return crtc->pipe;
1755 }
1756
1757 static void intel_enable_pipe(const struct intel_crtc_state *new_crtc_state)
1758 {
1759         struct intel_crtc *crtc = to_intel_crtc(new_crtc_state->base.crtc);
1760         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1761         enum transcoder cpu_transcoder = new_crtc_state->cpu_transcoder;
1762         enum pipe pipe = crtc->pipe;
1763         i915_reg_t reg;
1764         u32 val;
1765
1766         DRM_DEBUG_KMS("enabling pipe %c\n", pipe_name(pipe));
1767
1768         assert_planes_disabled(crtc);
1769
1770         /*
1771          * A pipe without a PLL won't actually be able to drive bits from
1772          * a plane.  On ILK+ the pipe PLLs are integrated, so we don't
1773          * need the check.
1774          */
1775         if (HAS_GMCH_DISPLAY(dev_priv)) {
1776                 if (intel_crtc_has_type(new_crtc_state, INTEL_OUTPUT_DSI))
1777                         assert_dsi_pll_enabled(dev_priv);
1778                 else
1779                         assert_pll_enabled(dev_priv, pipe);
1780         } else {
1781                 if (new_crtc_state->has_pch_encoder) {
1782                         /* if driving the PCH, we need FDI enabled */
1783                         assert_fdi_rx_pll_enabled(dev_priv,
1784                                                   intel_crtc_pch_transcoder(crtc));
1785                         assert_fdi_tx_pll_enabled(dev_priv,
1786                                                   (enum pipe) cpu_transcoder);
1787                 }
1788                 /* FIXME: assert CPU port conditions for SNB+ */
1789         }
1790
1791         reg = PIPECONF(cpu_transcoder);
1792         val = I915_READ(reg);
1793         if (val & PIPECONF_ENABLE) {
1794                 /* we keep both pipes enabled on 830 */
1795                 WARN_ON(!IS_I830(dev_priv));
1796                 return;
1797         }
1798
1799         I915_WRITE(reg, val | PIPECONF_ENABLE);
1800         POSTING_READ(reg);
1801
1802         /*
1803          * Until the pipe starts PIPEDSL reads will return a stale value,
1804          * which causes an apparent vblank timestamp jump when PIPEDSL
1805          * resets to its proper value. That also messes up the frame count
1806          * when it's derived from the timestamps. So let's wait for the
1807          * pipe to start properly before we call drm_crtc_vblank_on()
1808          */
1809         if (dev_priv->drm.max_vblank_count == 0)
1810                 intel_wait_for_pipe_scanline_moving(crtc);
1811 }
1812
1813 static void intel_disable_pipe(const struct intel_crtc_state *old_crtc_state)
1814 {
1815         struct intel_crtc *crtc = to_intel_crtc(old_crtc_state->base.crtc);
1816         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1817         enum transcoder cpu_transcoder = old_crtc_state->cpu_transcoder;
1818         enum pipe pipe = crtc->pipe;
1819         i915_reg_t reg;
1820         u32 val;
1821
1822         DRM_DEBUG_KMS("disabling pipe %c\n", pipe_name(pipe));
1823
1824         /*
1825          * Make sure planes won't keep trying to pump pixels to us,
1826          * or we might hang the display.
1827          */
1828         assert_planes_disabled(crtc);
1829
1830         reg = PIPECONF(cpu_transcoder);
1831         val = I915_READ(reg);
1832         if ((val & PIPECONF_ENABLE) == 0)
1833                 return;
1834
1835         /*
1836          * Double wide has implications for planes
1837          * so best keep it disabled when not needed.
1838          */
1839         if (old_crtc_state->double_wide)
1840                 val &= ~PIPECONF_DOUBLE_WIDE;
1841
1842         /* Don't disable pipe or pipe PLLs if needed */
1843         if (!IS_I830(dev_priv))
1844                 val &= ~PIPECONF_ENABLE;
1845
1846         I915_WRITE(reg, val);
1847         if ((val & PIPECONF_ENABLE) == 0)
1848                 intel_wait_for_pipe_off(old_crtc_state);
1849 }
1850
1851 static unsigned int intel_tile_size(const struct drm_i915_private *dev_priv)
1852 {
1853         return IS_GEN(dev_priv, 2) ? 2048 : 4096;
1854 }
1855
1856 static unsigned int
1857 intel_tile_width_bytes(const struct drm_framebuffer *fb, int color_plane)
1858 {
1859         struct drm_i915_private *dev_priv = to_i915(fb->dev);
1860         unsigned int cpp = fb->format->cpp[color_plane];
1861
1862         switch (fb->modifier) {
1863         case DRM_FORMAT_MOD_LINEAR:
1864                 return cpp;
1865         case I915_FORMAT_MOD_X_TILED:
1866                 if (IS_GEN(dev_priv, 2))
1867                         return 128;
1868                 else
1869                         return 512;
1870         case I915_FORMAT_MOD_Y_TILED_CCS:
1871                 if (color_plane == 1)
1872                         return 128;
1873                 /* fall through */
1874         case I915_FORMAT_MOD_Y_TILED:
1875                 if (IS_GEN(dev_priv, 2) || HAS_128_BYTE_Y_TILING(dev_priv))
1876                         return 128;
1877                 else
1878                         return 512;
1879         case I915_FORMAT_MOD_Yf_TILED_CCS:
1880                 if (color_plane == 1)
1881                         return 128;
1882                 /* fall through */
1883         case I915_FORMAT_MOD_Yf_TILED:
1884                 switch (cpp) {
1885                 case 1:
1886                         return 64;
1887                 case 2:
1888                 case 4:
1889                         return 128;
1890                 case 8:
1891                 case 16:
1892                         return 256;
1893                 default:
1894                         MISSING_CASE(cpp);
1895                         return cpp;
1896                 }
1897                 break;
1898         default:
1899                 MISSING_CASE(fb->modifier);
1900                 return cpp;
1901         }
1902 }
1903
1904 static unsigned int
1905 intel_tile_height(const struct drm_framebuffer *fb, int color_plane)
1906 {
1907         if (fb->modifier == DRM_FORMAT_MOD_LINEAR)
1908                 return 1;
1909         else
1910                 return intel_tile_size(to_i915(fb->dev)) /
1911                         intel_tile_width_bytes(fb, color_plane);
1912 }
1913
1914 /* Return the tile dimensions in pixel units */
1915 static void intel_tile_dims(const struct drm_framebuffer *fb, int color_plane,
1916                             unsigned int *tile_width,
1917                             unsigned int *tile_height)
1918 {
1919         unsigned int tile_width_bytes = intel_tile_width_bytes(fb, color_plane);
1920         unsigned int cpp = fb->format->cpp[color_plane];
1921
1922         *tile_width = tile_width_bytes / cpp;
1923         *tile_height = intel_tile_size(to_i915(fb->dev)) / tile_width_bytes;
1924 }
1925
1926 unsigned int
1927 intel_fb_align_height(const struct drm_framebuffer *fb,
1928                       int color_plane, unsigned int height)
1929 {
1930         unsigned int tile_height = intel_tile_height(fb, color_plane);
1931
1932         return ALIGN(height, tile_height);
1933 }
1934
1935 unsigned int intel_rotation_info_size(const struct intel_rotation_info *rot_info)
1936 {
1937         unsigned int size = 0;
1938         int i;
1939
1940         for (i = 0 ; i < ARRAY_SIZE(rot_info->plane); i++)
1941                 size += rot_info->plane[i].width * rot_info->plane[i].height;
1942
1943         return size;
1944 }
1945
1946 static void
1947 intel_fill_fb_ggtt_view(struct i915_ggtt_view *view,
1948                         const struct drm_framebuffer *fb,
1949                         unsigned int rotation)
1950 {
1951         view->type = I915_GGTT_VIEW_NORMAL;
1952         if (drm_rotation_90_or_270(rotation)) {
1953                 view->type = I915_GGTT_VIEW_ROTATED;
1954                 view->rotated = to_intel_framebuffer(fb)->rot_info;
1955         }
1956 }
1957
1958 static unsigned int intel_cursor_alignment(const struct drm_i915_private *dev_priv)
1959 {
1960         if (IS_I830(dev_priv))
1961                 return 16 * 1024;
1962         else if (IS_I85X(dev_priv))
1963                 return 256;
1964         else if (IS_I845G(dev_priv) || IS_I865G(dev_priv))
1965                 return 32;
1966         else
1967                 return 4 * 1024;
1968 }
1969
1970 static unsigned int intel_linear_alignment(const struct drm_i915_private *dev_priv)
1971 {
1972         if (INTEL_GEN(dev_priv) >= 9)
1973                 return 256 * 1024;
1974         else if (IS_I965G(dev_priv) || IS_I965GM(dev_priv) ||
1975                  IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
1976                 return 128 * 1024;
1977         else if (INTEL_GEN(dev_priv) >= 4)
1978                 return 4 * 1024;
1979         else
1980                 return 0;
1981 }
1982
1983 static unsigned int intel_surf_alignment(const struct drm_framebuffer *fb,
1984                                          int color_plane)
1985 {
1986         struct drm_i915_private *dev_priv = to_i915(fb->dev);
1987
1988         /* AUX_DIST needs only 4K alignment */
1989         if (color_plane == 1)
1990                 return 4096;
1991
1992         switch (fb->modifier) {
1993         case DRM_FORMAT_MOD_LINEAR:
1994                 return intel_linear_alignment(dev_priv);
1995         case I915_FORMAT_MOD_X_TILED:
1996                 if (INTEL_GEN(dev_priv) >= 9)
1997                         return 256 * 1024;
1998                 return 0;
1999         case I915_FORMAT_MOD_Y_TILED_CCS:
2000         case I915_FORMAT_MOD_Yf_TILED_CCS:
2001         case I915_FORMAT_MOD_Y_TILED:
2002         case I915_FORMAT_MOD_Yf_TILED:
2003                 return 1 * 1024 * 1024;
2004         default:
2005                 MISSING_CASE(fb->modifier);
2006                 return 0;
2007         }
2008 }
2009
2010 static bool intel_plane_uses_fence(const struct intel_plane_state *plane_state)
2011 {
2012         struct intel_plane *plane = to_intel_plane(plane_state->base.plane);
2013         struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
2014
2015         return INTEL_GEN(dev_priv) < 4 || plane->has_fbc;
2016 }
2017
2018 struct i915_vma *
2019 intel_pin_and_fence_fb_obj(struct drm_framebuffer *fb,
2020                            const struct i915_ggtt_view *view,
2021                            bool uses_fence,
2022                            unsigned long *out_flags)
2023 {
2024         struct drm_device *dev = fb->dev;
2025         struct drm_i915_private *dev_priv = to_i915(dev);
2026         struct drm_i915_gem_object *obj = intel_fb_obj(fb);
2027         struct i915_vma *vma;
2028         unsigned int pinctl;
2029         u32 alignment;
2030
2031         WARN_ON(!mutex_is_locked(&dev->struct_mutex));
2032
2033         alignment = intel_surf_alignment(fb, 0);
2034
2035         /* Note that the w/a also requires 64 PTE of padding following the
2036          * bo. We currently fill all unused PTE with the shadow page and so
2037          * we should always have valid PTE following the scanout preventing
2038          * the VT-d warning.
2039          */
2040         if (intel_scanout_needs_vtd_wa(dev_priv) && alignment < 256 * 1024)
2041                 alignment = 256 * 1024;
2042
2043         /*
2044          * Global gtt pte registers are special registers which actually forward
2045          * writes to a chunk of system memory. Which means that there is no risk
2046          * that the register values disappear as soon as we call
2047          * intel_runtime_pm_put(), so it is correct to wrap only the
2048          * pin/unpin/fence and not more.
2049          */
2050         intel_runtime_pm_get(dev_priv);
2051
2052         atomic_inc(&dev_priv->gpu_error.pending_fb_pin);
2053
2054         pinctl = 0;
2055
2056         /* Valleyview is definitely limited to scanning out the first
2057          * 512MiB. Lets presume this behaviour was inherited from the
2058          * g4x display engine and that all earlier gen are similarly
2059          * limited. Testing suggests that it is a little more
2060          * complicated than this. For example, Cherryview appears quite
2061          * happy to scanout from anywhere within its global aperture.
2062          */
2063         if (HAS_GMCH_DISPLAY(dev_priv))
2064                 pinctl |= PIN_MAPPABLE;
2065
2066         vma = i915_gem_object_pin_to_display_plane(obj,
2067                                                    alignment, view, pinctl);
2068         if (IS_ERR(vma))
2069                 goto err;
2070
2071         if (uses_fence && i915_vma_is_map_and_fenceable(vma)) {
2072                 int ret;
2073
2074                 /* Install a fence for tiled scan-out. Pre-i965 always needs a
2075                  * fence, whereas 965+ only requires a fence if using
2076                  * framebuffer compression.  For simplicity, we always, when
2077                  * possible, install a fence as the cost is not that onerous.
2078                  *
2079                  * If we fail to fence the tiled scanout, then either the
2080                  * modeset will reject the change (which is highly unlikely as
2081                  * the affected systems, all but one, do not have unmappable
2082                  * space) or we will not be able to enable full powersaving
2083                  * techniques (also likely not to apply due to various limits
2084                  * FBC and the like impose on the size of the buffer, which
2085                  * presumably we violated anyway with this unmappable buffer).
2086                  * Anyway, it is presumably better to stumble onwards with
2087                  * something and try to run the system in a "less than optimal"
2088                  * mode that matches the user configuration.
2089                  */
2090                 ret = i915_vma_pin_fence(vma);
2091                 if (ret != 0 && INTEL_GEN(dev_priv) < 4) {
2092                         i915_gem_object_unpin_from_display_plane(vma);
2093                         vma = ERR_PTR(ret);
2094                         goto err;
2095                 }
2096
2097                 if (ret == 0 && vma->fence)
2098                         *out_flags |= PLANE_HAS_FENCE;
2099         }
2100
2101         i915_vma_get(vma);
2102 err:
2103         atomic_dec(&dev_priv->gpu_error.pending_fb_pin);
2104
2105         intel_runtime_pm_put(dev_priv);
2106         return vma;
2107 }
2108
2109 void intel_unpin_fb_vma(struct i915_vma *vma, unsigned long flags)
2110 {
2111         lockdep_assert_held(&vma->vm->i915->drm.struct_mutex);
2112
2113         if (flags & PLANE_HAS_FENCE)
2114                 i915_vma_unpin_fence(vma);
2115         i915_gem_object_unpin_from_display_plane(vma);
2116         i915_vma_put(vma);
2117 }
2118
2119 static int intel_fb_pitch(const struct drm_framebuffer *fb, int color_plane,
2120                           unsigned int rotation)
2121 {
2122         if (drm_rotation_90_or_270(rotation))
2123                 return to_intel_framebuffer(fb)->rotated[color_plane].pitch;
2124         else
2125                 return fb->pitches[color_plane];
2126 }
2127
2128 /*
2129  * Convert the x/y offsets into a linear offset.
2130  * Only valid with 0/180 degree rotation, which is fine since linear
2131  * offset is only used with linear buffers on pre-hsw and tiled buffers
2132  * with gen2/3, and 90/270 degree rotations isn't supported on any of them.
2133  */
2134 u32 intel_fb_xy_to_linear(int x, int y,
2135                           const struct intel_plane_state *state,
2136                           int color_plane)
2137 {
2138         const struct drm_framebuffer *fb = state->base.fb;
2139         unsigned int cpp = fb->format->cpp[color_plane];
2140         unsigned int pitch = state->color_plane[color_plane].stride;
2141
2142         return y * pitch + x * cpp;
2143 }
2144
2145 /*
2146  * Add the x/y offsets derived from fb->offsets[] to the user
2147  * specified plane src x/y offsets. The resulting x/y offsets
2148  * specify the start of scanout from the beginning of the gtt mapping.
2149  */
2150 void intel_add_fb_offsets(int *x, int *y,
2151                           const struct intel_plane_state *state,
2152                           int color_plane)
2153
2154 {
2155         const struct intel_framebuffer *intel_fb = to_intel_framebuffer(state->base.fb);
2156         unsigned int rotation = state->base.rotation;
2157
2158         if (drm_rotation_90_or_270(rotation)) {
2159                 *x += intel_fb->rotated[color_plane].x;
2160                 *y += intel_fb->rotated[color_plane].y;
2161         } else {
2162                 *x += intel_fb->normal[color_plane].x;
2163                 *y += intel_fb->normal[color_plane].y;
2164         }
2165 }
2166
2167 static u32 intel_adjust_tile_offset(int *x, int *y,
2168                                     unsigned int tile_width,
2169                                     unsigned int tile_height,
2170                                     unsigned int tile_size,
2171                                     unsigned int pitch_tiles,
2172                                     u32 old_offset,
2173                                     u32 new_offset)
2174 {
2175         unsigned int pitch_pixels = pitch_tiles * tile_width;
2176         unsigned int tiles;
2177
2178         WARN_ON(old_offset & (tile_size - 1));
2179         WARN_ON(new_offset & (tile_size - 1));
2180         WARN_ON(new_offset > old_offset);
2181
2182         tiles = (old_offset - new_offset) / tile_size;
2183
2184         *y += tiles / pitch_tiles * tile_height;
2185         *x += tiles % pitch_tiles * tile_width;
2186
2187         /* minimize x in case it got needlessly big */
2188         *y += *x / pitch_pixels * tile_height;
2189         *x %= pitch_pixels;
2190
2191         return new_offset;
2192 }
2193
2194 static bool is_surface_linear(u64 modifier, int color_plane)
2195 {
2196         return modifier == DRM_FORMAT_MOD_LINEAR;
2197 }
2198
2199 static u32 intel_adjust_aligned_offset(int *x, int *y,
2200                                        const struct drm_framebuffer *fb,
2201                                        int color_plane,
2202                                        unsigned int rotation,
2203                                        unsigned int pitch,
2204                                        u32 old_offset, u32 new_offset)
2205 {
2206         struct drm_i915_private *dev_priv = to_i915(fb->dev);
2207         unsigned int cpp = fb->format->cpp[color_plane];
2208
2209         WARN_ON(new_offset > old_offset);
2210
2211         if (!is_surface_linear(fb->modifier, color_plane)) {
2212                 unsigned int tile_size, tile_width, tile_height;
2213                 unsigned int pitch_tiles;
2214
2215                 tile_size = intel_tile_size(dev_priv);
2216                 intel_tile_dims(fb, color_plane, &tile_width, &tile_height);
2217
2218                 if (drm_rotation_90_or_270(rotation)) {
2219                         pitch_tiles = pitch / tile_height;
2220                         swap(tile_width, tile_height);
2221                 } else {
2222                         pitch_tiles = pitch / (tile_width * cpp);
2223                 }
2224
2225                 intel_adjust_tile_offset(x, y, tile_width, tile_height,
2226                                          tile_size, pitch_tiles,
2227                                          old_offset, new_offset);
2228         } else {
2229                 old_offset += *y * pitch + *x * cpp;
2230
2231                 *y = (old_offset - new_offset) / pitch;
2232                 *x = ((old_offset - new_offset) - *y * pitch) / cpp;
2233         }
2234
2235         return new_offset;
2236 }
2237
2238 /*
2239  * Adjust the tile offset by moving the difference into
2240  * the x/y offsets.
2241  */
2242 static u32 intel_plane_adjust_aligned_offset(int *x, int *y,
2243                                              const struct intel_plane_state *state,
2244                                              int color_plane,
2245                                              u32 old_offset, u32 new_offset)
2246 {
2247         return intel_adjust_aligned_offset(x, y, state->base.fb, color_plane,
2248                                            state->base.rotation,
2249                                            state->color_plane[color_plane].stride,
2250                                            old_offset, new_offset);
2251 }
2252
2253 /*
2254  * Computes the aligned offset to the base tile and adjusts
2255  * x, y. bytes per pixel is assumed to be a power-of-two.
2256  *
2257  * In the 90/270 rotated case, x and y are assumed
2258  * to be already rotated to match the rotated GTT view, and
2259  * pitch is the tile_height aligned framebuffer height.
2260  *
2261  * This function is used when computing the derived information
2262  * under intel_framebuffer, so using any of that information
2263  * here is not allowed. Anything under drm_framebuffer can be
2264  * used. This is why the user has to pass in the pitch since it
2265  * is specified in the rotated orientation.
2266  */
2267 static u32 intel_compute_aligned_offset(struct drm_i915_private *dev_priv,
2268                                         int *x, int *y,
2269                                         const struct drm_framebuffer *fb,
2270                                         int color_plane,
2271                                         unsigned int pitch,
2272                                         unsigned int rotation,
2273                                         u32 alignment)
2274 {
2275         unsigned int cpp = fb->format->cpp[color_plane];
2276         u32 offset, offset_aligned;
2277
2278         if (alignment)
2279                 alignment--;
2280
2281         if (!is_surface_linear(fb->modifier, color_plane)) {
2282                 unsigned int tile_size, tile_width, tile_height;
2283                 unsigned int tile_rows, tiles, pitch_tiles;
2284
2285                 tile_size = intel_tile_size(dev_priv);
2286                 intel_tile_dims(fb, color_plane, &tile_width, &tile_height);
2287
2288                 if (drm_rotation_90_or_270(rotation)) {
2289                         pitch_tiles = pitch / tile_height;
2290                         swap(tile_width, tile_height);
2291                 } else {
2292                         pitch_tiles = pitch / (tile_width * cpp);
2293                 }
2294
2295                 tile_rows = *y / tile_height;
2296                 *y %= tile_height;
2297
2298                 tiles = *x / tile_width;
2299                 *x %= tile_width;
2300
2301                 offset = (tile_rows * pitch_tiles + tiles) * tile_size;
2302                 offset_aligned = offset & ~alignment;
2303
2304                 intel_adjust_tile_offset(x, y, tile_width, tile_height,
2305                                          tile_size, pitch_tiles,
2306                                          offset, offset_aligned);
2307         } else {
2308                 offset = *y * pitch + *x * cpp;
2309                 offset_aligned = offset & ~alignment;
2310
2311                 *y = (offset & alignment) / pitch;
2312                 *x = ((offset & alignment) - *y * pitch) / cpp;
2313         }
2314
2315         return offset_aligned;
2316 }
2317
2318 static u32 intel_plane_compute_aligned_offset(int *x, int *y,
2319                                               const struct intel_plane_state *state,
2320                                               int color_plane)
2321 {
2322         struct intel_plane *intel_plane = to_intel_plane(state->base.plane);
2323         struct drm_i915_private *dev_priv = to_i915(intel_plane->base.dev);
2324         const struct drm_framebuffer *fb = state->base.fb;
2325         unsigned int rotation = state->base.rotation;
2326         int pitch = state->color_plane[color_plane].stride;
2327         u32 alignment;
2328
2329         if (intel_plane->id == PLANE_CURSOR)
2330                 alignment = intel_cursor_alignment(dev_priv);
2331         else
2332                 alignment = intel_surf_alignment(fb, color_plane);
2333
2334         return intel_compute_aligned_offset(dev_priv, x, y, fb, color_plane,
2335                                             pitch, rotation, alignment);
2336 }
2337
2338 /* Convert the fb->offset[] into x/y offsets */
2339 static int intel_fb_offset_to_xy(int *x, int *y,
2340                                  const struct drm_framebuffer *fb,
2341                                  int color_plane)
2342 {
2343         struct drm_i915_private *dev_priv = to_i915(fb->dev);
2344         unsigned int height;
2345
2346         if (fb->modifier != DRM_FORMAT_MOD_LINEAR &&
2347             fb->offsets[color_plane] % intel_tile_size(dev_priv)) {
2348                 DRM_DEBUG_KMS("Misaligned offset 0x%08x for color plane %d\n",
2349                               fb->offsets[color_plane], color_plane);
2350                 return -EINVAL;
2351         }
2352
2353         height = drm_framebuffer_plane_height(fb->height, fb, color_plane);
2354         height = ALIGN(height, intel_tile_height(fb, color_plane));
2355
2356         /* Catch potential overflows early */
2357         if (add_overflows_t(u32, mul_u32_u32(height, fb->pitches[color_plane]),
2358                             fb->offsets[color_plane])) {
2359                 DRM_DEBUG_KMS("Bad offset 0x%08x or pitch %d for color plane %d\n",
2360                               fb->offsets[color_plane], fb->pitches[color_plane],
2361                               color_plane);
2362                 return -ERANGE;
2363         }
2364
2365         *x = 0;
2366         *y = 0;
2367
2368         intel_adjust_aligned_offset(x, y,
2369                                     fb, color_plane, DRM_MODE_ROTATE_0,
2370                                     fb->pitches[color_plane],
2371                                     fb->offsets[color_plane], 0);
2372
2373         return 0;
2374 }
2375
2376 static unsigned int intel_fb_modifier_to_tiling(uint64_t fb_modifier)
2377 {
2378         switch (fb_modifier) {
2379         case I915_FORMAT_MOD_X_TILED:
2380                 return I915_TILING_X;
2381         case I915_FORMAT_MOD_Y_TILED:
2382         case I915_FORMAT_MOD_Y_TILED_CCS:
2383                 return I915_TILING_Y;
2384         default:
2385                 return I915_TILING_NONE;
2386         }
2387 }
2388
2389 /*
2390  * From the Sky Lake PRM:
2391  * "The Color Control Surface (CCS) contains the compression status of
2392  *  the cache-line pairs. The compression state of the cache-line pair
2393  *  is specified by 2 bits in the CCS. Each CCS cache-line represents
2394  *  an area on the main surface of 16 x16 sets of 128 byte Y-tiled
2395  *  cache-line-pairs. CCS is always Y tiled."
2396  *
2397  * Since cache line pairs refers to horizontally adjacent cache lines,
2398  * each cache line in the CCS corresponds to an area of 32x16 cache
2399  * lines on the main surface. Since each pixel is 4 bytes, this gives
2400  * us a ratio of one byte in the CCS for each 8x16 pixels in the
2401  * main surface.
2402  */
2403 static const struct drm_format_info ccs_formats[] = {
2404         { .format = DRM_FORMAT_XRGB8888, .depth = 24, .num_planes = 2, .cpp = { 4, 1, }, .hsub = 8, .vsub = 16, },
2405         { .format = DRM_FORMAT_XBGR8888, .depth = 24, .num_planes = 2, .cpp = { 4, 1, }, .hsub = 8, .vsub = 16, },
2406         { .format = DRM_FORMAT_ARGB8888, .depth = 32, .num_planes = 2, .cpp = { 4, 1, }, .hsub = 8, .vsub = 16, },
2407         { .format = DRM_FORMAT_ABGR8888, .depth = 32, .num_planes = 2, .cpp = { 4, 1, }, .hsub = 8, .vsub = 16, },
2408 };
2409
2410 static const struct drm_format_info *
2411 lookup_format_info(const struct drm_format_info formats[],
2412                    int num_formats, u32 format)
2413 {
2414         int i;
2415
2416         for (i = 0; i < num_formats; i++) {
2417                 if (formats[i].format == format)
2418                         return &formats[i];
2419         }
2420
2421         return NULL;
2422 }
2423
2424 static const struct drm_format_info *
2425 intel_get_format_info(const struct drm_mode_fb_cmd2 *cmd)
2426 {
2427         switch (cmd->modifier[0]) {
2428         case I915_FORMAT_MOD_Y_TILED_CCS:
2429         case I915_FORMAT_MOD_Yf_TILED_CCS:
2430                 return lookup_format_info(ccs_formats,
2431                                           ARRAY_SIZE(ccs_formats),
2432                                           cmd->pixel_format);
2433         default:
2434                 return NULL;
2435         }
2436 }
2437
2438 bool is_ccs_modifier(u64 modifier)
2439 {
2440         return modifier == I915_FORMAT_MOD_Y_TILED_CCS ||
2441                modifier == I915_FORMAT_MOD_Yf_TILED_CCS;
2442 }
2443
2444 static int
2445 intel_fill_fb_info(struct drm_i915_private *dev_priv,
2446                    struct drm_framebuffer *fb)
2447 {
2448         struct intel_framebuffer *intel_fb = to_intel_framebuffer(fb);
2449         struct intel_rotation_info *rot_info = &intel_fb->rot_info;
2450         struct drm_i915_gem_object *obj = intel_fb_obj(fb);
2451         u32 gtt_offset_rotated = 0;
2452         unsigned int max_size = 0;
2453         int i, num_planes = fb->format->num_planes;
2454         unsigned int tile_size = intel_tile_size(dev_priv);
2455
2456         for (i = 0; i < num_planes; i++) {
2457                 unsigned int width, height;
2458                 unsigned int cpp, size;
2459                 u32 offset;
2460                 int x, y;
2461                 int ret;
2462
2463                 cpp = fb->format->cpp[i];
2464                 width = drm_framebuffer_plane_width(fb->width, fb, i);
2465                 height = drm_framebuffer_plane_height(fb->height, fb, i);
2466
2467                 ret = intel_fb_offset_to_xy(&x, &y, fb, i);
2468                 if (ret) {
2469                         DRM_DEBUG_KMS("bad fb plane %d offset: 0x%x\n",
2470                                       i, fb->offsets[i]);
2471                         return ret;
2472                 }
2473
2474                 if (is_ccs_modifier(fb->modifier) && i == 1) {
2475                         int hsub = fb->format->hsub;
2476                         int vsub = fb->format->vsub;
2477                         int tile_width, tile_height;
2478                         int main_x, main_y;
2479                         int ccs_x, ccs_y;
2480
2481                         intel_tile_dims(fb, i, &tile_width, &tile_height);
2482                         tile_width *= hsub;
2483                         tile_height *= vsub;
2484
2485                         ccs_x = (x * hsub) % tile_width;
2486                         ccs_y = (y * vsub) % tile_height;
2487                         main_x = intel_fb->normal[0].x % tile_width;
2488                         main_y = intel_fb->normal[0].y % tile_height;
2489
2490                         /*
2491                          * CCS doesn't have its own x/y offset register, so the intra CCS tile
2492                          * x/y offsets must match between CCS and the main surface.
2493                          */
2494                         if (main_x != ccs_x || main_y != ccs_y) {
2495                                 DRM_DEBUG_KMS("Bad CCS x/y (main %d,%d ccs %d,%d) full (main %d,%d ccs %d,%d)\n",
2496                                               main_x, main_y,
2497                                               ccs_x, ccs_y,
2498                                               intel_fb->normal[0].x,
2499                                               intel_fb->normal[0].y,
2500                                               x, y);
2501                                 return -EINVAL;
2502                         }
2503                 }
2504
2505                 /*
2506                  * The fence (if used) is aligned to the start of the object
2507                  * so having the framebuffer wrap around across the edge of the
2508                  * fenced region doesn't really work. We have no API to configure
2509                  * the fence start offset within the object (nor could we probably
2510                  * on gen2/3). So it's just easier if we just require that the
2511                  * fb layout agrees with the fence layout. We already check that the
2512                  * fb stride matches the fence stride elsewhere.
2513                  */
2514                 if (i == 0 && i915_gem_object_is_tiled(obj) &&
2515                     (x + width) * cpp > fb->pitches[i]) {
2516                         DRM_DEBUG_KMS("bad fb plane %d offset: 0x%x\n",
2517                                       i, fb->offsets[i]);
2518                         return -EINVAL;
2519                 }
2520
2521                 /*
2522                  * First pixel of the framebuffer from
2523                  * the start of the normal gtt mapping.
2524                  */
2525                 intel_fb->normal[i].x = x;
2526                 intel_fb->normal[i].y = y;
2527
2528                 offset = intel_compute_aligned_offset(dev_priv, &x, &y, fb, i,
2529                                                       fb->pitches[i],
2530                                                       DRM_MODE_ROTATE_0,
2531                                                       tile_size);
2532                 offset /= tile_size;
2533
2534                 if (!is_surface_linear(fb->modifier, i)) {
2535                         unsigned int tile_width, tile_height;
2536                         unsigned int pitch_tiles;
2537                         struct drm_rect r;
2538
2539                         intel_tile_dims(fb, i, &tile_width, &tile_height);
2540
2541                         rot_info->plane[i].offset = offset;
2542                         rot_info->plane[i].stride = DIV_ROUND_UP(fb->pitches[i], tile_width * cpp);
2543                         rot_info->plane[i].width = DIV_ROUND_UP(x + width, tile_width);
2544                         rot_info->plane[i].height = DIV_ROUND_UP(y + height, tile_height);
2545
2546                         intel_fb->rotated[i].pitch =
2547                                 rot_info->plane[i].height * tile_height;
2548
2549                         /* how many tiles does this plane need */
2550                         size = rot_info->plane[i].stride * rot_info->plane[i].height;
2551                         /*
2552                          * If the plane isn't horizontally tile aligned,
2553                          * we need one more tile.
2554                          */
2555                         if (x != 0)
2556                                 size++;
2557
2558                         /* rotate the x/y offsets to match the GTT view */
2559                         r.x1 = x;
2560                         r.y1 = y;
2561                         r.x2 = x + width;
2562                         r.y2 = y + height;
2563                         drm_rect_rotate(&r,
2564                                         rot_info->plane[i].width * tile_width,
2565                                         rot_info->plane[i].height * tile_height,
2566                                         DRM_MODE_ROTATE_270);
2567                         x = r.x1;
2568                         y = r.y1;
2569
2570                         /* rotate the tile dimensions to match the GTT view */
2571                         pitch_tiles = intel_fb->rotated[i].pitch / tile_height;
2572                         swap(tile_width, tile_height);
2573
2574                         /*
2575                          * We only keep the x/y offsets, so push all of the
2576                          * gtt offset into the x/y offsets.
2577                          */
2578                         intel_adjust_tile_offset(&x, &y,
2579                                                  tile_width, tile_height,
2580                                                  tile_size, pitch_tiles,
2581                                                  gtt_offset_rotated * tile_size, 0);
2582
2583                         gtt_offset_rotated += rot_info->plane[i].width * rot_info->plane[i].height;
2584
2585                         /*
2586                          * First pixel of the framebuffer from
2587                          * the start of the rotated gtt mapping.
2588                          */
2589                         intel_fb->rotated[i].x = x;
2590                         intel_fb->rotated[i].y = y;
2591                 } else {
2592                         size = DIV_ROUND_UP((y + height) * fb->pitches[i] +
2593                                             x * cpp, tile_size);
2594                 }
2595
2596                 /* how many tiles in total needed in the bo */
2597                 max_size = max(max_size, offset + size);
2598         }
2599
2600         if (mul_u32_u32(max_size, tile_size) > obj->base.size) {
2601                 DRM_DEBUG_KMS("fb too big for bo (need %llu bytes, have %zu bytes)\n",
2602                               mul_u32_u32(max_size, tile_size), obj->base.size);
2603                 return -EINVAL;
2604         }
2605
2606         return 0;
2607 }
2608
2609 static int i9xx_format_to_fourcc(int format)
2610 {
2611         switch (format) {
2612         case DISPPLANE_8BPP:
2613                 return DRM_FORMAT_C8;
2614         case DISPPLANE_BGRX555:
2615                 return DRM_FORMAT_XRGB1555;
2616         case DISPPLANE_BGRX565:
2617                 return DRM_FORMAT_RGB565;
2618         default:
2619         case DISPPLANE_BGRX888:
2620                 return DRM_FORMAT_XRGB8888;
2621         case DISPPLANE_RGBX888:
2622                 return DRM_FORMAT_XBGR8888;
2623         case DISPPLANE_BGRX101010:
2624                 return DRM_FORMAT_XRGB2101010;
2625         case DISPPLANE_RGBX101010:
2626                 return DRM_FORMAT_XBGR2101010;
2627         }
2628 }
2629
2630 int skl_format_to_fourcc(int format, bool rgb_order, bool alpha)
2631 {
2632         switch (format) {
2633         case PLANE_CTL_FORMAT_RGB_565:
2634                 return DRM_FORMAT_RGB565;
2635         case PLANE_CTL_FORMAT_NV12:
2636                 return DRM_FORMAT_NV12;
2637         default:
2638         case PLANE_CTL_FORMAT_XRGB_8888:
2639                 if (rgb_order) {
2640                         if (alpha)
2641                                 return DRM_FORMAT_ABGR8888;
2642                         else
2643                                 return DRM_FORMAT_XBGR8888;
2644                 } else {
2645                         if (alpha)
2646                                 return DRM_FORMAT_ARGB8888;
2647                         else
2648                                 return DRM_FORMAT_XRGB8888;
2649                 }
2650         case PLANE_CTL_FORMAT_XRGB_2101010:
2651                 if (rgb_order)
2652                         return DRM_FORMAT_XBGR2101010;
2653                 else
2654                         return DRM_FORMAT_XRGB2101010;
2655         }
2656 }
2657
2658 static bool
2659 intel_alloc_initial_plane_obj(struct intel_crtc *crtc,
2660                               struct intel_initial_plane_config *plane_config)
2661 {
2662         struct drm_device *dev = crtc->base.dev;
2663         struct drm_i915_private *dev_priv = to_i915(dev);
2664         struct drm_i915_gem_object *obj = NULL;
2665         struct drm_mode_fb_cmd2 mode_cmd = { 0 };
2666         struct drm_framebuffer *fb = &plane_config->fb->base;
2667         u32 base_aligned = round_down(plane_config->base, PAGE_SIZE);
2668         u32 size_aligned = round_up(plane_config->base + plane_config->size,
2669                                     PAGE_SIZE);
2670
2671         size_aligned -= base_aligned;
2672
2673         if (plane_config->size == 0)
2674                 return false;
2675
2676         /* If the FB is too big, just don't use it since fbdev is not very
2677          * important and we should probably use that space with FBC or other
2678          * features. */
2679         if (size_aligned * 2 > dev_priv->stolen_usable_size)
2680                 return false;
2681
2682         switch (fb->modifier) {
2683         case DRM_FORMAT_MOD_LINEAR:
2684         case I915_FORMAT_MOD_X_TILED:
2685         case I915_FORMAT_MOD_Y_TILED:
2686                 break;
2687         default:
2688                 DRM_DEBUG_DRIVER("Unsupported modifier for initial FB: 0x%llx\n",
2689                                  fb->modifier);
2690                 return false;
2691         }
2692
2693         mutex_lock(&dev->struct_mutex);
2694         obj = i915_gem_object_create_stolen_for_preallocated(dev_priv,
2695                                                              base_aligned,
2696                                                              base_aligned,
2697                                                              size_aligned);
2698         mutex_unlock(&dev->struct_mutex);
2699         if (!obj)
2700                 return false;
2701
2702         switch (plane_config->tiling) {
2703         case I915_TILING_NONE:
2704                 break;
2705         case I915_TILING_X:
2706         case I915_TILING_Y:
2707                 obj->tiling_and_stride = fb->pitches[0] | plane_config->tiling;
2708                 break;
2709         default:
2710                 MISSING_CASE(plane_config->tiling);
2711                 return false;
2712         }
2713
2714         mode_cmd.pixel_format = fb->format->format;
2715         mode_cmd.width = fb->width;
2716         mode_cmd.height = fb->height;
2717         mode_cmd.pitches[0] = fb->pitches[0];
2718         mode_cmd.modifier[0] = fb->modifier;
2719         mode_cmd.flags = DRM_MODE_FB_MODIFIERS;
2720
2721         if (intel_framebuffer_init(to_intel_framebuffer(fb), obj, &mode_cmd)) {
2722                 DRM_DEBUG_KMS("intel fb init failed\n");
2723                 goto out_unref_obj;
2724         }
2725
2726
2727         DRM_DEBUG_KMS("initial plane fb obj %p\n", obj);
2728         return true;
2729
2730 out_unref_obj:
2731         i915_gem_object_put(obj);
2732         return false;
2733 }
2734
2735 static void
2736 intel_set_plane_visible(struct intel_crtc_state *crtc_state,
2737                         struct intel_plane_state *plane_state,
2738                         bool visible)
2739 {
2740         struct intel_plane *plane = to_intel_plane(plane_state->base.plane);
2741
2742         plane_state->base.visible = visible;
2743
2744         if (visible)
2745                 crtc_state->base.plane_mask |= drm_plane_mask(&plane->base);
2746         else
2747                 crtc_state->base.plane_mask &= ~drm_plane_mask(&plane->base);
2748 }
2749
2750 static void fixup_active_planes(struct intel_crtc_state *crtc_state)
2751 {
2752         struct drm_i915_private *dev_priv = to_i915(crtc_state->base.crtc->dev);
2753         struct drm_plane *plane;
2754
2755         /*
2756          * Active_planes aliases if multiple "primary" or cursor planes
2757          * have been used on the same (or wrong) pipe. plane_mask uses
2758          * unique ids, hence we can use that to reconstruct active_planes.
2759          */
2760         crtc_state->active_planes = 0;
2761
2762         drm_for_each_plane_mask(plane, &dev_priv->drm,
2763                                 crtc_state->base.plane_mask)
2764                 crtc_state->active_planes |= BIT(to_intel_plane(plane)->id);
2765 }
2766
2767 static void intel_plane_disable_noatomic(struct intel_crtc *crtc,
2768                                          struct intel_plane *plane)
2769 {
2770         struct intel_crtc_state *crtc_state =
2771                 to_intel_crtc_state(crtc->base.state);
2772         struct intel_plane_state *plane_state =
2773                 to_intel_plane_state(plane->base.state);
2774
2775         DRM_DEBUG_KMS("Disabling [PLANE:%d:%s] on [CRTC:%d:%s]\n",
2776                       plane->base.base.id, plane->base.name,
2777                       crtc->base.base.id, crtc->base.name);
2778
2779         intel_set_plane_visible(crtc_state, plane_state, false);
2780         fixup_active_planes(crtc_state);
2781
2782         if (plane->id == PLANE_PRIMARY)
2783                 intel_pre_disable_primary_noatomic(&crtc->base);
2784
2785         trace_intel_disable_plane(&plane->base, crtc);
2786         plane->disable_plane(plane, crtc_state);
2787 }
2788
2789 static void
2790 intel_find_initial_plane_obj(struct intel_crtc *intel_crtc,
2791                              struct intel_initial_plane_config *plane_config)
2792 {
2793         struct drm_device *dev = intel_crtc->base.dev;
2794         struct drm_i915_private *dev_priv = to_i915(dev);
2795         struct drm_crtc *c;
2796         struct drm_i915_gem_object *obj;
2797         struct drm_plane *primary = intel_crtc->base.primary;
2798         struct drm_plane_state *plane_state = primary->state;
2799         struct intel_plane *intel_plane = to_intel_plane(primary);
2800         struct intel_plane_state *intel_state =
2801                 to_intel_plane_state(plane_state);
2802         struct drm_framebuffer *fb;
2803
2804         if (!plane_config->fb)
2805                 return;
2806
2807         if (intel_alloc_initial_plane_obj(intel_crtc, plane_config)) {
2808                 fb = &plane_config->fb->base;
2809                 goto valid_fb;
2810         }
2811
2812         kfree(plane_config->fb);
2813
2814         /*
2815          * Failed to alloc the obj, check to see if we should share
2816          * an fb with another CRTC instead
2817          */
2818         for_each_crtc(dev, c) {
2819                 struct intel_plane_state *state;
2820
2821                 if (c == &intel_crtc->base)
2822                         continue;
2823
2824                 if (!to_intel_crtc(c)->active)
2825                         continue;
2826
2827                 state = to_intel_plane_state(c->primary->state);
2828                 if (!state->vma)
2829                         continue;
2830
2831                 if (intel_plane_ggtt_offset(state) == plane_config->base) {
2832                         fb = state->base.fb;
2833                         drm_framebuffer_get(fb);
2834                         goto valid_fb;
2835                 }
2836         }
2837
2838         /*
2839          * We've failed to reconstruct the BIOS FB.  Current display state
2840          * indicates that the primary plane is visible, but has a NULL FB,
2841          * which will lead to problems later if we don't fix it up.  The
2842          * simplest solution is to just disable the primary plane now and
2843          * pretend the BIOS never had it enabled.
2844          */
2845         intel_plane_disable_noatomic(intel_crtc, intel_plane);
2846
2847         return;
2848
2849 valid_fb:
2850         intel_state->base.rotation = plane_config->rotation;
2851         intel_fill_fb_ggtt_view(&intel_state->view, fb,
2852                                 intel_state->base.rotation);
2853         intel_state->color_plane[0].stride =
2854                 intel_fb_pitch(fb, 0, intel_state->base.rotation);
2855
2856         mutex_lock(&dev->struct_mutex);
2857         intel_state->vma =
2858                 intel_pin_and_fence_fb_obj(fb,
2859                                            &intel_state->view,
2860                                            intel_plane_uses_fence(intel_state),
2861                                            &intel_state->flags);
2862         mutex_unlock(&dev->struct_mutex);
2863         if (IS_ERR(intel_state->vma)) {
2864                 DRM_ERROR("failed to pin boot fb on pipe %d: %li\n",
2865                           intel_crtc->pipe, PTR_ERR(intel_state->vma));
2866
2867                 intel_state->vma = NULL;
2868                 drm_framebuffer_put(fb);
2869                 return;
2870         }
2871
2872         obj = intel_fb_obj(fb);
2873         intel_fb_obj_flush(obj, ORIGIN_DIRTYFB);
2874
2875         plane_state->src_x = 0;
2876         plane_state->src_y = 0;
2877         plane_state->src_w = fb->width << 16;
2878         plane_state->src_h = fb->height << 16;
2879
2880         plane_state->crtc_x = 0;
2881         plane_state->crtc_y = 0;
2882         plane_state->crtc_w = fb->width;
2883         plane_state->crtc_h = fb->height;
2884
2885         intel_state->base.src = drm_plane_state_src(plane_state);
2886         intel_state->base.dst = drm_plane_state_dest(plane_state);
2887
2888         if (i915_gem_object_is_tiled(obj))
2889                 dev_priv->preserve_bios_swizzle = true;
2890
2891         plane_state->fb = fb;
2892         plane_state->crtc = &intel_crtc->base;
2893
2894         atomic_or(to_intel_plane(primary)->frontbuffer_bit,
2895                   &obj->frontbuffer_bits);
2896 }
2897
2898 static int skl_max_plane_width(const struct drm_framebuffer *fb,
2899                                int color_plane,
2900                                unsigned int rotation)
2901 {
2902         int cpp = fb->format->cpp[color_plane];
2903
2904         switch (fb->modifier) {
2905         case DRM_FORMAT_MOD_LINEAR:
2906         case I915_FORMAT_MOD_X_TILED:
2907                 switch (cpp) {
2908                 case 8:
2909                         return 4096;
2910                 case 4:
2911                 case 2:
2912                 case 1:
2913                         return 8192;
2914                 default:
2915                         MISSING_CASE(cpp);
2916                         break;
2917                 }
2918                 break;
2919         case I915_FORMAT_MOD_Y_TILED_CCS:
2920         case I915_FORMAT_MOD_Yf_TILED_CCS:
2921                 /* FIXME AUX plane? */
2922         case I915_FORMAT_MOD_Y_TILED:
2923         case I915_FORMAT_MOD_Yf_TILED:
2924                 switch (cpp) {
2925                 case 8:
2926                         return 2048;
2927                 case 4:
2928                         return 4096;
2929                 case 2:
2930                 case 1:
2931                         return 8192;
2932                 default:
2933                         MISSING_CASE(cpp);
2934                         break;
2935                 }
2936                 break;
2937         default:
2938                 MISSING_CASE(fb->modifier);
2939         }
2940
2941         return 2048;
2942 }
2943
2944 static bool skl_check_main_ccs_coordinates(struct intel_plane_state *plane_state,
2945                                            int main_x, int main_y, u32 main_offset)
2946 {
2947         const struct drm_framebuffer *fb = plane_state->base.fb;
2948         int hsub = fb->format->hsub;
2949         int vsub = fb->format->vsub;
2950         int aux_x = plane_state->color_plane[1].x;
2951         int aux_y = plane_state->color_plane[1].y;
2952         u32 aux_offset = plane_state->color_plane[1].offset;
2953         u32 alignment = intel_surf_alignment(fb, 1);
2954
2955         while (aux_offset >= main_offset && aux_y <= main_y) {
2956                 int x, y;
2957
2958                 if (aux_x == main_x && aux_y == main_y)
2959                         break;
2960
2961                 if (aux_offset == 0)
2962                         break;
2963
2964                 x = aux_x / hsub;
2965                 y = aux_y / vsub;
2966                 aux_offset = intel_plane_adjust_aligned_offset(&x, &y, plane_state, 1,
2967                                                                aux_offset, aux_offset - alignment);
2968                 aux_x = x * hsub + aux_x % hsub;
2969                 aux_y = y * vsub + aux_y % vsub;
2970         }
2971
2972         if (aux_x != main_x || aux_y != main_y)
2973                 return false;
2974
2975         plane_state->color_plane[1].offset = aux_offset;
2976         plane_state->color_plane[1].x = aux_x;
2977         plane_state->color_plane[1].y = aux_y;
2978
2979         return true;
2980 }
2981
2982 static int skl_check_main_surface(struct intel_plane_state *plane_state)
2983 {
2984         const struct drm_framebuffer *fb = plane_state->base.fb;
2985         unsigned int rotation = plane_state->base.rotation;
2986         int x = plane_state->base.src.x1 >> 16;
2987         int y = plane_state->base.src.y1 >> 16;
2988         int w = drm_rect_width(&plane_state->base.src) >> 16;
2989         int h = drm_rect_height(&plane_state->base.src) >> 16;
2990         int max_width = skl_max_plane_width(fb, 0, rotation);
2991         int max_height = 4096;
2992         u32 alignment, offset, aux_offset = plane_state->color_plane[1].offset;
2993
2994         if (w > max_width || h > max_height) {
2995                 DRM_DEBUG_KMS("requested Y/RGB source size %dx%d too big (limit %dx%d)\n",
2996                               w, h, max_width, max_height);
2997                 return -EINVAL;
2998         }
2999
3000         intel_add_fb_offsets(&x, &y, plane_state, 0);
3001         offset = intel_plane_compute_aligned_offset(&x, &y, plane_state, 0);
3002         alignment = intel_surf_alignment(fb, 0);
3003
3004         /*
3005          * AUX surface offset is specified as the distance from the
3006          * main surface offset, and it must be non-negative. Make
3007          * sure that is what we will get.
3008          */
3009         if (offset > aux_offset)
3010                 offset = intel_plane_adjust_aligned_offset(&x, &y, plane_state, 0,
3011                                                            offset, aux_offset & ~(alignment - 1));
3012
3013         /*
3014          * When using an X-tiled surface, the plane blows up
3015          * if the x offset + width exceed the stride.
3016          *
3017          * TODO: linear and Y-tiled seem fine, Yf untested,
3018          */
3019         if (fb->modifier == I915_FORMAT_MOD_X_TILED) {
3020                 int cpp = fb->format->cpp[0];
3021
3022                 while ((x + w) * cpp > plane_state->color_plane[0].stride) {
3023                         if (offset == 0) {
3024                                 DRM_DEBUG_KMS("Unable to find suitable display surface offset due to X-tiling\n");
3025                                 return -EINVAL;
3026                         }
3027
3028                         offset = intel_plane_adjust_aligned_offset(&x, &y, plane_state, 0,
3029                                                                    offset, offset - alignment);
3030                 }
3031         }
3032
3033         /*
3034          * CCS AUX surface doesn't have its own x/y offsets, we must make sure
3035          * they match with the main surface x/y offsets.
3036          */
3037         if (is_ccs_modifier(fb->modifier)) {
3038                 while (!skl_check_main_ccs_coordinates(plane_state, x, y, offset)) {
3039                         if (offset == 0)
3040                                 break;
3041
3042                         offset = intel_plane_adjust_aligned_offset(&x, &y, plane_state, 0,
3043                                                                    offset, offset - alignment);
3044                 }
3045
3046                 if (x != plane_state->color_plane[1].x || y != plane_state->color_plane[1].y) {
3047                         DRM_DEBUG_KMS("Unable to find suitable display surface offset due to CCS\n");
3048                         return -EINVAL;
3049                 }
3050         }
3051
3052         plane_state->color_plane[0].offset = offset;
3053         plane_state->color_plane[0].x = x;
3054         plane_state->color_plane[0].y = y;
3055
3056         return 0;
3057 }
3058
3059 static int skl_check_nv12_aux_surface(struct intel_plane_state *plane_state)
3060 {
3061         const struct drm_framebuffer *fb = plane_state->base.fb;
3062         unsigned int rotation = plane_state->base.rotation;
3063         int max_width = skl_max_plane_width(fb, 1, rotation);
3064         int max_height = 4096;
3065         int x = plane_state->base.src.x1 >> 17;
3066         int y = plane_state->base.src.y1 >> 17;
3067         int w = drm_rect_width(&plane_state->base.src) >> 17;
3068         int h = drm_rect_height(&plane_state->base.src) >> 17;
3069         u32 offset;
3070
3071         intel_add_fb_offsets(&x, &y, plane_state, 1);
3072         offset = intel_plane_compute_aligned_offset(&x, &y, plane_state, 1);
3073
3074         /* FIXME not quite sure how/if these apply to the chroma plane */
3075         if (w > max_width || h > max_height) {
3076                 DRM_DEBUG_KMS("CbCr source size %dx%d too big (limit %dx%d)\n",
3077                               w, h, max_width, max_height);
3078                 return -EINVAL;
3079         }
3080
3081         plane_state->color_plane[1].offset = offset;
3082         plane_state->color_plane[1].x = x;
3083         plane_state->color_plane[1].y = y;
3084
3085         return 0;
3086 }
3087
3088 static int skl_check_ccs_aux_surface(struct intel_plane_state *plane_state)
3089 {
3090         const struct drm_framebuffer *fb = plane_state->base.fb;
3091         int src_x = plane_state->base.src.x1 >> 16;
3092         int src_y = plane_state->base.src.y1 >> 16;
3093         int hsub = fb->format->hsub;
3094         int vsub = fb->format->vsub;
3095         int x = src_x / hsub;
3096         int y = src_y / vsub;
3097         u32 offset;
3098
3099         intel_add_fb_offsets(&x, &y, plane_state, 1);
3100         offset = intel_plane_compute_aligned_offset(&x, &y, plane_state, 1);
3101
3102         plane_state->color_plane[1].offset = offset;
3103         plane_state->color_plane[1].x = x * hsub + src_x % hsub;
3104         plane_state->color_plane[1].y = y * vsub + src_y % vsub;
3105
3106         return 0;
3107 }
3108
3109 int skl_check_plane_surface(struct intel_plane_state *plane_state)
3110 {
3111         const struct drm_framebuffer *fb = plane_state->base.fb;
3112         unsigned int rotation = plane_state->base.rotation;
3113         int ret;
3114
3115         intel_fill_fb_ggtt_view(&plane_state->view, fb, rotation);
3116         plane_state->color_plane[0].stride = intel_fb_pitch(fb, 0, rotation);
3117         plane_state->color_plane[1].stride = intel_fb_pitch(fb, 1, rotation);
3118
3119         ret = intel_plane_check_stride(plane_state);
3120         if (ret)
3121                 return ret;
3122
3123         if (!plane_state->base.visible)
3124                 return 0;
3125
3126         /* Rotate src coordinates to match rotated GTT view */
3127         if (drm_rotation_90_or_270(rotation))
3128                 drm_rect_rotate(&plane_state->base.src,
3129                                 fb->width << 16, fb->height << 16,
3130                                 DRM_MODE_ROTATE_270);
3131
3132         /*
3133          * Handle the AUX surface first since
3134          * the main surface setup depends on it.
3135          */
3136         if (fb->format->format == DRM_FORMAT_NV12) {
3137                 ret = skl_check_nv12_aux_surface(plane_state);
3138                 if (ret)
3139                         return ret;
3140         } else if (is_ccs_modifier(fb->modifier)) {
3141                 ret = skl_check_ccs_aux_surface(plane_state);
3142                 if (ret)
3143                         return ret;
3144         } else {
3145                 plane_state->color_plane[1].offset = ~0xfff;
3146                 plane_state->color_plane[1].x = 0;
3147                 plane_state->color_plane[1].y = 0;
3148         }
3149
3150         ret = skl_check_main_surface(plane_state);
3151         if (ret)
3152                 return ret;
3153
3154         return 0;
3155 }
3156
3157 unsigned int
3158 i9xx_plane_max_stride(struct intel_plane *plane,
3159                       u32 pixel_format, u64 modifier,
3160                       unsigned int rotation)
3161 {
3162         struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
3163
3164         if (!HAS_GMCH_DISPLAY(dev_priv)) {
3165                 return 32*1024;
3166         } else if (INTEL_GEN(dev_priv) >= 4) {
3167                 if (modifier == I915_FORMAT_MOD_X_TILED)
3168                         return 16*1024;
3169                 else
3170                         return 32*1024;
3171         } else if (INTEL_GEN(dev_priv) >= 3) {
3172                 if (modifier == I915_FORMAT_MOD_X_TILED)
3173                         return 8*1024;
3174                 else
3175                         return 16*1024;
3176         } else {
3177                 if (plane->i9xx_plane == PLANE_C)
3178                         return 4*1024;
3179                 else
3180                         return 8*1024;
3181         }
3182 }
3183
3184 static u32 i9xx_plane_ctl(const struct intel_crtc_state *crtc_state,
3185                           const struct intel_plane_state *plane_state)
3186 {
3187         struct drm_i915_private *dev_priv =
3188                 to_i915(plane_state->base.plane->dev);
3189         struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
3190         const struct drm_framebuffer *fb = plane_state->base.fb;
3191         unsigned int rotation = plane_state->base.rotation;
3192         u32 dspcntr;
3193
3194         dspcntr = DISPLAY_PLANE_ENABLE | DISPPLANE_GAMMA_ENABLE;
3195
3196         if (IS_G4X(dev_priv) || IS_GEN(dev_priv, 5) ||
3197             IS_GEN(dev_priv, 6) || IS_IVYBRIDGE(dev_priv))
3198                 dspcntr |= DISPPLANE_TRICKLE_FEED_DISABLE;
3199
3200         if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
3201                 dspcntr |= DISPPLANE_PIPE_CSC_ENABLE;
3202
3203         if (INTEL_GEN(dev_priv) < 5)
3204                 dspcntr |= DISPPLANE_SEL_PIPE(crtc->pipe);
3205
3206         switch (fb->format->format) {
3207         case DRM_FORMAT_C8:
3208                 dspcntr |= DISPPLANE_8BPP;
3209                 break;
3210         case DRM_FORMAT_XRGB1555:
3211                 dspcntr |= DISPPLANE_BGRX555;
3212                 break;
3213         case DRM_FORMAT_RGB565:
3214                 dspcntr |= DISPPLANE_BGRX565;
3215                 break;
3216         case DRM_FORMAT_XRGB8888:
3217                 dspcntr |= DISPPLANE_BGRX888;
3218                 break;
3219         case DRM_FORMAT_XBGR8888:
3220                 dspcntr |= DISPPLANE_RGBX888;
3221                 break;
3222         case DRM_FORMAT_XRGB2101010:
3223                 dspcntr |= DISPPLANE_BGRX101010;
3224                 break;
3225         case DRM_FORMAT_XBGR2101010:
3226                 dspcntr |= DISPPLANE_RGBX101010;
3227                 break;
3228         default:
3229                 MISSING_CASE(fb->format->format);
3230                 return 0;
3231         }
3232
3233         if (INTEL_GEN(dev_priv) >= 4 &&
3234             fb->modifier == I915_FORMAT_MOD_X_TILED)
3235                 dspcntr |= DISPPLANE_TILED;
3236
3237         if (rotation & DRM_MODE_ROTATE_180)
3238                 dspcntr |= DISPPLANE_ROTATE_180;
3239
3240         if (rotation & DRM_MODE_REFLECT_X)
3241                 dspcntr |= DISPPLANE_MIRROR;
3242
3243         return dspcntr;
3244 }
3245
3246 int i9xx_check_plane_surface(struct intel_plane_state *plane_state)
3247 {
3248         struct drm_i915_private *dev_priv =
3249                 to_i915(plane_state->base.plane->dev);
3250         const struct drm_framebuffer *fb = plane_state->base.fb;
3251         unsigned int rotation = plane_state->base.rotation;
3252         int src_x = plane_state->base.src.x1 >> 16;
3253         int src_y = plane_state->base.src.y1 >> 16;
3254         u32 offset;
3255         int ret;
3256
3257         intel_fill_fb_ggtt_view(&plane_state->view, fb, rotation);
3258         plane_state->color_plane[0].stride = intel_fb_pitch(fb, 0, rotation);
3259
3260         ret = intel_plane_check_stride(plane_state);
3261         if (ret)
3262                 return ret;
3263
3264         intel_add_fb_offsets(&src_x, &src_y, plane_state, 0);
3265
3266         if (INTEL_GEN(dev_priv) >= 4)
3267                 offset = intel_plane_compute_aligned_offset(&src_x, &src_y,
3268                                                             plane_state, 0);
3269         else
3270                 offset = 0;
3271
3272         /* HSW/BDW do this automagically in hardware */
3273         if (!IS_HASWELL(dev_priv) && !IS_BROADWELL(dev_priv)) {
3274                 int src_w = drm_rect_width(&plane_state->base.src) >> 16;
3275                 int src_h = drm_rect_height(&plane_state->base.src) >> 16;
3276
3277                 if (rotation & DRM_MODE_ROTATE_180) {
3278                         src_x += src_w - 1;
3279                         src_y += src_h - 1;
3280                 } else if (rotation & DRM_MODE_REFLECT_X) {
3281                         src_x += src_w - 1;
3282                 }
3283         }
3284
3285         plane_state->color_plane[0].offset = offset;
3286         plane_state->color_plane[0].x = src_x;
3287         plane_state->color_plane[0].y = src_y;
3288
3289         return 0;
3290 }
3291
3292 static int
3293 i9xx_plane_check(struct intel_crtc_state *crtc_state,
3294                  struct intel_plane_state *plane_state)
3295 {
3296         int ret;
3297
3298         ret = chv_plane_check_rotation(plane_state);
3299         if (ret)
3300                 return ret;
3301
3302         ret = drm_atomic_helper_check_plane_state(&plane_state->base,
3303                                                   &crtc_state->base,
3304                                                   DRM_PLANE_HELPER_NO_SCALING,
3305                                                   DRM_PLANE_HELPER_NO_SCALING,
3306                                                   false, true);
3307         if (ret)
3308                 return ret;
3309
3310         if (!plane_state->base.visible)
3311                 return 0;
3312
3313         ret = intel_plane_check_src_coordinates(plane_state);
3314         if (ret)
3315                 return ret;
3316
3317         ret = i9xx_check_plane_surface(plane_state);
3318         if (ret)
3319                 return ret;
3320
3321         plane_state->ctl = i9xx_plane_ctl(crtc_state, plane_state);
3322
3323         return 0;
3324 }
3325
3326 static void i9xx_update_plane(struct intel_plane *plane,
3327                               const struct intel_crtc_state *crtc_state,
3328                               const struct intel_plane_state *plane_state)
3329 {
3330         struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
3331         enum i9xx_plane_id i9xx_plane = plane->i9xx_plane;
3332         u32 linear_offset;
3333         u32 dspcntr = plane_state->ctl;
3334         int x = plane_state->color_plane[0].x;
3335         int y = plane_state->color_plane[0].y;
3336         unsigned long irqflags;
3337         u32 dspaddr_offset;
3338
3339         linear_offset = intel_fb_xy_to_linear(x, y, plane_state, 0);
3340
3341         if (INTEL_GEN(dev_priv) >= 4)
3342                 dspaddr_offset = plane_state->color_plane[0].offset;
3343         else
3344                 dspaddr_offset = linear_offset;
3345
3346         spin_lock_irqsave(&dev_priv->uncore.lock, irqflags);
3347
3348         I915_WRITE_FW(DSPSTRIDE(i9xx_plane), plane_state->color_plane[0].stride);
3349
3350         if (INTEL_GEN(dev_priv) < 4) {
3351                 /* pipesrc and dspsize control the size that is scaled from,
3352                  * which should always be the user's requested size.
3353                  */
3354                 I915_WRITE_FW(DSPPOS(i9xx_plane), 0);
3355                 I915_WRITE_FW(DSPSIZE(i9xx_plane),
3356                               ((crtc_state->pipe_src_h - 1) << 16) |
3357                               (crtc_state->pipe_src_w - 1));
3358         } else if (IS_CHERRYVIEW(dev_priv) && i9xx_plane == PLANE_B) {
3359                 I915_WRITE_FW(PRIMPOS(i9xx_plane), 0);
3360                 I915_WRITE_FW(PRIMSIZE(i9xx_plane),
3361                               ((crtc_state->pipe_src_h - 1) << 16) |
3362                               (crtc_state->pipe_src_w - 1));
3363                 I915_WRITE_FW(PRIMCNSTALPHA(i9xx_plane), 0);
3364         }
3365
3366         if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv)) {
3367                 I915_WRITE_FW(DSPOFFSET(i9xx_plane), (y << 16) | x);
3368         } else if (INTEL_GEN(dev_priv) >= 4) {
3369                 I915_WRITE_FW(DSPLINOFF(i9xx_plane), linear_offset);
3370                 I915_WRITE_FW(DSPTILEOFF(i9xx_plane), (y << 16) | x);
3371         }
3372
3373         /*
3374          * The control register self-arms if the plane was previously
3375          * disabled. Try to make the plane enable atomic by writing
3376          * the control register just before the surface register.
3377          */
3378         I915_WRITE_FW(DSPCNTR(i9xx_plane), dspcntr);
3379         if (INTEL_GEN(dev_priv) >= 4)
3380                 I915_WRITE_FW(DSPSURF(i9xx_plane),
3381                               intel_plane_ggtt_offset(plane_state) +
3382                               dspaddr_offset);
3383         else
3384                 I915_WRITE_FW(DSPADDR(i9xx_plane),
3385                               intel_plane_ggtt_offset(plane_state) +
3386                               dspaddr_offset);
3387
3388         spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags);
3389 }
3390
3391 static void i9xx_disable_plane(struct intel_plane *plane,
3392                                const struct intel_crtc_state *crtc_state)
3393 {
3394         struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
3395         enum i9xx_plane_id i9xx_plane = plane->i9xx_plane;
3396         unsigned long irqflags;
3397
3398         spin_lock_irqsave(&dev_priv->uncore.lock, irqflags);
3399
3400         I915_WRITE_FW(DSPCNTR(i9xx_plane), 0);
3401         if (INTEL_GEN(dev_priv) >= 4)
3402                 I915_WRITE_FW(DSPSURF(i9xx_plane), 0);
3403         else
3404                 I915_WRITE_FW(DSPADDR(i9xx_plane), 0);
3405
3406         spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags);
3407 }
3408
3409 static bool i9xx_plane_get_hw_state(struct intel_plane *plane,
3410                                     enum pipe *pipe)
3411 {
3412         struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
3413         enum intel_display_power_domain power_domain;
3414         enum i9xx_plane_id i9xx_plane = plane->i9xx_plane;
3415         bool ret;
3416         u32 val;
3417
3418         /*
3419          * Not 100% correct for planes that can move between pipes,
3420          * but that's only the case for gen2-4 which don't have any
3421          * display power wells.
3422          */
3423         power_domain = POWER_DOMAIN_PIPE(plane->pipe);
3424         if (!intel_display_power_get_if_enabled(dev_priv, power_domain))
3425                 return false;
3426
3427         val = I915_READ(DSPCNTR(i9xx_plane));
3428
3429         ret = val & DISPLAY_PLANE_ENABLE;
3430
3431         if (INTEL_GEN(dev_priv) >= 5)
3432                 *pipe = plane->pipe;
3433         else
3434                 *pipe = (val & DISPPLANE_SEL_PIPE_MASK) >>
3435                         DISPPLANE_SEL_PIPE_SHIFT;
3436
3437         intel_display_power_put(dev_priv, power_domain);
3438
3439         return ret;
3440 }
3441
3442 static u32
3443 intel_fb_stride_alignment(const struct drm_framebuffer *fb, int color_plane)
3444 {
3445         if (fb->modifier == DRM_FORMAT_MOD_LINEAR)
3446                 return 64;
3447         else
3448                 return intel_tile_width_bytes(fb, color_plane);
3449 }
3450
3451 static void skl_detach_scaler(struct intel_crtc *intel_crtc, int id)
3452 {
3453         struct drm_device *dev = intel_crtc->base.dev;
3454         struct drm_i915_private *dev_priv = to_i915(dev);
3455
3456         I915_WRITE(SKL_PS_CTRL(intel_crtc->pipe, id), 0);
3457         I915_WRITE(SKL_PS_WIN_POS(intel_crtc->pipe, id), 0);
3458         I915_WRITE(SKL_PS_WIN_SZ(intel_crtc->pipe, id), 0);
3459 }
3460
3461 /*
3462  * This function detaches (aka. unbinds) unused scalers in hardware
3463  */
3464 static void skl_detach_scalers(const struct intel_crtc_state *crtc_state)
3465 {
3466         struct intel_crtc *intel_crtc = to_intel_crtc(crtc_state->base.crtc);
3467         const struct intel_crtc_scaler_state *scaler_state =
3468                 &crtc_state->scaler_state;
3469         int i;
3470
3471         /* loop through and disable scalers that aren't in use */
3472         for (i = 0; i < intel_crtc->num_scalers; i++) {
3473                 if (!scaler_state->scalers[i].in_use)
3474                         skl_detach_scaler(intel_crtc, i);
3475         }
3476 }
3477
3478 static unsigned int skl_plane_stride_mult(const struct drm_framebuffer *fb,
3479                                           int color_plane, unsigned int rotation)
3480 {
3481         /*
3482          * The stride is either expressed as a multiple of 64 bytes chunks for
3483          * linear buffers or in number of tiles for tiled buffers.
3484          */
3485         if (fb->modifier == DRM_FORMAT_MOD_LINEAR)
3486                 return 64;
3487         else if (drm_rotation_90_or_270(rotation))
3488                 return intel_tile_height(fb, color_plane);
3489         else
3490                 return intel_tile_width_bytes(fb, color_plane);
3491 }
3492
3493 u32 skl_plane_stride(const struct intel_plane_state *plane_state,
3494                      int color_plane)
3495 {
3496         const struct drm_framebuffer *fb = plane_state->base.fb;
3497         unsigned int rotation = plane_state->base.rotation;
3498         u32 stride = plane_state->color_plane[color_plane].stride;
3499
3500         if (color_plane >= fb->format->num_planes)
3501                 return 0;
3502
3503         return stride / skl_plane_stride_mult(fb, color_plane, rotation);
3504 }
3505
3506 static u32 skl_plane_ctl_format(uint32_t pixel_format)
3507 {
3508         switch (pixel_format) {
3509         case DRM_FORMAT_C8:
3510                 return PLANE_CTL_FORMAT_INDEXED;
3511         case DRM_FORMAT_RGB565:
3512                 return PLANE_CTL_FORMAT_RGB_565;
3513         case DRM_FORMAT_XBGR8888:
3514         case DRM_FORMAT_ABGR8888:
3515                 return PLANE_CTL_FORMAT_XRGB_8888 | PLANE_CTL_ORDER_RGBX;
3516         case DRM_FORMAT_XRGB8888:
3517         case DRM_FORMAT_ARGB8888:
3518                 return PLANE_CTL_FORMAT_XRGB_8888;
3519         case DRM_FORMAT_XRGB2101010:
3520                 return PLANE_CTL_FORMAT_XRGB_2101010;
3521         case DRM_FORMAT_XBGR2101010:
3522                 return PLANE_CTL_ORDER_RGBX | PLANE_CTL_FORMAT_XRGB_2101010;
3523         case DRM_FORMAT_YUYV:
3524                 return PLANE_CTL_FORMAT_YUV422 | PLANE_CTL_YUV422_YUYV;
3525         case DRM_FORMAT_YVYU:
3526                 return PLANE_CTL_FORMAT_YUV422 | PLANE_CTL_YUV422_YVYU;
3527         case DRM_FORMAT_UYVY:
3528                 return PLANE_CTL_FORMAT_YUV422 | PLANE_CTL_YUV422_UYVY;
3529         case DRM_FORMAT_VYUY:
3530                 return PLANE_CTL_FORMAT_YUV422 | PLANE_CTL_YUV422_VYUY;
3531         case DRM_FORMAT_NV12:
3532                 return PLANE_CTL_FORMAT_NV12;
3533         default:
3534                 MISSING_CASE(pixel_format);
3535         }
3536
3537         return 0;
3538 }
3539
3540 static u32 skl_plane_ctl_alpha(const struct intel_plane_state *plane_state)
3541 {
3542         if (!plane_state->base.fb->format->has_alpha)
3543                 return PLANE_CTL_ALPHA_DISABLE;
3544
3545         switch (plane_state->base.pixel_blend_mode) {
3546         case DRM_MODE_BLEND_PIXEL_NONE:
3547                 return PLANE_CTL_ALPHA_DISABLE;
3548         case DRM_MODE_BLEND_PREMULTI:
3549                 return PLANE_CTL_ALPHA_SW_PREMULTIPLY;
3550         case DRM_MODE_BLEND_COVERAGE:
3551                 return PLANE_CTL_ALPHA_HW_PREMULTIPLY;
3552         default:
3553                 MISSING_CASE(plane_state->base.pixel_blend_mode);
3554                 return PLANE_CTL_ALPHA_DISABLE;
3555         }
3556 }
3557
3558 static u32 glk_plane_color_ctl_alpha(const struct intel_plane_state *plane_state)
3559 {
3560         if (!plane_state->base.fb->format->has_alpha)
3561                 return PLANE_COLOR_ALPHA_DISABLE;
3562
3563         switch (plane_state->base.pixel_blend_mode) {
3564         case DRM_MODE_BLEND_PIXEL_NONE:
3565                 return PLANE_COLOR_ALPHA_DISABLE;
3566         case DRM_MODE_BLEND_PREMULTI:
3567                 return PLANE_COLOR_ALPHA_SW_PREMULTIPLY;
3568         case DRM_MODE_BLEND_COVERAGE:
3569                 return PLANE_COLOR_ALPHA_HW_PREMULTIPLY;
3570         default:
3571                 MISSING_CASE(plane_state->base.pixel_blend_mode);
3572                 return PLANE_COLOR_ALPHA_DISABLE;
3573         }
3574 }
3575
3576 static u32 skl_plane_ctl_tiling(uint64_t fb_modifier)
3577 {
3578         switch (fb_modifier) {
3579         case DRM_FORMAT_MOD_LINEAR:
3580                 break;
3581         case I915_FORMAT_MOD_X_TILED:
3582                 return PLANE_CTL_TILED_X;
3583         case I915_FORMAT_MOD_Y_TILED:
3584                 return PLANE_CTL_TILED_Y;
3585         case I915_FORMAT_MOD_Y_TILED_CCS:
3586                 return PLANE_CTL_TILED_Y | PLANE_CTL_RENDER_DECOMPRESSION_ENABLE;
3587         case I915_FORMAT_MOD_Yf_TILED:
3588                 return PLANE_CTL_TILED_YF;
3589         case I915_FORMAT_MOD_Yf_TILED_CCS:
3590                 return PLANE_CTL_TILED_YF | PLANE_CTL_RENDER_DECOMPRESSION_ENABLE;
3591         default:
3592                 MISSING_CASE(fb_modifier);
3593         }
3594
3595         return 0;
3596 }
3597
3598 static u32 skl_plane_ctl_rotate(unsigned int rotate)
3599 {
3600         switch (rotate) {
3601         case DRM_MODE_ROTATE_0:
3602                 break;
3603         /*
3604          * DRM_MODE_ROTATE_ is counter clockwise to stay compatible with Xrandr
3605          * while i915 HW rotation is clockwise, thats why this swapping.
3606          */
3607         case DRM_MODE_ROTATE_90:
3608                 return PLANE_CTL_ROTATE_270;
3609         case DRM_MODE_ROTATE_180:
3610                 return PLANE_CTL_ROTATE_180;
3611         case DRM_MODE_ROTATE_270:
3612                 return PLANE_CTL_ROTATE_90;
3613         default:
3614                 MISSING_CASE(rotate);
3615         }
3616
3617         return 0;
3618 }
3619
3620 static u32 cnl_plane_ctl_flip(unsigned int reflect)
3621 {
3622         switch (reflect) {
3623         case 0:
3624                 break;
3625         case DRM_MODE_REFLECT_X:
3626                 return PLANE_CTL_FLIP_HORIZONTAL;
3627         case DRM_MODE_REFLECT_Y:
3628         default:
3629                 MISSING_CASE(reflect);
3630         }
3631
3632         return 0;
3633 }
3634
3635 u32 skl_plane_ctl(const struct intel_crtc_state *crtc_state,
3636                   const struct intel_plane_state *plane_state)
3637 {
3638         struct drm_i915_private *dev_priv =
3639                 to_i915(plane_state->base.plane->dev);
3640         const struct drm_framebuffer *fb = plane_state->base.fb;
3641         unsigned int rotation = plane_state->base.rotation;
3642         const struct drm_intel_sprite_colorkey *key = &plane_state->ckey;
3643         u32 plane_ctl;
3644
3645         plane_ctl = PLANE_CTL_ENABLE;
3646
3647         if (INTEL_GEN(dev_priv) < 10 && !IS_GEMINILAKE(dev_priv)) {
3648                 plane_ctl |= skl_plane_ctl_alpha(plane_state);
3649                 plane_ctl |=
3650                         PLANE_CTL_PIPE_GAMMA_ENABLE |
3651                         PLANE_CTL_PIPE_CSC_ENABLE |
3652                         PLANE_CTL_PLANE_GAMMA_DISABLE;
3653
3654                 if (plane_state->base.color_encoding == DRM_COLOR_YCBCR_BT709)
3655                         plane_ctl |= PLANE_CTL_YUV_TO_RGB_CSC_FORMAT_BT709;
3656
3657                 if (plane_state->base.color_range == DRM_COLOR_YCBCR_FULL_RANGE)
3658                         plane_ctl |= PLANE_CTL_YUV_RANGE_CORRECTION_DISABLE;
3659         }
3660
3661         plane_ctl |= skl_plane_ctl_format(fb->format->format);
3662         plane_ctl |= skl_plane_ctl_tiling(fb->modifier);
3663         plane_ctl |= skl_plane_ctl_rotate(rotation & DRM_MODE_ROTATE_MASK);
3664
3665         if (INTEL_GEN(dev_priv) >= 10)
3666                 plane_ctl |= cnl_plane_ctl_flip(rotation &
3667                                                 DRM_MODE_REFLECT_MASK);
3668
3669         if (key->flags & I915_SET_COLORKEY_DESTINATION)
3670                 plane_ctl |= PLANE_CTL_KEY_ENABLE_DESTINATION;
3671         else if (key->flags & I915_SET_COLORKEY_SOURCE)
3672                 plane_ctl |= PLANE_CTL_KEY_ENABLE_SOURCE;
3673
3674         return plane_ctl;
3675 }
3676
3677 u32 glk_plane_color_ctl(const struct intel_crtc_state *crtc_state,
3678                         const struct intel_plane_state *plane_state)
3679 {
3680         struct drm_i915_private *dev_priv =
3681                 to_i915(plane_state->base.plane->dev);
3682         const struct drm_framebuffer *fb = plane_state->base.fb;
3683         struct intel_plane *plane = to_intel_plane(plane_state->base.plane);
3684         u32 plane_color_ctl = 0;
3685
3686         if (INTEL_GEN(dev_priv) < 11) {
3687                 plane_color_ctl |= PLANE_COLOR_PIPE_GAMMA_ENABLE;
3688                 plane_color_ctl |= PLANE_COLOR_PIPE_CSC_ENABLE;
3689         }
3690         plane_color_ctl |= PLANE_COLOR_PLANE_GAMMA_DISABLE;
3691         plane_color_ctl |= glk_plane_color_ctl_alpha(plane_state);
3692
3693         if (fb->format->is_yuv && !icl_is_hdr_plane(plane)) {
3694                 if (plane_state->base.color_encoding == DRM_COLOR_YCBCR_BT709)
3695                         plane_color_ctl |= PLANE_COLOR_CSC_MODE_YUV709_TO_RGB709;
3696                 else
3697                         plane_color_ctl |= PLANE_COLOR_CSC_MODE_YUV601_TO_RGB709;
3698
3699                 if (plane_state->base.color_range == DRM_COLOR_YCBCR_FULL_RANGE)
3700                         plane_color_ctl |= PLANE_COLOR_YUV_RANGE_CORRECTION_DISABLE;
3701         } else if (fb->format->is_yuv) {
3702                 plane_color_ctl |= PLANE_COLOR_INPUT_CSC_ENABLE;
3703         }
3704
3705         return plane_color_ctl;
3706 }
3707
3708 static int
3709 __intel_display_resume(struct drm_device *dev,
3710                        struct drm_atomic_state *state,
3711                        struct drm_modeset_acquire_ctx *ctx)
3712 {
3713         struct drm_crtc_state *crtc_state;
3714         struct drm_crtc *crtc;
3715         int i, ret;
3716
3717         intel_modeset_setup_hw_state(dev, ctx);
3718         i915_redisable_vga(to_i915(dev));
3719
3720         if (!state)
3721                 return 0;
3722
3723         /*
3724          * We've duplicated the state, pointers to the old state are invalid.
3725          *
3726          * Don't attempt to use the old state until we commit the duplicated state.
3727          */
3728         for_each_new_crtc_in_state(state, crtc, crtc_state, i) {
3729                 /*
3730                  * Force recalculation even if we restore
3731                  * current state. With fast modeset this may not result
3732                  * in a modeset when the state is compatible.
3733                  */
3734                 crtc_state->mode_changed = true;
3735         }
3736
3737         /* ignore any reset values/BIOS leftovers in the WM registers */
3738         if (!HAS_GMCH_DISPLAY(to_i915(dev)))
3739                 to_intel_atomic_state(state)->skip_intermediate_wm = true;
3740
3741         ret = drm_atomic_helper_commit_duplicated_state(state, ctx);
3742
3743         WARN_ON(ret == -EDEADLK);
3744         return ret;
3745 }
3746
3747 static bool gpu_reset_clobbers_display(struct drm_i915_private *dev_priv)
3748 {
3749         return intel_has_gpu_reset(dev_priv) &&
3750                 INTEL_GEN(dev_priv) < 5 && !IS_G4X(dev_priv);
3751 }
3752
3753 void intel_prepare_reset(struct drm_i915_private *dev_priv)
3754 {
3755         struct drm_device *dev = &dev_priv->drm;
3756         struct drm_modeset_acquire_ctx *ctx = &dev_priv->reset_ctx;
3757         struct drm_atomic_state *state;
3758         int ret;
3759
3760         /* reset doesn't touch the display */
3761         if (!i915_modparams.force_reset_modeset_test &&
3762             !gpu_reset_clobbers_display(dev_priv))
3763                 return;
3764
3765         /* We have a modeset vs reset deadlock, defensively unbreak it. */
3766         set_bit(I915_RESET_MODESET, &dev_priv->gpu_error.flags);
3767         wake_up_all(&dev_priv->gpu_error.wait_queue);
3768
3769         if (atomic_read(&dev_priv->gpu_error.pending_fb_pin)) {
3770                 DRM_DEBUG_KMS("Modeset potentially stuck, unbreaking through wedging\n");
3771                 i915_gem_set_wedged(dev_priv);
3772         }
3773
3774         /*
3775          * Need mode_config.mutex so that we don't
3776          * trample ongoing ->detect() and whatnot.
3777          */
3778         mutex_lock(&dev->mode_config.mutex);
3779         drm_modeset_acquire_init(ctx, 0);
3780         while (1) {
3781                 ret = drm_modeset_lock_all_ctx(dev, ctx);
3782                 if (ret != -EDEADLK)
3783                         break;
3784
3785                 drm_modeset_backoff(ctx);
3786         }
3787         /*
3788          * Disabling the crtcs gracefully seems nicer. Also the
3789          * g33 docs say we should at least disable all the planes.
3790          */
3791         state = drm_atomic_helper_duplicate_state(dev, ctx);
3792         if (IS_ERR(state)) {
3793                 ret = PTR_ERR(state);
3794                 DRM_ERROR("Duplicating state failed with %i\n", ret);
3795                 return;
3796         }
3797
3798         ret = drm_atomic_helper_disable_all(dev, ctx);
3799         if (ret) {
3800                 DRM_ERROR("Suspending crtc's failed with %i\n", ret);
3801                 drm_atomic_state_put(state);
3802                 return;
3803         }
3804
3805         dev_priv->modeset_restore_state = state;
3806         state->acquire_ctx = ctx;
3807 }
3808
3809 void intel_finish_reset(struct drm_i915_private *dev_priv)
3810 {
3811         struct drm_device *dev = &dev_priv->drm;
3812         struct drm_modeset_acquire_ctx *ctx = &dev_priv->reset_ctx;
3813         struct drm_atomic_state *state;
3814         int ret;
3815
3816         /* reset doesn't touch the display */
3817         if (!test_bit(I915_RESET_MODESET, &dev_priv->gpu_error.flags))
3818                 return;
3819
3820         state = fetch_and_zero(&dev_priv->modeset_restore_state);
3821         if (!state)
3822                 goto unlock;
3823
3824         /* reset doesn't touch the display */
3825         if (!gpu_reset_clobbers_display(dev_priv)) {
3826                 /* for testing only restore the display */
3827                 ret = __intel_display_resume(dev, state, ctx);
3828                 if (ret)
3829                         DRM_ERROR("Restoring old state failed with %i\n", ret);
3830         } else {
3831                 /*
3832                  * The display has been reset as well,
3833                  * so need a full re-initialization.
3834                  */
3835                 intel_runtime_pm_disable_interrupts(dev_priv);
3836                 intel_runtime_pm_enable_interrupts(dev_priv);
3837
3838                 intel_pps_unlock_regs_wa(dev_priv);
3839                 intel_modeset_init_hw(dev);
3840                 intel_init_clock_gating(dev_priv);
3841
3842                 spin_lock_irq(&dev_priv->irq_lock);
3843                 if (dev_priv->display.hpd_irq_setup)
3844                         dev_priv->display.hpd_irq_setup(dev_priv);
3845                 spin_unlock_irq(&dev_priv->irq_lock);
3846
3847                 ret = __intel_display_resume(dev, state, ctx);
3848                 if (ret)
3849                         DRM_ERROR("Restoring old state failed with %i\n", ret);
3850
3851                 intel_hpd_init(dev_priv);
3852         }
3853
3854         drm_atomic_state_put(state);
3855 unlock:
3856         drm_modeset_drop_locks(ctx);
3857         drm_modeset_acquire_fini(ctx);
3858         mutex_unlock(&dev->mode_config.mutex);
3859
3860         clear_bit(I915_RESET_MODESET, &dev_priv->gpu_error.flags);
3861 }
3862
3863 static void intel_update_pipe_config(const struct intel_crtc_state *old_crtc_state,
3864                                      const struct intel_crtc_state *new_crtc_state)
3865 {
3866         struct intel_crtc *crtc = to_intel_crtc(new_crtc_state->base.crtc);
3867         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
3868
3869         /* drm_atomic_helper_update_legacy_modeset_state might not be called. */
3870         crtc->base.mode = new_crtc_state->base.mode;
3871
3872         /*
3873          * Update pipe size and adjust fitter if needed: the reason for this is
3874          * that in compute_mode_changes we check the native mode (not the pfit
3875          * mode) to see if we can flip rather than do a full mode set. In the
3876          * fastboot case, we'll flip, but if we don't update the pipesrc and
3877          * pfit state, we'll end up with a big fb scanned out into the wrong
3878          * sized surface.
3879          */
3880
3881         I915_WRITE(PIPESRC(crtc->pipe),
3882                    ((new_crtc_state->pipe_src_w - 1) << 16) |
3883                    (new_crtc_state->pipe_src_h - 1));
3884
3885         /* on skylake this is done by detaching scalers */
3886         if (INTEL_GEN(dev_priv) >= 9) {
3887                 skl_detach_scalers(new_crtc_state);
3888
3889                 if (new_crtc_state->pch_pfit.enabled)
3890                         skylake_pfit_enable(new_crtc_state);
3891         } else if (HAS_PCH_SPLIT(dev_priv)) {
3892                 if (new_crtc_state->pch_pfit.enabled)
3893                         ironlake_pfit_enable(new_crtc_state);
3894                 else if (old_crtc_state->pch_pfit.enabled)
3895                         ironlake_pfit_disable(old_crtc_state);
3896         }
3897 }
3898
3899 static void intel_fdi_normal_train(struct intel_crtc *crtc)
3900 {
3901         struct drm_device *dev = crtc->base.dev;
3902         struct drm_i915_private *dev_priv = to_i915(dev);
3903         int pipe = crtc->pipe;
3904         i915_reg_t reg;
3905         u32 temp;
3906
3907         /* enable normal train */
3908         reg = FDI_TX_CTL(pipe);
3909         temp = I915_READ(reg);
3910         if (IS_IVYBRIDGE(dev_priv)) {
3911                 temp &= ~FDI_LINK_TRAIN_NONE_IVB;
3912                 temp |= FDI_LINK_TRAIN_NONE_IVB | FDI_TX_ENHANCE_FRAME_ENABLE;
3913         } else {
3914                 temp &= ~FDI_LINK_TRAIN_NONE;
3915                 temp |= FDI_LINK_TRAIN_NONE | FDI_TX_ENHANCE_FRAME_ENABLE;
3916         }
3917         I915_WRITE(reg, temp);
3918
3919         reg = FDI_RX_CTL(pipe);
3920         temp = I915_READ(reg);
3921         if (HAS_PCH_CPT(dev_priv)) {
3922                 temp &= ~FDI_LINK_TRAIN_PATTERN_MASK_CPT;
3923                 temp |= FDI_LINK_TRAIN_NORMAL_CPT;
3924         } else {
3925                 temp &= ~FDI_LINK_TRAIN_NONE;
3926                 temp |= FDI_LINK_TRAIN_NONE;
3927         }
3928         I915_WRITE(reg, temp | FDI_RX_ENHANCE_FRAME_ENABLE);
3929
3930         /* wait one idle pattern time */
3931         POSTING_READ(reg);
3932         udelay(1000);
3933
3934         /* IVB wants error correction enabled */
3935         if (IS_IVYBRIDGE(dev_priv))
3936                 I915_WRITE(reg, I915_READ(reg) | FDI_FS_ERRC_ENABLE |
3937                            FDI_FE_ERRC_ENABLE);
3938 }
3939
3940 /* The FDI link training functions for ILK/Ibexpeak. */
3941 static void ironlake_fdi_link_train(struct intel_crtc *crtc,
3942                                     const struct intel_crtc_state *crtc_state)
3943 {
3944         struct drm_device *dev = crtc->base.dev;
3945         struct drm_i915_private *dev_priv = to_i915(dev);
3946         int pipe = crtc->pipe;
3947         i915_reg_t reg;
3948         u32 temp, tries;
3949
3950         /* FDI needs bits from pipe first */
3951         assert_pipe_enabled(dev_priv, pipe);
3952
3953         /* Train 1: umask FDI RX Interrupt symbol_lock and bit_lock bit
3954            for train result */
3955         reg = FDI_RX_IMR(pipe);
3956         temp = I915_READ(reg);
3957         temp &= ~FDI_RX_SYMBOL_LOCK;
3958         temp &= ~FDI_RX_BIT_LOCK;
3959         I915_WRITE(reg, temp);
3960         I915_READ(reg);
3961         udelay(150);
3962
3963         /* enable CPU FDI TX and PCH FDI RX */
3964         reg = FDI_TX_CTL(pipe);
3965         temp = I915_READ(reg);
3966         temp &= ~FDI_DP_PORT_WIDTH_MASK;
3967         temp |= FDI_DP_PORT_WIDTH(crtc_state->fdi_lanes);
3968         temp &= ~FDI_LINK_TRAIN_NONE;
3969         temp |= FDI_LINK_TRAIN_PATTERN_1;
3970         I915_WRITE(reg, temp | FDI_TX_ENABLE);
3971
3972         reg = FDI_RX_CTL(pipe);
3973         temp = I915_READ(reg);
3974         temp &= ~FDI_LINK_TRAIN_NONE;
3975         temp |= FDI_LINK_TRAIN_PATTERN_1;
3976         I915_WRITE(reg, temp | FDI_RX_ENABLE);
3977
3978         POSTING_READ(reg);
3979         udelay(150);
3980
3981         /* Ironlake workaround, enable clock pointer after FDI enable*/
3982         I915_WRITE(FDI_RX_CHICKEN(pipe), FDI_RX_PHASE_SYNC_POINTER_OVR);
3983         I915_WRITE(FDI_RX_CHICKEN(pipe), FDI_RX_PHASE_SYNC_POINTER_OVR |
3984                    FDI_RX_PHASE_SYNC_POINTER_EN);
3985
3986         reg = FDI_RX_IIR(pipe);
3987         for (tries = 0; tries < 5; tries++) {
3988                 temp = I915_READ(reg);
3989                 DRM_DEBUG_KMS("FDI_RX_IIR 0x%x\n", temp);
3990
3991                 if ((temp & FDI_RX_BIT_LOCK)) {
3992                         DRM_DEBUG_KMS("FDI train 1 done.\n");
3993                         I915_WRITE(reg, temp | FDI_RX_BIT_LOCK);
3994                         break;
3995                 }
3996         }
3997         if (tries == 5)
3998                 DRM_ERROR("FDI train 1 fail!\n");
3999
4000         /* Train 2 */
4001         reg = FDI_TX_CTL(pipe);
4002         temp = I915_READ(reg);
4003         temp &= ~FDI_LINK_TRAIN_NONE;
4004         temp |= FDI_LINK_TRAIN_PATTERN_2;
4005         I915_WRITE(reg, temp);
4006
4007         reg = FDI_RX_CTL(pipe);
4008         temp = I915_READ(reg);
4009         temp &= ~FDI_LINK_TRAIN_NONE;
4010         temp |= FDI_LINK_TRAIN_PATTERN_2;
4011         I915_WRITE(reg, temp);
4012
4013         POSTING_READ(reg);
4014         udelay(150);
4015
4016         reg = FDI_RX_IIR(pipe);
4017         for (tries = 0; tries < 5; tries++) {
4018                 temp = I915_READ(reg);
4019                 DRM_DEBUG_KMS("FDI_RX_IIR 0x%x\n", temp);
4020
4021                 if (temp & FDI_RX_SYMBOL_LOCK) {
4022                         I915_WRITE(reg, temp | FDI_RX_SYMBOL_LOCK);
4023                         DRM_DEBUG_KMS("FDI train 2 done.\n");
4024                         break;
4025                 }
4026         }
4027         if (tries == 5)
4028                 DRM_ERROR("FDI train 2 fail!\n");
4029
4030         DRM_DEBUG_KMS("FDI train done\n");
4031
4032 }
4033
4034 static const int snb_b_fdi_train_param[] = {
4035         FDI_LINK_TRAIN_400MV_0DB_SNB_B,
4036         FDI_LINK_TRAIN_400MV_6DB_SNB_B,
4037         FDI_LINK_TRAIN_600MV_3_5DB_SNB_B,
4038         FDI_LINK_TRAIN_800MV_0DB_SNB_B,
4039 };
4040
4041 /* The FDI link training functions for SNB/Cougarpoint. */
4042 static void gen6_fdi_link_train(struct intel_crtc *crtc,
4043                                 const struct intel_crtc_state *crtc_state)
4044 {
4045         struct drm_device *dev = crtc->base.dev;
4046         struct drm_i915_private *dev_priv = to_i915(dev);
4047         int pipe = crtc->pipe;
4048         i915_reg_t reg;
4049         u32 temp, i, retry;
4050
4051         /* Train 1: umask FDI RX Interrupt symbol_lock and bit_lock bit
4052            for train result */
4053         reg = FDI_RX_IMR(pipe);
4054         temp = I915_READ(reg);
4055         temp &= ~FDI_RX_SYMBOL_LOCK;
4056         temp &= ~FDI_RX_BIT_LOCK;
4057         I915_WRITE(reg, temp);
4058
4059         POSTING_READ(reg);
4060         udelay(150);
4061
4062         /* enable CPU FDI TX and PCH FDI RX */
4063         reg = FDI_TX_CTL(pipe);
4064         temp = I915_READ(reg);
4065         temp &= ~FDI_DP_PORT_WIDTH_MASK;
4066         temp |= FDI_DP_PORT_WIDTH(crtc_state->fdi_lanes);
4067         temp &= ~FDI_LINK_TRAIN_NONE;
4068         temp |= FDI_LINK_TRAIN_PATTERN_1;
4069         temp &= ~FDI_LINK_TRAIN_VOL_EMP_MASK;
4070         /* SNB-B */
4071         temp |= FDI_LINK_TRAIN_400MV_0DB_SNB_B;
4072         I915_WRITE(reg, temp | FDI_TX_ENABLE);
4073
4074         I915_WRITE(FDI_RX_MISC(pipe),
4075                    FDI_RX_TP1_TO_TP2_48 | FDI_RX_FDI_DELAY_90);
4076
4077         reg = FDI_RX_CTL(pipe);
4078         temp = I915_READ(reg);
4079         if (HAS_PCH_CPT(dev_priv)) {
4080                 temp &= ~FDI_LINK_TRAIN_PATTERN_MASK_CPT;
4081                 temp |= FDI_LINK_TRAIN_PATTERN_1_CPT;
4082         } else {
4083                 temp &= ~FDI_LINK_TRAIN_NONE;
4084                 temp |= FDI_LINK_TRAIN_PATTERN_1;
4085         }
4086         I915_WRITE(reg, temp | FDI_RX_ENABLE);
4087
4088         POSTING_READ(reg);
4089         udelay(150);
4090
4091         for (i = 0; i < 4; i++) {
4092                 reg = FDI_TX_CTL(pipe);
4093                 temp = I915_READ(reg);
4094                 temp &= ~FDI_LINK_TRAIN_VOL_EMP_MASK;
4095                 temp |= snb_b_fdi_train_param[i];
4096                 I915_WRITE(reg, temp);
4097
4098                 POSTING_READ(reg);
4099                 udelay(500);
4100
4101                 for (retry = 0; retry < 5; retry++) {
4102                         reg = FDI_RX_IIR(pipe);
4103                         temp = I915_READ(reg);
4104                         DRM_DEBUG_KMS("FDI_RX_IIR 0x%x\n", temp);
4105                         if (temp & FDI_RX_BIT_LOCK) {
4106                                 I915_WRITE(reg, temp | FDI_RX_BIT_LOCK);
4107                                 DRM_DEBUG_KMS("FDI train 1 done.\n");
4108                                 break;
4109                         }
4110                         udelay(50);
4111                 }
4112                 if (retry < 5)
4113                         break;
4114         }
4115         if (i == 4)
4116                 DRM_ERROR("FDI train 1 fail!\n");
4117
4118         /* Train 2 */
4119         reg = FDI_TX_CTL(pipe);
4120         temp = I915_READ(reg);
4121         temp &= ~FDI_LINK_TRAIN_NONE;
4122         temp |= FDI_LINK_TRAIN_PATTERN_2;
4123         if (IS_GEN(dev_priv, 6)) {
4124                 temp &= ~FDI_LINK_TRAIN_VOL_EMP_MASK;
4125                 /* SNB-B */
4126                 temp |= FDI_LINK_TRAIN_400MV_0DB_SNB_B;
4127         }
4128         I915_WRITE(reg, temp);
4129
4130         reg = FDI_RX_CTL(pipe);
4131         temp = I915_READ(reg);
4132         if (HAS_PCH_CPT(dev_priv)) {
4133                 temp &= ~FDI_LINK_TRAIN_PATTERN_MASK_CPT;
4134                 temp |= FDI_LINK_TRAIN_PATTERN_2_CPT;
4135         } else {
4136                 temp &= ~FDI_LINK_TRAIN_NONE;
4137                 temp |= FDI_LINK_TRAIN_PATTERN_2;
4138         }
4139         I915_WRITE(reg, temp);
4140
4141         POSTING_READ(reg);
4142         udelay(150);
4143
4144         for (i = 0; i < 4; i++) {
4145                 reg = FDI_TX_CTL(pipe);
4146                 temp = I915_READ(reg);
4147                 temp &= ~FDI_LINK_TRAIN_VOL_EMP_MASK;
4148                 temp |= snb_b_fdi_train_param[i];
4149                 I915_WRITE(reg, temp);
4150
4151                 POSTING_READ(reg);
4152                 udelay(500);
4153
4154                 for (retry = 0; retry < 5; retry++) {
4155                         reg = FDI_RX_IIR(pipe);
4156                         temp = I915_READ(reg);
4157                         DRM_DEBUG_KMS("FDI_RX_IIR 0x%x\n", temp);
4158                         if (temp & FDI_RX_SYMBOL_LOCK) {
4159                                 I915_WRITE(reg, temp | FDI_RX_SYMBOL_LOCK);
4160                                 DRM_DEBUG_KMS("FDI train 2 done.\n");
4161                                 break;
4162                         }
4163                         udelay(50);
4164                 }
4165                 if (retry < 5)
4166                         break;
4167         }
4168         if (i == 4)
4169                 DRM_ERROR("FDI train 2 fail!\n");
4170
4171         DRM_DEBUG_KMS("FDI train done.\n");
4172 }
4173
4174 /* Manual link training for Ivy Bridge A0 parts */
4175 static void ivb_manual_fdi_link_train(struct intel_crtc *crtc,
4176                                       const struct intel_crtc_state *crtc_state)
4177 {
4178         struct drm_device *dev = crtc->base.dev;
4179         struct drm_i915_private *dev_priv = to_i915(dev);
4180         int pipe = crtc->pipe;
4181         i915_reg_t reg;
4182         u32 temp, i, j;
4183
4184         /* Train 1: umask FDI RX Interrupt symbol_lock and bit_lock bit
4185            for train result */
4186         reg = FDI_RX_IMR(pipe);
4187         temp = I915_READ(reg);
4188         temp &= ~FDI_RX_SYMBOL_LOCK;
4189         temp &= ~FDI_RX_BIT_LOCK;
4190         I915_WRITE(reg, temp);
4191
4192         POSTING_READ(reg);
4193         udelay(150);
4194
4195         DRM_DEBUG_KMS("FDI_RX_IIR before link train 0x%x\n",
4196                       I915_READ(FDI_RX_IIR(pipe)));
4197
4198         /* Try each vswing and preemphasis setting twice before moving on */
4199         for (j = 0; j < ARRAY_SIZE(snb_b_fdi_train_param) * 2; j++) {
4200                 /* disable first in case we need to retry */
4201                 reg = FDI_TX_CTL(pipe);
4202                 temp = I915_READ(reg);
4203                 temp &= ~(FDI_LINK_TRAIN_AUTO | FDI_LINK_TRAIN_NONE_IVB);
4204                 temp &= ~FDI_TX_ENABLE;
4205                 I915_WRITE(reg, temp);
4206
4207                 reg = FDI_RX_CTL(pipe);
4208                 temp = I915_READ(reg);
4209                 temp &= ~FDI_LINK_TRAIN_AUTO;
4210                 temp &= ~FDI_LINK_TRAIN_PATTERN_MASK_CPT;
4211                 temp &= ~FDI_RX_ENABLE;
4212                 I915_WRITE(reg, temp);
4213
4214                 /* enable CPU FDI TX and PCH FDI RX */
4215                 reg = FDI_TX_CTL(pipe);
4216                 temp = I915_READ(reg);
4217                 temp &= ~FDI_DP_PORT_WIDTH_MASK;
4218                 temp |= FDI_DP_PORT_WIDTH(crtc_state->fdi_lanes);
4219                 temp |= FDI_LINK_TRAIN_PATTERN_1_IVB;
4220                 temp &= ~FDI_LINK_TRAIN_VOL_EMP_MASK;
4221                 temp |= snb_b_fdi_train_param[j/2];
4222                 temp |= FDI_COMPOSITE_SYNC;
4223                 I915_WRITE(reg, temp | FDI_TX_ENABLE);
4224
4225                 I915_WRITE(FDI_RX_MISC(pipe),
4226                            FDI_RX_TP1_TO_TP2_48 | FDI_RX_FDI_DELAY_90);
4227
4228                 reg = FDI_RX_CTL(pipe);
4229                 temp = I915_READ(reg);
4230                 temp |= FDI_LINK_TRAIN_PATTERN_1_CPT;
4231                 temp |= FDI_COMPOSITE_SYNC;
4232                 I915_WRITE(reg, temp | FDI_RX_ENABLE);
4233
4234                 POSTING_READ(reg);
4235                 udelay(1); /* should be 0.5us */
4236
4237                 for (i = 0; i < 4; i++) {
4238                         reg = FDI_RX_IIR(pipe);
4239                         temp = I915_READ(reg);
4240                         DRM_DEBUG_KMS("FDI_RX_IIR 0x%x\n", temp);
4241
4242                         if (temp & FDI_RX_BIT_LOCK ||
4243                             (I915_READ(reg) & FDI_RX_BIT_LOCK)) {
4244                                 I915_WRITE(reg, temp | FDI_RX_BIT_LOCK);
4245                                 DRM_DEBUG_KMS("FDI train 1 done, level %i.\n",
4246                                               i);
4247                                 break;
4248                         }
4249                         udelay(1); /* should be 0.5us */
4250                 }
4251                 if (i == 4) {
4252                         DRM_DEBUG_KMS("FDI train 1 fail on vswing %d\n", j / 2);
4253                         continue;
4254                 }
4255
4256                 /* Train 2 */
4257                 reg = FDI_TX_CTL(pipe);
4258                 temp = I915_READ(reg);
4259                 temp &= ~FDI_LINK_TRAIN_NONE_IVB;
4260                 temp |= FDI_LINK_TRAIN_PATTERN_2_IVB;
4261                 I915_WRITE(reg, temp);
4262
4263                 reg = FDI_RX_CTL(pipe);
4264                 temp = I915_READ(reg);
4265                 temp &= ~FDI_LINK_TRAIN_PATTERN_MASK_CPT;
4266                 temp |= FDI_LINK_TRAIN_PATTERN_2_CPT;
4267                 I915_WRITE(reg, temp);
4268
4269                 POSTING_READ(reg);
4270                 udelay(2); /* should be 1.5us */
4271
4272                 for (i = 0; i < 4; i++) {
4273                         reg = FDI_RX_IIR(pipe);
4274                         temp = I915_READ(reg);
4275                         DRM_DEBUG_KMS("FDI_RX_IIR 0x%x\n", temp);
4276
4277                         if (temp & FDI_RX_SYMBOL_LOCK ||
4278                             (I915_READ(reg) & FDI_RX_SYMBOL_LOCK)) {
4279                                 I915_WRITE(reg, temp | FDI_RX_SYMBOL_LOCK);
4280                                 DRM_DEBUG_KMS("FDI train 2 done, level %i.\n",
4281                                               i);
4282                                 goto train_done;
4283                         }
4284                         udelay(2); /* should be 1.5us */
4285                 }
4286                 if (i == 4)
4287                         DRM_DEBUG_KMS("FDI train 2 fail on vswing %d\n", j / 2);
4288         }
4289
4290 train_done:
4291         DRM_DEBUG_KMS("FDI train done.\n");
4292 }
4293
4294 static void ironlake_fdi_pll_enable(const struct intel_crtc_state *crtc_state)
4295 {
4296         struct intel_crtc *intel_crtc = to_intel_crtc(crtc_state->base.crtc);
4297         struct drm_i915_private *dev_priv = to_i915(intel_crtc->base.dev);
4298         int pipe = intel_crtc->pipe;
4299         i915_reg_t reg;
4300         u32 temp;
4301
4302         /* enable PCH FDI RX PLL, wait warmup plus DMI latency */
4303         reg = FDI_RX_CTL(pipe);
4304         temp = I915_READ(reg);
4305         temp &= ~(FDI_DP_PORT_WIDTH_MASK | (0x7 << 16));
4306         temp |= FDI_DP_PORT_WIDTH(crtc_state->fdi_lanes);
4307         temp |= (I915_READ(PIPECONF(pipe)) & PIPECONF_BPC_MASK) << 11;
4308         I915_WRITE(reg, temp | FDI_RX_PLL_ENABLE);
4309
4310         POSTING_READ(reg);
4311         udelay(200);
4312
4313         /* Switch from Rawclk to PCDclk */
4314         temp = I915_READ(reg);
4315         I915_WRITE(reg, temp | FDI_PCDCLK);
4316
4317         POSTING_READ(reg);
4318         udelay(200);
4319
4320         /* Enable CPU FDI TX PLL, always on for Ironlake */
4321         reg = FDI_TX_CTL(pipe);
4322         temp = I915_READ(reg);
4323         if ((temp & FDI_TX_PLL_ENABLE) == 0) {
4324                 I915_WRITE(reg, temp | FDI_TX_PLL_ENABLE);
4325
4326                 POSTING_READ(reg);
4327                 udelay(100);
4328         }
4329 }
4330
4331 static void ironlake_fdi_pll_disable(struct intel_crtc *intel_crtc)
4332 {
4333         struct drm_device *dev = intel_crtc->base.dev;
4334         struct drm_i915_private *dev_priv = to_i915(dev);
4335         int pipe = intel_crtc->pipe;
4336         i915_reg_t reg;
4337         u32 temp;
4338
4339         /* Switch from PCDclk to Rawclk */
4340         reg = FDI_RX_CTL(pipe);
4341         temp = I915_READ(reg);
4342         I915_WRITE(reg, temp & ~FDI_PCDCLK);
4343
4344         /* Disable CPU FDI TX PLL */
4345         reg = FDI_TX_CTL(pipe);
4346         temp = I915_READ(reg);
4347         I915_WRITE(reg, temp & ~FDI_TX_PLL_ENABLE);
4348
4349         POSTING_READ(reg);
4350         udelay(100);
4351
4352         reg = FDI_RX_CTL(pipe);
4353         temp = I915_READ(reg);
4354         I915_WRITE(reg, temp & ~FDI_RX_PLL_ENABLE);
4355
4356         /* Wait for the clocks to turn off. */
4357         POSTING_READ(reg);
4358         udelay(100);
4359 }
4360
4361 static void ironlake_fdi_disable(struct drm_crtc *crtc)
4362 {
4363         struct drm_device *dev = crtc->dev;
4364         struct drm_i915_private *dev_priv = to_i915(dev);
4365         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
4366         int pipe = intel_crtc->pipe;
4367         i915_reg_t reg;
4368         u32 temp;
4369
4370         /* disable CPU FDI tx and PCH FDI rx */
4371         reg = FDI_TX_CTL(pipe);
4372         temp = I915_READ(reg);
4373         I915_WRITE(reg, temp & ~FDI_TX_ENABLE);
4374         POSTING_READ(reg);
4375
4376         reg = FDI_RX_CTL(pipe);
4377         temp = I915_READ(reg);
4378         temp &= ~(0x7 << 16);
4379         temp |= (I915_READ(PIPECONF(pipe)) & PIPECONF_BPC_MASK) << 11;
4380         I915_WRITE(reg, temp & ~FDI_RX_ENABLE);
4381
4382         POSTING_READ(reg);
4383         udelay(100);
4384
4385         /* Ironlake workaround, disable clock pointer after downing FDI */
4386         if (HAS_PCH_IBX(dev_priv))
4387                 I915_WRITE(FDI_RX_CHICKEN(pipe), FDI_RX_PHASE_SYNC_POINTER_OVR);
4388
4389         /* still set train pattern 1 */
4390         reg = FDI_TX_CTL(pipe);
4391         temp = I915_READ(reg);
4392         temp &= ~FDI_LINK_TRAIN_NONE;
4393         temp |= FDI_LINK_TRAIN_PATTERN_1;
4394         I915_WRITE(reg, temp);
4395
4396         reg = FDI_RX_CTL(pipe);
4397         temp = I915_READ(reg);
4398         if (HAS_PCH_CPT(dev_priv)) {
4399                 temp &= ~FDI_LINK_TRAIN_PATTERN_MASK_CPT;
4400                 temp |= FDI_LINK_TRAIN_PATTERN_1_CPT;
4401         } else {
4402                 temp &= ~FDI_LINK_TRAIN_NONE;
4403                 temp |= FDI_LINK_TRAIN_PATTERN_1;
4404         }
4405         /* BPC in FDI rx is consistent with that in PIPECONF */
4406         temp &= ~(0x07 << 16);
4407         temp |= (I915_READ(PIPECONF(pipe)) & PIPECONF_BPC_MASK) << 11;
4408         I915_WRITE(reg, temp);
4409
4410         POSTING_READ(reg);
4411         udelay(100);
4412 }
4413
4414 bool intel_has_pending_fb_unpin(struct drm_i915_private *dev_priv)
4415 {
4416         struct drm_crtc *crtc;
4417         bool cleanup_done;
4418
4419         drm_for_each_crtc(crtc, &dev_priv->drm) {
4420                 struct drm_crtc_commit *commit;
4421                 spin_lock(&crtc->commit_lock);
4422                 commit = list_first_entry_or_null(&crtc->commit_list,
4423                                                   struct drm_crtc_commit, commit_entry);
4424                 cleanup_done = commit ?
4425                         try_wait_for_completion(&commit->cleanup_done) : true;
4426                 spin_unlock(&crtc->commit_lock);
4427
4428                 if (cleanup_done)
4429                         continue;
4430
4431                 drm_crtc_wait_one_vblank(crtc);
4432
4433                 return true;
4434         }
4435
4436         return false;
4437 }
4438
4439 void lpt_disable_iclkip(struct drm_i915_private *dev_priv)
4440 {
4441         u32 temp;
4442
4443         I915_WRITE(PIXCLK_GATE, PIXCLK_GATE_GATE);
4444
4445         mutex_lock(&dev_priv->sb_lock);
4446
4447         temp = intel_sbi_read(dev_priv, SBI_SSCCTL6, SBI_ICLK);
4448         temp |= SBI_SSCCTL_DISABLE;
4449         intel_sbi_write(dev_priv, SBI_SSCCTL6, temp, SBI_ICLK);
4450
4451         mutex_unlock(&dev_priv->sb_lock);
4452 }
4453
4454 /* Program iCLKIP clock to the desired frequency */
4455 static void lpt_program_iclkip(const struct intel_crtc_state *crtc_state)
4456 {
4457         struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
4458         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
4459         int clock = crtc_state->base.adjusted_mode.crtc_clock;
4460         u32 divsel, phaseinc, auxdiv, phasedir = 0;
4461         u32 temp;
4462
4463         lpt_disable_iclkip(dev_priv);
4464
4465         /* The iCLK virtual clock root frequency is in MHz,
4466          * but the adjusted_mode->crtc_clock in in KHz. To get the
4467          * divisors, it is necessary to divide one by another, so we
4468          * convert the virtual clock precision to KHz here for higher
4469          * precision.
4470          */
4471         for (auxdiv = 0; auxdiv < 2; auxdiv++) {
4472                 u32 iclk_virtual_root_freq = 172800 * 1000;
4473                 u32 iclk_pi_range = 64;
4474                 u32 desired_divisor;
4475
4476                 desired_divisor = DIV_ROUND_CLOSEST(iclk_virtual_root_freq,
4477                                                     clock << auxdiv);
4478                 divsel = (desired_divisor / iclk_pi_range) - 2;
4479                 phaseinc = desired_divisor % iclk_pi_range;
4480
4481                 /*
4482                  * Near 20MHz is a corner case which is
4483                  * out of range for the 7-bit divisor
4484                  */
4485                 if (divsel <= 0x7f)
4486                         break;
4487         }
4488
4489         /* This should not happen with any sane values */
4490         WARN_ON(SBI_SSCDIVINTPHASE_DIVSEL(divsel) &
4491                 ~SBI_SSCDIVINTPHASE_DIVSEL_MASK);
4492         WARN_ON(SBI_SSCDIVINTPHASE_DIR(phasedir) &
4493                 ~SBI_SSCDIVINTPHASE_INCVAL_MASK);
4494
4495         DRM_DEBUG_KMS("iCLKIP clock: found settings for %dKHz refresh rate: auxdiv=%x, divsel=%x, phasedir=%x, phaseinc=%x\n",
4496                         clock,
4497                         auxdiv,
4498                         divsel,
4499                         phasedir,
4500                         phaseinc);
4501
4502         mutex_lock(&dev_priv->sb_lock);
4503
4504         /* Program SSCDIVINTPHASE6 */
4505         temp = intel_sbi_read(dev_priv, SBI_SSCDIVINTPHASE6, SBI_ICLK);
4506         temp &= ~SBI_SSCDIVINTPHASE_DIVSEL_MASK;
4507         temp |= SBI_SSCDIVINTPHASE_DIVSEL(divsel);
4508         temp &= ~SBI_SSCDIVINTPHASE_INCVAL_MASK;
4509         temp |= SBI_SSCDIVINTPHASE_INCVAL(phaseinc);
4510         temp |= SBI_SSCDIVINTPHASE_DIR(phasedir);
4511         temp |= SBI_SSCDIVINTPHASE_PROPAGATE;
4512         intel_sbi_write(dev_priv, SBI_SSCDIVINTPHASE6, temp, SBI_ICLK);
4513
4514         /* Program SSCAUXDIV */
4515         temp = intel_sbi_read(dev_priv, SBI_SSCAUXDIV6, SBI_ICLK);
4516         temp &= ~SBI_SSCAUXDIV_FINALDIV2SEL(1);
4517         temp |= SBI_SSCAUXDIV_FINALDIV2SEL(auxdiv);
4518         intel_sbi_write(dev_priv, SBI_SSCAUXDIV6, temp, SBI_ICLK);
4519
4520         /* Enable modulator and associated divider */
4521         temp = intel_sbi_read(dev_priv, SBI_SSCCTL6, SBI_ICLK);
4522         temp &= ~SBI_SSCCTL_DISABLE;
4523         intel_sbi_write(dev_priv, SBI_SSCCTL6, temp, SBI_ICLK);
4524
4525         mutex_unlock(&dev_priv->sb_lock);
4526
4527         /* Wait for initialization time */
4528         udelay(24);
4529
4530         I915_WRITE(PIXCLK_GATE, PIXCLK_GATE_UNGATE);
4531 }
4532
4533 int lpt_get_iclkip(struct drm_i915_private *dev_priv)
4534 {
4535         u32 divsel, phaseinc, auxdiv;
4536         u32 iclk_virtual_root_freq = 172800 * 1000;
4537         u32 iclk_pi_range = 64;
4538         u32 desired_divisor;
4539         u32 temp;
4540
4541         if ((I915_READ(PIXCLK_GATE) & PIXCLK_GATE_UNGATE) == 0)
4542                 return 0;
4543
4544         mutex_lock(&dev_priv->sb_lock);
4545
4546         temp = intel_sbi_read(dev_priv, SBI_SSCCTL6, SBI_ICLK);
4547         if (temp & SBI_SSCCTL_DISABLE) {
4548                 mutex_unlock(&dev_priv->sb_lock);
4549                 return 0;
4550         }
4551
4552         temp = intel_sbi_read(dev_priv, SBI_SSCDIVINTPHASE6, SBI_ICLK);
4553         divsel = (temp & SBI_SSCDIVINTPHASE_DIVSEL_MASK) >>
4554                 SBI_SSCDIVINTPHASE_DIVSEL_SHIFT;
4555         phaseinc = (temp & SBI_SSCDIVINTPHASE_INCVAL_MASK) >>
4556                 SBI_SSCDIVINTPHASE_INCVAL_SHIFT;
4557
4558         temp = intel_sbi_read(dev_priv, SBI_SSCAUXDIV6, SBI_ICLK);
4559         auxdiv = (temp & SBI_SSCAUXDIV_FINALDIV2SEL_MASK) >>
4560                 SBI_SSCAUXDIV_FINALDIV2SEL_SHIFT;
4561
4562         mutex_unlock(&dev_priv->sb_lock);
4563
4564         desired_divisor = (divsel + 2) * iclk_pi_range + phaseinc;
4565
4566         return DIV_ROUND_CLOSEST(iclk_virtual_root_freq,
4567                                  desired_divisor << auxdiv);
4568 }
4569
4570 static void ironlake_pch_transcoder_set_timings(const struct intel_crtc_state *crtc_state,
4571                                                 enum pipe pch_transcoder)
4572 {
4573         struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
4574         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
4575         enum transcoder cpu_transcoder = crtc_state->cpu_transcoder;
4576
4577         I915_WRITE(PCH_TRANS_HTOTAL(pch_transcoder),
4578                    I915_READ(HTOTAL(cpu_transcoder)));
4579         I915_WRITE(PCH_TRANS_HBLANK(pch_transcoder),
4580                    I915_READ(HBLANK(cpu_transcoder)));
4581         I915_WRITE(PCH_TRANS_HSYNC(pch_transcoder),
4582                    I915_READ(HSYNC(cpu_transcoder)));
4583
4584         I915_WRITE(PCH_TRANS_VTOTAL(pch_transcoder),
4585                    I915_READ(VTOTAL(cpu_transcoder)));
4586         I915_WRITE(PCH_TRANS_VBLANK(pch_transcoder),
4587                    I915_READ(VBLANK(cpu_transcoder)));
4588         I915_WRITE(PCH_TRANS_VSYNC(pch_transcoder),
4589                    I915_READ(VSYNC(cpu_transcoder)));
4590         I915_WRITE(PCH_TRANS_VSYNCSHIFT(pch_transcoder),
4591                    I915_READ(VSYNCSHIFT(cpu_transcoder)));
4592 }
4593
4594 static void cpt_set_fdi_bc_bifurcation(struct drm_i915_private *dev_priv, bool enable)
4595 {
4596         uint32_t temp;
4597
4598         temp = I915_READ(SOUTH_CHICKEN1);
4599         if (!!(temp & FDI_BC_BIFURCATION_SELECT) == enable)
4600                 return;
4601
4602         WARN_ON(I915_READ(FDI_RX_CTL(PIPE_B)) & FDI_RX_ENABLE);
4603         WARN_ON(I915_READ(FDI_RX_CTL(PIPE_C)) & FDI_RX_ENABLE);
4604
4605         temp &= ~FDI_BC_BIFURCATION_SELECT;
4606         if (enable)
4607                 temp |= FDI_BC_BIFURCATION_SELECT;
4608
4609         DRM_DEBUG_KMS("%sabling fdi C rx\n", enable ? "en" : "dis");
4610         I915_WRITE(SOUTH_CHICKEN1, temp);
4611         POSTING_READ(SOUTH_CHICKEN1);
4612 }
4613
4614 static void ivybridge_update_fdi_bc_bifurcation(const struct intel_crtc_state *crtc_state)
4615 {
4616         struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
4617         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
4618
4619         switch (crtc->pipe) {
4620         case PIPE_A:
4621                 break;
4622         case PIPE_B:
4623                 if (crtc_state->fdi_lanes > 2)
4624                         cpt_set_fdi_bc_bifurcation(dev_priv, false);
4625                 else
4626                         cpt_set_fdi_bc_bifurcation(dev_priv, true);
4627
4628                 break;
4629         case PIPE_C:
4630                 cpt_set_fdi_bc_bifurcation(dev_priv, true);
4631
4632                 break;
4633         default:
4634                 BUG();
4635         }
4636 }
4637
4638 /*
4639  * Finds the encoder associated with the given CRTC. This can only be
4640  * used when we know that the CRTC isn't feeding multiple encoders!
4641  */
4642 static struct intel_encoder *
4643 intel_get_crtc_new_encoder(const struct intel_atomic_state *state,
4644                            const struct intel_crtc_state *crtc_state)
4645 {
4646         struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
4647         const struct drm_connector_state *connector_state;
4648         const struct drm_connector *connector;
4649         struct intel_encoder *encoder = NULL;
4650         int num_encoders = 0;
4651         int i;
4652
4653         for_each_new_connector_in_state(&state->base, connector, connector_state, i) {
4654                 if (connector_state->crtc != &crtc->base)
4655                         continue;
4656
4657                 encoder = to_intel_encoder(connector_state->best_encoder);
4658                 num_encoders++;
4659         }
4660
4661         WARN(num_encoders != 1, "%d encoders for pipe %c\n",
4662              num_encoders, pipe_name(crtc->pipe));
4663
4664         return encoder;
4665 }
4666
4667 /*
4668  * Enable PCH resources required for PCH ports:
4669  *   - PCH PLLs
4670  *   - FDI training & RX/TX
4671  *   - update transcoder timings
4672  *   - DP transcoding bits
4673  *   - transcoder
4674  */
4675 static void ironlake_pch_enable(const struct intel_atomic_state *state,
4676                                 const struct intel_crtc_state *crtc_state)
4677 {
4678         struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
4679         struct drm_device *dev = crtc->base.dev;
4680         struct drm_i915_private *dev_priv = to_i915(dev);
4681         int pipe = crtc->pipe;
4682         u32 temp;
4683
4684         assert_pch_transcoder_disabled(dev_priv, pipe);
4685
4686         if (IS_IVYBRIDGE(dev_priv))
4687                 ivybridge_update_fdi_bc_bifurcation(crtc_state);
4688
4689         /* Write the TU size bits before fdi link training, so that error
4690          * detection works. */
4691         I915_WRITE(FDI_RX_TUSIZE1(pipe),
4692                    I915_READ(PIPE_DATA_M1(pipe)) & TU_SIZE_MASK);
4693
4694         /* For PCH output, training FDI link */
4695         dev_priv->display.fdi_link_train(crtc, crtc_state);
4696
4697         /* We need to program the right clock selection before writing the pixel
4698          * mutliplier into the DPLL. */
4699         if (HAS_PCH_CPT(dev_priv)) {
4700                 u32 sel;
4701
4702                 temp = I915_READ(PCH_DPLL_SEL);
4703                 temp |= TRANS_DPLL_ENABLE(pipe);
4704                 sel = TRANS_DPLLB_SEL(pipe);
4705                 if (crtc_state->shared_dpll ==
4706                     intel_get_shared_dpll_by_id(dev_priv, DPLL_ID_PCH_PLL_B))
4707                         temp |= sel;
4708                 else
4709                         temp &= ~sel;
4710                 I915_WRITE(PCH_DPLL_SEL, temp);
4711         }
4712
4713         /* XXX: pch pll's can be enabled any time before we enable the PCH
4714          * transcoder, and we actually should do this to not upset any PCH
4715          * transcoder that already use the clock when we share it.
4716          *
4717          * Note that enable_shared_dpll tries to do the right thing, but
4718          * get_shared_dpll unconditionally resets the pll - we need that to have
4719          * the right LVDS enable sequence. */
4720         intel_enable_shared_dpll(crtc_state);
4721
4722         /* set transcoder timing, panel must allow it */
4723         assert_panel_unlocked(dev_priv, pipe);
4724         ironlake_pch_transcoder_set_timings(crtc_state, pipe);
4725
4726         intel_fdi_normal_train(crtc);
4727
4728         /* For PCH DP, enable TRANS_DP_CTL */
4729         if (HAS_PCH_CPT(dev_priv) &&
4730             intel_crtc_has_dp_encoder(crtc_state)) {
4731                 const struct drm_display_mode *adjusted_mode =
4732                         &crtc_state->base.adjusted_mode;
4733                 u32 bpc = (I915_READ(PIPECONF(pipe)) & PIPECONF_BPC_MASK) >> 5;
4734                 i915_reg_t reg = TRANS_DP_CTL(pipe);
4735                 enum port port;
4736
4737                 temp = I915_READ(reg);
4738                 temp &= ~(TRANS_DP_PORT_SEL_MASK |
4739                           TRANS_DP_SYNC_MASK |
4740                           TRANS_DP_BPC_MASK);
4741                 temp |= TRANS_DP_OUTPUT_ENABLE;
4742                 temp |= bpc << 9; /* same format but at 11:9 */
4743
4744                 if (adjusted_mode->flags & DRM_MODE_FLAG_PHSYNC)
4745                         temp |= TRANS_DP_HSYNC_ACTIVE_HIGH;
4746                 if (adjusted_mode->flags & DRM_MODE_FLAG_PVSYNC)
4747                         temp |= TRANS_DP_VSYNC_ACTIVE_HIGH;
4748
4749                 port = intel_get_crtc_new_encoder(state, crtc_state)->port;
4750                 WARN_ON(port < PORT_B || port > PORT_D);
4751                 temp |= TRANS_DP_PORT_SEL(port);
4752
4753                 I915_WRITE(reg, temp);
4754         }
4755
4756         ironlake_enable_pch_transcoder(crtc_state);
4757 }
4758
4759 static void lpt_pch_enable(const struct intel_atomic_state *state,
4760                            const struct intel_crtc_state *crtc_state)
4761 {
4762         struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
4763         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
4764         enum transcoder cpu_transcoder = crtc_state->cpu_transcoder;
4765
4766         assert_pch_transcoder_disabled(dev_priv, PIPE_A);
4767
4768         lpt_program_iclkip(crtc_state);
4769
4770         /* Set transcoder timing. */
4771         ironlake_pch_transcoder_set_timings(crtc_state, PIPE_A);
4772
4773         lpt_enable_pch_transcoder(dev_priv, cpu_transcoder);
4774 }
4775
4776 static void cpt_verify_modeset(struct drm_device *dev, int pipe)
4777 {
4778         struct drm_i915_private *dev_priv = to_i915(dev);
4779         i915_reg_t dslreg = PIPEDSL(pipe);
4780         u32 temp;
4781
4782         temp = I915_READ(dslreg);
4783         udelay(500);
4784         if (wait_for(I915_READ(dslreg) != temp, 5)) {
4785                 if (wait_for(I915_READ(dslreg) != temp, 5))
4786                         DRM_ERROR("mode set failed: pipe %c stuck\n", pipe_name(pipe));
4787         }
4788 }
4789
4790 /*
4791  * The hardware phase 0.0 refers to the center of the pixel.
4792  * We want to start from the top/left edge which is phase
4793  * -0.5. That matches how the hardware calculates the scaling
4794  * factors (from top-left of the first pixel to bottom-right
4795  * of the last pixel, as opposed to the pixel centers).
4796  *
4797  * For 4:2:0 subsampled chroma planes we obviously have to
4798  * adjust that so that the chroma sample position lands in
4799  * the right spot.
4800  *
4801  * Note that for packed YCbCr 4:2:2 formats there is no way to
4802  * control chroma siting. The hardware simply replicates the
4803  * chroma samples for both of the luma samples, and thus we don't
4804  * actually get the expected MPEG2 chroma siting convention :(
4805  * The same behaviour is observed on pre-SKL platforms as well.
4806  *
4807  * Theory behind the formula (note that we ignore sub-pixel
4808  * source coordinates):
4809  * s = source sample position
4810  * d = destination sample position
4811  *
4812  * Downscaling 4:1:
4813  * -0.5
4814  * | 0.0
4815  * | |     1.5 (initial phase)
4816  * | |     |
4817  * v v     v
4818  * | s | s | s | s |
4819  * |       d       |
4820  *
4821  * Upscaling 1:4:
4822  * -0.5
4823  * | -0.375 (initial phase)
4824  * | |     0.0
4825  * | |     |
4826  * v v     v
4827  * |       s       |
4828  * | d | d | d | d |
4829  */
4830 u16 skl_scaler_calc_phase(int sub, int scale, bool chroma_cosited)
4831 {
4832         int phase = -0x8000;
4833         u16 trip = 0;
4834
4835         if (chroma_cosited)
4836                 phase += (sub - 1) * 0x8000 / sub;
4837
4838         phase += scale / (2 * sub);
4839
4840         /*
4841          * Hardware initial phase limited to [-0.5:1.5].
4842          * Since the max hardware scale factor is 3.0, we
4843          * should never actually excdeed 1.0 here.
4844          */
4845         WARN_ON(phase < -0x8000 || phase > 0x18000);
4846
4847         if (phase < 0)
4848                 phase = 0x10000 + phase;
4849         else
4850                 trip = PS_PHASE_TRIP;
4851
4852         return ((phase >> 2) & PS_PHASE_MASK) | trip;
4853 }
4854
4855 static int
4856 skl_update_scaler(struct intel_crtc_state *crtc_state, bool force_detach,
4857                   unsigned int scaler_user, int *scaler_id,
4858                   int src_w, int src_h, int dst_w, int dst_h,
4859                   const struct drm_format_info *format, bool need_scaler)
4860 {
4861         struct intel_crtc_scaler_state *scaler_state =
4862                 &crtc_state->scaler_state;
4863         struct intel_crtc *intel_crtc =
4864                 to_intel_crtc(crtc_state->base.crtc);
4865         struct drm_i915_private *dev_priv = to_i915(intel_crtc->base.dev);
4866         const struct drm_display_mode *adjusted_mode =
4867                 &crtc_state->base.adjusted_mode;
4868
4869         /*
4870          * Src coordinates are already rotated by 270 degrees for
4871          * the 90/270 degree plane rotation cases (to match the
4872          * GTT mapping), hence no need to account for rotation here.
4873          */
4874         if (src_w != dst_w || src_h != dst_h)
4875                 need_scaler = true;
4876
4877         /*
4878          * Scaling/fitting not supported in IF-ID mode in GEN9+
4879          * TODO: Interlace fetch mode doesn't support YUV420 planar formats.
4880          * Once NV12 is enabled, handle it here while allocating scaler
4881          * for NV12.
4882          */
4883         if (INTEL_GEN(dev_priv) >= 9 && crtc_state->base.enable &&
4884             need_scaler && adjusted_mode->flags & DRM_MODE_FLAG_INTERLACE) {
4885                 DRM_DEBUG_KMS("Pipe/Plane scaling not supported with IF-ID mode\n");
4886                 return -EINVAL;
4887         }
4888
4889         /*
4890          * if plane is being disabled or scaler is no more required or force detach
4891          *  - free scaler binded to this plane/crtc
4892          *  - in order to do this, update crtc->scaler_usage
4893          *
4894          * Here scaler state in crtc_state is set free so that
4895          * scaler can be assigned to other user. Actual register
4896          * update to free the scaler is done in plane/panel-fit programming.
4897          * For this purpose crtc/plane_state->scaler_id isn't reset here.
4898          */
4899         if (force_detach || !need_scaler) {
4900                 if (*scaler_id >= 0) {
4901                         scaler_state->scaler_users &= ~(1 << scaler_user);
4902                         scaler_state->scalers[*scaler_id].in_use = 0;
4903
4904                         DRM_DEBUG_KMS("scaler_user index %u.%u: "
4905                                 "Staged freeing scaler id %d scaler_users = 0x%x\n",
4906                                 intel_crtc->pipe, scaler_user, *scaler_id,
4907                                 scaler_state->scaler_users);
4908                         *scaler_id = -1;
4909                 }
4910                 return 0;
4911         }
4912
4913         if (format && format->format == DRM_FORMAT_NV12 &&
4914             (src_h < SKL_MIN_YUV_420_SRC_H || src_w < SKL_MIN_YUV_420_SRC_W)) {
4915                 DRM_DEBUG_KMS("NV12: src dimensions not met\n");
4916                 return -EINVAL;
4917         }
4918
4919         /* range checks */
4920         if (src_w < SKL_MIN_SRC_W || src_h < SKL_MIN_SRC_H ||
4921             dst_w < SKL_MIN_DST_W || dst_h < SKL_MIN_DST_H ||
4922             (IS_GEN(dev_priv, 11) &&
4923              (src_w > ICL_MAX_SRC_W || src_h > ICL_MAX_SRC_H ||
4924               dst_w > ICL_MAX_DST_W || dst_h > ICL_MAX_DST_H)) ||
4925             (!IS_GEN(dev_priv, 11) &&
4926              (src_w > SKL_MAX_SRC_W || src_h > SKL_MAX_SRC_H ||
4927               dst_w > SKL_MAX_DST_W || dst_h > SKL_MAX_DST_H))) {
4928                 DRM_DEBUG_KMS("scaler_user index %u.%u: src %ux%u dst %ux%u "
4929                         "size is out of scaler range\n",
4930                         intel_crtc->pipe, scaler_user, src_w, src_h, dst_w, dst_h);
4931                 return -EINVAL;
4932         }
4933
4934         /* mark this plane as a scaler user in crtc_state */
4935         scaler_state->scaler_users |= (1 << scaler_user);
4936         DRM_DEBUG_KMS("scaler_user index %u.%u: "
4937                 "staged scaling request for %ux%u->%ux%u scaler_users = 0x%x\n",
4938                 intel_crtc->pipe, scaler_user, src_w, src_h, dst_w, dst_h,
4939                 scaler_state->scaler_users);
4940
4941         return 0;
4942 }
4943
4944 /**
4945  * skl_update_scaler_crtc - Stages update to scaler state for a given crtc.
4946  *
4947  * @state: crtc's scaler state
4948  *
4949  * Return
4950  *     0 - scaler_usage updated successfully
4951  *    error - requested scaling cannot be supported or other error condition
4952  */
4953 int skl_update_scaler_crtc(struct intel_crtc_state *state)
4954 {
4955         const struct drm_display_mode *adjusted_mode = &state->base.adjusted_mode;
4956         bool need_scaler = false;
4957
4958         if (state->output_format == INTEL_OUTPUT_FORMAT_YCBCR420)
4959                 need_scaler = true;
4960
4961         return skl_update_scaler(state, !state->base.active, SKL_CRTC_INDEX,
4962                                  &state->scaler_state.scaler_id,
4963                                  state->pipe_src_w, state->pipe_src_h,
4964                                  adjusted_mode->crtc_hdisplay,
4965                                  adjusted_mode->crtc_vdisplay, NULL, need_scaler);
4966 }
4967
4968 /**
4969  * skl_update_scaler_plane - Stages update to scaler state for a given plane.
4970  * @crtc_state: crtc's scaler state
4971  * @plane_state: atomic plane state to update
4972  *
4973  * Return
4974  *     0 - scaler_usage updated successfully
4975  *    error - requested scaling cannot be supported or other error condition
4976  */
4977 static int skl_update_scaler_plane(struct intel_crtc_state *crtc_state,
4978                                    struct intel_plane_state *plane_state)
4979 {
4980         struct intel_plane *intel_plane =
4981                 to_intel_plane(plane_state->base.plane);
4982         struct drm_framebuffer *fb = plane_state->base.fb;
4983         int ret;
4984         bool force_detach = !fb || !plane_state->base.visible;
4985         bool need_scaler = false;
4986
4987         /* Pre-gen11 and SDR planes always need a scaler for planar formats. */
4988         if (!icl_is_hdr_plane(intel_plane) &&
4989             fb && fb->format->format == DRM_FORMAT_NV12)
4990                 need_scaler = true;
4991
4992         ret = skl_update_scaler(crtc_state, force_detach,
4993                                 drm_plane_index(&intel_plane->base),
4994                                 &plane_state->scaler_id,
4995                                 drm_rect_width(&plane_state->base.src) >> 16,
4996                                 drm_rect_height(&plane_state->base.src) >> 16,
4997                                 drm_rect_width(&plane_state->base.dst),
4998                                 drm_rect_height(&plane_state->base.dst),
4999                                 fb ? fb->format : NULL, need_scaler);
5000
5001         if (ret || plane_state->scaler_id < 0)
5002                 return ret;
5003
5004         /* check colorkey */
5005         if (plane_state->ckey.flags) {
5006                 DRM_DEBUG_KMS("[PLANE:%d:%s] scaling with color key not allowed",
5007                               intel_plane->base.base.id,
5008                               intel_plane->base.name);
5009                 return -EINVAL;
5010         }
5011
5012         /* Check src format */
5013         switch (fb->format->format) {
5014         case DRM_FORMAT_RGB565:
5015         case DRM_FORMAT_XBGR8888:
5016         case DRM_FORMAT_XRGB8888:
5017         case DRM_FORMAT_ABGR8888:
5018         case DRM_FORMAT_ARGB8888:
5019         case DRM_FORMAT_XRGB2101010:
5020         case DRM_FORMAT_XBGR2101010:
5021         case DRM_FORMAT_YUYV:
5022         case DRM_FORMAT_YVYU:
5023         case DRM_FORMAT_UYVY:
5024         case DRM_FORMAT_VYUY:
5025         case DRM_FORMAT_NV12:
5026                 break;
5027         default:
5028                 DRM_DEBUG_KMS("[PLANE:%d:%s] FB:%d unsupported scaling format 0x%x\n",
5029                               intel_plane->base.base.id, intel_plane->base.name,
5030                               fb->base.id, fb->format->format);
5031                 return -EINVAL;
5032         }
5033
5034         return 0;
5035 }
5036
5037 static void skylake_scaler_disable(struct intel_crtc *crtc)
5038 {
5039         int i;
5040
5041         for (i = 0; i < crtc->num_scalers; i++)
5042                 skl_detach_scaler(crtc, i);
5043 }
5044
5045 static void skylake_pfit_enable(const struct intel_crtc_state *crtc_state)
5046 {
5047         struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
5048         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
5049         enum pipe pipe = crtc->pipe;
5050         const struct intel_crtc_scaler_state *scaler_state =
5051                 &crtc_state->scaler_state;
5052
5053         if (crtc_state->pch_pfit.enabled) {
5054                 u16 uv_rgb_hphase, uv_rgb_vphase;
5055                 int pfit_w, pfit_h, hscale, vscale;
5056                 int id;
5057
5058                 if (WARN_ON(crtc_state->scaler_state.scaler_id < 0))
5059                         return;
5060
5061                 pfit_w = (crtc_state->pch_pfit.size >> 16) & 0xFFFF;
5062                 pfit_h = crtc_state->pch_pfit.size & 0xFFFF;
5063
5064                 hscale = (crtc_state->pipe_src_w << 16) / pfit_w;
5065                 vscale = (crtc_state->pipe_src_h << 16) / pfit_h;
5066
5067                 uv_rgb_hphase = skl_scaler_calc_phase(1, hscale, false);
5068                 uv_rgb_vphase = skl_scaler_calc_phase(1, vscale, false);
5069
5070                 id = scaler_state->scaler_id;
5071                 I915_WRITE(SKL_PS_CTRL(pipe, id), PS_SCALER_EN |
5072                         PS_FILTER_MEDIUM | scaler_state->scalers[id].mode);
5073                 I915_WRITE_FW(SKL_PS_VPHASE(pipe, id),
5074                               PS_Y_PHASE(0) | PS_UV_RGB_PHASE(uv_rgb_vphase));
5075                 I915_WRITE_FW(SKL_PS_HPHASE(pipe, id),
5076                               PS_Y_PHASE(0) | PS_UV_RGB_PHASE(uv_rgb_hphase));
5077                 I915_WRITE(SKL_PS_WIN_POS(pipe, id), crtc_state->pch_pfit.pos);
5078                 I915_WRITE(SKL_PS_WIN_SZ(pipe, id), crtc_state->pch_pfit.size);
5079         }
5080 }
5081
5082 static void ironlake_pfit_enable(const struct intel_crtc_state *crtc_state)
5083 {
5084         struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
5085         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
5086         int pipe = crtc->pipe;
5087
5088         if (crtc_state->pch_pfit.enabled) {
5089                 /* Force use of hard-coded filter coefficients
5090                  * as some pre-programmed values are broken,
5091                  * e.g. x201.
5092                  */
5093                 if (IS_IVYBRIDGE(dev_priv) || IS_HASWELL(dev_priv))
5094                         I915_WRITE(PF_CTL(pipe), PF_ENABLE | PF_FILTER_MED_3x3 |
5095                                                  PF_PIPE_SEL_IVB(pipe));
5096                 else
5097                         I915_WRITE(PF_CTL(pipe), PF_ENABLE | PF_FILTER_MED_3x3);
5098                 I915_WRITE(PF_WIN_POS(pipe), crtc_state->pch_pfit.pos);
5099                 I915_WRITE(PF_WIN_SZ(pipe), crtc_state->pch_pfit.size);
5100         }
5101 }
5102
5103 void hsw_enable_ips(const struct intel_crtc_state *crtc_state)
5104 {
5105         struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
5106         struct drm_device *dev = crtc->base.dev;
5107         struct drm_i915_private *dev_priv = to_i915(dev);
5108
5109         if (!crtc_state->ips_enabled)
5110                 return;
5111
5112         /*
5113          * We can only enable IPS after we enable a plane and wait for a vblank
5114          * This function is called from post_plane_update, which is run after
5115          * a vblank wait.
5116          */
5117         WARN_ON(!(crtc_state->active_planes & ~BIT(PLANE_CURSOR)));
5118
5119         if (IS_BROADWELL(dev_priv)) {
5120                 mutex_lock(&dev_priv->pcu_lock);
5121                 WARN_ON(sandybridge_pcode_write(dev_priv, DISPLAY_IPS_CONTROL,
5122                                                 IPS_ENABLE | IPS_PCODE_CONTROL));
5123                 mutex_unlock(&dev_priv->pcu_lock);
5124                 /* Quoting Art Runyan: "its not safe to expect any particular
5125                  * value in IPS_CTL bit 31 after enabling IPS through the
5126                  * mailbox." Moreover, the mailbox may return a bogus state,
5127                  * so we need to just enable it and continue on.
5128                  */
5129         } else {
5130                 I915_WRITE(IPS_CTL, IPS_ENABLE);
5131                 /* The bit only becomes 1 in the next vblank, so this wait here
5132                  * is essentially intel_wait_for_vblank. If we don't have this
5133                  * and don't wait for vblanks until the end of crtc_enable, then
5134                  * the HW state readout code will complain that the expected
5135                  * IPS_CTL value is not the one we read. */
5136                 if (intel_wait_for_register(dev_priv,
5137                                             IPS_CTL, IPS_ENABLE, IPS_ENABLE,
5138                                             50))
5139                         DRM_ERROR("Timed out waiting for IPS enable\n");
5140         }
5141 }
5142
5143 void hsw_disable_ips(const struct intel_crtc_state *crtc_state)
5144 {
5145         struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
5146         struct drm_device *dev = crtc->base.dev;
5147         struct drm_i915_private *dev_priv = to_i915(dev);
5148
5149         if (!crtc_state->ips_enabled)
5150                 return;
5151
5152         if (IS_BROADWELL(dev_priv)) {
5153                 mutex_lock(&dev_priv->pcu_lock);
5154                 WARN_ON(sandybridge_pcode_write(dev_priv, DISPLAY_IPS_CONTROL, 0));
5155                 mutex_unlock(&dev_priv->pcu_lock);
5156                 /*
5157                  * Wait for PCODE to finish disabling IPS. The BSpec specified
5158                  * 42ms timeout value leads to occasional timeouts so use 100ms
5159                  * instead.
5160                  */
5161                 if (intel_wait_for_register(dev_priv,
5162                                             IPS_CTL, IPS_ENABLE, 0,
5163                                             100))
5164                         DRM_ERROR("Timed out waiting for IPS disable\n");
5165         } else {
5166                 I915_WRITE(IPS_CTL, 0);
5167                 POSTING_READ(IPS_CTL);
5168         }
5169
5170         /* We need to wait for a vblank before we can disable the plane. */
5171         intel_wait_for_vblank(dev_priv, crtc->pipe);
5172 }
5173
5174 static void intel_crtc_dpms_overlay_disable(struct intel_crtc *intel_crtc)
5175 {
5176         if (intel_crtc->overlay) {
5177                 struct drm_device *dev = intel_crtc->base.dev;
5178
5179                 mutex_lock(&dev->struct_mutex);
5180                 (void) intel_overlay_switch_off(intel_crtc->overlay);
5181                 mutex_unlock(&dev->struct_mutex);
5182         }
5183
5184         /* Let userspace switch the overlay on again. In most cases userspace
5185          * has to recompute where to put it anyway.
5186          */
5187 }
5188
5189 /**
5190  * intel_post_enable_primary - Perform operations after enabling primary plane
5191  * @crtc: the CRTC whose primary plane was just enabled
5192  * @new_crtc_state: the enabling state
5193  *
5194  * Performs potentially sleeping operations that must be done after the primary
5195  * plane is enabled, such as updating FBC and IPS.  Note that this may be
5196  * called due to an explicit primary plane update, or due to an implicit
5197  * re-enable that is caused when a sprite plane is updated to no longer
5198  * completely hide the primary plane.
5199  */
5200 static void
5201 intel_post_enable_primary(struct drm_crtc *crtc,
5202                           const struct intel_crtc_state *new_crtc_state)
5203 {
5204         struct drm_device *dev = crtc->dev;
5205         struct drm_i915_private *dev_priv = to_i915(dev);
5206         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
5207         int pipe = intel_crtc->pipe;
5208
5209         /*
5210          * Gen2 reports pipe underruns whenever all planes are disabled.
5211          * So don't enable underrun reporting before at least some planes
5212          * are enabled.
5213          * FIXME: Need to fix the logic to work when we turn off all planes
5214          * but leave the pipe running.
5215          */
5216         if (IS_GEN(dev_priv, 2))
5217                 intel_set_cpu_fifo_underrun_reporting(dev_priv, pipe, true);
5218
5219         /* Underruns don't always raise interrupts, so check manually. */
5220         intel_check_cpu_fifo_underruns(dev_priv);
5221         intel_check_pch_fifo_underruns(dev_priv);
5222 }
5223
5224 /* FIXME get rid of this and use pre_plane_update */
5225 static void
5226 intel_pre_disable_primary_noatomic(struct drm_crtc *crtc)
5227 {
5228         struct drm_device *dev = crtc->dev;
5229         struct drm_i915_private *dev_priv = to_i915(dev);
5230         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
5231         int pipe = intel_crtc->pipe;
5232
5233         /*
5234          * Gen2 reports pipe underruns whenever all planes are disabled.
5235          * So disable underrun reporting before all the planes get disabled.
5236          */
5237         if (IS_GEN(dev_priv, 2))
5238                 intel_set_cpu_fifo_underrun_reporting(dev_priv, pipe, false);
5239
5240         hsw_disable_ips(to_intel_crtc_state(crtc->state));
5241
5242         /*
5243          * Vblank time updates from the shadow to live plane control register
5244          * are blocked if the memory self-refresh mode is active at that
5245          * moment. So to make sure the plane gets truly disabled, disable
5246          * first the self-refresh mode. The self-refresh enable bit in turn
5247          * will be checked/applied by the HW only at the next frame start
5248          * event which is after the vblank start event, so we need to have a
5249          * wait-for-vblank between disabling the plane and the pipe.
5250          */
5251         if (HAS_GMCH_DISPLAY(dev_priv) &&
5252             intel_set_memory_cxsr(dev_priv, false))
5253                 intel_wait_for_vblank(dev_priv, pipe);
5254 }
5255
5256 static bool hsw_pre_update_disable_ips(const struct intel_crtc_state *old_crtc_state,
5257                                        const struct intel_crtc_state *new_crtc_state)
5258 {
5259         if (!old_crtc_state->ips_enabled)
5260                 return false;
5261
5262         if (needs_modeset(&new_crtc_state->base))
5263                 return true;
5264
5265         return !new_crtc_state->ips_enabled;
5266 }
5267
5268 static bool hsw_post_update_enable_ips(const struct intel_crtc_state *old_crtc_state,
5269                                        const struct intel_crtc_state *new_crtc_state)
5270 {
5271         if (!new_crtc_state->ips_enabled)
5272                 return false;
5273
5274         if (needs_modeset(&new_crtc_state->base))
5275                 return true;
5276
5277         /*
5278          * We can't read out IPS on broadwell, assume the worst and
5279          * forcibly enable IPS on the first fastset.
5280          */
5281         if (new_crtc_state->update_pipe &&
5282             old_crtc_state->base.adjusted_mode.private_flags & I915_MODE_FLAG_INHERITED)
5283                 return true;
5284
5285         return !old_crtc_state->ips_enabled;
5286 }
5287
5288 static bool needs_nv12_wa(struct drm_i915_private *dev_priv,
5289                           const struct intel_crtc_state *crtc_state)
5290 {
5291         if (!crtc_state->nv12_planes)
5292                 return false;
5293
5294         /* WA Display #0827: Gen9:all */
5295         if (IS_GEN(dev_priv, 9) && !IS_GEMINILAKE(dev_priv))
5296                 return true;
5297
5298         return false;
5299 }
5300
5301 static void intel_post_plane_update(struct intel_crtc_state *old_crtc_state)
5302 {
5303         struct intel_crtc *crtc = to_intel_crtc(old_crtc_state->base.crtc);
5304         struct drm_device *dev = crtc->base.dev;
5305         struct drm_i915_private *dev_priv = to_i915(dev);
5306         struct drm_atomic_state *old_state = old_crtc_state->base.state;
5307         struct intel_crtc_state *pipe_config =
5308                 intel_atomic_get_new_crtc_state(to_intel_atomic_state(old_state),
5309                                                 crtc);
5310         struct drm_plane *primary = crtc->base.primary;
5311         struct drm_plane_state *old_primary_state =
5312                 drm_atomic_get_old_plane_state(old_state, primary);
5313
5314         intel_frontbuffer_flip(to_i915(crtc->base.dev), pipe_config->fb_bits);
5315
5316         if (pipe_config->update_wm_post && pipe_config->base.active)
5317                 intel_update_watermarks(crtc);
5318
5319         if (hsw_post_update_enable_ips(old_crtc_state, pipe_config))
5320                 hsw_enable_ips(pipe_config);
5321
5322         if (old_primary_state) {
5323                 struct drm_plane_state *new_primary_state =
5324                         drm_atomic_get_new_plane_state(old_state, primary);
5325
5326                 intel_fbc_post_update(crtc);
5327
5328                 if (new_primary_state->visible &&
5329                     (needs_modeset(&pipe_config->base) ||
5330                      !old_primary_state->visible))
5331                         intel_post_enable_primary(&crtc->base, pipe_config);
5332         }
5333
5334         /* Display WA 827 */
5335         if (needs_nv12_wa(dev_priv, old_crtc_state) &&
5336             !needs_nv12_wa(dev_priv, pipe_config)) {
5337                 skl_wa_clkgate(dev_priv, crtc->pipe, false);
5338         }
5339 }
5340
5341 static void intel_pre_plane_update(struct intel_crtc_state *old_crtc_state,
5342                                    struct intel_crtc_state *pipe_config)
5343 {
5344         struct intel_crtc *crtc = to_intel_crtc(old_crtc_state->base.crtc);
5345         struct drm_device *dev = crtc->base.dev;
5346         struct drm_i915_private *dev_priv = to_i915(dev);
5347         struct drm_atomic_state *old_state = old_crtc_state->base.state;
5348         struct drm_plane *primary = crtc->base.primary;
5349         struct drm_plane_state *old_primary_state =
5350                 drm_atomic_get_old_plane_state(old_state, primary);
5351         bool modeset = needs_modeset(&pipe_config->base);
5352         struct intel_atomic_state *old_intel_state =
5353                 to_intel_atomic_state(old_state);
5354
5355         if (hsw_pre_update_disable_ips(old_crtc_state, pipe_config))
5356                 hsw_disable_ips(old_crtc_state);
5357
5358         if (old_primary_state) {
5359                 struct intel_plane_state *new_primary_state =
5360                         intel_atomic_get_new_plane_state(old_intel_state,
5361                                                          to_intel_plane(primary));
5362
5363                 intel_fbc_pre_update(crtc, pipe_config, new_primary_state);
5364                 /*
5365                  * Gen2 reports pipe underruns whenever all planes are disabled.
5366                  * So disable underrun reporting before all the planes get disabled.
5367                  */
5368                 if (IS_GEN(dev_priv, 2) && old_primary_state->visible &&
5369                     (modeset || !new_primary_state->base.visible))
5370                         intel_set_cpu_fifo_underrun_reporting(dev_priv, crtc->pipe, false);
5371         }
5372
5373         /* Display WA 827 */
5374         if (!needs_nv12_wa(dev_priv, old_crtc_state) &&
5375             needs_nv12_wa(dev_priv, pipe_config)) {
5376                 skl_wa_clkgate(dev_priv, crtc->pipe, true);
5377         }
5378
5379         /*
5380          * Vblank time updates from the shadow to live plane control register
5381          * are blocked if the memory self-refresh mode is active at that
5382          * moment. So to make sure the plane gets truly disabled, disable
5383          * first the self-refresh mode. The self-refresh enable bit in turn
5384          * will be checked/applied by the HW only at the next frame start
5385          * event which is after the vblank start event, so we need to have a
5386          * wait-for-vblank between disabling the plane and the pipe.
5387          */
5388         if (HAS_GMCH_DISPLAY(dev_priv) && old_crtc_state->base.active &&
5389             pipe_config->disable_cxsr && intel_set_memory_cxsr(dev_priv, false))
5390                 intel_wait_for_vblank(dev_priv, crtc->pipe);
5391
5392         /*
5393          * IVB workaround: must disable low power watermarks for at least
5394          * one frame before enabling scaling.  LP watermarks can be re-enabled
5395          * when scaling is disabled.
5396          *
5397          * WaCxSRDisabledForSpriteScaling:ivb
5398          */
5399         if (pipe_config->disable_lp_wm && ilk_disable_lp_wm(dev) &&
5400             old_crtc_state->base.active)
5401                 intel_wait_for_vblank(dev_priv, crtc->pipe);
5402
5403         /*
5404          * If we're doing a modeset, we're done.  No need to do any pre-vblank
5405          * watermark programming here.
5406          */
5407         if (needs_modeset(&pipe_config->base))
5408                 return;
5409
5410         /*
5411          * For platforms that support atomic watermarks, program the
5412          * 'intermediate' watermarks immediately.  On pre-gen9 platforms, these
5413          * will be the intermediate values that are safe for both pre- and
5414          * post- vblank; when vblank happens, the 'active' values will be set
5415          * to the final 'target' values and we'll do this again to get the
5416          * optimal watermarks.  For gen9+ platforms, the values we program here
5417          * will be the final target values which will get automatically latched
5418          * at vblank time; no further programming will be necessary.
5419          *
5420          * If a platform hasn't been transitioned to atomic watermarks yet,
5421          * we'll continue to update watermarks the old way, if flags tell
5422          * us to.
5423          */
5424         if (dev_priv->display.initial_watermarks != NULL)
5425                 dev_priv->display.initial_watermarks(old_intel_state,
5426                                                      pipe_config);
5427         else if (pipe_config->update_wm_pre)
5428                 intel_update_watermarks(crtc);
5429 }
5430
5431 static void intel_crtc_disable_planes(struct intel_atomic_state *state,
5432                                       struct intel_crtc *crtc)
5433 {
5434         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
5435         const struct intel_crtc_state *new_crtc_state =
5436                 intel_atomic_get_new_crtc_state(state, crtc);
5437         unsigned int update_mask = new_crtc_state->update_planes;
5438         const struct intel_plane_state *old_plane_state;
5439         struct intel_plane *plane;
5440         unsigned fb_bits = 0;
5441         int i;
5442
5443         intel_crtc_dpms_overlay_disable(crtc);
5444
5445         for_each_old_intel_plane_in_state(state, plane, old_plane_state, i) {
5446                 if (crtc->pipe != plane->pipe ||
5447                     !(update_mask & BIT(plane->id)))
5448                         continue;
5449
5450                 plane->disable_plane(plane, new_crtc_state);
5451
5452                 if (old_plane_state->base.visible)
5453                         fb_bits |= plane->frontbuffer_bit;
5454         }
5455
5456         intel_frontbuffer_flip(dev_priv, fb_bits);
5457 }
5458
5459 static void intel_encoders_pre_pll_enable(struct drm_crtc *crtc,
5460                                           struct intel_crtc_state *crtc_state,
5461                                           struct drm_atomic_state *old_state)
5462 {
5463         struct drm_connector_state *conn_state;
5464         struct drm_connector *conn;
5465         int i;
5466
5467         for_each_new_connector_in_state(old_state, conn, conn_state, i) {
5468                 struct intel_encoder *encoder =
5469                         to_intel_encoder(conn_state->best_encoder);
5470
5471                 if (conn_state->crtc != crtc)
5472                         continue;
5473
5474                 if (encoder->pre_pll_enable)
5475                         encoder->pre_pll_enable(encoder, crtc_state, conn_state);
5476         }
5477 }
5478
5479 static void intel_encoders_pre_enable(struct drm_crtc *crtc,
5480                                       struct intel_crtc_state *crtc_state,
5481                                       struct drm_atomic_state *old_state)
5482 {
5483         struct drm_connector_state *conn_state;
5484         struct drm_connector *conn;
5485         int i;
5486
5487         for_each_new_connector_in_state(old_state, conn, conn_state, i) {
5488                 struct intel_encoder *encoder =
5489                         to_intel_encoder(conn_state->best_encoder);
5490
5491                 if (conn_state->crtc != crtc)
5492                         continue;
5493
5494                 if (encoder->pre_enable)
5495                         encoder->pre_enable(encoder, crtc_state, conn_state);
5496         }
5497 }
5498
5499 static void intel_encoders_enable(struct drm_crtc *crtc,
5500                                   struct intel_crtc_state *crtc_state,
5501                                   struct drm_atomic_state *old_state)
5502 {
5503         struct drm_connector_state *conn_state;
5504         struct drm_connector *conn;
5505         int i;
5506
5507         for_each_new_connector_in_state(old_state, conn, conn_state, i) {
5508                 struct intel_encoder *encoder =
5509                         to_intel_encoder(conn_state->best_encoder);
5510
5511                 if (conn_state->crtc != crtc)
5512                         continue;
5513
5514                 if (encoder->enable)
5515                         encoder->enable(encoder, crtc_state, conn_state);
5516                 intel_opregion_notify_encoder(encoder, true);
5517         }
5518 }
5519
5520 static void intel_encoders_disable(struct drm_crtc *crtc,
5521                                    struct intel_crtc_state *old_crtc_state,
5522                                    struct drm_atomic_state *old_state)
5523 {
5524         struct drm_connector_state *old_conn_state;
5525         struct drm_connector *conn;
5526         int i;
5527
5528         for_each_old_connector_in_state(old_state, conn, old_conn_state, i) {
5529                 struct intel_encoder *encoder =
5530                         to_intel_encoder(old_conn_state->best_encoder);
5531
5532                 if (old_conn_state->crtc != crtc)
5533                         continue;
5534
5535                 intel_opregion_notify_encoder(encoder, false);
5536                 if (encoder->disable)
5537                         encoder->disable(encoder, old_crtc_state, old_conn_state);
5538         }
5539 }
5540
5541 static void intel_encoders_post_disable(struct drm_crtc *crtc,
5542                                         struct intel_crtc_state *old_crtc_state,
5543                                         struct drm_atomic_state *old_state)
5544 {
5545         struct drm_connector_state *old_conn_state;
5546         struct drm_connector *conn;
5547         int i;
5548
5549         for_each_old_connector_in_state(old_state, conn, old_conn_state, i) {
5550                 struct intel_encoder *encoder =
5551                         to_intel_encoder(old_conn_state->best_encoder);
5552
5553                 if (old_conn_state->crtc != crtc)
5554                         continue;
5555
5556                 if (encoder->post_disable)
5557                         encoder->post_disable(encoder, old_crtc_state, old_conn_state);
5558         }
5559 }
5560
5561 static void intel_encoders_post_pll_disable(struct drm_crtc *crtc,
5562                                             struct intel_crtc_state *old_crtc_state,
5563                                             struct drm_atomic_state *old_state)
5564 {
5565         struct drm_connector_state *old_conn_state;
5566         struct drm_connector *conn;
5567         int i;
5568
5569         for_each_old_connector_in_state(old_state, conn, old_conn_state, i) {
5570                 struct intel_encoder *encoder =
5571                         to_intel_encoder(old_conn_state->best_encoder);
5572
5573                 if (old_conn_state->crtc != crtc)
5574                         continue;
5575
5576                 if (encoder->post_pll_disable)
5577                         encoder->post_pll_disable(encoder, old_crtc_state, old_conn_state);
5578         }
5579 }
5580
5581 static void ironlake_crtc_enable(struct intel_crtc_state *pipe_config,
5582                                  struct drm_atomic_state *old_state)
5583 {
5584         struct drm_crtc *crtc = pipe_config->base.crtc;
5585         struct drm_device *dev = crtc->dev;
5586         struct drm_i915_private *dev_priv = to_i915(dev);
5587         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
5588         int pipe = intel_crtc->pipe;
5589         struct intel_atomic_state *old_intel_state =
5590                 to_intel_atomic_state(old_state);
5591
5592         if (WARN_ON(intel_crtc->active))
5593                 return;
5594
5595         /*
5596          * Sometimes spurious CPU pipe underruns happen during FDI
5597          * training, at least with VGA+HDMI cloning. Suppress them.
5598          *
5599          * On ILK we get an occasional spurious CPU pipe underruns
5600          * between eDP port A enable and vdd enable. Also PCH port
5601          * enable seems to result in the occasional CPU pipe underrun.
5602          *
5603          * Spurious PCH underruns also occur during PCH enabling.
5604          */
5605         intel_set_cpu_fifo_underrun_reporting(dev_priv, pipe, false);
5606         intel_set_pch_fifo_underrun_reporting(dev_priv, pipe, false);
5607
5608         if (pipe_config->has_pch_encoder)
5609                 intel_prepare_shared_dpll(pipe_config);
5610
5611         if (intel_crtc_has_dp_encoder(pipe_config))
5612                 intel_dp_set_m_n(pipe_config, M1_N1);
5613
5614         intel_set_pipe_timings(pipe_config);
5615         intel_set_pipe_src_size(pipe_config);
5616
5617         if (pipe_config->has_pch_encoder) {
5618                 intel_cpu_transcoder_set_m_n(pipe_config,
5619                                              &pipe_config->fdi_m_n, NULL);
5620         }
5621
5622         ironlake_set_pipeconf(pipe_config);
5623
5624         intel_crtc->active = true;
5625
5626         intel_encoders_pre_enable(crtc, pipe_config, old_state);
5627
5628         if (pipe_config->has_pch_encoder) {
5629                 /* Note: FDI PLL enabling _must_ be done before we enable the
5630                  * cpu pipes, hence this is separate from all the other fdi/pch
5631                  * enabling. */
5632                 ironlake_fdi_pll_enable(pipe_config);
5633         } else {
5634                 assert_fdi_tx_disabled(dev_priv, pipe);
5635                 assert_fdi_rx_disabled(dev_priv, pipe);
5636         }
5637
5638         ironlake_pfit_enable(pipe_config);
5639
5640         /*
5641          * On ILK+ LUT must be loaded before the pipe is running but with
5642          * clocks enabled
5643          */
5644         intel_color_load_luts(pipe_config);
5645
5646         if (dev_priv->display.initial_watermarks != NULL)
5647                 dev_priv->display.initial_watermarks(old_intel_state, pipe_config);
5648         intel_enable_pipe(pipe_config);
5649
5650         if (pipe_config->has_pch_encoder)
5651                 ironlake_pch_enable(old_intel_state, pipe_config);
5652
5653         assert_vblank_disabled(crtc);
5654         drm_crtc_vblank_on(crtc);
5655
5656         intel_encoders_enable(crtc, pipe_config, old_state);
5657
5658         if (HAS_PCH_CPT(dev_priv))
5659                 cpt_verify_modeset(dev, intel_crtc->pipe);
5660
5661         /*
5662          * Must wait for vblank to avoid spurious PCH FIFO underruns.
5663          * And a second vblank wait is needed at least on ILK with
5664          * some interlaced HDMI modes. Let's do the double wait always
5665          * in case there are more corner cases we don't know about.
5666          */
5667         if (pipe_config->has_pch_encoder) {
5668                 intel_wait_for_vblank(dev_priv, pipe);
5669                 intel_wait_for_vblank(dev_priv, pipe);
5670         }
5671         intel_set_cpu_fifo_underrun_reporting(dev_priv, pipe, true);
5672         intel_set_pch_fifo_underrun_reporting(dev_priv, pipe, true);
5673 }
5674
5675 /* IPS only exists on ULT machines and is tied to pipe A. */
5676 static bool hsw_crtc_supports_ips(struct intel_crtc *crtc)
5677 {
5678         return HAS_IPS(to_i915(crtc->base.dev)) && crtc->pipe == PIPE_A;
5679 }
5680
5681 static void glk_pipe_scaler_clock_gating_wa(struct drm_i915_private *dev_priv,
5682                                             enum pipe pipe, bool apply)
5683 {
5684         u32 val = I915_READ(CLKGATE_DIS_PSL(pipe));
5685         u32 mask = DPF_GATING_DIS | DPF_RAM_GATING_DIS | DPFR_GATING_DIS;
5686
5687         if (apply)
5688                 val |= mask;
5689         else
5690                 val &= ~mask;
5691
5692         I915_WRITE(CLKGATE_DIS_PSL(pipe), val);
5693 }
5694
5695 static void icl_pipe_mbus_enable(struct intel_crtc *crtc)
5696 {
5697         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
5698         enum pipe pipe = crtc->pipe;
5699         uint32_t val;
5700
5701         val = MBUS_DBOX_A_CREDIT(2);
5702         val |= MBUS_DBOX_BW_CREDIT(1);
5703         val |= MBUS_DBOX_B_CREDIT(8);
5704
5705         I915_WRITE(PIPE_MBUS_DBOX_CTL(pipe), val);
5706 }
5707
5708 static void haswell_crtc_enable(struct intel_crtc_state *pipe_config,
5709                                 struct drm_atomic_state *old_state)
5710 {
5711         struct drm_crtc *crtc = pipe_config->base.crtc;
5712         struct drm_i915_private *dev_priv = to_i915(crtc->dev);
5713         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
5714         int pipe = intel_crtc->pipe, hsw_workaround_pipe;
5715         enum transcoder cpu_transcoder = pipe_config->cpu_transcoder;
5716         struct intel_atomic_state *old_intel_state =
5717                 to_intel_atomic_state(old_state);
5718         bool psl_clkgate_wa;
5719         u32 pipe_chicken;
5720
5721         if (WARN_ON(intel_crtc->active))
5722                 return;
5723
5724         intel_encoders_pre_pll_enable(crtc, pipe_config, old_state);
5725
5726         if (pipe_config->shared_dpll)
5727                 intel_enable_shared_dpll(pipe_config);
5728
5729         intel_encoders_pre_enable(crtc, pipe_config, old_state);
5730
5731         if (intel_crtc_has_dp_encoder(pipe_config))
5732                 intel_dp_set_m_n(pipe_config, M1_N1);
5733
5734         if (!transcoder_is_dsi(cpu_transcoder))
5735                 intel_set_pipe_timings(pipe_config);
5736
5737         intel_set_pipe_src_size(pipe_config);
5738
5739         if (cpu_transcoder != TRANSCODER_EDP &&
5740             !transcoder_is_dsi(cpu_transcoder)) {
5741                 I915_WRITE(PIPE_MULT(cpu_transcoder),
5742                            pipe_config->pixel_multiplier - 1);
5743         }
5744
5745         if (pipe_config->has_pch_encoder) {
5746                 intel_cpu_transcoder_set_m_n(pipe_config,
5747                                              &pipe_config->fdi_m_n, NULL);
5748         }
5749
5750         if (!transcoder_is_dsi(cpu_transcoder))
5751                 haswell_set_pipeconf(pipe_config);
5752
5753         haswell_set_pipemisc(pipe_config);
5754
5755         intel_color_set_csc(pipe_config);
5756
5757         intel_crtc->active = true;
5758
5759         /* Display WA #1180: WaDisableScalarClockGating: glk, cnl */
5760         psl_clkgate_wa = (IS_GEMINILAKE(dev_priv) || IS_CANNONLAKE(dev_priv)) &&
5761                          pipe_config->pch_pfit.enabled;
5762         if (psl_clkgate_wa)
5763                 glk_pipe_scaler_clock_gating_wa(dev_priv, pipe, true);
5764
5765         if (INTEL_GEN(dev_priv) >= 9)
5766                 skylake_pfit_enable(pipe_config);
5767         else
5768                 ironlake_pfit_enable(pipe_config);
5769
5770         /*
5771          * On ILK+ LUT must be loaded before the pipe is running but with
5772          * clocks enabled
5773          */
5774         intel_color_load_luts(pipe_config);
5775
5776         /*
5777          * Display WA #1153: enable hardware to bypass the alpha math
5778          * and rounding for per-pixel values 00 and 0xff
5779          */
5780         if (INTEL_GEN(dev_priv) >= 11) {
5781                 pipe_chicken = I915_READ(PIPE_CHICKEN(pipe));
5782                 if (!(pipe_chicken & PER_PIXEL_ALPHA_BYPASS_EN))
5783                         I915_WRITE_FW(PIPE_CHICKEN(pipe),
5784                                       pipe_chicken | PER_PIXEL_ALPHA_BYPASS_EN);
5785         }
5786
5787         intel_ddi_set_pipe_settings(pipe_config);
5788         if (!transcoder_is_dsi(cpu_transcoder))
5789                 intel_ddi_enable_transcoder_func(pipe_config);
5790
5791         if (dev_priv->display.initial_watermarks != NULL)
5792                 dev_priv->display.initial_watermarks(old_intel_state, pipe_config);
5793
5794         if (INTEL_GEN(dev_priv) >= 11)
5795                 icl_pipe_mbus_enable(intel_crtc);
5796
5797         /* XXX: Do the pipe assertions at the right place for BXT DSI. */
5798         if (!transcoder_is_dsi(cpu_transcoder))
5799                 intel_enable_pipe(pipe_config);
5800
5801         if (pipe_config->has_pch_encoder)
5802                 lpt_pch_enable(old_intel_state, pipe_config);
5803
5804         if (intel_crtc_has_type(pipe_config, INTEL_OUTPUT_DP_MST))
5805                 intel_ddi_set_vc_payload_alloc(pipe_config, true);
5806
5807         assert_vblank_disabled(crtc);
5808         drm_crtc_vblank_on(crtc);
5809
5810         intel_encoders_enable(crtc, pipe_config, old_state);
5811
5812         if (psl_clkgate_wa) {
5813                 intel_wait_for_vblank(dev_priv, pipe);
5814                 glk_pipe_scaler_clock_gating_wa(dev_priv, pipe, false);
5815         }
5816
5817         /* If we change the relative order between pipe/planes enabling, we need
5818          * to change the workaround. */
5819         hsw_workaround_pipe = pipe_config->hsw_workaround_pipe;
5820         if (IS_HASWELL(dev_priv) && hsw_workaround_pipe != INVALID_PIPE) {
5821                 intel_wait_for_vblank(dev_priv, hsw_workaround_pipe);
5822                 intel_wait_for_vblank(dev_priv, hsw_workaround_pipe);
5823         }
5824 }
5825
5826 static void ironlake_pfit_disable(const struct intel_crtc_state *old_crtc_state)
5827 {
5828         struct intel_crtc *crtc = to_intel_crtc(old_crtc_state->base.crtc);
5829         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
5830         enum pipe pipe = crtc->pipe;
5831
5832         /* To avoid upsetting the power well on haswell only disable the pfit if
5833          * it's in use. The hw state code will make sure we get this right. */
5834         if (old_crtc_state->pch_pfit.enabled) {
5835                 I915_WRITE(PF_CTL(pipe), 0);
5836                 I915_WRITE(PF_WIN_POS(pipe), 0);
5837                 I915_WRITE(PF_WIN_SZ(pipe), 0);
5838         }
5839 }
5840
5841 static void ironlake_crtc_disable(struct intel_crtc_state *old_crtc_state,
5842                                   struct drm_atomic_state *old_state)
5843 {
5844         struct drm_crtc *crtc = old_crtc_state->base.crtc;
5845         struct drm_device *dev = crtc->dev;
5846         struct drm_i915_private *dev_priv = to_i915(dev);
5847         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
5848         int pipe = intel_crtc->pipe;
5849
5850         /*
5851          * Sometimes spurious CPU pipe underruns happen when the
5852          * pipe is already disabled, but FDI RX/TX is still enabled.
5853          * Happens at least with VGA+HDMI cloning. Suppress them.
5854          */
5855         intel_set_cpu_fifo_underrun_reporting(dev_priv, pipe, false);
5856         intel_set_pch_fifo_underrun_reporting(dev_priv, pipe, false);
5857
5858         intel_encoders_disable(crtc, old_crtc_state, old_state);
5859
5860         drm_crtc_vblank_off(crtc);
5861         assert_vblank_disabled(crtc);
5862
5863         intel_disable_pipe(old_crtc_state);
5864
5865         ironlake_pfit_disable(old_crtc_state);
5866
5867         if (old_crtc_state->has_pch_encoder)
5868                 ironlake_fdi_disable(crtc);
5869
5870         intel_encoders_post_disable(crtc, old_crtc_state, old_state);
5871
5872         if (old_crtc_state->has_pch_encoder) {
5873                 ironlake_disable_pch_transcoder(dev_priv, pipe);
5874
5875                 if (HAS_PCH_CPT(dev_priv)) {
5876                         i915_reg_t reg;
5877                         u32 temp;
5878
5879                         /* disable TRANS_DP_CTL */
5880                         reg = TRANS_DP_CTL(pipe);
5881                         temp = I915_READ(reg);
5882                         temp &= ~(TRANS_DP_OUTPUT_ENABLE |
5883                                   TRANS_DP_PORT_SEL_MASK);
5884                         temp |= TRANS_DP_PORT_SEL_NONE;
5885                         I915_WRITE(reg, temp);
5886
5887                         /* disable DPLL_SEL */
5888                         temp = I915_READ(PCH_DPLL_SEL);
5889                         temp &= ~(TRANS_DPLL_ENABLE(pipe) | TRANS_DPLLB_SEL(pipe));
5890                         I915_WRITE(PCH_DPLL_SEL, temp);
5891                 }
5892
5893                 ironlake_fdi_pll_disable(intel_crtc);
5894         }
5895
5896         intel_set_cpu_fifo_underrun_reporting(dev_priv, pipe, true);
5897         intel_set_pch_fifo_underrun_reporting(dev_priv, pipe, true);
5898 }
5899
5900 static void haswell_crtc_disable(struct intel_crtc_state *old_crtc_state,
5901                                  struct drm_atomic_state *old_state)
5902 {
5903         struct drm_crtc *crtc = old_crtc_state->base.crtc;
5904         struct drm_i915_private *dev_priv = to_i915(crtc->dev);
5905         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
5906         enum transcoder cpu_transcoder = old_crtc_state->cpu_transcoder;
5907
5908         intel_encoders_disable(crtc, old_crtc_state, old_state);
5909
5910         drm_crtc_vblank_off(crtc);
5911         assert_vblank_disabled(crtc);
5912
5913         /* XXX: Do the pipe assertions at the right place for BXT DSI. */
5914         if (!transcoder_is_dsi(cpu_transcoder))
5915                 intel_disable_pipe(old_crtc_state);
5916
5917         if (intel_crtc_has_type(old_crtc_state, INTEL_OUTPUT_DP_MST))
5918                 intel_ddi_set_vc_payload_alloc(old_crtc_state, false);
5919
5920         if (!transcoder_is_dsi(cpu_transcoder))
5921                 intel_ddi_disable_transcoder_func(old_crtc_state);
5922
5923         intel_dsc_disable(old_crtc_state);
5924
5925         if (INTEL_GEN(dev_priv) >= 9)
5926                 skylake_scaler_disable(intel_crtc);
5927         else
5928                 ironlake_pfit_disable(old_crtc_state);
5929
5930         intel_encoders_post_disable(crtc, old_crtc_state, old_state);
5931
5932         intel_encoders_post_pll_disable(crtc, old_crtc_state, old_state);
5933 }
5934
5935 static void i9xx_pfit_enable(const struct intel_crtc_state *crtc_state)
5936 {
5937         struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
5938         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
5939
5940         if (!crtc_state->gmch_pfit.control)
5941                 return;
5942
5943         /*
5944          * The panel fitter should only be adjusted whilst the pipe is disabled,
5945          * according to register description and PRM.
5946          */
5947         WARN_ON(I915_READ(PFIT_CONTROL) & PFIT_ENABLE);
5948         assert_pipe_disabled(dev_priv, crtc->pipe);
5949
5950         I915_WRITE(PFIT_PGM_RATIOS, crtc_state->gmch_pfit.pgm_ratios);
5951         I915_WRITE(PFIT_CONTROL, crtc_state->gmch_pfit.control);
5952
5953         /* Border color in case we don't scale up to the full screen. Black by
5954          * default, change to something else for debugging. */
5955         I915_WRITE(BCLRPAT(crtc->pipe), 0);
5956 }
5957
5958 bool intel_port_is_combophy(struct drm_i915_private *dev_priv, enum port port)
5959 {
5960         if (port == PORT_NONE)
5961                 return false;
5962
5963         if (IS_ICELAKE(dev_priv))
5964                 return port <= PORT_B;
5965
5966         return false;
5967 }
5968
5969 bool intel_port_is_tc(struct drm_i915_private *dev_priv, enum port port)
5970 {
5971         if (IS_ICELAKE(dev_priv))
5972                 return port >= PORT_C && port <= PORT_F;
5973
5974         return false;
5975 }
5976
5977 enum tc_port intel_port_to_tc(struct drm_i915_private *dev_priv, enum port port)
5978 {
5979         if (!intel_port_is_tc(dev_priv, port))
5980                 return PORT_TC_NONE;
5981
5982         return port - PORT_C;
5983 }
5984
5985 enum intel_display_power_domain intel_port_to_power_domain(enum port port)
5986 {
5987         switch (port) {
5988         case PORT_A:
5989                 return POWER_DOMAIN_PORT_DDI_A_LANES;
5990         case PORT_B:
5991                 return POWER_DOMAIN_PORT_DDI_B_LANES;
5992         case PORT_C:
5993                 return POWER_DOMAIN_PORT_DDI_C_LANES;
5994         case PORT_D:
5995                 return POWER_DOMAIN_PORT_DDI_D_LANES;
5996         case PORT_E:
5997                 return POWER_DOMAIN_PORT_DDI_E_LANES;
5998         case PORT_F:
5999                 return POWER_DOMAIN_PORT_DDI_F_LANES;
6000         default:
6001                 MISSING_CASE(port);
6002                 return POWER_DOMAIN_PORT_OTHER;
6003         }
6004 }
6005
6006 enum intel_display_power_domain
6007 intel_aux_power_domain(struct intel_digital_port *dig_port)
6008 {
6009         switch (dig_port->aux_ch) {
6010         case AUX_CH_A:
6011                 return POWER_DOMAIN_AUX_A;
6012         case AUX_CH_B:
6013                 return POWER_DOMAIN_AUX_B;
6014         case AUX_CH_C:
6015                 return POWER_DOMAIN_AUX_C;
6016         case AUX_CH_D:
6017                 return POWER_DOMAIN_AUX_D;
6018         case AUX_CH_E:
6019                 return POWER_DOMAIN_AUX_E;
6020         case AUX_CH_F:
6021                 return POWER_DOMAIN_AUX_F;
6022         default:
6023                 MISSING_CASE(dig_port->aux_ch);
6024                 return POWER_DOMAIN_AUX_A;
6025         }
6026 }
6027
6028 static u64 get_crtc_power_domains(struct drm_crtc *crtc,
6029                                   struct intel_crtc_state *crtc_state)
6030 {
6031         struct drm_device *dev = crtc->dev;
6032         struct drm_i915_private *dev_priv = to_i915(dev);
6033         struct drm_encoder *encoder;
6034         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
6035         enum pipe pipe = intel_crtc->pipe;
6036         u64 mask;
6037         enum transcoder transcoder = crtc_state->cpu_transcoder;
6038
6039         if (!crtc_state->base.active)
6040                 return 0;
6041
6042         mask = BIT_ULL(POWER_DOMAIN_PIPE(pipe));
6043         mask |= BIT_ULL(POWER_DOMAIN_TRANSCODER(transcoder));
6044         if (crtc_state->pch_pfit.enabled ||
6045             crtc_state->pch_pfit.force_thru)
6046                 mask |= BIT_ULL(POWER_DOMAIN_PIPE_PANEL_FITTER(pipe));
6047
6048         drm_for_each_encoder_mask(encoder, dev, crtc_state->base.encoder_mask) {
6049                 struct intel_encoder *intel_encoder = to_intel_encoder(encoder);
6050
6051                 mask |= BIT_ULL(intel_encoder->power_domain);
6052         }
6053
6054         if (HAS_DDI(dev_priv) && crtc_state->has_audio)
6055                 mask |= BIT_ULL(POWER_DOMAIN_AUDIO);
6056
6057         if (crtc_state->shared_dpll)
6058                 mask |= BIT_ULL(POWER_DOMAIN_PLLS);
6059
6060         return mask;
6061 }
6062
6063 static u64
6064 modeset_get_crtc_power_domains(struct drm_crtc *crtc,
6065                                struct intel_crtc_state *crtc_state)
6066 {
6067         struct drm_i915_private *dev_priv = to_i915(crtc->dev);
6068         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
6069         enum intel_display_power_domain domain;
6070         u64 domains, new_domains, old_domains;
6071
6072         old_domains = intel_crtc->enabled_power_domains;
6073         intel_crtc->enabled_power_domains = new_domains =
6074                 get_crtc_power_domains(crtc, crtc_state);
6075
6076         domains = new_domains & ~old_domains;
6077
6078         for_each_power_domain(domain, domains)
6079                 intel_display_power_get(dev_priv, domain);
6080
6081         return old_domains & ~new_domains;
6082 }
6083
6084 static void modeset_put_power_domains(struct drm_i915_private *dev_priv,
6085                                       u64 domains)
6086 {
6087         enum intel_display_power_domain domain;
6088
6089         for_each_power_domain(domain, domains)
6090                 intel_display_power_put(dev_priv, domain);
6091 }
6092
6093 static void valleyview_crtc_enable(struct intel_crtc_state *pipe_config,
6094                                    struct drm_atomic_state *old_state)
6095 {
6096         struct intel_atomic_state *old_intel_state =
6097                 to_intel_atomic_state(old_state);
6098         struct drm_crtc *crtc = pipe_config->base.crtc;
6099         struct drm_device *dev = crtc->dev;
6100         struct drm_i915_private *dev_priv = to_i915(dev);
6101         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
6102         int pipe = intel_crtc->pipe;
6103
6104         if (WARN_ON(intel_crtc->active))
6105                 return;
6106
6107         if (intel_crtc_has_dp_encoder(pipe_config))
6108                 intel_dp_set_m_n(pipe_config, M1_N1);
6109
6110         intel_set_pipe_timings(pipe_config);
6111         intel_set_pipe_src_size(pipe_config);
6112
6113         if (IS_CHERRYVIEW(dev_priv) && pipe == PIPE_B) {
6114                 I915_WRITE(CHV_BLEND(pipe), CHV_BLEND_LEGACY);
6115                 I915_WRITE(CHV_CANVAS(pipe), 0);
6116         }
6117
6118         i9xx_set_pipeconf(pipe_config);
6119
6120         intel_color_set_csc(pipe_config);
6121
6122         intel_crtc->active = true;
6123
6124         intel_set_cpu_fifo_underrun_reporting(dev_priv, pipe, true);
6125
6126         intel_encoders_pre_pll_enable(crtc, pipe_config, old_state);
6127
6128         if (IS_CHERRYVIEW(dev_priv)) {
6129                 chv_prepare_pll(intel_crtc, pipe_config);
6130                 chv_enable_pll(intel_crtc, pipe_config);
6131         } else {
6132                 vlv_prepare_pll(intel_crtc, pipe_config);
6133                 vlv_enable_pll(intel_crtc, pipe_config);
6134         }
6135
6136         intel_encoders_pre_enable(crtc, pipe_config, old_state);
6137
6138         i9xx_pfit_enable(pipe_config);
6139
6140         intel_color_load_luts(pipe_config);
6141
6142         dev_priv->display.initial_watermarks(old_intel_state,
6143                                              pipe_config);
6144         intel_enable_pipe(pipe_config);
6145
6146         assert_vblank_disabled(crtc);
6147         drm_crtc_vblank_on(crtc);
6148
6149         intel_encoders_enable(crtc, pipe_config, old_state);
6150 }
6151
6152 static void i9xx_set_pll_dividers(const struct intel_crtc_state *crtc_state)
6153 {
6154         struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
6155         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
6156
6157         I915_WRITE(FP0(crtc->pipe), crtc_state->dpll_hw_state.fp0);
6158         I915_WRITE(FP1(crtc->pipe), crtc_state->dpll_hw_state.fp1);
6159 }
6160
6161 static void i9xx_crtc_enable(struct intel_crtc_state *pipe_config,
6162                              struct drm_atomic_state *old_state)
6163 {
6164         struct intel_atomic_state *old_intel_state =
6165                 to_intel_atomic_state(old_state);
6166         struct drm_crtc *crtc = pipe_config->base.crtc;
6167         struct drm_device *dev = crtc->dev;
6168         struct drm_i915_private *dev_priv = to_i915(dev);
6169         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
6170         enum pipe pipe = intel_crtc->pipe;
6171
6172         if (WARN_ON(intel_crtc->active))
6173                 return;
6174
6175         i9xx_set_pll_dividers(pipe_config);
6176
6177         if (intel_crtc_has_dp_encoder(pipe_config))
6178                 intel_dp_set_m_n(pipe_config, M1_N1);
6179
6180         intel_set_pipe_timings(pipe_config);
6181         intel_set_pipe_src_size(pipe_config);
6182
6183         i9xx_set_pipeconf(pipe_config);
6184
6185         intel_crtc->active = true;
6186
6187         if (!IS_GEN(dev_priv, 2))
6188                 intel_set_cpu_fifo_underrun_reporting(dev_priv, pipe, true);
6189
6190         intel_encoders_pre_enable(crtc, pipe_config, old_state);
6191
6192         i9xx_enable_pll(intel_crtc, pipe_config);
6193
6194         i9xx_pfit_enable(pipe_config);
6195
6196         intel_color_load_luts(pipe_config);
6197
6198         if (dev_priv->display.initial_watermarks != NULL)
6199                 dev_priv->display.initial_watermarks(old_intel_state,
6200                                                      pipe_config);
6201         else
6202                 intel_update_watermarks(intel_crtc);
6203         intel_enable_pipe(pipe_config);
6204
6205         assert_vblank_disabled(crtc);
6206         drm_crtc_vblank_on(crtc);
6207
6208         intel_encoders_enable(crtc, pipe_config, old_state);
6209 }
6210
6211 static void i9xx_pfit_disable(const struct intel_crtc_state *old_crtc_state)
6212 {
6213         struct intel_crtc *crtc = to_intel_crtc(old_crtc_state->base.crtc);
6214         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
6215
6216         if (!old_crtc_state->gmch_pfit.control)
6217                 return;
6218
6219         assert_pipe_disabled(dev_priv, crtc->pipe);
6220
6221         DRM_DEBUG_KMS("disabling pfit, current: 0x%08x\n",
6222                       I915_READ(PFIT_CONTROL));
6223         I915_WRITE(PFIT_CONTROL, 0);
6224 }
6225
6226 static void i9xx_crtc_disable(struct intel_crtc_state *old_crtc_state,
6227                               struct drm_atomic_state *old_state)
6228 {
6229         struct drm_crtc *crtc = old_crtc_state->base.crtc;
6230         struct drm_device *dev = crtc->dev;
6231         struct drm_i915_private *dev_priv = to_i915(dev);
6232         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
6233         int pipe = intel_crtc->pipe;
6234
6235         /*
6236          * On gen2 planes are double buffered but the pipe isn't, so we must
6237          * wait for planes to fully turn off before disabling the pipe.
6238          */
6239         if (IS_GEN(dev_priv, 2))
6240                 intel_wait_for_vblank(dev_priv, pipe);
6241
6242         intel_encoders_disable(crtc, old_crtc_state, old_state);
6243
6244         drm_crtc_vblank_off(crtc);
6245         assert_vblank_disabled(crtc);
6246
6247         intel_disable_pipe(old_crtc_state);
6248
6249         i9xx_pfit_disable(old_crtc_state);
6250
6251         intel_encoders_post_disable(crtc, old_crtc_state, old_state);
6252
6253         if (!intel_crtc_has_type(old_crtc_state, INTEL_OUTPUT_DSI)) {
6254                 if (IS_CHERRYVIEW(dev_priv))
6255                         chv_disable_pll(dev_priv, pipe);
6256                 else if (IS_VALLEYVIEW(dev_priv))
6257                         vlv_disable_pll(dev_priv, pipe);
6258                 else
6259                         i9xx_disable_pll(old_crtc_state);
6260         }
6261
6262         intel_encoders_post_pll_disable(crtc, old_crtc_state, old_state);
6263
6264         if (!IS_GEN(dev_priv, 2))
6265                 intel_set_cpu_fifo_underrun_reporting(dev_priv, pipe, false);
6266
6267         if (!dev_priv->display.initial_watermarks)
6268                 intel_update_watermarks(intel_crtc);
6269
6270         /* clock the pipe down to 640x480@60 to potentially save power */
6271         if (IS_I830(dev_priv))
6272                 i830_enable_pipe(dev_priv, pipe);
6273 }
6274
6275 static void intel_crtc_disable_noatomic(struct drm_crtc *crtc,
6276                                         struct drm_modeset_acquire_ctx *ctx)
6277 {
6278         struct intel_encoder *encoder;
6279         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
6280         struct drm_i915_private *dev_priv = to_i915(crtc->dev);
6281         enum intel_display_power_domain domain;
6282         struct intel_plane *plane;
6283         u64 domains;
6284         struct drm_atomic_state *state;
6285         struct intel_crtc_state *crtc_state;
6286         int ret;
6287
6288         if (!intel_crtc->active)
6289                 return;
6290
6291         for_each_intel_plane_on_crtc(&dev_priv->drm, intel_crtc, plane) {
6292                 const struct intel_plane_state *plane_state =
6293                         to_intel_plane_state(plane->base.state);
6294
6295                 if (plane_state->base.visible)
6296                         intel_plane_disable_noatomic(intel_crtc, plane);
6297         }
6298
6299         state = drm_atomic_state_alloc(crtc->dev);
6300         if (!state) {
6301                 DRM_DEBUG_KMS("failed to disable [CRTC:%d:%s], out of memory",
6302                               crtc->base.id, crtc->name);
6303                 return;
6304         }
6305
6306         state->acquire_ctx = ctx;
6307
6308         /* Everything's already locked, -EDEADLK can't happen. */
6309         crtc_state = intel_atomic_get_crtc_state(state, intel_crtc);
6310         ret = drm_atomic_add_affected_connectors(state, crtc);
6311
6312         WARN_ON(IS_ERR(crtc_state) || ret);
6313
6314         dev_priv->display.crtc_disable(crtc_state, state);
6315
6316         drm_atomic_state_put(state);
6317
6318         DRM_DEBUG_KMS("[CRTC:%d:%s] hw state adjusted, was enabled, now disabled\n",
6319                       crtc->base.id, crtc->name);
6320
6321         WARN_ON(drm_atomic_set_mode_for_crtc(crtc->state, NULL) < 0);
6322         crtc->state->active = false;
6323         intel_crtc->active = false;
6324         crtc->enabled = false;
6325         crtc->state->connector_mask = 0;
6326         crtc->state->encoder_mask = 0;
6327
6328         for_each_encoder_on_crtc(crtc->dev, crtc, encoder)
6329                 encoder->base.crtc = NULL;
6330
6331         intel_fbc_disable(intel_crtc);
6332         intel_update_watermarks(intel_crtc);
6333         intel_disable_shared_dpll(to_intel_crtc_state(crtc->state));
6334
6335         domains = intel_crtc->enabled_power_domains;
6336         for_each_power_domain(domain, domains)
6337                 intel_display_power_put(dev_priv, domain);
6338         intel_crtc->enabled_power_domains = 0;
6339
6340         dev_priv->active_crtcs &= ~(1 << intel_crtc->pipe);
6341         dev_priv->min_cdclk[intel_crtc->pipe] = 0;
6342         dev_priv->min_voltage_level[intel_crtc->pipe] = 0;
6343 }
6344
6345 /*
6346  * turn all crtc's off, but do not adjust state
6347  * This has to be paired with a call to intel_modeset_setup_hw_state.
6348  */
6349 int intel_display_suspend(struct drm_device *dev)
6350 {
6351         struct drm_i915_private *dev_priv = to_i915(dev);
6352         struct drm_atomic_state *state;
6353         int ret;
6354
6355         state = drm_atomic_helper_suspend(dev);
6356         ret = PTR_ERR_OR_ZERO(state);
6357         if (ret)
6358                 DRM_ERROR("Suspending crtc's failed with %i\n", ret);
6359         else
6360                 dev_priv->modeset_restore_state = state;
6361         return ret;
6362 }
6363
6364 void intel_encoder_destroy(struct drm_encoder *encoder)
6365 {
6366         struct intel_encoder *intel_encoder = to_intel_encoder(encoder);
6367
6368         drm_encoder_cleanup(encoder);
6369         kfree(intel_encoder);
6370 }
6371
6372 /* Cross check the actual hw state with our own modeset state tracking (and it's
6373  * internal consistency). */
6374 static void intel_connector_verify_state(struct drm_crtc_state *crtc_state,
6375                                          struct drm_connector_state *conn_state)
6376 {
6377         struct intel_connector *connector = to_intel_connector(conn_state->connector);
6378
6379         DRM_DEBUG_KMS("[CONNECTOR:%d:%s]\n",
6380                       connector->base.base.id,
6381                       connector->base.name);
6382
6383         if (connector->get_hw_state(connector)) {
6384                 struct intel_encoder *encoder = connector->encoder;
6385
6386                 I915_STATE_WARN(!crtc_state,
6387                          "connector enabled without attached crtc\n");
6388
6389                 if (!crtc_state)
6390                         return;
6391
6392                 I915_STATE_WARN(!crtc_state->active,
6393                       "connector is active, but attached crtc isn't\n");
6394
6395                 if (!encoder || encoder->type == INTEL_OUTPUT_DP_MST)
6396                         return;
6397
6398                 I915_STATE_WARN(conn_state->best_encoder != &encoder->base,
6399                         "atomic encoder doesn't match attached encoder\n");
6400
6401                 I915_STATE_WARN(conn_state->crtc != encoder->base.crtc,
6402                         "attached encoder crtc differs from connector crtc\n");
6403         } else {
6404                 I915_STATE_WARN(crtc_state && crtc_state->active,
6405                         "attached crtc is active, but connector isn't\n");
6406                 I915_STATE_WARN(!crtc_state && conn_state->best_encoder,
6407                         "best encoder set without crtc!\n");
6408         }
6409 }
6410
6411 static int pipe_required_fdi_lanes(struct intel_crtc_state *crtc_state)
6412 {
6413         if (crtc_state->base.enable && crtc_state->has_pch_encoder)
6414                 return crtc_state->fdi_lanes;
6415
6416         return 0;
6417 }
6418
6419 static int ironlake_check_fdi_lanes(struct drm_device *dev, enum pipe pipe,
6420                                      struct intel_crtc_state *pipe_config)
6421 {
6422         struct drm_i915_private *dev_priv = to_i915(dev);
6423         struct drm_atomic_state *state = pipe_config->base.state;
6424         struct intel_crtc *other_crtc;
6425         struct intel_crtc_state *other_crtc_state;
6426
6427         DRM_DEBUG_KMS("checking fdi config on pipe %c, lanes %i\n",
6428                       pipe_name(pipe), pipe_config->fdi_lanes);
6429         if (pipe_config->fdi_lanes > 4) {
6430                 DRM_DEBUG_KMS("invalid fdi lane config on pipe %c: %i lanes\n",
6431                               pipe_name(pipe), pipe_config->fdi_lanes);
6432                 return -EINVAL;
6433         }
6434
6435         if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv)) {
6436                 if (pipe_config->fdi_lanes > 2) {
6437                         DRM_DEBUG_KMS("only 2 lanes on haswell, required: %i lanes\n",
6438                                       pipe_config->fdi_lanes);
6439                         return -EINVAL;
6440                 } else {
6441                         return 0;
6442                 }
6443         }
6444
6445         if (INTEL_INFO(dev_priv)->num_pipes == 2)
6446                 return 0;
6447
6448         /* Ivybridge 3 pipe is really complicated */
6449         switch (pipe) {
6450         case PIPE_A:
6451                 return 0;
6452         case PIPE_B:
6453                 if (pipe_config->fdi_lanes <= 2)
6454                         return 0;
6455
6456                 other_crtc = intel_get_crtc_for_pipe(dev_priv, PIPE_C);
6457                 other_crtc_state =
6458                         intel_atomic_get_crtc_state(state, other_crtc);
6459                 if (IS_ERR(other_crtc_state))
6460                         return PTR_ERR(other_crtc_state);
6461
6462                 if (pipe_required_fdi_lanes(other_crtc_state) > 0) {
6463                         DRM_DEBUG_KMS("invalid shared fdi lane config on pipe %c: %i lanes\n",
6464                                       pipe_name(pipe), pipe_config->fdi_lanes);
6465                         return -EINVAL;
6466                 }
6467                 return 0;
6468         case PIPE_C:
6469                 if (pipe_config->fdi_lanes > 2) {
6470                         DRM_DEBUG_KMS("only 2 lanes on pipe %c: required %i lanes\n",
6471                                       pipe_name(pipe), pipe_config->fdi_lanes);
6472                         return -EINVAL;
6473                 }
6474
6475                 other_crtc = intel_get_crtc_for_pipe(dev_priv, PIPE_B);
6476                 other_crtc_state =
6477                         intel_atomic_get_crtc_state(state, other_crtc);
6478                 if (IS_ERR(other_crtc_state))
6479                         return PTR_ERR(other_crtc_state);
6480
6481                 if (pipe_required_fdi_lanes(other_crtc_state) > 2) {
6482                         DRM_DEBUG_KMS("fdi link B uses too many lanes to enable link C\n");
6483                         return -EINVAL;
6484                 }
6485                 return 0;
6486         default:
6487                 BUG();
6488         }
6489 }
6490
6491 #define RETRY 1
6492 static int ironlake_fdi_compute_config(struct intel_crtc *intel_crtc,
6493                                        struct intel_crtc_state *pipe_config)
6494 {
6495         struct drm_device *dev = intel_crtc->base.dev;
6496         const struct drm_display_mode *adjusted_mode = &pipe_config->base.adjusted_mode;
6497         int lane, link_bw, fdi_dotclock, ret;
6498         bool needs_recompute = false;
6499
6500 retry:
6501         /* FDI is a binary signal running at ~2.7GHz, encoding
6502          * each output octet as 10 bits. The actual frequency
6503          * is stored as a divider into a 100MHz clock, and the
6504          * mode pixel clock is stored in units of 1KHz.
6505          * Hence the bw of each lane in terms of the mode signal
6506          * is:
6507          */
6508         link_bw = intel_fdi_link_freq(to_i915(dev), pipe_config);
6509
6510         fdi_dotclock = adjusted_mode->crtc_clock;
6511
6512         lane = ironlake_get_lanes_required(fdi_dotclock, link_bw,
6513                                            pipe_config->pipe_bpp);
6514
6515         pipe_config->fdi_lanes = lane;
6516
6517         intel_link_compute_m_n(pipe_config->pipe_bpp, lane, fdi_dotclock,
6518                                link_bw, &pipe_config->fdi_m_n, false);
6519
6520         ret = ironlake_check_fdi_lanes(dev, intel_crtc->pipe, pipe_config);
6521         if (ret == -EDEADLK)
6522                 return ret;
6523
6524         if (ret == -EINVAL && pipe_config->pipe_bpp > 6*3) {
6525                 pipe_config->pipe_bpp -= 2*3;
6526                 DRM_DEBUG_KMS("fdi link bw constraint, reducing pipe bpp to %i\n",
6527                               pipe_config->pipe_bpp);
6528                 needs_recompute = true;
6529                 pipe_config->bw_constrained = true;
6530
6531                 goto retry;
6532         }
6533
6534         if (needs_recompute)
6535                 return RETRY;
6536
6537         return ret;
6538 }
6539
6540 bool hsw_crtc_state_ips_capable(const struct intel_crtc_state *crtc_state)
6541 {
6542         struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
6543         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
6544
6545         /* IPS only exists on ULT machines and is tied to pipe A. */
6546         if (!hsw_crtc_supports_ips(crtc))
6547                 return false;
6548
6549         if (!i915_modparams.enable_ips)
6550                 return false;
6551
6552         if (crtc_state->pipe_bpp > 24)
6553                 return false;
6554
6555         /*
6556          * We compare against max which means we must take
6557          * the increased cdclk requirement into account when
6558          * calculating the new cdclk.
6559          *
6560          * Should measure whether using a lower cdclk w/o IPS
6561          */
6562         if (IS_BROADWELL(dev_priv) &&
6563             crtc_state->pixel_rate > dev_priv->max_cdclk_freq * 95 / 100)
6564                 return false;
6565
6566         return true;
6567 }
6568
6569 static bool hsw_compute_ips_config(struct intel_crtc_state *crtc_state)
6570 {
6571         struct drm_i915_private *dev_priv =
6572                 to_i915(crtc_state->base.crtc->dev);
6573         struct intel_atomic_state *intel_state =
6574                 to_intel_atomic_state(crtc_state->base.state);
6575
6576         if (!hsw_crtc_state_ips_capable(crtc_state))
6577                 return false;
6578
6579         if (crtc_state->ips_force_disable)
6580                 return false;
6581
6582         /* IPS should be fine as long as at least one plane is enabled. */
6583         if (!(crtc_state->active_planes & ~BIT(PLANE_CURSOR)))
6584                 return false;
6585
6586         /* pixel rate mustn't exceed 95% of cdclk with IPS on BDW */
6587         if (IS_BROADWELL(dev_priv) &&
6588             crtc_state->pixel_rate > intel_state->cdclk.logical.cdclk * 95 / 100)
6589                 return false;
6590
6591         return true;
6592 }
6593
6594 static bool intel_crtc_supports_double_wide(const struct intel_crtc *crtc)
6595 {
6596         const struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
6597
6598         /* GDG double wide on either pipe, otherwise pipe A only */
6599         return INTEL_GEN(dev_priv) < 4 &&
6600                 (crtc->pipe == PIPE_A || IS_I915G(dev_priv));
6601 }
6602
6603 static uint32_t ilk_pipe_pixel_rate(const struct intel_crtc_state *pipe_config)
6604 {
6605         uint32_t pixel_rate;
6606
6607         pixel_rate = pipe_config->base.adjusted_mode.crtc_clock;
6608
6609         /*
6610          * We only use IF-ID interlacing. If we ever use
6611          * PF-ID we'll need to adjust the pixel_rate here.
6612          */
6613
6614         if (pipe_config->pch_pfit.enabled) {
6615                 uint64_t pipe_w, pipe_h, pfit_w, pfit_h;
6616                 uint32_t pfit_size = pipe_config->pch_pfit.size;
6617
6618                 pipe_w = pipe_config->pipe_src_w;
6619                 pipe_h = pipe_config->pipe_src_h;
6620
6621                 pfit_w = (pfit_size >> 16) & 0xFFFF;
6622                 pfit_h = pfit_size & 0xFFFF;
6623                 if (pipe_w < pfit_w)
6624                         pipe_w = pfit_w;
6625                 if (pipe_h < pfit_h)
6626                         pipe_h = pfit_h;
6627
6628                 if (WARN_ON(!pfit_w || !pfit_h))
6629                         return pixel_rate;
6630
6631                 pixel_rate = div_u64((uint64_t) pixel_rate * pipe_w * pipe_h,
6632                                      pfit_w * pfit_h);
6633         }
6634
6635         return pixel_rate;
6636 }
6637
6638 static void intel_crtc_compute_pixel_rate(struct intel_crtc_state *crtc_state)
6639 {
6640         struct drm_i915_private *dev_priv = to_i915(crtc_state->base.crtc->dev);
6641
6642         if (HAS_GMCH_DISPLAY(dev_priv))
6643                 /* FIXME calculate proper pipe pixel rate for GMCH pfit */
6644                 crtc_state->pixel_rate =
6645                         crtc_state->base.adjusted_mode.crtc_clock;
6646         else
6647                 crtc_state->pixel_rate =
6648                         ilk_pipe_pixel_rate(crtc_state);
6649 }
6650
6651 static int intel_crtc_compute_config(struct intel_crtc *crtc,
6652                                      struct intel_crtc_state *pipe_config)
6653 {
6654         struct drm_device *dev = crtc->base.dev;
6655         struct drm_i915_private *dev_priv = to_i915(dev);
6656         const struct drm_display_mode *adjusted_mode = &pipe_config->base.adjusted_mode;
6657         int clock_limit = dev_priv->max_dotclk_freq;
6658
6659         if (INTEL_GEN(dev_priv) < 4) {
6660                 clock_limit = dev_priv->max_cdclk_freq * 9 / 10;
6661
6662                 /*
6663                  * Enable double wide mode when the dot clock
6664                  * is > 90% of the (display) core speed.
6665                  */
6666                 if (intel_crtc_supports_double_wide(crtc) &&
6667                     adjusted_mode->crtc_clock > clock_limit) {
6668                         clock_limit = dev_priv->max_dotclk_freq;
6669                         pipe_config->double_wide = true;
6670                 }
6671         }
6672
6673         if (adjusted_mode->crtc_clock > clock_limit) {
6674                 DRM_DEBUG_KMS("requested pixel clock (%d kHz) too high (max: %d kHz, double wide: %s)\n",
6675                               adjusted_mode->crtc_clock, clock_limit,
6676                               yesno(pipe_config->double_wide));
6677                 return -EINVAL;
6678         }
6679
6680         if ((pipe_config->output_format == INTEL_OUTPUT_FORMAT_YCBCR420 ||
6681              pipe_config->output_format == INTEL_OUTPUT_FORMAT_YCBCR444) &&
6682              pipe_config->base.ctm) {
6683                 /*
6684                  * There is only one pipe CSC unit per pipe, and we need that
6685                  * for output conversion from RGB->YCBCR. So if CTM is already
6686                  * applied we can't support YCBCR420 output.
6687                  */
6688                 DRM_DEBUG_KMS("YCBCR420 and CTM together are not possible\n");
6689                 return -EINVAL;
6690         }
6691
6692         /*
6693          * Pipe horizontal size must be even in:
6694          * - DVO ganged mode
6695          * - LVDS dual channel mode
6696          * - Double wide pipe
6697          */
6698         if (pipe_config->pipe_src_w & 1) {
6699                 if (pipe_config->double_wide) {
6700                         DRM_DEBUG_KMS("Odd pipe source width not supported with double wide pipe\n");
6701                         return -EINVAL;
6702                 }
6703
6704                 if (intel_crtc_has_type(pipe_config, INTEL_OUTPUT_LVDS) &&
6705                     intel_is_dual_link_lvds(dev)) {
6706                         DRM_DEBUG_KMS("Odd pipe source width not supported with dual link LVDS\n");
6707                         return -EINVAL;
6708                 }
6709         }
6710
6711         /* Cantiga+ cannot handle modes with a hsync front porch of 0.
6712          * WaPruneModeWithIncorrectHsyncOffset:ctg,elk,ilk,snb,ivb,vlv,hsw.
6713          */
6714         if ((INTEL_GEN(dev_priv) > 4 || IS_G4X(dev_priv)) &&
6715                 adjusted_mode->crtc_hsync_start == adjusted_mode->crtc_hdisplay)
6716                 return -EINVAL;
6717
6718         intel_crtc_compute_pixel_rate(pipe_config);
6719
6720         if (pipe_config->has_pch_encoder)
6721                 return ironlake_fdi_compute_config(crtc, pipe_config);
6722
6723         return 0;
6724 }
6725
6726 static void
6727 intel_reduce_m_n_ratio(uint32_t *num, uint32_t *den)
6728 {
6729         while (*num > DATA_LINK_M_N_MASK ||
6730                *den > DATA_LINK_M_N_MASK) {
6731                 *num >>= 1;
6732                 *den >>= 1;
6733         }
6734 }
6735
6736 static void compute_m_n(unsigned int m, unsigned int n,
6737                         uint32_t *ret_m, uint32_t *ret_n,
6738                         bool constant_n)
6739 {
6740         /*
6741          * Several DP dongles in particular seem to be fussy about
6742          * too large link M/N values. Give N value as 0x8000 that
6743          * should be acceptable by specific devices. 0x8000 is the
6744          * specified fixed N value for asynchronous clock mode,
6745          * which the devices expect also in synchronous clock mode.
6746          */
6747         if (constant_n)
6748                 *ret_n = 0x8000;
6749         else
6750                 *ret_n = min_t(unsigned int, roundup_pow_of_two(n), DATA_LINK_N_MAX);
6751
6752         *ret_m = div_u64((uint64_t) m * *ret_n, n);
6753         intel_reduce_m_n_ratio(ret_m, ret_n);
6754 }
6755
6756 void
6757 intel_link_compute_m_n(u16 bits_per_pixel, int nlanes,
6758                        int pixel_clock, int link_clock,
6759                        struct intel_link_m_n *m_n,
6760                        bool constant_n)
6761 {
6762         m_n->tu = 64;
6763
6764         compute_m_n(bits_per_pixel * pixel_clock,
6765                     link_clock * nlanes * 8,
6766                     &m_n->gmch_m, &m_n->gmch_n,
6767                     constant_n);
6768
6769         compute_m_n(pixel_clock, link_clock,
6770                     &m_n->link_m, &m_n->link_n,
6771                     constant_n);
6772 }
6773
6774 static inline bool intel_panel_use_ssc(struct drm_i915_private *dev_priv)
6775 {
6776         if (i915_modparams.panel_use_ssc >= 0)
6777                 return i915_modparams.panel_use_ssc != 0;
6778         return dev_priv->vbt.lvds_use_ssc
6779                 && !(dev_priv->quirks & QUIRK_LVDS_SSC_DISABLE);
6780 }
6781
6782 static uint32_t pnv_dpll_compute_fp(struct dpll *dpll)
6783 {
6784         return (1 << dpll->n) << 16 | dpll->m2;
6785 }
6786
6787 static uint32_t i9xx_dpll_compute_fp(struct dpll *dpll)
6788 {
6789         return dpll->n << 16 | dpll->m1 << 8 | dpll->m2;
6790 }
6791
6792 static void i9xx_update_pll_dividers(struct intel_crtc *crtc,
6793                                      struct intel_crtc_state *crtc_state,
6794                                      struct dpll *reduced_clock)
6795 {
6796         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
6797         u32 fp, fp2 = 0;
6798
6799         if (IS_PINEVIEW(dev_priv)) {
6800                 fp = pnv_dpll_compute_fp(&crtc_state->dpll);
6801                 if (reduced_clock)
6802                         fp2 = pnv_dpll_compute_fp(reduced_clock);
6803         } else {
6804                 fp = i9xx_dpll_compute_fp(&crtc_state->dpll);
6805                 if (reduced_clock)
6806                         fp2 = i9xx_dpll_compute_fp(reduced_clock);
6807         }
6808
6809         crtc_state->dpll_hw_state.fp0 = fp;
6810
6811         if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_LVDS) &&
6812             reduced_clock) {
6813                 crtc_state->dpll_hw_state.fp1 = fp2;
6814         } else {
6815                 crtc_state->dpll_hw_state.fp1 = fp;
6816         }
6817 }
6818
6819 static void vlv_pllb_recal_opamp(struct drm_i915_private *dev_priv, enum pipe
6820                 pipe)
6821 {
6822         u32 reg_val;
6823
6824         /*
6825          * PLLB opamp always calibrates to max value of 0x3f, force enable it
6826          * and set it to a reasonable value instead.
6827          */
6828         reg_val = vlv_dpio_read(dev_priv, pipe, VLV_PLL_DW9(1));
6829         reg_val &= 0xffffff00;
6830         reg_val |= 0x00000030;
6831         vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW9(1), reg_val);
6832
6833         reg_val = vlv_dpio_read(dev_priv, pipe, VLV_REF_DW13);
6834         reg_val &= 0x00ffffff;
6835         reg_val |= 0x8c000000;
6836         vlv_dpio_write(dev_priv, pipe, VLV_REF_DW13, reg_val);
6837
6838         reg_val = vlv_dpio_read(dev_priv, pipe, VLV_PLL_DW9(1));
6839         reg_val &= 0xffffff00;
6840         vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW9(1), reg_val);
6841
6842         reg_val = vlv_dpio_read(dev_priv, pipe, VLV_REF_DW13);
6843         reg_val &= 0x00ffffff;
6844         reg_val |= 0xb0000000;
6845         vlv_dpio_write(dev_priv, pipe, VLV_REF_DW13, reg_val);
6846 }
6847
6848 static void intel_pch_transcoder_set_m_n(const struct intel_crtc_state *crtc_state,
6849                                          const struct intel_link_m_n *m_n)
6850 {
6851         struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
6852         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
6853         enum pipe pipe = crtc->pipe;
6854
6855         I915_WRITE(PCH_TRANS_DATA_M1(pipe), TU_SIZE(m_n->tu) | m_n->gmch_m);
6856         I915_WRITE(PCH_TRANS_DATA_N1(pipe), m_n->gmch_n);
6857         I915_WRITE(PCH_TRANS_LINK_M1(pipe), m_n->link_m);
6858         I915_WRITE(PCH_TRANS_LINK_N1(pipe), m_n->link_n);
6859 }
6860
6861 static bool transcoder_has_m2_n2(struct drm_i915_private *dev_priv,
6862                                  enum transcoder transcoder)
6863 {
6864         if (IS_HASWELL(dev_priv))
6865                 return transcoder == TRANSCODER_EDP;
6866
6867         /*
6868          * Strictly speaking some registers are available before
6869          * gen7, but we only support DRRS on gen7+
6870          */
6871         return IS_GEN(dev_priv, 7) || IS_CHERRYVIEW(dev_priv);
6872 }
6873
6874 static void intel_cpu_transcoder_set_m_n(const struct intel_crtc_state *crtc_state,
6875                                          const struct intel_link_m_n *m_n,
6876                                          const struct intel_link_m_n *m2_n2)
6877 {
6878         struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
6879         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
6880         enum pipe pipe = crtc->pipe;
6881         enum transcoder transcoder = crtc_state->cpu_transcoder;
6882
6883         if (INTEL_GEN(dev_priv) >= 5) {
6884                 I915_WRITE(PIPE_DATA_M1(transcoder), TU_SIZE(m_n->tu) | m_n->gmch_m);
6885                 I915_WRITE(PIPE_DATA_N1(transcoder), m_n->gmch_n);
6886                 I915_WRITE(PIPE_LINK_M1(transcoder), m_n->link_m);
6887                 I915_WRITE(PIPE_LINK_N1(transcoder), m_n->link_n);
6888                 /*
6889                  *  M2_N2 registers are set only if DRRS is supported
6890                  * (to make sure the registers are not unnecessarily accessed).
6891                  */
6892                 if (m2_n2 && crtc_state->has_drrs &&
6893                     transcoder_has_m2_n2(dev_priv, transcoder)) {
6894                         I915_WRITE(PIPE_DATA_M2(transcoder),
6895                                         TU_SIZE(m2_n2->tu) | m2_n2->gmch_m);
6896                         I915_WRITE(PIPE_DATA_N2(transcoder), m2_n2->gmch_n);
6897                         I915_WRITE(PIPE_LINK_M2(transcoder), m2_n2->link_m);
6898                         I915_WRITE(PIPE_LINK_N2(transcoder), m2_n2->link_n);
6899                 }
6900         } else {
6901                 I915_WRITE(PIPE_DATA_M_G4X(pipe), TU_SIZE(m_n->tu) | m_n->gmch_m);
6902                 I915_WRITE(PIPE_DATA_N_G4X(pipe), m_n->gmch_n);
6903                 I915_WRITE(PIPE_LINK_M_G4X(pipe), m_n->link_m);
6904                 I915_WRITE(PIPE_LINK_N_G4X(pipe), m_n->link_n);
6905         }
6906 }
6907
6908 void intel_dp_set_m_n(const struct intel_crtc_state *crtc_state, enum link_m_n_set m_n)
6909 {
6910         const struct intel_link_m_n *dp_m_n, *dp_m2_n2 = NULL;
6911
6912         if (m_n == M1_N1) {
6913                 dp_m_n = &crtc_state->dp_m_n;
6914                 dp_m2_n2 = &crtc_state->dp_m2_n2;
6915         } else if (m_n == M2_N2) {
6916
6917                 /*
6918                  * M2_N2 registers are not supported. Hence m2_n2 divider value
6919                  * needs to be programmed into M1_N1.
6920                  */
6921                 dp_m_n = &crtc_state->dp_m2_n2;
6922         } else {
6923                 DRM_ERROR("Unsupported divider value\n");
6924                 return;
6925         }
6926
6927         if (crtc_state->has_pch_encoder)
6928                 intel_pch_transcoder_set_m_n(crtc_state, &crtc_state->dp_m_n);
6929         else
6930                 intel_cpu_transcoder_set_m_n(crtc_state, dp_m_n, dp_m2_n2);
6931 }
6932
6933 static void vlv_compute_dpll(struct intel_crtc *crtc,
6934                              struct intel_crtc_state *pipe_config)
6935 {
6936         pipe_config->dpll_hw_state.dpll = DPLL_INTEGRATED_REF_CLK_VLV |
6937                 DPLL_REF_CLK_ENABLE_VLV | DPLL_VGA_MODE_DIS;
6938         if (crtc->pipe != PIPE_A)
6939                 pipe_config->dpll_hw_state.dpll |= DPLL_INTEGRATED_CRI_CLK_VLV;
6940
6941         /* DPLL not used with DSI, but still need the rest set up */
6942         if (!intel_crtc_has_type(pipe_config, INTEL_OUTPUT_DSI))
6943                 pipe_config->dpll_hw_state.dpll |= DPLL_VCO_ENABLE |
6944                         DPLL_EXT_BUFFER_ENABLE_VLV;
6945
6946         pipe_config->dpll_hw_state.dpll_md =
6947                 (pipe_config->pixel_multiplier - 1) << DPLL_MD_UDI_MULTIPLIER_SHIFT;
6948 }
6949
6950 static void chv_compute_dpll(struct intel_crtc *crtc,
6951                              struct intel_crtc_state *pipe_config)
6952 {
6953         pipe_config->dpll_hw_state.dpll = DPLL_SSC_REF_CLK_CHV |
6954                 DPLL_REF_CLK_ENABLE_VLV | DPLL_VGA_MODE_DIS;
6955         if (crtc->pipe != PIPE_A)
6956                 pipe_config->dpll_hw_state.dpll |= DPLL_INTEGRATED_CRI_CLK_VLV;
6957
6958         /* DPLL not used with DSI, but still need the rest set up */
6959         if (!intel_crtc_has_type(pipe_config, INTEL_OUTPUT_DSI))
6960                 pipe_config->dpll_hw_state.dpll |= DPLL_VCO_ENABLE;
6961
6962         pipe_config->dpll_hw_state.dpll_md =
6963                 (pipe_config->pixel_multiplier - 1) << DPLL_MD_UDI_MULTIPLIER_SHIFT;
6964 }
6965
6966 static void vlv_prepare_pll(struct intel_crtc *crtc,
6967                             const struct intel_crtc_state *pipe_config)
6968 {
6969         struct drm_device *dev = crtc->base.dev;
6970         struct drm_i915_private *dev_priv = to_i915(dev);
6971         enum pipe pipe = crtc->pipe;
6972         u32 mdiv;
6973         u32 bestn, bestm1, bestm2, bestp1, bestp2;
6974         u32 coreclk, reg_val;
6975
6976         /* Enable Refclk */
6977         I915_WRITE(DPLL(pipe),
6978                    pipe_config->dpll_hw_state.dpll &
6979                    ~(DPLL_VCO_ENABLE | DPLL_EXT_BUFFER_ENABLE_VLV));
6980
6981         /* No need to actually set up the DPLL with DSI */
6982         if ((pipe_config->dpll_hw_state.dpll & DPLL_VCO_ENABLE) == 0)
6983                 return;
6984
6985         mutex_lock(&dev_priv->sb_lock);
6986
6987         bestn = pipe_config->dpll.n;
6988         bestm1 = pipe_config->dpll.m1;
6989         bestm2 = pipe_config->dpll.m2;
6990         bestp1 = pipe_config->dpll.p1;
6991         bestp2 = pipe_config->dpll.p2;
6992
6993         /* See eDP HDMI DPIO driver vbios notes doc */
6994
6995         /* PLL B needs special handling */
6996         if (pipe == PIPE_B)
6997                 vlv_pllb_recal_opamp(dev_priv, pipe);
6998
6999         /* Set up Tx target for periodic Rcomp update */
7000         vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW9_BCAST, 0x0100000f);
7001
7002         /* Disable target IRef on PLL */
7003         reg_val = vlv_dpio_read(dev_priv, pipe, VLV_PLL_DW8(pipe));
7004         reg_val &= 0x00ffffff;
7005         vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW8(pipe), reg_val);
7006
7007         /* Disable fast lock */
7008         vlv_dpio_write(dev_priv, pipe, VLV_CMN_DW0, 0x610);
7009
7010         /* Set idtafcrecal before PLL is enabled */
7011         mdiv = ((bestm1 << DPIO_M1DIV_SHIFT) | (bestm2 & DPIO_M2DIV_MASK));
7012         mdiv |= ((bestp1 << DPIO_P1_SHIFT) | (bestp2 << DPIO_P2_SHIFT));
7013         mdiv |= ((bestn << DPIO_N_SHIFT));
7014         mdiv |= (1 << DPIO_K_SHIFT);
7015
7016         /*
7017          * Post divider depends on pixel clock rate, DAC vs digital (and LVDS,
7018          * but we don't support that).
7019          * Note: don't use the DAC post divider as it seems unstable.
7020          */
7021         mdiv |= (DPIO_POST_DIV_HDMIDP << DPIO_POST_DIV_SHIFT);
7022         vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW3(pipe), mdiv);
7023
7024         mdiv |= DPIO_ENABLE_CALIBRATION;
7025         vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW3(pipe), mdiv);
7026
7027         /* Set HBR and RBR LPF coefficients */
7028         if (pipe_config->port_clock == 162000 ||
7029             intel_crtc_has_type(pipe_config, INTEL_OUTPUT_ANALOG) ||
7030             intel_crtc_has_type(pipe_config, INTEL_OUTPUT_HDMI))
7031                 vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW10(pipe),
7032                                  0x009f0003);
7033         else
7034                 vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW10(pipe),
7035                                  0x00d0000f);
7036
7037         if (intel_crtc_has_dp_encoder(pipe_config)) {
7038                 /* Use SSC source */
7039                 if (pipe == PIPE_A)
7040                         vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW5(pipe),
7041                                          0x0df40000);
7042                 else
7043                         vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW5(pipe),
7044                                          0x0df70000);
7045         } else { /* HDMI or VGA */
7046                 /* Use bend source */
7047                 if (pipe == PIPE_A)
7048                         vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW5(pipe),
7049                                          0x0df70000);
7050                 else
7051                         vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW5(pipe),
7052                                          0x0df40000);
7053         }
7054
7055         coreclk = vlv_dpio_read(dev_priv, pipe, VLV_PLL_DW7(pipe));
7056         coreclk = (coreclk & 0x0000ff00) | 0x01c00000;
7057         if (intel_crtc_has_dp_encoder(pipe_config))
7058                 coreclk |= 0x01000000;
7059         vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW7(pipe), coreclk);
7060
7061         vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW11(pipe), 0x87871000);
7062         mutex_unlock(&dev_priv->sb_lock);
7063 }
7064
7065 static void chv_prepare_pll(struct intel_crtc *crtc,
7066                             const struct intel_crtc_state *pipe_config)
7067 {
7068         struct drm_device *dev = crtc->base.dev;
7069         struct drm_i915_private *dev_priv = to_i915(dev);
7070         enum pipe pipe = crtc->pipe;
7071         enum dpio_channel port = vlv_pipe_to_channel(pipe);
7072         u32 loopfilter, tribuf_calcntr;
7073         u32 bestn, bestm1, bestm2, bestp1, bestp2, bestm2_frac;
7074         u32 dpio_val;
7075         int vco;
7076
7077         /* Enable Refclk and SSC */
7078         I915_WRITE(DPLL(pipe),
7079                    pipe_config->dpll_hw_state.dpll & ~DPLL_VCO_ENABLE);
7080
7081         /* No need to actually set up the DPLL with DSI */
7082         if ((pipe_config->dpll_hw_state.dpll & DPLL_VCO_ENABLE) == 0)
7083                 return;
7084
7085         bestn = pipe_config->dpll.n;
7086         bestm2_frac = pipe_config->dpll.m2 & 0x3fffff;
7087         bestm1 = pipe_config->dpll.m1;
7088         bestm2 = pipe_config->dpll.m2 >> 22;
7089         bestp1 = pipe_config->dpll.p1;
7090         bestp2 = pipe_config->dpll.p2;
7091         vco = pipe_config->dpll.vco;
7092         dpio_val = 0;
7093         loopfilter = 0;
7094
7095         mutex_lock(&dev_priv->sb_lock);
7096
7097         /* p1 and p2 divider */
7098         vlv_dpio_write(dev_priv, pipe, CHV_CMN_DW13(port),
7099                         5 << DPIO_CHV_S1_DIV_SHIFT |
7100                         bestp1 << DPIO_CHV_P1_DIV_SHIFT |
7101                         bestp2 << DPIO_CHV_P2_DIV_SHIFT |
7102                         1 << DPIO_CHV_K_DIV_SHIFT);
7103
7104         /* Feedback post-divider - m2 */
7105         vlv_dpio_write(dev_priv, pipe, CHV_PLL_DW0(port), bestm2);
7106
7107         /* Feedback refclk divider - n and m1 */
7108         vlv_dpio_write(dev_priv, pipe, CHV_PLL_DW1(port),
7109                         DPIO_CHV_M1_DIV_BY_2 |
7110                         1 << DPIO_CHV_N_DIV_SHIFT);
7111
7112         /* M2 fraction division */
7113         vlv_dpio_write(dev_priv, pipe, CHV_PLL_DW2(port), bestm2_frac);
7114
7115         /* M2 fraction division enable */
7116         dpio_val = vlv_dpio_read(dev_priv, pipe, CHV_PLL_DW3(port));
7117         dpio_val &= ~(DPIO_CHV_FEEDFWD_GAIN_MASK | DPIO_CHV_FRAC_DIV_EN);
7118         dpio_val |= (2 << DPIO_CHV_FEEDFWD_GAIN_SHIFT);
7119         if (bestm2_frac)
7120                 dpio_val |= DPIO_CHV_FRAC_DIV_EN;
7121         vlv_dpio_write(dev_priv, pipe, CHV_PLL_DW3(port), dpio_val);
7122
7123         /* Program digital lock detect threshold */
7124         dpio_val = vlv_dpio_read(dev_priv, pipe, CHV_PLL_DW9(port));
7125         dpio_val &= ~(DPIO_CHV_INT_LOCK_THRESHOLD_MASK |
7126                                         DPIO_CHV_INT_LOCK_THRESHOLD_SEL_COARSE);
7127         dpio_val |= (0x5 << DPIO_CHV_INT_LOCK_THRESHOLD_SHIFT);
7128         if (!bestm2_frac)
7129                 dpio_val |= DPIO_CHV_INT_LOCK_THRESHOLD_SEL_COARSE;
7130         vlv_dpio_write(dev_priv, pipe, CHV_PLL_DW9(port), dpio_val);
7131
7132         /* Loop filter */
7133         if (vco == 5400000) {
7134                 loopfilter |= (0x3 << DPIO_CHV_PROP_COEFF_SHIFT);
7135                 loopfilter |= (0x8 << DPIO_CHV_INT_COEFF_SHIFT);
7136                 loopfilter |= (0x1 << DPIO_CHV_GAIN_CTRL_SHIFT);
7137                 tribuf_calcntr = 0x9;
7138         } else if (vco <= 6200000) {
7139                 loopfilter |= (0x5 << DPIO_CHV_PROP_COEFF_SHIFT);
7140                 loopfilter |= (0xB << DPIO_CHV_INT_COEFF_SHIFT);
7141                 loopfilter |= (0x3 << DPIO_CHV_GAIN_CTRL_SHIFT);
7142                 tribuf_calcntr = 0x9;
7143         } else if (vco <= 6480000) {
7144                 loopfilter |= (0x4 << DPIO_CHV_PROP_COEFF_SHIFT);
7145                 loopfilter |= (0x9 << DPIO_CHV_INT_COEFF_SHIFT);
7146                 loopfilter |= (0x3 << DPIO_CHV_GAIN_CTRL_SHIFT);
7147                 tribuf_calcntr = 0x8;
7148         } else {
7149                 /* Not supported. Apply the same limits as in the max case */
7150                 loopfilter |= (0x4 << DPIO_CHV_PROP_COEFF_SHIFT);
7151                 loopfilter |= (0x9 << DPIO_CHV_INT_COEFF_SHIFT);
7152                 loopfilter |= (0x3 << DPIO_CHV_GAIN_CTRL_SHIFT);
7153                 tribuf_calcntr = 0;
7154         }
7155         vlv_dpio_write(dev_priv, pipe, CHV_PLL_DW6(port), loopfilter);
7156
7157         dpio_val = vlv_dpio_read(dev_priv, pipe, CHV_PLL_DW8(port));
7158         dpio_val &= ~DPIO_CHV_TDC_TARGET_CNT_MASK;
7159         dpio_val |= (tribuf_calcntr << DPIO_CHV_TDC_TARGET_CNT_SHIFT);
7160         vlv_dpio_write(dev_priv, pipe, CHV_PLL_DW8(port), dpio_val);
7161
7162         /* AFC Recal */
7163         vlv_dpio_write(dev_priv, pipe, CHV_CMN_DW14(port),
7164                         vlv_dpio_read(dev_priv, pipe, CHV_CMN_DW14(port)) |
7165                         DPIO_AFC_RECAL);
7166
7167         mutex_unlock(&dev_priv->sb_lock);
7168 }
7169
7170 /**
7171  * vlv_force_pll_on - forcibly enable just the PLL
7172  * @dev_priv: i915 private structure
7173  * @pipe: pipe PLL to enable
7174  * @dpll: PLL configuration
7175  *
7176  * Enable the PLL for @pipe using the supplied @dpll config. To be used
7177  * in cases where we need the PLL enabled even when @pipe is not going to
7178  * be enabled.
7179  */
7180 int vlv_force_pll_on(struct drm_i915_private *dev_priv, enum pipe pipe,
7181                      const struct dpll *dpll)
7182 {
7183         struct intel_crtc *crtc = intel_get_crtc_for_pipe(dev_priv, pipe);
7184         struct intel_crtc_state *pipe_config;
7185
7186         pipe_config = kzalloc(sizeof(*pipe_config), GFP_KERNEL);
7187         if (!pipe_config)
7188                 return -ENOMEM;
7189
7190         pipe_config->base.crtc = &crtc->base;
7191         pipe_config->pixel_multiplier = 1;
7192         pipe_config->dpll = *dpll;
7193
7194         if (IS_CHERRYVIEW(dev_priv)) {
7195                 chv_compute_dpll(crtc, pipe_config);
7196                 chv_prepare_pll(crtc, pipe_config);
7197                 chv_enable_pll(crtc, pipe_config);
7198         } else {
7199                 vlv_compute_dpll(crtc, pipe_config);
7200                 vlv_prepare_pll(crtc, pipe_config);
7201                 vlv_enable_pll(crtc, pipe_config);
7202         }
7203
7204         kfree(pipe_config);
7205
7206         return 0;
7207 }
7208
7209 /**
7210  * vlv_force_pll_off - forcibly disable just the PLL
7211  * @dev_priv: i915 private structure
7212  * @pipe: pipe PLL to disable
7213  *
7214  * Disable the PLL for @pipe. To be used in cases where we need
7215  * the PLL enabled even when @pipe is not going to be enabled.
7216  */
7217 void vlv_force_pll_off(struct drm_i915_private *dev_priv, enum pipe pipe)
7218 {
7219         if (IS_CHERRYVIEW(dev_priv))
7220                 chv_disable_pll(dev_priv, pipe);
7221         else
7222                 vlv_disable_pll(dev_priv, pipe);
7223 }
7224
7225 static void i9xx_compute_dpll(struct intel_crtc *crtc,
7226                               struct intel_crtc_state *crtc_state,
7227                               struct dpll *reduced_clock)
7228 {
7229         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
7230         u32 dpll;
7231         struct dpll *clock = &crtc_state->dpll;
7232
7233         i9xx_update_pll_dividers(crtc, crtc_state, reduced_clock);
7234
7235         dpll = DPLL_VGA_MODE_DIS;
7236
7237         if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_LVDS))
7238                 dpll |= DPLLB_MODE_LVDS;
7239         else
7240                 dpll |= DPLLB_MODE_DAC_SERIAL;
7241
7242         if (IS_I945G(dev_priv) || IS_I945GM(dev_priv) ||
7243             IS_G33(dev_priv) || IS_PINEVIEW(dev_priv)) {
7244                 dpll |= (crtc_state->pixel_multiplier - 1)
7245                         << SDVO_MULTIPLIER_SHIFT_HIRES;
7246         }
7247
7248         if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_SDVO) ||
7249             intel_crtc_has_type(crtc_state, INTEL_OUTPUT_HDMI))
7250                 dpll |= DPLL_SDVO_HIGH_SPEED;
7251
7252         if (intel_crtc_has_dp_encoder(crtc_state))
7253                 dpll |= DPLL_SDVO_HIGH_SPEED;
7254
7255         /* compute bitmask from p1 value */
7256         if (IS_PINEVIEW(dev_priv))
7257                 dpll |= (1 << (clock->p1 - 1)) << DPLL_FPA01_P1_POST_DIV_SHIFT_PINEVIEW;
7258         else {
7259                 dpll |= (1 << (clock->p1 - 1)) << DPLL_FPA01_P1_POST_DIV_SHIFT;
7260                 if (IS_G4X(dev_priv) && reduced_clock)
7261                         dpll |= (1 << (reduced_clock->p1 - 1)) << DPLL_FPA1_P1_POST_DIV_SHIFT;
7262         }
7263         switch (clock->p2) {
7264         case 5:
7265                 dpll |= DPLL_DAC_SERIAL_P2_CLOCK_DIV_5;
7266                 break;
7267         case 7:
7268                 dpll |= DPLLB_LVDS_P2_CLOCK_DIV_7;
7269                 break;
7270         case 10:
7271                 dpll |= DPLL_DAC_SERIAL_P2_CLOCK_DIV_10;
7272                 break;
7273         case 14:
7274                 dpll |= DPLLB_LVDS_P2_CLOCK_DIV_14;
7275                 break;
7276         }
7277         if (INTEL_GEN(dev_priv) >= 4)
7278                 dpll |= (6 << PLL_LOAD_PULSE_PHASE_SHIFT);
7279
7280         if (crtc_state->sdvo_tv_clock)
7281                 dpll |= PLL_REF_INPUT_TVCLKINBC;
7282         else if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_LVDS) &&
7283                  intel_panel_use_ssc(dev_priv))
7284                 dpll |= PLLB_REF_INPUT_SPREADSPECTRUMIN;
7285         else
7286                 dpll |= PLL_REF_INPUT_DREFCLK;
7287
7288         dpll |= DPLL_VCO_ENABLE;
7289         crtc_state->dpll_hw_state.dpll = dpll;
7290
7291         if (INTEL_GEN(dev_priv) >= 4) {
7292                 u32 dpll_md = (crtc_state->pixel_multiplier - 1)
7293                         << DPLL_MD_UDI_MULTIPLIER_SHIFT;
7294                 crtc_state->dpll_hw_state.dpll_md = dpll_md;
7295         }
7296 }
7297
7298 static void i8xx_compute_dpll(struct intel_crtc *crtc,
7299                               struct intel_crtc_state *crtc_state,
7300                               struct dpll *reduced_clock)
7301 {
7302         struct drm_device *dev = crtc->base.dev;
7303         struct drm_i915_private *dev_priv = to_i915(dev);
7304         u32 dpll;
7305         struct dpll *clock = &crtc_state->dpll;
7306
7307         i9xx_update_pll_dividers(crtc, crtc_state, reduced_clock);
7308
7309         dpll = DPLL_VGA_MODE_DIS;
7310
7311         if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_LVDS)) {
7312                 dpll |= (1 << (clock->p1 - 1)) << DPLL_FPA01_P1_POST_DIV_SHIFT;
7313         } else {
7314                 if (clock->p1 == 2)
7315                         dpll |= PLL_P1_DIVIDE_BY_TWO;
7316                 else
7317                         dpll |= (clock->p1 - 2) << DPLL_FPA01_P1_POST_DIV_SHIFT;
7318                 if (clock->p2 == 4)
7319                         dpll |= PLL_P2_DIVIDE_BY_4;
7320         }
7321
7322         if (!IS_I830(dev_priv) &&
7323             intel_crtc_has_type(crtc_state, INTEL_OUTPUT_DVO))
7324                 dpll |= DPLL_DVO_2X_MODE;
7325
7326         if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_LVDS) &&
7327             intel_panel_use_ssc(dev_priv))
7328                 dpll |= PLLB_REF_INPUT_SPREADSPECTRUMIN;
7329         else
7330                 dpll |= PLL_REF_INPUT_DREFCLK;
7331
7332         dpll |= DPLL_VCO_ENABLE;
7333         crtc_state->dpll_hw_state.dpll = dpll;
7334 }
7335
7336 static void intel_set_pipe_timings(const struct intel_crtc_state *crtc_state)
7337 {
7338         struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
7339         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
7340         enum pipe pipe = crtc->pipe;
7341         enum transcoder cpu_transcoder = crtc_state->cpu_transcoder;
7342         const struct drm_display_mode *adjusted_mode = &crtc_state->base.adjusted_mode;
7343         uint32_t crtc_vtotal, crtc_vblank_end;
7344         int vsyncshift = 0;
7345
7346         /* We need to be careful not to changed the adjusted mode, for otherwise
7347          * the hw state checker will get angry at the mismatch. */
7348         crtc_vtotal = adjusted_mode->crtc_vtotal;
7349         crtc_vblank_end = adjusted_mode->crtc_vblank_end;
7350
7351         if (adjusted_mode->flags & DRM_MODE_FLAG_INTERLACE) {
7352                 /* the chip adds 2 halflines automatically */
7353                 crtc_vtotal -= 1;
7354                 crtc_vblank_end -= 1;
7355
7356                 if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_SDVO))
7357                         vsyncshift = (adjusted_mode->crtc_htotal - 1) / 2;
7358                 else
7359                         vsyncshift = adjusted_mode->crtc_hsync_start -
7360                                 adjusted_mode->crtc_htotal / 2;
7361                 if (vsyncshift < 0)
7362                         vsyncshift += adjusted_mode->crtc_htotal;
7363         }
7364
7365         if (INTEL_GEN(dev_priv) > 3)
7366                 I915_WRITE(VSYNCSHIFT(cpu_transcoder), vsyncshift);
7367
7368         I915_WRITE(HTOTAL(cpu_transcoder),
7369                    (adjusted_mode->crtc_hdisplay - 1) |
7370                    ((adjusted_mode->crtc_htotal - 1) << 16));
7371         I915_WRITE(HBLANK(cpu_transcoder),
7372                    (adjusted_mode->crtc_hblank_start - 1) |
7373                    ((adjusted_mode->crtc_hblank_end - 1) << 16));
7374         I915_WRITE(HSYNC(cpu_transcoder),
7375                    (adjusted_mode->crtc_hsync_start - 1) |
7376                    ((adjusted_mode->crtc_hsync_end - 1) << 16));
7377
7378         I915_WRITE(VTOTAL(cpu_transcoder),
7379                    (adjusted_mode->crtc_vdisplay - 1) |
7380                    ((crtc_vtotal - 1) << 16));
7381         I915_WRITE(VBLANK(cpu_transcoder),
7382                    (adjusted_mode->crtc_vblank_start - 1) |
7383                    ((crtc_vblank_end - 1) << 16));
7384         I915_WRITE(VSYNC(cpu_transcoder),
7385                    (adjusted_mode->crtc_vsync_start - 1) |
7386                    ((adjusted_mode->crtc_vsync_end - 1) << 16));
7387
7388         /* Workaround: when the EDP input selection is B, the VTOTAL_B must be
7389          * programmed with the VTOTAL_EDP value. Same for VTOTAL_C. This is
7390          * documented on the DDI_FUNC_CTL register description, EDP Input Select
7391          * bits. */
7392         if (IS_HASWELL(dev_priv) && cpu_transcoder == TRANSCODER_EDP &&
7393             (pipe == PIPE_B || pipe == PIPE_C))
7394                 I915_WRITE(VTOTAL(pipe), I915_READ(VTOTAL(cpu_transcoder)));
7395
7396 }
7397
7398 static void intel_set_pipe_src_size(const struct intel_crtc_state *crtc_state)
7399 {
7400         struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
7401         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
7402         enum pipe pipe = crtc->pipe;
7403
7404         /* pipesrc controls the size that is scaled from, which should
7405          * always be the user's requested size.
7406          */
7407         I915_WRITE(PIPESRC(pipe),
7408                    ((crtc_state->pipe_src_w - 1) << 16) |
7409                    (crtc_state->pipe_src_h - 1));
7410 }
7411
7412 static void intel_get_pipe_timings(struct intel_crtc *crtc,
7413                                    struct intel_crtc_state *pipe_config)
7414 {
7415         struct drm_device *dev = crtc->base.dev;
7416         struct drm_i915_private *dev_priv = to_i915(dev);
7417         enum transcoder cpu_transcoder = pipe_config->cpu_transcoder;
7418         uint32_t tmp;
7419
7420         tmp = I915_READ(HTOTAL(cpu_transcoder));
7421         pipe_config->base.adjusted_mode.crtc_hdisplay = (tmp & 0xffff) + 1;
7422         pipe_config->base.adjusted_mode.crtc_htotal = ((tmp >> 16) & 0xffff) + 1;
7423         tmp = I915_READ(HBLANK(cpu_transcoder));
7424         pipe_config->base.adjusted_mode.crtc_hblank_start = (tmp & 0xffff) + 1;
7425         pipe_config->base.adjusted_mode.crtc_hblank_end = ((tmp >> 16) & 0xffff) + 1;
7426         tmp = I915_READ(HSYNC(cpu_transcoder));
7427         pipe_config->base.adjusted_mode.crtc_hsync_start = (tmp & 0xffff) + 1;
7428         pipe_config->base.adjusted_mode.crtc_hsync_end = ((tmp >> 16) & 0xffff) + 1;
7429
7430         tmp = I915_READ(VTOTAL(cpu_transcoder));
7431         pipe_config->base.adjusted_mode.crtc_vdisplay = (tmp & 0xffff) + 1;
7432         pipe_config->base.adjusted_mode.crtc_vtotal = ((tmp >> 16) & 0xffff) + 1;
7433         tmp = I915_READ(VBLANK(cpu_transcoder));
7434         pipe_config->base.adjusted_mode.crtc_vblank_start = (tmp & 0xffff) + 1;
7435         pipe_config->base.adjusted_mode.crtc_vblank_end = ((tmp >> 16) & 0xffff) + 1;
7436         tmp = I915_READ(VSYNC(cpu_transcoder));
7437         pipe_config->base.adjusted_mode.crtc_vsync_start = (tmp & 0xffff) + 1;
7438         pipe_config->base.adjusted_mode.crtc_vsync_end = ((tmp >> 16) & 0xffff) + 1;
7439
7440         if (I915_READ(PIPECONF(cpu_transcoder)) & PIPECONF_INTERLACE_MASK) {
7441                 pipe_config->base.adjusted_mode.flags |= DRM_MODE_FLAG_INTERLACE;
7442                 pipe_config->base.adjusted_mode.crtc_vtotal += 1;
7443                 pipe_config->base.adjusted_mode.crtc_vblank_end += 1;
7444         }
7445 }
7446
7447 static void intel_get_pipe_src_size(struct intel_crtc *crtc,
7448                                     struct intel_crtc_state *pipe_config)
7449 {
7450         struct drm_device *dev = crtc->base.dev;
7451         struct drm_i915_private *dev_priv = to_i915(dev);
7452         u32 tmp;
7453
7454         tmp = I915_READ(PIPESRC(crtc->pipe));
7455         pipe_config->pipe_src_h = (tmp & 0xffff) + 1;
7456         pipe_config->pipe_src_w = ((tmp >> 16) & 0xffff) + 1;
7457
7458         pipe_config->base.mode.vdisplay = pipe_config->pipe_src_h;
7459         pipe_config->base.mode.hdisplay = pipe_config->pipe_src_w;
7460 }
7461
7462 void intel_mode_from_pipe_config(struct drm_display_mode *mode,
7463                                  struct intel_crtc_state *pipe_config)
7464 {
7465         mode->hdisplay = pipe_config->base.adjusted_mode.crtc_hdisplay;
7466         mode->htotal = pipe_config->base.adjusted_mode.crtc_htotal;
7467         mode->hsync_start = pipe_config->base.adjusted_mode.crtc_hsync_start;
7468         mode->hsync_end = pipe_config->base.adjusted_mode.crtc_hsync_end;
7469
7470         mode->vdisplay = pipe_config->base.adjusted_mode.crtc_vdisplay;
7471         mode->vtotal = pipe_config->base.adjusted_mode.crtc_vtotal;
7472         mode->vsync_start = pipe_config->base.adjusted_mode.crtc_vsync_start;
7473         mode->vsync_end = pipe_config->base.adjusted_mode.crtc_vsync_end;
7474
7475         mode->flags = pipe_config->base.adjusted_mode.flags;
7476         mode->type = DRM_MODE_TYPE_DRIVER;
7477
7478         mode->clock = pipe_config->base.adjusted_mode.crtc_clock;
7479
7480         mode->hsync = drm_mode_hsync(mode);
7481         mode->vrefresh = drm_mode_vrefresh(mode);
7482         drm_mode_set_name(mode);
7483 }
7484
7485 static void i9xx_set_pipeconf(const struct intel_crtc_state *crtc_state)
7486 {
7487         struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
7488         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
7489         uint32_t pipeconf;
7490
7491         pipeconf = 0;
7492
7493         /* we keep both pipes enabled on 830 */
7494         if (IS_I830(dev_priv))
7495                 pipeconf |= I915_READ(PIPECONF(crtc->pipe)) & PIPECONF_ENABLE;
7496
7497         if (crtc_state->double_wide)
7498                 pipeconf |= PIPECONF_DOUBLE_WIDE;
7499
7500         /* only g4x and later have fancy bpc/dither controls */
7501         if (IS_G4X(dev_priv) || IS_VALLEYVIEW(dev_priv) ||
7502             IS_CHERRYVIEW(dev_priv)) {
7503                 /* Bspec claims that we can't use dithering for 30bpp pipes. */
7504                 if (crtc_state->dither && crtc_state->pipe_bpp != 30)
7505                         pipeconf |= PIPECONF_DITHER_EN |
7506                                     PIPECONF_DITHER_TYPE_SP;
7507
7508                 switch (crtc_state->pipe_bpp) {
7509                 case 18:
7510                         pipeconf |= PIPECONF_6BPC;
7511                         break;
7512                 case 24:
7513                         pipeconf |= PIPECONF_8BPC;
7514                         break;
7515                 case 30:
7516                         pipeconf |= PIPECONF_10BPC;
7517                         break;
7518                 default:
7519                         /* Case prevented by intel_choose_pipe_bpp_dither. */
7520                         BUG();
7521                 }
7522         }
7523
7524         if (crtc_state->base.adjusted_mode.flags & DRM_MODE_FLAG_INTERLACE) {
7525                 if (INTEL_GEN(dev_priv) < 4 ||
7526                     intel_crtc_has_type(crtc_state, INTEL_OUTPUT_SDVO))
7527                         pipeconf |= PIPECONF_INTERLACE_W_FIELD_INDICATION;
7528                 else
7529                         pipeconf |= PIPECONF_INTERLACE_W_SYNC_SHIFT;
7530         } else
7531                 pipeconf |= PIPECONF_PROGRESSIVE;
7532
7533         if ((IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) &&
7534              crtc_state->limited_color_range)
7535                 pipeconf |= PIPECONF_COLOR_RANGE_SELECT;
7536
7537         I915_WRITE(PIPECONF(crtc->pipe), pipeconf);
7538         POSTING_READ(PIPECONF(crtc->pipe));
7539 }
7540
7541 static int i8xx_crtc_compute_clock(struct intel_crtc *crtc,
7542                                    struct intel_crtc_state *crtc_state)
7543 {
7544         struct drm_device *dev = crtc->base.dev;
7545         struct drm_i915_private *dev_priv = to_i915(dev);
7546         const struct intel_limit *limit;
7547         int refclk = 48000;
7548
7549         memset(&crtc_state->dpll_hw_state, 0,
7550                sizeof(crtc_state->dpll_hw_state));
7551
7552         if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_LVDS)) {
7553                 if (intel_panel_use_ssc(dev_priv)) {
7554                         refclk = dev_priv->vbt.lvds_ssc_freq;
7555                         DRM_DEBUG_KMS("using SSC reference clock of %d kHz\n", refclk);
7556                 }
7557
7558                 limit = &intel_limits_i8xx_lvds;
7559         } else if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_DVO)) {
7560                 limit = &intel_limits_i8xx_dvo;
7561         } else {
7562                 limit = &intel_limits_i8xx_dac;
7563         }
7564
7565         if (!crtc_state->clock_set &&
7566             !i9xx_find_best_dpll(limit, crtc_state, crtc_state->port_clock,
7567                                  refclk, NULL, &crtc_state->dpll)) {
7568                 DRM_ERROR("Couldn't find PLL settings for mode!\n");
7569                 return -EINVAL;
7570         }
7571
7572         i8xx_compute_dpll(crtc, crtc_state, NULL);
7573
7574         return 0;
7575 }
7576
7577 static int g4x_crtc_compute_clock(struct intel_crtc *crtc,
7578                                   struct intel_crtc_state *crtc_state)
7579 {
7580         struct drm_device *dev = crtc->base.dev;
7581         struct drm_i915_private *dev_priv = to_i915(dev);
7582         const struct intel_limit *limit;
7583         int refclk = 96000;
7584
7585         memset(&crtc_state->dpll_hw_state, 0,
7586                sizeof(crtc_state->dpll_hw_state));
7587
7588         if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_LVDS)) {
7589                 if (intel_panel_use_ssc(dev_priv)) {
7590                         refclk = dev_priv->vbt.lvds_ssc_freq;
7591                         DRM_DEBUG_KMS("using SSC reference clock of %d kHz\n", refclk);
7592                 }
7593
7594                 if (intel_is_dual_link_lvds(dev))
7595                         limit = &intel_limits_g4x_dual_channel_lvds;
7596                 else
7597                         limit = &intel_limits_g4x_single_channel_lvds;
7598         } else if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_HDMI) ||
7599                    intel_crtc_has_type(crtc_state, INTEL_OUTPUT_ANALOG)) {
7600                 limit = &intel_limits_g4x_hdmi;
7601         } else if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_SDVO)) {
7602                 limit = &intel_limits_g4x_sdvo;
7603         } else {
7604                 /* The option is for other outputs */
7605                 limit = &intel_limits_i9xx_sdvo;
7606         }
7607
7608         if (!crtc_state->clock_set &&
7609             !g4x_find_best_dpll(limit, crtc_state, crtc_state->port_clock,
7610                                 refclk, NULL, &crtc_state->dpll)) {
7611                 DRM_ERROR("Couldn't find PLL settings for mode!\n");
7612                 return -EINVAL;
7613         }
7614
7615         i9xx_compute_dpll(crtc, crtc_state, NULL);
7616
7617         return 0;
7618 }
7619
7620 static int pnv_crtc_compute_clock(struct intel_crtc *crtc,
7621                                   struct intel_crtc_state *crtc_state)
7622 {
7623         struct drm_device *dev = crtc->base.dev;
7624         struct drm_i915_private *dev_priv = to_i915(dev);
7625         const struct intel_limit *limit;
7626         int refclk = 96000;
7627
7628         memset(&crtc_state->dpll_hw_state, 0,
7629                sizeof(crtc_state->dpll_hw_state));
7630
7631         if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_LVDS)) {
7632                 if (intel_panel_use_ssc(dev_priv)) {
7633                         refclk = dev_priv->vbt.lvds_ssc_freq;
7634                         DRM_DEBUG_KMS("using SSC reference clock of %d kHz\n", refclk);
7635                 }
7636
7637                 limit = &intel_limits_pineview_lvds;
7638         } else {
7639                 limit = &intel_limits_pineview_sdvo;
7640         }
7641
7642         if (!crtc_state->clock_set &&
7643             !pnv_find_best_dpll(limit, crtc_state, crtc_state->port_clock,
7644                                 refclk, NULL, &crtc_state->dpll)) {
7645                 DRM_ERROR("Couldn't find PLL settings for mode!\n");
7646                 return -EINVAL;
7647         }
7648
7649         i9xx_compute_dpll(crtc, crtc_state, NULL);
7650
7651         return 0;
7652 }
7653
7654 static int i9xx_crtc_compute_clock(struct intel_crtc *crtc,
7655                                    struct intel_crtc_state *crtc_state)
7656 {
7657         struct drm_device *dev = crtc->base.dev;
7658         struct drm_i915_private *dev_priv = to_i915(dev);
7659         const struct intel_limit *limit;
7660         int refclk = 96000;
7661
7662         memset(&crtc_state->dpll_hw_state, 0,
7663                sizeof(crtc_state->dpll_hw_state));
7664
7665         if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_LVDS)) {
7666                 if (intel_panel_use_ssc(dev_priv)) {
7667                         refclk = dev_priv->vbt.lvds_ssc_freq;
7668                         DRM_DEBUG_KMS("using SSC reference clock of %d kHz\n", refclk);
7669                 }
7670
7671                 limit = &intel_limits_i9xx_lvds;
7672         } else {
7673                 limit = &intel_limits_i9xx_sdvo;
7674         }
7675
7676         if (!crtc_state->clock_set &&
7677             !i9xx_find_best_dpll(limit, crtc_state, crtc_state->port_clock,
7678                                  refclk, NULL, &crtc_state->dpll)) {
7679                 DRM_ERROR("Couldn't find PLL settings for mode!\n");
7680                 return -EINVAL;
7681         }
7682
7683         i9xx_compute_dpll(crtc, crtc_state, NULL);
7684
7685         return 0;
7686 }
7687
7688 static int chv_crtc_compute_clock(struct intel_crtc *crtc,
7689                                   struct intel_crtc_state *crtc_state)
7690 {
7691         int refclk = 100000;
7692         const struct intel_limit *limit = &intel_limits_chv;
7693
7694         memset(&crtc_state->dpll_hw_state, 0,
7695                sizeof(crtc_state->dpll_hw_state));
7696
7697         if (!crtc_state->clock_set &&
7698             !chv_find_best_dpll(limit, crtc_state, crtc_state->port_clock,
7699                                 refclk, NULL, &crtc_state->dpll)) {
7700                 DRM_ERROR("Couldn't find PLL settings for mode!\n");
7701                 return -EINVAL;
7702         }
7703
7704         chv_compute_dpll(crtc, crtc_state);
7705
7706         return 0;
7707 }
7708
7709 static int vlv_crtc_compute_clock(struct intel_crtc *crtc,
7710                                   struct intel_crtc_state *crtc_state)
7711 {
7712         int refclk = 100000;
7713         const struct intel_limit *limit = &intel_limits_vlv;
7714
7715         memset(&crtc_state->dpll_hw_state, 0,
7716                sizeof(crtc_state->dpll_hw_state));
7717
7718         if (!crtc_state->clock_set &&
7719             !vlv_find_best_dpll(limit, crtc_state, crtc_state->port_clock,
7720                                 refclk, NULL, &crtc_state->dpll)) {
7721                 DRM_ERROR("Couldn't find PLL settings for mode!\n");
7722                 return -EINVAL;
7723         }
7724
7725         vlv_compute_dpll(crtc, crtc_state);
7726
7727         return 0;
7728 }
7729
7730 static void i9xx_get_pfit_config(struct intel_crtc *crtc,
7731                                  struct intel_crtc_state *pipe_config)
7732 {
7733         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
7734         uint32_t tmp;
7735
7736         if (INTEL_GEN(dev_priv) <= 3 &&
7737             (IS_I830(dev_priv) || !IS_MOBILE(dev_priv)))
7738                 return;
7739
7740         tmp = I915_READ(PFIT_CONTROL);
7741         if (!(tmp & PFIT_ENABLE))
7742                 return;
7743
7744         /* Check whether the pfit is attached to our pipe. */
7745         if (INTEL_GEN(dev_priv) < 4) {
7746                 if (crtc->pipe != PIPE_B)
7747                         return;
7748         } else {
7749                 if ((tmp & PFIT_PIPE_MASK) != (crtc->pipe << PFIT_PIPE_SHIFT))
7750                         return;
7751         }
7752
7753         pipe_config->gmch_pfit.control = tmp;
7754         pipe_config->gmch_pfit.pgm_ratios = I915_READ(PFIT_PGM_RATIOS);
7755 }
7756
7757 static void vlv_crtc_clock_get(struct intel_crtc *crtc,
7758                                struct intel_crtc_state *pipe_config)
7759 {
7760         struct drm_device *dev = crtc->base.dev;
7761         struct drm_i915_private *dev_priv = to_i915(dev);
7762         int pipe = pipe_config->cpu_transcoder;
7763         struct dpll clock;
7764         u32 mdiv;
7765         int refclk = 100000;
7766
7767         /* In case of DSI, DPLL will not be used */
7768         if ((pipe_config->dpll_hw_state.dpll & DPLL_VCO_ENABLE) == 0)
7769                 return;
7770
7771         mutex_lock(&dev_priv->sb_lock);
7772         mdiv = vlv_dpio_read(dev_priv, pipe, VLV_PLL_DW3(pipe));
7773         mutex_unlock(&dev_priv->sb_lock);
7774
7775         clock.m1 = (mdiv >> DPIO_M1DIV_SHIFT) & 7;
7776         clock.m2 = mdiv & DPIO_M2DIV_MASK;
7777         clock.n = (mdiv >> DPIO_N_SHIFT) & 0xf;
7778         clock.p1 = (mdiv >> DPIO_P1_SHIFT) & 7;
7779         clock.p2 = (mdiv >> DPIO_P2_SHIFT) & 0x1f;
7780
7781         pipe_config->port_clock = vlv_calc_dpll_params(refclk, &clock);
7782 }
7783
7784 static void
7785 i9xx_get_initial_plane_config(struct intel_crtc *crtc,
7786                               struct intel_initial_plane_config *plane_config)
7787 {
7788         struct drm_device *dev = crtc->base.dev;
7789         struct drm_i915_private *dev_priv = to_i915(dev);
7790         struct intel_plane *plane = to_intel_plane(crtc->base.primary);
7791         enum i9xx_plane_id i9xx_plane = plane->i9xx_plane;
7792         enum pipe pipe;
7793         u32 val, base, offset;
7794         int fourcc, pixel_format;
7795         unsigned int aligned_height;
7796         struct drm_framebuffer *fb;
7797         struct intel_framebuffer *intel_fb;
7798
7799         if (!plane->get_hw_state(plane, &pipe))
7800                 return;
7801
7802         WARN_ON(pipe != crtc->pipe);
7803
7804         intel_fb = kzalloc(sizeof(*intel_fb), GFP_KERNEL);
7805         if (!intel_fb) {
7806                 DRM_DEBUG_KMS("failed to alloc fb\n");
7807                 return;
7808         }
7809
7810         fb = &intel_fb->base;
7811
7812         fb->dev = dev;
7813
7814         val = I915_READ(DSPCNTR(i9xx_plane));
7815
7816         if (INTEL_GEN(dev_priv) >= 4) {
7817                 if (val & DISPPLANE_TILED) {
7818                         plane_config->tiling = I915_TILING_X;
7819                         fb->modifier = I915_FORMAT_MOD_X_TILED;
7820                 }
7821
7822                 if (val & DISPPLANE_ROTATE_180)
7823                         plane_config->rotation = DRM_MODE_ROTATE_180;
7824         }
7825
7826         if (IS_CHERRYVIEW(dev_priv) && pipe == PIPE_B &&
7827             val & DISPPLANE_MIRROR)
7828                 plane_config->rotation |= DRM_MODE_REFLECT_X;
7829
7830         pixel_format = val & DISPPLANE_PIXFORMAT_MASK;
7831         fourcc = i9xx_format_to_fourcc(pixel_format);
7832         fb->format = drm_format_info(fourcc);
7833
7834         if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv)) {
7835                 offset = I915_READ(DSPOFFSET(i9xx_plane));
7836                 base = I915_READ(DSPSURF(i9xx_plane)) & 0xfffff000;
7837         } else if (INTEL_GEN(dev_priv) >= 4) {
7838                 if (plane_config->tiling)
7839                         offset = I915_READ(DSPTILEOFF(i9xx_plane));
7840                 else
7841                         offset = I915_READ(DSPLINOFF(i9xx_plane));
7842                 base = I915_READ(DSPSURF(i9xx_plane)) & 0xfffff000;
7843         } else {
7844                 base = I915_READ(DSPADDR(i9xx_plane));
7845         }
7846         plane_config->base = base;
7847
7848         val = I915_READ(PIPESRC(pipe));
7849         fb->width = ((val >> 16) & 0xfff) + 1;
7850         fb->height = ((val >> 0) & 0xfff) + 1;
7851
7852         val = I915_READ(DSPSTRIDE(i9xx_plane));
7853         fb->pitches[0] = val & 0xffffffc0;
7854
7855         aligned_height = intel_fb_align_height(fb, 0, fb->height);
7856
7857         plane_config->size = fb->pitches[0] * aligned_height;
7858
7859         DRM_DEBUG_KMS("%s/%s with fb: size=%dx%d@%d, offset=%x, pitch %d, size 0x%x\n",
7860                       crtc->base.name, plane->base.name, fb->width, fb->height,
7861                       fb->format->cpp[0] * 8, base, fb->pitches[0],
7862                       plane_config->size);
7863
7864         plane_config->fb = intel_fb;
7865 }
7866
7867 static void chv_crtc_clock_get(struct intel_crtc *crtc,
7868                                struct intel_crtc_state *pipe_config)
7869 {
7870         struct drm_device *dev = crtc->base.dev;
7871         struct drm_i915_private *dev_priv = to_i915(dev);
7872         int pipe = pipe_config->cpu_transcoder;
7873         enum dpio_channel port = vlv_pipe_to_channel(pipe);
7874         struct dpll clock;
7875         u32 cmn_dw13, pll_dw0, pll_dw1, pll_dw2, pll_dw3;
7876         int refclk = 100000;
7877
7878         /* In case of DSI, DPLL will not be used */
7879         if ((pipe_config->dpll_hw_state.dpll & DPLL_VCO_ENABLE) == 0)
7880                 return;
7881
7882         mutex_lock(&dev_priv->sb_lock);
7883         cmn_dw13 = vlv_dpio_read(dev_priv, pipe, CHV_CMN_DW13(port));
7884         pll_dw0 = vlv_dpio_read(dev_priv, pipe, CHV_PLL_DW0(port));
7885         pll_dw1 = vlv_dpio_read(dev_priv, pipe, CHV_PLL_DW1(port));
7886         pll_dw2 = vlv_dpio_read(dev_priv, pipe, CHV_PLL_DW2(port));
7887         pll_dw3 = vlv_dpio_read(dev_priv, pipe, CHV_PLL_DW3(port));
7888         mutex_unlock(&dev_priv->sb_lock);
7889
7890         clock.m1 = (pll_dw1 & 0x7) == DPIO_CHV_M1_DIV_BY_2 ? 2 : 0;
7891         clock.m2 = (pll_dw0 & 0xff) << 22;
7892         if (pll_dw3 & DPIO_CHV_FRAC_DIV_EN)
7893                 clock.m2 |= pll_dw2 & 0x3fffff;
7894         clock.n = (pll_dw1 >> DPIO_CHV_N_DIV_SHIFT) & 0xf;
7895         clock.p1 = (cmn_dw13 >> DPIO_CHV_P1_DIV_SHIFT) & 0x7;
7896         clock.p2 = (cmn_dw13 >> DPIO_CHV_P2_DIV_SHIFT) & 0x1f;
7897
7898         pipe_config->port_clock = chv_calc_dpll_params(refclk, &clock);
7899 }
7900
7901 static void intel_get_crtc_ycbcr_config(struct intel_crtc *crtc,
7902                                         struct intel_crtc_state *pipe_config)
7903 {
7904         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
7905         enum intel_output_format output = INTEL_OUTPUT_FORMAT_RGB;
7906
7907         pipe_config->lspcon_downsampling = false;
7908
7909         if (IS_BROADWELL(dev_priv) || INTEL_GEN(dev_priv) >= 9) {
7910                 u32 tmp = I915_READ(PIPEMISC(crtc->pipe));
7911
7912                 if (tmp & PIPEMISC_OUTPUT_COLORSPACE_YUV) {
7913                         bool ycbcr420_enabled = tmp & PIPEMISC_YUV420_ENABLE;
7914                         bool blend = tmp & PIPEMISC_YUV420_MODE_FULL_BLEND;
7915
7916                         if (ycbcr420_enabled) {
7917                                 /* We support 4:2:0 in full blend mode only */
7918                                 if (!blend)
7919                                         output = INTEL_OUTPUT_FORMAT_INVALID;
7920                                 else if (!(IS_GEMINILAKE(dev_priv) ||
7921                                            INTEL_GEN(dev_priv) >= 10))
7922                                         output = INTEL_OUTPUT_FORMAT_INVALID;
7923                                 else
7924                                         output = INTEL_OUTPUT_FORMAT_YCBCR420;
7925                         } else {
7926                                 /*
7927                                  * Currently there is no interface defined to
7928                                  * check user preference between RGB/YCBCR444
7929                                  * or YCBCR420. So the only possible case for
7930                                  * YCBCR444 usage is driving YCBCR420 output
7931                                  * with LSPCON, when pipe is configured for
7932                                  * YCBCR444 output and LSPCON takes care of
7933                                  * downsampling it.
7934                                  */
7935                                 pipe_config->lspcon_downsampling = true;
7936                                 output = INTEL_OUTPUT_FORMAT_YCBCR444;
7937                         }
7938                 }
7939         }
7940
7941         pipe_config->output_format = output;
7942 }
7943
7944 static bool i9xx_get_pipe_config(struct intel_crtc *crtc,
7945                                  struct intel_crtc_state *pipe_config)
7946 {
7947         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
7948         enum intel_display_power_domain power_domain;
7949         uint32_t tmp;
7950         bool ret;
7951
7952         power_domain = POWER_DOMAIN_PIPE(crtc->pipe);
7953         if (!intel_display_power_get_if_enabled(dev_priv, power_domain))
7954                 return false;
7955
7956         pipe_config->output_format = INTEL_OUTPUT_FORMAT_RGB;
7957         pipe_config->cpu_transcoder = (enum transcoder) crtc->pipe;
7958         pipe_config->shared_dpll = NULL;
7959
7960         ret = false;
7961
7962         tmp = I915_READ(PIPECONF(crtc->pipe));
7963         if (!(tmp & PIPECONF_ENABLE))
7964                 goto out;
7965
7966         if (IS_G4X(dev_priv) || IS_VALLEYVIEW(dev_priv) ||
7967             IS_CHERRYVIEW(dev_priv)) {
7968                 switch (tmp & PIPECONF_BPC_MASK) {
7969                 case PIPECONF_6BPC:
7970                         pipe_config->pipe_bpp = 18;
7971                         break;
7972                 case PIPECONF_8BPC:
7973                         pipe_config->pipe_bpp = 24;
7974                         break;
7975                 case PIPECONF_10BPC:
7976                         pipe_config->pipe_bpp = 30;
7977                         break;
7978                 default:
7979                         break;
7980                 }
7981         }
7982
7983         if ((IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) &&
7984             (tmp & PIPECONF_COLOR_RANGE_SELECT))
7985                 pipe_config->limited_color_range = true;
7986
7987         if (INTEL_GEN(dev_priv) < 4)
7988                 pipe_config->double_wide = tmp & PIPECONF_DOUBLE_WIDE;
7989
7990         intel_get_pipe_timings(crtc, pipe_config);
7991         intel_get_pipe_src_size(crtc, pipe_config);
7992
7993         i9xx_get_pfit_config(crtc, pipe_config);
7994
7995         if (INTEL_GEN(dev_priv) >= 4) {
7996                 /* No way to read it out on pipes B and C */
7997                 if (IS_CHERRYVIEW(dev_priv) && crtc->pipe != PIPE_A)
7998                         tmp = dev_priv->chv_dpll_md[crtc->pipe];
7999                 else
8000                         tmp = I915_READ(DPLL_MD(crtc->pipe));
8001                 pipe_config->pixel_multiplier =
8002                         ((tmp & DPLL_MD_UDI_MULTIPLIER_MASK)
8003                          >> DPLL_MD_UDI_MULTIPLIER_SHIFT) + 1;
8004                 pipe_config->dpll_hw_state.dpll_md = tmp;
8005         } else if (IS_I945G(dev_priv) || IS_I945GM(dev_priv) ||
8006                    IS_G33(dev_priv) || IS_PINEVIEW(dev_priv)) {
8007                 tmp = I915_READ(DPLL(crtc->pipe));
8008                 pipe_config->pixel_multiplier =
8009                         ((tmp & SDVO_MULTIPLIER_MASK)
8010                          >> SDVO_MULTIPLIER_SHIFT_HIRES) + 1;
8011         } else {
8012                 /* Note that on i915G/GM the pixel multiplier is in the sdvo
8013                  * port and will be fixed up in the encoder->get_config
8014                  * function. */
8015                 pipe_config->pixel_multiplier = 1;
8016         }
8017         pipe_config->dpll_hw_state.dpll = I915_READ(DPLL(crtc->pipe));
8018         if (!IS_VALLEYVIEW(dev_priv) && !IS_CHERRYVIEW(dev_priv)) {
8019                 /*
8020                  * DPLL_DVO_2X_MODE must be enabled for both DPLLs
8021                  * on 830. Filter it out here so that we don't
8022                  * report errors due to that.
8023                  */
8024                 if (IS_I830(dev_priv))
8025                         pipe_config->dpll_hw_state.dpll &= ~DPLL_DVO_2X_MODE;
8026
8027                 pipe_config->dpll_hw_state.fp0 = I915_READ(FP0(crtc->pipe));
8028                 pipe_config->dpll_hw_state.fp1 = I915_READ(FP1(crtc->pipe));
8029         } else {
8030                 /* Mask out read-only status bits. */
8031                 pipe_config->dpll_hw_state.dpll &= ~(DPLL_LOCK_VLV |
8032                                                      DPLL_PORTC_READY_MASK |
8033                                                      DPLL_PORTB_READY_MASK);
8034         }
8035
8036         if (IS_CHERRYVIEW(dev_priv))
8037                 chv_crtc_clock_get(crtc, pipe_config);
8038         else if (IS_VALLEYVIEW(dev_priv))
8039                 vlv_crtc_clock_get(crtc, pipe_config);
8040         else
8041                 i9xx_crtc_clock_get(crtc, pipe_config);
8042
8043         /*
8044          * Normally the dotclock is filled in by the encoder .get_config()
8045          * but in case the pipe is enabled w/o any ports we need a sane
8046          * default.
8047          */
8048         pipe_config->base.adjusted_mode.crtc_clock =
8049                 pipe_config->port_clock / pipe_config->pixel_multiplier;
8050
8051         ret = true;
8052
8053 out:
8054         intel_display_power_put(dev_priv, power_domain);
8055
8056         return ret;
8057 }
8058
8059 static void ironlake_init_pch_refclk(struct drm_i915_private *dev_priv)
8060 {
8061         struct intel_encoder *encoder;
8062         int i;
8063         u32 val, final;
8064         bool has_lvds = false;
8065         bool has_cpu_edp = false;
8066         bool has_panel = false;
8067         bool has_ck505 = false;
8068         bool can_ssc = false;
8069         bool using_ssc_source = false;
8070
8071         /* We need to take the global config into account */
8072         for_each_intel_encoder(&dev_priv->drm, encoder) {
8073                 switch (encoder->type) {
8074                 case INTEL_OUTPUT_LVDS:
8075                         has_panel = true;
8076                         has_lvds = true;
8077                         break;
8078                 case INTEL_OUTPUT_EDP:
8079                         has_panel = true;
8080                         if (encoder->port == PORT_A)
8081                                 has_cpu_edp = true;
8082                         break;
8083                 default:
8084                         break;
8085                 }
8086         }
8087
8088         if (HAS_PCH_IBX(dev_priv)) {
8089                 has_ck505 = dev_priv->vbt.display_clock_mode;
8090                 can_ssc = has_ck505;
8091         } else {
8092                 has_ck505 = false;
8093                 can_ssc = true;
8094         }
8095
8096         /* Check if any DPLLs are using the SSC source */
8097         for (i = 0; i < dev_priv->num_shared_dpll; i++) {
8098                 u32 temp = I915_READ(PCH_DPLL(i));
8099
8100                 if (!(temp & DPLL_VCO_ENABLE))
8101                         continue;
8102
8103                 if ((temp & PLL_REF_INPUT_MASK) ==
8104                     PLLB_REF_INPUT_SPREADSPECTRUMIN) {
8105                         using_ssc_source = true;
8106                         break;
8107                 }
8108         }
8109
8110         DRM_DEBUG_KMS("has_panel %d has_lvds %d has_ck505 %d using_ssc_source %d\n",
8111                       has_panel, has_lvds, has_ck505, using_ssc_source);
8112
8113         /* Ironlake: try to setup display ref clock before DPLL
8114          * enabling. This is only under driver's control after
8115          * PCH B stepping, previous chipset stepping should be
8116          * ignoring this setting.
8117          */
8118         val = I915_READ(PCH_DREF_CONTROL);
8119
8120         /* As we must carefully and slowly disable/enable each source in turn,
8121          * compute the final state we want first and check if we need to
8122          * make any changes at all.
8123          */
8124         final = val;
8125         final &= ~DREF_NONSPREAD_SOURCE_MASK;
8126         if (has_ck505)
8127                 final |= DREF_NONSPREAD_CK505_ENABLE;
8128         else
8129                 final |= DREF_NONSPREAD_SOURCE_ENABLE;
8130
8131         final &= ~DREF_SSC_SOURCE_MASK;
8132         final &= ~DREF_CPU_SOURCE_OUTPUT_MASK;
8133         final &= ~DREF_SSC1_ENABLE;
8134
8135         if (has_panel) {
8136                 final |= DREF_SSC_SOURCE_ENABLE;
8137
8138                 if (intel_panel_use_ssc(dev_priv) && can_ssc)
8139                         final |= DREF_SSC1_ENABLE;
8140
8141                 if (has_cpu_edp) {
8142                         if (intel_panel_use_ssc(dev_priv) && can_ssc)
8143                                 final |= DREF_CPU_SOURCE_OUTPUT_DOWNSPREAD;
8144                         else
8145                                 final |= DREF_CPU_SOURCE_OUTPUT_NONSPREAD;
8146                 } else
8147                         final |= DREF_CPU_SOURCE_OUTPUT_DISABLE;
8148         } else if (using_ssc_source) {
8149                 final |= DREF_SSC_SOURCE_ENABLE;
8150                 final |= DREF_SSC1_ENABLE;
8151         }
8152
8153         if (final == val)
8154                 return;
8155
8156         /* Always enable nonspread source */
8157         val &= ~DREF_NONSPREAD_SOURCE_MASK;
8158
8159         if (has_ck505)
8160                 val |= DREF_NONSPREAD_CK505_ENABLE;
8161         else
8162                 val |= DREF_NONSPREAD_SOURCE_ENABLE;
8163
8164         if (has_panel) {
8165                 val &= ~DREF_SSC_SOURCE_MASK;
8166                 val |= DREF_SSC_SOURCE_ENABLE;
8167
8168                 /* SSC must be turned on before enabling the CPU output  */
8169                 if (intel_panel_use_ssc(dev_priv) && can_ssc) {
8170                         DRM_DEBUG_KMS("Using SSC on panel\n");
8171                         val |= DREF_SSC1_ENABLE;
8172                 } else
8173                         val &= ~DREF_SSC1_ENABLE;
8174
8175                 /* Get SSC going before enabling the outputs */
8176                 I915_WRITE(PCH_DREF_CONTROL, val);
8177                 POSTING_READ(PCH_DREF_CONTROL);
8178                 udelay(200);
8179
8180                 val &= ~DREF_CPU_SOURCE_OUTPUT_MASK;
8181
8182                 /* Enable CPU source on CPU attached eDP */
8183                 if (has_cpu_edp) {
8184                         if (intel_panel_use_ssc(dev_priv) && can_ssc) {
8185                                 DRM_DEBUG_KMS("Using SSC on eDP\n");
8186                                 val |= DREF_CPU_SOURCE_OUTPUT_DOWNSPREAD;
8187                         } else
8188                                 val |= DREF_CPU_SOURCE_OUTPUT_NONSPREAD;
8189                 } else
8190                         val |= DREF_CPU_SOURCE_OUTPUT_DISABLE;
8191
8192                 I915_WRITE(PCH_DREF_CONTROL, val);
8193                 POSTING_READ(PCH_DREF_CONTROL);
8194                 udelay(200);
8195         } else {
8196                 DRM_DEBUG_KMS("Disabling CPU source output\n");
8197
8198                 val &= ~DREF_CPU_SOURCE_OUTPUT_MASK;
8199
8200                 /* Turn off CPU output */
8201                 val |= DREF_CPU_SOURCE_OUTPUT_DISABLE;
8202
8203                 I915_WRITE(PCH_DREF_CONTROL, val);
8204                 POSTING_READ(PCH_DREF_CONTROL);
8205                 udelay(200);
8206
8207                 if (!using_ssc_source) {
8208                         DRM_DEBUG_KMS("Disabling SSC source\n");
8209
8210                         /* Turn off the SSC source */
8211                         val &= ~DREF_SSC_SOURCE_MASK;
8212                         val |= DREF_SSC_SOURCE_DISABLE;
8213
8214                         /* Turn off SSC1 */
8215                         val &= ~DREF_SSC1_ENABLE;
8216
8217                         I915_WRITE(PCH_DREF_CONTROL, val);
8218                         POSTING_READ(PCH_DREF_CONTROL);
8219                         udelay(200);
8220                 }
8221         }
8222
8223         BUG_ON(val != final);
8224 }
8225
8226 static void lpt_reset_fdi_mphy(struct drm_i915_private *dev_priv)
8227 {
8228         uint32_t tmp;
8229
8230         tmp = I915_READ(SOUTH_CHICKEN2);
8231         tmp |= FDI_MPHY_IOSFSB_RESET_CTL;
8232         I915_WRITE(SOUTH_CHICKEN2, tmp);
8233
8234         if (wait_for_us(I915_READ(SOUTH_CHICKEN2) &
8235                         FDI_MPHY_IOSFSB_RESET_STATUS, 100))
8236                 DRM_ERROR("FDI mPHY reset assert timeout\n");
8237
8238         tmp = I915_READ(SOUTH_CHICKEN2);
8239         tmp &= ~FDI_MPHY_IOSFSB_RESET_CTL;
8240         I915_WRITE(SOUTH_CHICKEN2, tmp);
8241
8242         if (wait_for_us((I915_READ(SOUTH_CHICKEN2) &
8243                          FDI_MPHY_IOSFSB_RESET_STATUS) == 0, 100))
8244                 DRM_ERROR("FDI mPHY reset de-assert timeout\n");
8245 }
8246
8247 /* WaMPhyProgramming:hsw */
8248 static void lpt_program_fdi_mphy(struct drm_i915_private *dev_priv)
8249 {
8250         uint32_t tmp;
8251
8252         tmp = intel_sbi_read(dev_priv, 0x8008, SBI_MPHY);
8253         tmp &= ~(0xFF << 24);
8254         tmp |= (0x12 << 24);
8255         intel_sbi_write(dev_priv, 0x8008, tmp, SBI_MPHY);
8256
8257         tmp = intel_sbi_read(dev_priv, 0x2008, SBI_MPHY);
8258         tmp |= (1 << 11);
8259         intel_sbi_write(dev_priv, 0x2008, tmp, SBI_MPHY);
8260
8261         tmp = intel_sbi_read(dev_priv, 0x2108, SBI_MPHY);
8262         tmp |= (1 << 11);
8263         intel_sbi_write(dev_priv, 0x2108, tmp, SBI_MPHY);
8264
8265         tmp = intel_sbi_read(dev_priv, 0x206C, SBI_MPHY);
8266         tmp |= (1 << 24) | (1 << 21) | (1 << 18);
8267         intel_sbi_write(dev_priv, 0x206C, tmp, SBI_MPHY);
8268
8269         tmp = intel_sbi_read(dev_priv, 0x216C, SBI_MPHY);
8270         tmp |= (1 << 24) | (1 << 21) | (1 << 18);
8271         intel_sbi_write(dev_priv, 0x216C, tmp, SBI_MPHY);
8272
8273         tmp = intel_sbi_read(dev_priv, 0x2080, SBI_MPHY);
8274         tmp &= ~(7 << 13);
8275         tmp |= (5 << 13);
8276         intel_sbi_write(dev_priv, 0x2080, tmp, SBI_MPHY);
8277
8278         tmp = intel_sbi_read(dev_priv, 0x2180, SBI_MPHY);
8279         tmp &= ~(7 << 13);
8280         tmp |= (5 << 13);
8281         intel_sbi_write(dev_priv, 0x2180, tmp, SBI_MPHY);
8282
8283         tmp = intel_sbi_read(dev_priv, 0x208C, SBI_MPHY);
8284         tmp &= ~0xFF;
8285         tmp |= 0x1C;
8286         intel_sbi_write(dev_priv, 0x208C, tmp, SBI_MPHY);
8287
8288         tmp = intel_sbi_read(dev_priv, 0x218C, SBI_MPHY);
8289         tmp &= ~0xFF;
8290         tmp |= 0x1C;
8291         intel_sbi_write(dev_priv, 0x218C, tmp, SBI_MPHY);
8292
8293         tmp = intel_sbi_read(dev_priv, 0x2098, SBI_MPHY);
8294         tmp &= ~(0xFF << 16);
8295         tmp |= (0x1C << 16);
8296         intel_sbi_write(dev_priv, 0x2098, tmp, SBI_MPHY);
8297
8298         tmp = intel_sbi_read(dev_priv, 0x2198, SBI_MPHY);
8299         tmp &= ~(0xFF << 16);
8300         tmp |= (0x1C << 16);
8301         intel_sbi_write(dev_priv, 0x2198, tmp, SBI_MPHY);
8302
8303         tmp = intel_sbi_read(dev_priv, 0x20C4, SBI_MPHY);
8304         tmp |= (1 << 27);
8305         intel_sbi_write(dev_priv, 0x20C4, tmp, SBI_MPHY);
8306
8307         tmp = intel_sbi_read(dev_priv, 0x21C4, SBI_MPHY);
8308         tmp |= (1 << 27);
8309         intel_sbi_write(dev_priv, 0x21C4, tmp, SBI_MPHY);
8310
8311         tmp = intel_sbi_read(dev_priv, 0x20EC, SBI_MPHY);
8312         tmp &= ~(0xF << 28);
8313         tmp |= (4 << 28);
8314         intel_sbi_write(dev_priv, 0x20EC, tmp, SBI_MPHY);
8315
8316         tmp = intel_sbi_read(dev_priv, 0x21EC, SBI_MPHY);
8317         tmp &= ~(0xF << 28);
8318         tmp |= (4 << 28);
8319         intel_sbi_write(dev_priv, 0x21EC, tmp, SBI_MPHY);
8320 }
8321
8322 /* Implements 3 different sequences from BSpec chapter "Display iCLK
8323  * Programming" based on the parameters passed:
8324  * - Sequence to enable CLKOUT_DP
8325  * - Sequence to enable CLKOUT_DP without spread
8326  * - Sequence to enable CLKOUT_DP for FDI usage and configure PCH FDI I/O
8327  */
8328 static void lpt_enable_clkout_dp(struct drm_i915_private *dev_priv,
8329                                  bool with_spread, bool with_fdi)
8330 {
8331         uint32_t reg, tmp;
8332
8333         if (WARN(with_fdi && !with_spread, "FDI requires downspread\n"))
8334                 with_spread = true;
8335         if (WARN(HAS_PCH_LPT_LP(dev_priv) &&
8336             with_fdi, "LP PCH doesn't have FDI\n"))
8337                 with_fdi = false;
8338
8339         mutex_lock(&dev_priv->sb_lock);
8340
8341         tmp = intel_sbi_read(dev_priv, SBI_SSCCTL, SBI_ICLK);
8342         tmp &= ~SBI_SSCCTL_DISABLE;
8343         tmp |= SBI_SSCCTL_PATHALT;
8344         intel_sbi_write(dev_priv, SBI_SSCCTL, tmp, SBI_ICLK);
8345
8346         udelay(24);
8347
8348         if (with_spread) {
8349                 tmp = intel_sbi_read(dev_priv, SBI_SSCCTL, SBI_ICLK);
8350                 tmp &= ~SBI_SSCCTL_PATHALT;
8351                 intel_sbi_write(dev_priv, SBI_SSCCTL, tmp, SBI_ICLK);
8352
8353                 if (with_fdi) {
8354                         lpt_reset_fdi_mphy(dev_priv);
8355                         lpt_program_fdi_mphy(dev_priv);
8356                 }
8357         }
8358
8359         reg = HAS_PCH_LPT_LP(dev_priv) ? SBI_GEN0 : SBI_DBUFF0;
8360         tmp = intel_sbi_read(dev_priv, reg, SBI_ICLK);
8361         tmp |= SBI_GEN0_CFG_BUFFENABLE_DISABLE;
8362         intel_sbi_write(dev_priv, reg, tmp, SBI_ICLK);
8363
8364         mutex_unlock(&dev_priv->sb_lock);
8365 }
8366
8367 /* Sequence to disable CLKOUT_DP */
8368 static void lpt_disable_clkout_dp(struct drm_i915_private *dev_priv)
8369 {
8370         uint32_t reg, tmp;
8371
8372         mutex_lock(&dev_priv->sb_lock);
8373
8374         reg = HAS_PCH_LPT_LP(dev_priv) ? SBI_GEN0 : SBI_DBUFF0;
8375         tmp = intel_sbi_read(dev_priv, reg, SBI_ICLK);
8376         tmp &= ~SBI_GEN0_CFG_BUFFENABLE_DISABLE;
8377         intel_sbi_write(dev_priv, reg, tmp, SBI_ICLK);
8378
8379         tmp = intel_sbi_read(dev_priv, SBI_SSCCTL, SBI_ICLK);
8380         if (!(tmp & SBI_SSCCTL_DISABLE)) {
8381                 if (!(tmp & SBI_SSCCTL_PATHALT)) {
8382                         tmp |= SBI_SSCCTL_PATHALT;
8383                         intel_sbi_write(dev_priv, SBI_SSCCTL, tmp, SBI_ICLK);
8384                         udelay(32);
8385                 }
8386                 tmp |= SBI_SSCCTL_DISABLE;
8387                 intel_sbi_write(dev_priv, SBI_SSCCTL, tmp, SBI_ICLK);
8388         }
8389
8390         mutex_unlock(&dev_priv->sb_lock);
8391 }
8392
8393 #define BEND_IDX(steps) ((50 + (steps)) / 5)
8394
8395 static const uint16_t sscdivintphase[] = {
8396         [BEND_IDX( 50)] = 0x3B23,
8397         [BEND_IDX( 45)] = 0x3B23,
8398         [BEND_IDX( 40)] = 0x3C23,
8399         [BEND_IDX( 35)] = 0x3C23,
8400         [BEND_IDX( 30)] = 0x3D23,
8401         [BEND_IDX( 25)] = 0x3D23,
8402         [BEND_IDX( 20)] = 0x3E23,
8403         [BEND_IDX( 15)] = 0x3E23,
8404         [BEND_IDX( 10)] = 0x3F23,
8405         [BEND_IDX(  5)] = 0x3F23,
8406         [BEND_IDX(  0)] = 0x0025,
8407         [BEND_IDX( -5)] = 0x0025,
8408         [BEND_IDX(-10)] = 0x0125,
8409         [BEND_IDX(-15)] = 0x0125,
8410         [BEND_IDX(-20)] = 0x0225,
8411         [BEND_IDX(-25)] = 0x0225,
8412         [BEND_IDX(-30)] = 0x0325,
8413         [BEND_IDX(-35)] = 0x0325,
8414         [BEND_IDX(-40)] = 0x0425,
8415         [BEND_IDX(-45)] = 0x0425,
8416         [BEND_IDX(-50)] = 0x0525,
8417 };
8418
8419 /*
8420  * Bend CLKOUT_DP
8421  * steps -50 to 50 inclusive, in steps of 5
8422  * < 0 slow down the clock, > 0 speed up the clock, 0 == no bend (135MHz)
8423  * change in clock period = -(steps / 10) * 5.787 ps
8424  */
8425 static void lpt_bend_clkout_dp(struct drm_i915_private *dev_priv, int steps)
8426 {
8427         uint32_t tmp;
8428         int idx = BEND_IDX(steps);
8429
8430         if (WARN_ON(steps % 5 != 0))
8431                 return;
8432
8433         if (WARN_ON(idx >= ARRAY_SIZE(sscdivintphase)))
8434                 return;
8435
8436         mutex_lock(&dev_priv->sb_lock);
8437
8438         if (steps % 10 != 0)
8439                 tmp = 0xAAAAAAAB;
8440         else
8441                 tmp = 0x00000000;
8442         intel_sbi_write(dev_priv, SBI_SSCDITHPHASE, tmp, SBI_ICLK);
8443
8444         tmp = intel_sbi_read(dev_priv, SBI_SSCDIVINTPHASE, SBI_ICLK);
8445         tmp &= 0xffff0000;
8446         tmp |= sscdivintphase[idx];
8447         intel_sbi_write(dev_priv, SBI_SSCDIVINTPHASE, tmp, SBI_ICLK);
8448
8449         mutex_unlock(&dev_priv->sb_lock);
8450 }
8451
8452 #undef BEND_IDX
8453
8454 static void lpt_init_pch_refclk(struct drm_i915_private *dev_priv)
8455 {
8456         struct intel_encoder *encoder;
8457         bool has_vga = false;
8458
8459         for_each_intel_encoder(&dev_priv->drm, encoder) {
8460                 switch (encoder->type) {
8461                 case INTEL_OUTPUT_ANALOG:
8462                         has_vga = true;
8463                         break;
8464                 default:
8465                         break;
8466                 }
8467         }
8468
8469         if (has_vga) {
8470                 lpt_bend_clkout_dp(dev_priv, 0);
8471                 lpt_enable_clkout_dp(dev_priv, true, true);
8472         } else {
8473                 lpt_disable_clkout_dp(dev_priv);
8474         }
8475 }
8476
8477 /*
8478  * Initialize reference clocks when the driver loads
8479  */
8480 void intel_init_pch_refclk(struct drm_i915_private *dev_priv)
8481 {
8482         if (HAS_PCH_IBX(dev_priv) || HAS_PCH_CPT(dev_priv))
8483                 ironlake_init_pch_refclk(dev_priv);
8484         else if (HAS_PCH_LPT(dev_priv))
8485                 lpt_init_pch_refclk(dev_priv);
8486 }
8487
8488 static void ironlake_set_pipeconf(const struct intel_crtc_state *crtc_state)
8489 {
8490         struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
8491         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
8492         enum pipe pipe = crtc->pipe;
8493         uint32_t val;
8494
8495         val = 0;
8496
8497         switch (crtc_state->pipe_bpp) {
8498         case 18:
8499                 val |= PIPECONF_6BPC;
8500                 break;
8501         case 24:
8502                 val |= PIPECONF_8BPC;
8503                 break;
8504         case 30:
8505                 val |= PIPECONF_10BPC;
8506                 break;
8507         case 36:
8508                 val |= PIPECONF_12BPC;
8509                 break;
8510         default:
8511                 /* Case prevented by intel_choose_pipe_bpp_dither. */
8512                 BUG();
8513         }
8514
8515         if (crtc_state->dither)
8516                 val |= (PIPECONF_DITHER_EN | PIPECONF_DITHER_TYPE_SP);
8517
8518         if (crtc_state->base.adjusted_mode.flags & DRM_MODE_FLAG_INTERLACE)
8519                 val |= PIPECONF_INTERLACED_ILK;
8520         else
8521                 val |= PIPECONF_PROGRESSIVE;
8522
8523         if (crtc_state->limited_color_range)
8524                 val |= PIPECONF_COLOR_RANGE_SELECT;
8525
8526         I915_WRITE(PIPECONF(pipe), val);
8527         POSTING_READ(PIPECONF(pipe));
8528 }
8529
8530 static void haswell_set_pipeconf(const struct intel_crtc_state *crtc_state)
8531 {
8532         struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
8533         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
8534         enum transcoder cpu_transcoder = crtc_state->cpu_transcoder;
8535         u32 val = 0;
8536
8537         if (IS_HASWELL(dev_priv) && crtc_state->dither)
8538                 val |= (PIPECONF_DITHER_EN | PIPECONF_DITHER_TYPE_SP);
8539
8540         if (crtc_state->base.adjusted_mode.flags & DRM_MODE_FLAG_INTERLACE)
8541                 val |= PIPECONF_INTERLACED_ILK;
8542         else
8543                 val |= PIPECONF_PROGRESSIVE;
8544
8545         I915_WRITE(PIPECONF(cpu_transcoder), val);
8546         POSTING_READ(PIPECONF(cpu_transcoder));
8547 }
8548
8549 static void haswell_set_pipemisc(const struct intel_crtc_state *crtc_state)
8550 {
8551         struct intel_crtc *intel_crtc = to_intel_crtc(crtc_state->base.crtc);
8552         struct drm_i915_private *dev_priv = to_i915(intel_crtc->base.dev);
8553
8554         if (IS_BROADWELL(dev_priv) || INTEL_GEN(dev_priv) >= 9) {
8555                 u32 val = 0;
8556
8557                 switch (crtc_state->pipe_bpp) {
8558                 case 18:
8559                         val |= PIPEMISC_DITHER_6_BPC;
8560                         break;
8561                 case 24:
8562                         val |= PIPEMISC_DITHER_8_BPC;
8563                         break;
8564                 case 30:
8565                         val |= PIPEMISC_DITHER_10_BPC;
8566                         break;
8567                 case 36:
8568                         val |= PIPEMISC_DITHER_12_BPC;
8569                         break;
8570                 default:
8571                         /* Case prevented by pipe_config_set_bpp. */
8572                         BUG();
8573                 }
8574
8575                 if (crtc_state->dither)
8576                         val |= PIPEMISC_DITHER_ENABLE | PIPEMISC_DITHER_TYPE_SP;
8577
8578                 if (crtc_state->output_format == INTEL_OUTPUT_FORMAT_YCBCR420 ||
8579                     crtc_state->output_format == INTEL_OUTPUT_FORMAT_YCBCR444)
8580                         val |= PIPEMISC_OUTPUT_COLORSPACE_YUV;
8581
8582                 if (crtc_state->output_format == INTEL_OUTPUT_FORMAT_YCBCR420)
8583                         val |= PIPEMISC_YUV420_ENABLE |
8584                                 PIPEMISC_YUV420_MODE_FULL_BLEND;
8585
8586                 I915_WRITE(PIPEMISC(intel_crtc->pipe), val);
8587         }
8588 }
8589
8590 int ironlake_get_lanes_required(int target_clock, int link_bw, int bpp)
8591 {
8592         /*
8593          * Account for spread spectrum to avoid
8594          * oversubscribing the link. Max center spread
8595          * is 2.5%; use 5% for safety's sake.
8596          */
8597         u32 bps = target_clock * bpp * 21 / 20;
8598         return DIV_ROUND_UP(bps, link_bw * 8);
8599 }
8600
8601 static bool ironlake_needs_fb_cb_tune(struct dpll *dpll, int factor)
8602 {
8603         return i9xx_dpll_compute_m(dpll) < factor * dpll->n;
8604 }
8605
8606 static void ironlake_compute_dpll(struct intel_crtc *intel_crtc,
8607                                   struct intel_crtc_state *crtc_state,
8608                                   struct dpll *reduced_clock)
8609 {
8610         struct drm_crtc *crtc = &intel_crtc->base;
8611         struct drm_device *dev = crtc->dev;
8612         struct drm_i915_private *dev_priv = to_i915(dev);
8613         u32 dpll, fp, fp2;
8614         int factor;
8615
8616         /* Enable autotuning of the PLL clock (if permissible) */
8617         factor = 21;
8618         if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_LVDS)) {
8619                 if ((intel_panel_use_ssc(dev_priv) &&
8620                      dev_priv->vbt.lvds_ssc_freq == 100000) ||
8621                     (HAS_PCH_IBX(dev_priv) && intel_is_dual_link_lvds(dev)))
8622                         factor = 25;
8623         } else if (crtc_state->sdvo_tv_clock)
8624                 factor = 20;
8625
8626         fp = i9xx_dpll_compute_fp(&crtc_state->dpll);
8627
8628         if (ironlake_needs_fb_cb_tune(&crtc_state->dpll, factor))
8629                 fp |= FP_CB_TUNE;
8630
8631         if (reduced_clock) {
8632                 fp2 = i9xx_dpll_compute_fp(reduced_clock);
8633
8634                 if (reduced_clock->m < factor * reduced_clock->n)
8635                         fp2 |= FP_CB_TUNE;
8636         } else {
8637                 fp2 = fp;
8638         }
8639
8640         dpll = 0;
8641
8642         if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_LVDS))
8643                 dpll |= DPLLB_MODE_LVDS;
8644         else
8645                 dpll |= DPLLB_MODE_DAC_SERIAL;
8646
8647         dpll |= (crtc_state->pixel_multiplier - 1)
8648                 << PLL_REF_SDVO_HDMI_MULTIPLIER_SHIFT;
8649
8650         if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_SDVO) ||
8651             intel_crtc_has_type(crtc_state, INTEL_OUTPUT_HDMI))
8652                 dpll |= DPLL_SDVO_HIGH_SPEED;
8653
8654         if (intel_crtc_has_dp_encoder(crtc_state))
8655                 dpll |= DPLL_SDVO_HIGH_SPEED;
8656
8657         /*
8658          * The high speed IO clock is only really required for
8659          * SDVO/HDMI/DP, but we also enable it for CRT to make it
8660          * possible to share the DPLL between CRT and HDMI. Enabling
8661          * the clock needlessly does no real harm, except use up a
8662          * bit of power potentially.
8663          *
8664          * We'll limit this to IVB with 3 pipes, since it has only two
8665          * DPLLs and so DPLL sharing is the only way to get three pipes
8666          * driving PCH ports at the same time. On SNB we could do this,
8667          * and potentially avoid enabling the second DPLL, but it's not
8668          * clear if it''s a win or loss power wise. No point in doing
8669          * this on ILK at all since it has a fixed DPLL<->pipe mapping.
8670          */
8671         if (INTEL_INFO(dev_priv)->num_pipes == 3 &&
8672             intel_crtc_has_type(crtc_state, INTEL_OUTPUT_ANALOG))
8673                 dpll |= DPLL_SDVO_HIGH_SPEED;
8674
8675         /* compute bitmask from p1 value */
8676         dpll |= (1 << (crtc_state->dpll.p1 - 1)) << DPLL_FPA01_P1_POST_DIV_SHIFT;
8677         /* also FPA1 */
8678         dpll |= (1 << (crtc_state->dpll.p1 - 1)) << DPLL_FPA1_P1_POST_DIV_SHIFT;
8679
8680         switch (crtc_state->dpll.p2) {
8681         case 5:
8682                 dpll |= DPLL_DAC_SERIAL_P2_CLOCK_DIV_5;
8683                 break;
8684         case 7:
8685                 dpll |= DPLLB_LVDS_P2_CLOCK_DIV_7;
8686                 break;
8687         case 10:
8688                 dpll |= DPLL_DAC_SERIAL_P2_CLOCK_DIV_10;
8689                 break;
8690         case 14:
8691                 dpll |= DPLLB_LVDS_P2_CLOCK_DIV_14;
8692                 break;
8693         }
8694
8695         if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_LVDS) &&
8696             intel_panel_use_ssc(dev_priv))
8697                 dpll |= PLLB_REF_INPUT_SPREADSPECTRUMIN;
8698         else
8699                 dpll |= PLL_REF_INPUT_DREFCLK;
8700
8701         dpll |= DPLL_VCO_ENABLE;
8702
8703         crtc_state->dpll_hw_state.dpll = dpll;
8704         crtc_state->dpll_hw_state.fp0 = fp;
8705         crtc_state->dpll_hw_state.fp1 = fp2;
8706 }
8707
8708 static int ironlake_crtc_compute_clock(struct intel_crtc *crtc,
8709                                        struct intel_crtc_state *crtc_state)
8710 {
8711         struct drm_device *dev = crtc->base.dev;
8712         struct drm_i915_private *dev_priv = to_i915(dev);
8713         const struct intel_limit *limit;
8714         int refclk = 120000;
8715
8716         memset(&crtc_state->dpll_hw_state, 0,
8717                sizeof(crtc_state->dpll_hw_state));
8718
8719         /* CPU eDP is the only output that doesn't need a PCH PLL of its own. */
8720         if (!crtc_state->has_pch_encoder)
8721                 return 0;
8722
8723         if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_LVDS)) {
8724                 if (intel_panel_use_ssc(dev_priv)) {
8725                         DRM_DEBUG_KMS("using SSC reference clock of %d kHz\n",
8726                                       dev_priv->vbt.lvds_ssc_freq);
8727                         refclk = dev_priv->vbt.lvds_ssc_freq;
8728                 }
8729
8730                 if (intel_is_dual_link_lvds(dev)) {
8731                         if (refclk == 100000)
8732                                 limit = &intel_limits_ironlake_dual_lvds_100m;
8733                         else
8734                                 limit = &intel_limits_ironlake_dual_lvds;
8735                 } else {
8736                         if (refclk == 100000)
8737                                 limit = &intel_limits_ironlake_single_lvds_100m;
8738                         else
8739                                 limit = &intel_limits_ironlake_single_lvds;
8740                 }
8741         } else {
8742                 limit = &intel_limits_ironlake_dac;
8743         }
8744
8745         if (!crtc_state->clock_set &&
8746             !g4x_find_best_dpll(limit, crtc_state, crtc_state->port_clock,
8747                                 refclk, NULL, &crtc_state->dpll)) {
8748                 DRM_ERROR("Couldn't find PLL settings for mode!\n");
8749                 return -EINVAL;
8750         }
8751
8752         ironlake_compute_dpll(crtc, crtc_state, NULL);
8753
8754         if (!intel_get_shared_dpll(crtc, crtc_state, NULL)) {
8755                 DRM_DEBUG_KMS("failed to find PLL for pipe %c\n",
8756                               pipe_name(crtc->pipe));
8757                 return -EINVAL;
8758         }
8759
8760         return 0;
8761 }
8762
8763 static void intel_pch_transcoder_get_m_n(struct intel_crtc *crtc,
8764                                          struct intel_link_m_n *m_n)
8765 {
8766         struct drm_device *dev = crtc->base.dev;
8767         struct drm_i915_private *dev_priv = to_i915(dev);
8768         enum pipe pipe = crtc->pipe;
8769
8770         m_n->link_m = I915_READ(PCH_TRANS_LINK_M1(pipe));
8771         m_n->link_n = I915_READ(PCH_TRANS_LINK_N1(pipe));
8772         m_n->gmch_m = I915_READ(PCH_TRANS_DATA_M1(pipe))
8773                 & ~TU_SIZE_MASK;
8774         m_n->gmch_n = I915_READ(PCH_TRANS_DATA_N1(pipe));
8775         m_n->tu = ((I915_READ(PCH_TRANS_DATA_M1(pipe))
8776                     & TU_SIZE_MASK) >> TU_SIZE_SHIFT) + 1;
8777 }
8778
8779 static void intel_cpu_transcoder_get_m_n(struct intel_crtc *crtc,
8780                                          enum transcoder transcoder,
8781                                          struct intel_link_m_n *m_n,
8782                                          struct intel_link_m_n *m2_n2)
8783 {
8784         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
8785         enum pipe pipe = crtc->pipe;
8786
8787         if (INTEL_GEN(dev_priv) >= 5) {
8788                 m_n->link_m = I915_READ(PIPE_LINK_M1(transcoder));
8789                 m_n->link_n = I915_READ(PIPE_LINK_N1(transcoder));
8790                 m_n->gmch_m = I915_READ(PIPE_DATA_M1(transcoder))
8791                         & ~TU_SIZE_MASK;
8792                 m_n->gmch_n = I915_READ(PIPE_DATA_N1(transcoder));
8793                 m_n->tu = ((I915_READ(PIPE_DATA_M1(transcoder))
8794                             & TU_SIZE_MASK) >> TU_SIZE_SHIFT) + 1;
8795
8796                 if (m2_n2 && transcoder_has_m2_n2(dev_priv, transcoder)) {
8797                         m2_n2->link_m = I915_READ(PIPE_LINK_M2(transcoder));
8798                         m2_n2->link_n = I915_READ(PIPE_LINK_N2(transcoder));
8799                         m2_n2->gmch_m = I915_READ(PIPE_DATA_M2(transcoder))
8800                                         & ~TU_SIZE_MASK;
8801                         m2_n2->gmch_n = I915_READ(PIPE_DATA_N2(transcoder));
8802                         m2_n2->tu = ((I915_READ(PIPE_DATA_M2(transcoder))
8803                                         & TU_SIZE_MASK) >> TU_SIZE_SHIFT) + 1;
8804                 }
8805         } else {
8806                 m_n->link_m = I915_READ(PIPE_LINK_M_G4X(pipe));
8807                 m_n->link_n = I915_READ(PIPE_LINK_N_G4X(pipe));
8808                 m_n->gmch_m = I915_READ(PIPE_DATA_M_G4X(pipe))
8809                         & ~TU_SIZE_MASK;
8810                 m_n->gmch_n = I915_READ(PIPE_DATA_N_G4X(pipe));
8811                 m_n->tu = ((I915_READ(PIPE_DATA_M_G4X(pipe))
8812                             & TU_SIZE_MASK) >> TU_SIZE_SHIFT) + 1;
8813         }
8814 }
8815
8816 void intel_dp_get_m_n(struct intel_crtc *crtc,
8817                       struct intel_crtc_state *pipe_config)
8818 {
8819         if (pipe_config->has_pch_encoder)
8820                 intel_pch_transcoder_get_m_n(crtc, &pipe_config->dp_m_n);
8821         else
8822                 intel_cpu_transcoder_get_m_n(crtc, pipe_config->cpu_transcoder,
8823                                              &pipe_config->dp_m_n,
8824                                              &pipe_config->dp_m2_n2);
8825 }
8826
8827 static void ironlake_get_fdi_m_n_config(struct intel_crtc *crtc,
8828                                         struct intel_crtc_state *pipe_config)
8829 {
8830         intel_cpu_transcoder_get_m_n(crtc, pipe_config->cpu_transcoder,
8831                                      &pipe_config->fdi_m_n, NULL);
8832 }
8833
8834 static void skylake_get_pfit_config(struct intel_crtc *crtc,
8835                                     struct intel_crtc_state *pipe_config)
8836 {
8837         struct drm_device *dev = crtc->base.dev;
8838         struct drm_i915_private *dev_priv = to_i915(dev);
8839         struct intel_crtc_scaler_state *scaler_state = &pipe_config->scaler_state;
8840         uint32_t ps_ctrl = 0;
8841         int id = -1;
8842         int i;
8843
8844         /* find scaler attached to this pipe */
8845         for (i = 0; i < crtc->num_scalers; i++) {
8846                 ps_ctrl = I915_READ(SKL_PS_CTRL(crtc->pipe, i));
8847                 if (ps_ctrl & PS_SCALER_EN && !(ps_ctrl & PS_PLANE_SEL_MASK)) {
8848                         id = i;
8849                         pipe_config->pch_pfit.enabled = true;
8850                         pipe_config->pch_pfit.pos = I915_READ(SKL_PS_WIN_POS(crtc->pipe, i));
8851                         pipe_config->pch_pfit.size = I915_READ(SKL_PS_WIN_SZ(crtc->pipe, i));
8852                         break;
8853                 }
8854         }
8855
8856         scaler_state->scaler_id = id;
8857         if (id >= 0) {
8858                 scaler_state->scaler_users |= (1 << SKL_CRTC_INDEX);
8859         } else {
8860                 scaler_state->scaler_users &= ~(1 << SKL_CRTC_INDEX);
8861         }
8862 }
8863
8864 static void
8865 skylake_get_initial_plane_config(struct intel_crtc *crtc,
8866                                  struct intel_initial_plane_config *plane_config)
8867 {
8868         struct drm_device *dev = crtc->base.dev;
8869         struct drm_i915_private *dev_priv = to_i915(dev);
8870         struct intel_plane *plane = to_intel_plane(crtc->base.primary);
8871         enum plane_id plane_id = plane->id;
8872         enum pipe pipe;
8873         u32 val, base, offset, stride_mult, tiling, alpha;
8874         int fourcc, pixel_format;
8875         unsigned int aligned_height;
8876         struct drm_framebuffer *fb;
8877         struct intel_framebuffer *intel_fb;
8878
8879         if (!plane->get_hw_state(plane, &pipe))
8880                 return;
8881
8882         WARN_ON(pipe != crtc->pipe);
8883
8884         intel_fb = kzalloc(sizeof(*intel_fb), GFP_KERNEL);
8885         if (!intel_fb) {
8886                 DRM_DEBUG_KMS("failed to alloc fb\n");
8887                 return;
8888         }
8889
8890         fb = &intel_fb->base;
8891
8892         fb->dev = dev;
8893
8894         val = I915_READ(PLANE_CTL(pipe, plane_id));
8895
8896         if (INTEL_GEN(dev_priv) >= 11)
8897                 pixel_format = val & ICL_PLANE_CTL_FORMAT_MASK;
8898         else
8899                 pixel_format = val & PLANE_CTL_FORMAT_MASK;
8900
8901         if (INTEL_GEN(dev_priv) >= 10 || IS_GEMINILAKE(dev_priv)) {
8902                 alpha = I915_READ(PLANE_COLOR_CTL(pipe, plane_id));
8903                 alpha &= PLANE_COLOR_ALPHA_MASK;
8904         } else {
8905                 alpha = val & PLANE_CTL_ALPHA_MASK;
8906         }
8907
8908         fourcc = skl_format_to_fourcc(pixel_format,
8909                                       val & PLANE_CTL_ORDER_RGBX, alpha);
8910         fb->format = drm_format_info(fourcc);
8911
8912         tiling = val & PLANE_CTL_TILED_MASK;
8913         switch (tiling) {
8914         case PLANE_CTL_TILED_LINEAR:
8915                 fb->modifier = DRM_FORMAT_MOD_LINEAR;
8916                 break;
8917         case PLANE_CTL_TILED_X:
8918                 plane_config->tiling = I915_TILING_X;
8919                 fb->modifier = I915_FORMAT_MOD_X_TILED;
8920                 break;
8921         case PLANE_CTL_TILED_Y:
8922                 plane_config->tiling = I915_TILING_Y;
8923                 if (val & PLANE_CTL_RENDER_DECOMPRESSION_ENABLE)
8924                         fb->modifier = I915_FORMAT_MOD_Y_TILED_CCS;
8925                 else
8926                         fb->modifier = I915_FORMAT_MOD_Y_TILED;
8927                 break;
8928         case PLANE_CTL_TILED_YF:
8929                 if (val & PLANE_CTL_RENDER_DECOMPRESSION_ENABLE)
8930                         fb->modifier = I915_FORMAT_MOD_Yf_TILED_CCS;
8931                 else
8932                         fb->modifier = I915_FORMAT_MOD_Yf_TILED;
8933                 break;
8934         default:
8935                 MISSING_CASE(tiling);
8936                 goto error;
8937         }
8938
8939         /*
8940          * DRM_MODE_ROTATE_ is counter clockwise to stay compatible with Xrandr
8941          * while i915 HW rotation is clockwise, thats why this swapping.
8942          */
8943         switch (val & PLANE_CTL_ROTATE_MASK) {
8944         case PLANE_CTL_ROTATE_0:
8945                 plane_config->rotation = DRM_MODE_ROTATE_0;
8946                 break;
8947         case PLANE_CTL_ROTATE_90:
8948                 plane_config->rotation = DRM_MODE_ROTATE_270;
8949                 break;
8950         case PLANE_CTL_ROTATE_180:
8951                 plane_config->rotation = DRM_MODE_ROTATE_180;
8952                 break;
8953         case PLANE_CTL_ROTATE_270:
8954                 plane_config->rotation = DRM_MODE_ROTATE_90;
8955                 break;
8956         }
8957
8958         if (INTEL_GEN(dev_priv) >= 10 &&
8959             val & PLANE_CTL_FLIP_HORIZONTAL)
8960                 plane_config->rotation |= DRM_MODE_REFLECT_X;
8961
8962         base = I915_READ(PLANE_SURF(pipe, plane_id)) & 0xfffff000;
8963         plane_config->base = base;
8964
8965         offset = I915_READ(PLANE_OFFSET(pipe, plane_id));
8966
8967         val = I915_READ(PLANE_SIZE(pipe, plane_id));
8968         fb->height = ((val >> 16) & 0xfff) + 1;
8969         fb->width = ((val >> 0) & 0x1fff) + 1;
8970
8971         val = I915_READ(PLANE_STRIDE(pipe, plane_id));
8972         stride_mult = skl_plane_stride_mult(fb, 0, DRM_MODE_ROTATE_0);
8973         fb->pitches[0] = (val & 0x3ff) * stride_mult;
8974
8975         aligned_height = intel_fb_align_height(fb, 0, fb->height);
8976
8977         plane_config->size = fb->pitches[0] * aligned_height;
8978
8979         DRM_DEBUG_KMS("%s/%s with fb: size=%dx%d@%d, offset=%x, pitch %d, size 0x%x\n",
8980                       crtc->base.name, plane->base.name, fb->width, fb->height,
8981                       fb->format->cpp[0] * 8, base, fb->pitches[0],
8982                       plane_config->size);
8983
8984         plane_config->fb = intel_fb;
8985         return;
8986
8987 error:
8988         kfree(intel_fb);
8989 }
8990
8991 static void ironlake_get_pfit_config(struct intel_crtc *crtc,
8992                                      struct intel_crtc_state *pipe_config)
8993 {
8994         struct drm_device *dev = crtc->base.dev;
8995         struct drm_i915_private *dev_priv = to_i915(dev);
8996         uint32_t tmp;
8997
8998         tmp = I915_READ(PF_CTL(crtc->pipe));
8999
9000         if (tmp & PF_ENABLE) {
9001                 pipe_config->pch_pfit.enabled = true;
9002                 pipe_config->pch_pfit.pos = I915_READ(PF_WIN_POS(crtc->pipe));
9003                 pipe_config->pch_pfit.size = I915_READ(PF_WIN_SZ(crtc->pipe));
9004
9005                 /* We currently do not free assignements of panel fitters on
9006                  * ivb/hsw (since we don't use the higher upscaling modes which
9007                  * differentiates them) so just WARN about this case for now. */
9008                 if (IS_GEN(dev_priv, 7)) {
9009                         WARN_ON((tmp & PF_PIPE_SEL_MASK_IVB) !=
9010                                 PF_PIPE_SEL_IVB(crtc->pipe));
9011                 }
9012         }
9013 }
9014
9015 static bool ironlake_get_pipe_config(struct intel_crtc *crtc,
9016                                      struct intel_crtc_state *pipe_config)
9017 {
9018         struct drm_device *dev = crtc->base.dev;
9019         struct drm_i915_private *dev_priv = to_i915(dev);
9020         enum intel_display_power_domain power_domain;
9021         uint32_t tmp;
9022         bool ret;
9023
9024         power_domain = POWER_DOMAIN_PIPE(crtc->pipe);
9025         if (!intel_display_power_get_if_enabled(dev_priv, power_domain))
9026                 return false;
9027
9028         pipe_config->output_format = INTEL_OUTPUT_FORMAT_RGB;
9029         pipe_config->cpu_transcoder = (enum transcoder) crtc->pipe;
9030         pipe_config->shared_dpll = NULL;
9031
9032         ret = false;
9033         tmp = I915_READ(PIPECONF(crtc->pipe));
9034         if (!(tmp & PIPECONF_ENABLE))
9035                 goto out;
9036
9037         switch (tmp & PIPECONF_BPC_MASK) {
9038         case PIPECONF_6BPC:
9039                 pipe_config->pipe_bpp = 18;
9040                 break;
9041         case PIPECONF_8BPC:
9042                 pipe_config->pipe_bpp = 24;
9043                 break;
9044         case PIPECONF_10BPC:
9045                 pipe_config->pipe_bpp = 30;
9046                 break;
9047         case PIPECONF_12BPC:
9048                 pipe_config->pipe_bpp = 36;
9049                 break;
9050         default:
9051                 break;
9052         }
9053
9054         if (tmp & PIPECONF_COLOR_RANGE_SELECT)
9055                 pipe_config->limited_color_range = true;
9056
9057         if (I915_READ(PCH_TRANSCONF(crtc->pipe)) & TRANS_ENABLE) {
9058                 struct intel_shared_dpll *pll;
9059                 enum intel_dpll_id pll_id;
9060
9061                 pipe_config->has_pch_encoder = true;
9062
9063                 tmp = I915_READ(FDI_RX_CTL(crtc->pipe));
9064                 pipe_config->fdi_lanes = ((FDI_DP_PORT_WIDTH_MASK & tmp) >>
9065                                           FDI_DP_PORT_WIDTH_SHIFT) + 1;
9066
9067                 ironlake_get_fdi_m_n_config(crtc, pipe_config);
9068
9069                 if (HAS_PCH_IBX(dev_priv)) {
9070                         /*
9071                          * The pipe->pch transcoder and pch transcoder->pll
9072                          * mapping is fixed.
9073                          */
9074                         pll_id = (enum intel_dpll_id) crtc->pipe;
9075                 } else {
9076                         tmp = I915_READ(PCH_DPLL_SEL);
9077                         if (tmp & TRANS_DPLLB_SEL(crtc->pipe))
9078                                 pll_id = DPLL_ID_PCH_PLL_B;
9079                         else
9080                                 pll_id= DPLL_ID_PCH_PLL_A;
9081                 }
9082
9083                 pipe_config->shared_dpll =
9084                         intel_get_shared_dpll_by_id(dev_priv, pll_id);
9085                 pll = pipe_config->shared_dpll;
9086
9087                 WARN_ON(!pll->info->funcs->get_hw_state(dev_priv, pll,
9088                                                 &pipe_config->dpll_hw_state));
9089
9090                 tmp = pipe_config->dpll_hw_state.dpll;
9091                 pipe_config->pixel_multiplier =
9092                         ((tmp & PLL_REF_SDVO_HDMI_MULTIPLIER_MASK)
9093                          >> PLL_REF_SDVO_HDMI_MULTIPLIER_SHIFT) + 1;
9094
9095                 ironlake_pch_clock_get(crtc, pipe_config);
9096         } else {
9097                 pipe_config->pixel_multiplier = 1;
9098         }
9099
9100         intel_get_pipe_timings(crtc, pipe_config);
9101         intel_get_pipe_src_size(crtc, pipe_config);
9102
9103         ironlake_get_pfit_config(crtc, pipe_config);
9104
9105         ret = true;
9106
9107 out:
9108         intel_display_power_put(dev_priv, power_domain);
9109
9110         return ret;
9111 }
9112
9113 static void assert_can_disable_lcpll(struct drm_i915_private *dev_priv)
9114 {
9115         struct drm_device *dev = &dev_priv->drm;
9116         struct intel_crtc *crtc;
9117
9118         for_each_intel_crtc(dev, crtc)
9119                 I915_STATE_WARN(crtc->active, "CRTC for pipe %c enabled\n",
9120                      pipe_name(crtc->pipe));
9121
9122         I915_STATE_WARN(I915_READ(HSW_PWR_WELL_CTL2),
9123                         "Display power well on\n");
9124         I915_STATE_WARN(I915_READ(SPLL_CTL) & SPLL_PLL_ENABLE, "SPLL enabled\n");
9125         I915_STATE_WARN(I915_READ(WRPLL_CTL(0)) & WRPLL_PLL_ENABLE, "WRPLL1 enabled\n");
9126         I915_STATE_WARN(I915_READ(WRPLL_CTL(1)) & WRPLL_PLL_ENABLE, "WRPLL2 enabled\n");
9127         I915_STATE_WARN(I915_READ(PP_STATUS(0)) & PP_ON, "Panel power on\n");
9128         I915_STATE_WARN(I915_READ(BLC_PWM_CPU_CTL2) & BLM_PWM_ENABLE,
9129              "CPU PWM1 enabled\n");
9130         if (IS_HASWELL(dev_priv))
9131                 I915_STATE_WARN(I915_READ(HSW_BLC_PWM2_CTL) & BLM_PWM_ENABLE,
9132                      "CPU PWM2 enabled\n");
9133         I915_STATE_WARN(I915_READ(BLC_PWM_PCH_CTL1) & BLM_PCH_PWM_ENABLE,
9134              "PCH PWM1 enabled\n");
9135         I915_STATE_WARN(I915_READ(UTIL_PIN_CTL) & UTIL_PIN_ENABLE,
9136              "Utility pin enabled\n");
9137         I915_STATE_WARN(I915_READ(PCH_GTC_CTL) & PCH_GTC_ENABLE, "PCH GTC enabled\n");
9138
9139         /*
9140          * In theory we can still leave IRQs enabled, as long as only the HPD
9141          * interrupts remain enabled. We used to check for that, but since it's
9142          * gen-specific and since we only disable LCPLL after we fully disable
9143          * the interrupts, the check below should be enough.
9144          */
9145         I915_STATE_WARN(intel_irqs_enabled(dev_priv), "IRQs enabled\n");
9146 }
9147
9148 static uint32_t hsw_read_dcomp(struct drm_i915_private *dev_priv)
9149 {
9150         if (IS_HASWELL(dev_priv))
9151                 return I915_READ(D_COMP_HSW);
9152         else
9153                 return I915_READ(D_COMP_BDW);
9154 }
9155
9156 static void hsw_write_dcomp(struct drm_i915_private *dev_priv, uint32_t val)
9157 {
9158         if (IS_HASWELL(dev_priv)) {
9159                 mutex_lock(&dev_priv->pcu_lock);
9160                 if (sandybridge_pcode_write(dev_priv, GEN6_PCODE_WRITE_D_COMP,
9161                                             val))
9162                         DRM_DEBUG_KMS("Failed to write to D_COMP\n");
9163                 mutex_unlock(&dev_priv->pcu_lock);
9164         } else {
9165                 I915_WRITE(D_COMP_BDW, val);
9166                 POSTING_READ(D_COMP_BDW);
9167         }
9168 }
9169
9170 /*
9171  * This function implements pieces of two sequences from BSpec:
9172  * - Sequence for display software to disable LCPLL
9173  * - Sequence for display software to allow package C8+
9174  * The steps implemented here are just the steps that actually touch the LCPLL
9175  * register. Callers should take care of disabling all the display engine
9176  * functions, doing the mode unset, fixing interrupts, etc.
9177  */
9178 static void hsw_disable_lcpll(struct drm_i915_private *dev_priv,
9179                               bool switch_to_fclk, bool allow_power_down)
9180 {
9181         uint32_t val;
9182
9183         assert_can_disable_lcpll(dev_priv);
9184
9185         val = I915_READ(LCPLL_CTL);
9186
9187         if (switch_to_fclk) {
9188                 val |= LCPLL_CD_SOURCE_FCLK;
9189                 I915_WRITE(LCPLL_CTL, val);
9190
9191                 if (wait_for_us(I915_READ(LCPLL_CTL) &
9192                                 LCPLL_CD_SOURCE_FCLK_DONE, 1))
9193                         DRM_ERROR("Switching to FCLK failed\n");
9194
9195                 val = I915_READ(LCPLL_CTL);
9196         }
9197
9198         val |= LCPLL_PLL_DISABLE;
9199         I915_WRITE(LCPLL_CTL, val);
9200         POSTING_READ(LCPLL_CTL);
9201
9202         if (intel_wait_for_register(dev_priv, LCPLL_CTL, LCPLL_PLL_LOCK, 0, 1))
9203                 DRM_ERROR("LCPLL still locked\n");
9204
9205         val = hsw_read_dcomp(dev_priv);
9206         val |= D_COMP_COMP_DISABLE;
9207         hsw_write_dcomp(dev_priv, val);
9208         ndelay(100);
9209
9210         if (wait_for((hsw_read_dcomp(dev_priv) & D_COMP_RCOMP_IN_PROGRESS) == 0,
9211                      1))
9212                 DRM_ERROR("D_COMP RCOMP still in progress\n");
9213
9214         if (allow_power_down) {
9215                 val = I915_READ(LCPLL_CTL);
9216                 val |= LCPLL_POWER_DOWN_ALLOW;
9217                 I915_WRITE(LCPLL_CTL, val);
9218                 POSTING_READ(LCPLL_CTL);
9219         }
9220 }
9221
9222 /*
9223  * Fully restores LCPLL, disallowing power down and switching back to LCPLL
9224  * source.
9225  */
9226 static void hsw_restore_lcpll(struct drm_i915_private *dev_priv)
9227 {
9228         uint32_t val;
9229
9230         val = I915_READ(LCPLL_CTL);
9231
9232         if ((val & (LCPLL_PLL_LOCK | LCPLL_PLL_DISABLE | LCPLL_CD_SOURCE_FCLK |
9233                     LCPLL_POWER_DOWN_ALLOW)) == LCPLL_PLL_LOCK)
9234                 return;
9235
9236         /*
9237          * Make sure we're not on PC8 state before disabling PC8, otherwise
9238          * we'll hang the machine. To prevent PC8 state, just enable force_wake.
9239          */
9240         intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);
9241
9242         if (val & LCPLL_POWER_DOWN_ALLOW) {
9243                 val &= ~LCPLL_POWER_DOWN_ALLOW;
9244                 I915_WRITE(LCPLL_CTL, val);
9245                 POSTING_READ(LCPLL_CTL);
9246         }
9247
9248         val = hsw_read_dcomp(dev_priv);
9249         val |= D_COMP_COMP_FORCE;
9250         val &= ~D_COMP_COMP_DISABLE;
9251         hsw_write_dcomp(dev_priv, val);
9252
9253         val = I915_READ(LCPLL_CTL);
9254         val &= ~LCPLL_PLL_DISABLE;
9255         I915_WRITE(LCPLL_CTL, val);
9256
9257         if (intel_wait_for_register(dev_priv,
9258                                     LCPLL_CTL, LCPLL_PLL_LOCK, LCPLL_PLL_LOCK,
9259                                     5))
9260                 DRM_ERROR("LCPLL not locked yet\n");
9261
9262         if (val & LCPLL_CD_SOURCE_FCLK) {
9263                 val = I915_READ(LCPLL_CTL);
9264                 val &= ~LCPLL_CD_SOURCE_FCLK;
9265                 I915_WRITE(LCPLL_CTL, val);
9266
9267                 if (wait_for_us((I915_READ(LCPLL_CTL) &
9268                                  LCPLL_CD_SOURCE_FCLK_DONE) == 0, 1))
9269                         DRM_ERROR("Switching back to LCPLL failed\n");
9270         }
9271
9272         intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
9273
9274         intel_update_cdclk(dev_priv);
9275         intel_dump_cdclk_state(&dev_priv->cdclk.hw, "Current CDCLK");
9276 }
9277
9278 /*
9279  * Package states C8 and deeper are really deep PC states that can only be
9280  * reached when all the devices on the system allow it, so even if the graphics
9281  * device allows PC8+, it doesn't mean the system will actually get to these
9282  * states. Our driver only allows PC8+ when going into runtime PM.
9283  *
9284  * The requirements for PC8+ are that all the outputs are disabled, the power
9285  * well is disabled and most interrupts are disabled, and these are also
9286  * requirements for runtime PM. When these conditions are met, we manually do
9287  * the other conditions: disable the interrupts, clocks and switch LCPLL refclk
9288  * to Fclk. If we're in PC8+ and we get an non-hotplug interrupt, we can hard
9289  * hang the machine.
9290  *
9291  * When we really reach PC8 or deeper states (not just when we allow it) we lose
9292  * the state of some registers, so when we come back from PC8+ we need to
9293  * restore this state. We don't get into PC8+ if we're not in RC6, so we don't
9294  * need to take care of the registers kept by RC6. Notice that this happens even
9295  * if we don't put the device in PCI D3 state (which is what currently happens
9296  * because of the runtime PM support).
9297  *
9298  * For more, read "Display Sequences for Package C8" on the hardware
9299  * documentation.
9300  */
9301 void hsw_enable_pc8(struct drm_i915_private *dev_priv)
9302 {
9303         uint32_t val;
9304
9305         DRM_DEBUG_KMS("Enabling package C8+\n");
9306
9307         if (HAS_PCH_LPT_LP(dev_priv)) {
9308                 val = I915_READ(SOUTH_DSPCLK_GATE_D);
9309                 val &= ~PCH_LP_PARTITION_LEVEL_DISABLE;
9310                 I915_WRITE(SOUTH_DSPCLK_GATE_D, val);
9311         }
9312
9313         lpt_disable_clkout_dp(dev_priv);
9314         hsw_disable_lcpll(dev_priv, true, true);
9315 }
9316
9317 void hsw_disable_pc8(struct drm_i915_private *dev_priv)
9318 {
9319         uint32_t val;
9320
9321         DRM_DEBUG_KMS("Disabling package C8+\n");
9322
9323         hsw_restore_lcpll(dev_priv);
9324         lpt_init_pch_refclk(dev_priv);
9325
9326         if (HAS_PCH_LPT_LP(dev_priv)) {
9327                 val = I915_READ(SOUTH_DSPCLK_GATE_D);
9328                 val |= PCH_LP_PARTITION_LEVEL_DISABLE;
9329                 I915_WRITE(SOUTH_DSPCLK_GATE_D, val);
9330         }
9331 }
9332
9333 static int haswell_crtc_compute_clock(struct intel_crtc *crtc,
9334                                       struct intel_crtc_state *crtc_state)
9335 {
9336         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
9337         struct intel_atomic_state *state =
9338                 to_intel_atomic_state(crtc_state->base.state);
9339
9340         if (!intel_crtc_has_type(crtc_state, INTEL_OUTPUT_DSI) ||
9341             IS_ICELAKE(dev_priv)) {
9342                 struct intel_encoder *encoder =
9343                         intel_get_crtc_new_encoder(state, crtc_state);
9344
9345                 if (!intel_get_shared_dpll(crtc, crtc_state, encoder)) {
9346                         DRM_DEBUG_KMS("failed to find PLL for pipe %c\n",
9347                                       pipe_name(crtc->pipe));
9348                         return -EINVAL;
9349                 }
9350         }
9351
9352         return 0;
9353 }
9354
9355 static void cannonlake_get_ddi_pll(struct drm_i915_private *dev_priv,
9356                                    enum port port,
9357                                    struct intel_crtc_state *pipe_config)
9358 {
9359         enum intel_dpll_id id;
9360         u32 temp;
9361
9362         temp = I915_READ(DPCLKA_CFGCR0) & DPCLKA_CFGCR0_DDI_CLK_SEL_MASK(port);
9363         id = temp >> DPCLKA_CFGCR0_DDI_CLK_SEL_SHIFT(port);
9364
9365         if (WARN_ON(id < SKL_DPLL0 || id > SKL_DPLL2))
9366                 return;
9367
9368         pipe_config->shared_dpll = intel_get_shared_dpll_by_id(dev_priv, id);
9369 }
9370
9371 static void icelake_get_ddi_pll(struct drm_i915_private *dev_priv,
9372                                 enum port port,
9373                                 struct intel_crtc_state *pipe_config)
9374 {
9375         enum intel_dpll_id id;
9376         u32 temp;
9377
9378         /* TODO: TBT pll not implemented. */
9379         if (intel_port_is_combophy(dev_priv, port)) {
9380                 temp = I915_READ(DPCLKA_CFGCR0_ICL) &
9381                        DPCLKA_CFGCR0_DDI_CLK_SEL_MASK(port);
9382                 id = temp >> DPCLKA_CFGCR0_DDI_CLK_SEL_SHIFT(port);
9383
9384                 if (WARN_ON(!intel_dpll_is_combophy(id)))
9385                         return;
9386         } else if (intel_port_is_tc(dev_priv, port)) {
9387                 id = icl_port_to_mg_pll_id(port);
9388         } else {
9389                 WARN(1, "Invalid port %x\n", port);
9390                 return;
9391         }
9392
9393         pipe_config->shared_dpll = intel_get_shared_dpll_by_id(dev_priv, id);
9394 }
9395
9396 static void bxt_get_ddi_pll(struct drm_i915_private *dev_priv,
9397                                 enum port port,
9398                                 struct intel_crtc_state *pipe_config)
9399 {
9400         enum intel_dpll_id id;
9401
9402         switch (port) {
9403         case PORT_A:
9404                 id = DPLL_ID_SKL_DPLL0;
9405                 break;
9406         case PORT_B:
9407                 id = DPLL_ID_SKL_DPLL1;
9408                 break;
9409         case PORT_C:
9410                 id = DPLL_ID_SKL_DPLL2;
9411                 break;
9412         default:
9413                 DRM_ERROR("Incorrect port type\n");
9414                 return;
9415         }
9416
9417         pipe_config->shared_dpll = intel_get_shared_dpll_by_id(dev_priv, id);
9418 }
9419
9420 static void skylake_get_ddi_pll(struct drm_i915_private *dev_priv,
9421                                 enum port port,
9422                                 struct intel_crtc_state *pipe_config)
9423 {
9424         enum intel_dpll_id id;
9425         u32 temp;
9426
9427         temp = I915_READ(DPLL_CTRL2) & DPLL_CTRL2_DDI_CLK_SEL_MASK(port);
9428         id = temp >> (port * 3 + 1);
9429
9430         if (WARN_ON(id < SKL_DPLL0 || id > SKL_DPLL3))
9431                 return;
9432
9433         pipe_config->shared_dpll = intel_get_shared_dpll_by_id(dev_priv, id);
9434 }
9435
9436 static void haswell_get_ddi_pll(struct drm_i915_private *dev_priv,
9437                                 enum port port,
9438                                 struct intel_crtc_state *pipe_config)
9439 {
9440         enum intel_dpll_id id;
9441         uint32_t ddi_pll_sel = I915_READ(PORT_CLK_SEL(port));
9442
9443         switch (ddi_pll_sel) {
9444         case PORT_CLK_SEL_WRPLL1:
9445                 id = DPLL_ID_WRPLL1;
9446                 break;
9447         case PORT_CLK_SEL_WRPLL2:
9448                 id = DPLL_ID_WRPLL2;
9449                 break;
9450         case PORT_CLK_SEL_SPLL:
9451                 id = DPLL_ID_SPLL;
9452                 break;
9453         case PORT_CLK_SEL_LCPLL_810:
9454                 id = DPLL_ID_LCPLL_810;
9455                 break;
9456         case PORT_CLK_SEL_LCPLL_1350:
9457                 id = DPLL_ID_LCPLL_1350;
9458                 break;
9459         case PORT_CLK_SEL_LCPLL_2700:
9460                 id = DPLL_ID_LCPLL_2700;
9461                 break;
9462         default:
9463                 MISSING_CASE(ddi_pll_sel);
9464                 /* fall through */
9465         case PORT_CLK_SEL_NONE:
9466                 return;
9467         }
9468
9469         pipe_config->shared_dpll = intel_get_shared_dpll_by_id(dev_priv, id);
9470 }
9471
9472 static bool hsw_get_transcoder_state(struct intel_crtc *crtc,
9473                                      struct intel_crtc_state *pipe_config,
9474                                      u64 *power_domain_mask)
9475 {
9476         struct drm_device *dev = crtc->base.dev;
9477         struct drm_i915_private *dev_priv = to_i915(dev);
9478         enum intel_display_power_domain power_domain;
9479         unsigned long panel_transcoder_mask = BIT(TRANSCODER_EDP);
9480         unsigned long enabled_panel_transcoders = 0;
9481         enum transcoder panel_transcoder;
9482         u32 tmp;
9483
9484         if (IS_ICELAKE(dev_priv))
9485                 panel_transcoder_mask |=
9486                         BIT(TRANSCODER_DSI_0) | BIT(TRANSCODER_DSI_1);
9487
9488         /*
9489          * The pipe->transcoder mapping is fixed with the exception of the eDP
9490          * and DSI transcoders handled below.
9491          */
9492         pipe_config->cpu_transcoder = (enum transcoder) crtc->pipe;
9493
9494         /*
9495          * XXX: Do intel_display_power_get_if_enabled before reading this (for
9496          * consistency and less surprising code; it's in always on power).
9497          */
9498         for_each_set_bit(panel_transcoder, &panel_transcoder_mask, 32) {
9499                 enum pipe trans_pipe;
9500
9501                 tmp = I915_READ(TRANS_DDI_FUNC_CTL(panel_transcoder));
9502                 if (!(tmp & TRANS_DDI_FUNC_ENABLE))
9503                         continue;
9504
9505                 /*
9506                  * Log all enabled ones, only use the first one.
9507                  *
9508                  * FIXME: This won't work for two separate DSI displays.
9509                  */
9510                 enabled_panel_transcoders |= BIT(panel_transcoder);
9511                 if (enabled_panel_transcoders != BIT(panel_transcoder))
9512                         continue;
9513
9514                 switch (tmp & TRANS_DDI_EDP_INPUT_MASK) {
9515                 default:
9516                         WARN(1, "unknown pipe linked to transcoder %s\n",
9517                              transcoder_name(panel_transcoder));
9518                         /* fall through */
9519                 case TRANS_DDI_EDP_INPUT_A_ONOFF:
9520                 case TRANS_DDI_EDP_INPUT_A_ON:
9521                         trans_pipe = PIPE_A;
9522                         break;
9523                 case TRANS_DDI_EDP_INPUT_B_ONOFF:
9524                         trans_pipe = PIPE_B;
9525                         break;
9526                 case TRANS_DDI_EDP_INPUT_C_ONOFF:
9527                         trans_pipe = PIPE_C;
9528                         break;
9529                 }
9530
9531                 if (trans_pipe == crtc->pipe)
9532                         pipe_config->cpu_transcoder = panel_transcoder;
9533         }
9534
9535         /*
9536          * Valid combos: none, eDP, DSI0, DSI1, DSI0+DSI1
9537          */
9538         WARN_ON((enabled_panel_transcoders & BIT(TRANSCODER_EDP)) &&
9539                 enabled_panel_transcoders != BIT(TRANSCODER_EDP));
9540
9541         power_domain = POWER_DOMAIN_TRANSCODER(pipe_config->cpu_transcoder);
9542         if (!intel_display_power_get_if_enabled(dev_priv, power_domain))
9543                 return false;
9544         *power_domain_mask |= BIT_ULL(power_domain);
9545
9546         tmp = I915_READ(PIPECONF(pipe_config->cpu_transcoder));
9547
9548         return tmp & PIPECONF_ENABLE;
9549 }
9550
9551 static bool bxt_get_dsi_transcoder_state(struct intel_crtc *crtc,
9552                                          struct intel_crtc_state *pipe_config,
9553                                          u64 *power_domain_mask)
9554 {
9555         struct drm_device *dev = crtc->base.dev;
9556         struct drm_i915_private *dev_priv = to_i915(dev);
9557         enum intel_display_power_domain power_domain;
9558         enum port port;
9559         enum transcoder cpu_transcoder;
9560         u32 tmp;
9561
9562         for_each_port_masked(port, BIT(PORT_A) | BIT(PORT_C)) {
9563                 if (port == PORT_A)
9564                         cpu_transcoder = TRANSCODER_DSI_A;
9565                 else
9566                         cpu_transcoder = TRANSCODER_DSI_C;
9567
9568                 power_domain = POWER_DOMAIN_TRANSCODER(cpu_transcoder);
9569                 if (!intel_display_power_get_if_enabled(dev_priv, power_domain))
9570                         continue;
9571                 *power_domain_mask |= BIT_ULL(power_domain);
9572
9573                 /*
9574                  * The PLL needs to be enabled with a valid divider
9575                  * configuration, otherwise accessing DSI registers will hang
9576                  * the machine. See BSpec North Display Engine
9577                  * registers/MIPI[BXT]. We can break out here early, since we
9578                  * need the same DSI PLL to be enabled for both DSI ports.
9579                  */
9580                 if (!bxt_dsi_pll_is_enabled(dev_priv))
9581                         break;
9582
9583                 /* XXX: this works for video mode only */
9584                 tmp = I915_READ(BXT_MIPI_PORT_CTRL(port));
9585                 if (!(tmp & DPI_ENABLE))
9586                         continue;
9587
9588                 tmp = I915_READ(MIPI_CTRL(port));
9589                 if ((tmp & BXT_PIPE_SELECT_MASK) != BXT_PIPE_SELECT(crtc->pipe))
9590                         continue;
9591
9592                 pipe_config->cpu_transcoder = cpu_transcoder;
9593                 break;
9594         }
9595
9596         return transcoder_is_dsi(pipe_config->cpu_transcoder);
9597 }
9598
9599 static void haswell_get_ddi_port_state(struct intel_crtc *crtc,
9600                                        struct intel_crtc_state *pipe_config)
9601 {
9602         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
9603         struct intel_shared_dpll *pll;
9604         enum port port;
9605         uint32_t tmp;
9606
9607         tmp = I915_READ(TRANS_DDI_FUNC_CTL(pipe_config->cpu_transcoder));
9608
9609         port = (tmp & TRANS_DDI_PORT_MASK) >> TRANS_DDI_PORT_SHIFT;
9610
9611         if (IS_ICELAKE(dev_priv))
9612                 icelake_get_ddi_pll(dev_priv, port, pipe_config);
9613         else if (IS_CANNONLAKE(dev_priv))
9614                 cannonlake_get_ddi_pll(dev_priv, port, pipe_config);
9615         else if (IS_GEN9_BC(dev_priv))
9616                 skylake_get_ddi_pll(dev_priv, port, pipe_config);
9617         else if (IS_GEN9_LP(dev_priv))
9618                 bxt_get_ddi_pll(dev_priv, port, pipe_config);
9619         else
9620                 haswell_get_ddi_pll(dev_priv, port, pipe_config);
9621
9622         pll = pipe_config->shared_dpll;
9623         if (pll) {
9624                 WARN_ON(!pll->info->funcs->get_hw_state(dev_priv, pll,
9625                                                 &pipe_config->dpll_hw_state));
9626         }
9627
9628         /*
9629          * Haswell has only FDI/PCH transcoder A. It is which is connected to
9630          * DDI E. So just check whether this pipe is wired to DDI E and whether
9631          * the PCH transcoder is on.
9632          */
9633         if (INTEL_GEN(dev_priv) < 9 &&
9634             (port == PORT_E) && I915_READ(LPT_TRANSCONF) & TRANS_ENABLE) {
9635                 pipe_config->has_pch_encoder = true;
9636
9637                 tmp = I915_READ(FDI_RX_CTL(PIPE_A));
9638                 pipe_config->fdi_lanes = ((FDI_DP_PORT_WIDTH_MASK & tmp) >>
9639                                           FDI_DP_PORT_WIDTH_SHIFT) + 1;
9640
9641                 ironlake_get_fdi_m_n_config(crtc, pipe_config);
9642         }
9643 }
9644
9645 static bool haswell_get_pipe_config(struct intel_crtc *crtc,
9646                                     struct intel_crtc_state *pipe_config)
9647 {
9648         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
9649         enum intel_display_power_domain power_domain;
9650         u64 power_domain_mask;
9651         bool active;
9652
9653         intel_crtc_init_scalers(crtc, pipe_config);
9654
9655         power_domain = POWER_DOMAIN_PIPE(crtc->pipe);
9656         if (!intel_display_power_get_if_enabled(dev_priv, power_domain))
9657                 return false;
9658         power_domain_mask = BIT_ULL(power_domain);
9659
9660         pipe_config->shared_dpll = NULL;
9661
9662         active = hsw_get_transcoder_state(crtc, pipe_config, &power_domain_mask);
9663
9664         if (IS_GEN9_LP(dev_priv) &&
9665             bxt_get_dsi_transcoder_state(crtc, pipe_config, &power_domain_mask)) {
9666                 WARN_ON(active);
9667                 active = true;
9668         }
9669
9670         if (!active)
9671                 goto out;
9672
9673         if (!transcoder_is_dsi(pipe_config->cpu_transcoder) ||
9674             IS_ICELAKE(dev_priv)) {
9675                 haswell_get_ddi_port_state(crtc, pipe_config);
9676                 intel_get_pipe_timings(crtc, pipe_config);
9677         }
9678
9679         intel_get_pipe_src_size(crtc, pipe_config);
9680         intel_get_crtc_ycbcr_config(crtc, pipe_config);
9681
9682         pipe_config->gamma_mode =
9683                 I915_READ(GAMMA_MODE(crtc->pipe)) & GAMMA_MODE_MODE_MASK;
9684
9685         power_domain = POWER_DOMAIN_PIPE_PANEL_FITTER(crtc->pipe);
9686         if (intel_display_power_get_if_enabled(dev_priv, power_domain)) {
9687                 power_domain_mask |= BIT_ULL(power_domain);
9688                 if (INTEL_GEN(dev_priv) >= 9)
9689                         skylake_get_pfit_config(crtc, pipe_config);
9690                 else
9691                         ironlake_get_pfit_config(crtc, pipe_config);
9692         }
9693
9694         if (hsw_crtc_supports_ips(crtc)) {
9695                 if (IS_HASWELL(dev_priv))
9696                         pipe_config->ips_enabled = I915_READ(IPS_CTL) & IPS_ENABLE;
9697                 else {
9698                         /*
9699                          * We cannot readout IPS state on broadwell, set to
9700                          * true so we can set it to a defined state on first
9701                          * commit.
9702                          */
9703                         pipe_config->ips_enabled = true;
9704                 }
9705         }
9706
9707         if (pipe_config->cpu_transcoder != TRANSCODER_EDP &&
9708             !transcoder_is_dsi(pipe_config->cpu_transcoder)) {
9709                 pipe_config->pixel_multiplier =
9710                         I915_READ(PIPE_MULT(pipe_config->cpu_transcoder)) + 1;
9711         } else {
9712                 pipe_config->pixel_multiplier = 1;
9713         }
9714
9715 out:
9716         for_each_power_domain(power_domain, power_domain_mask)
9717                 intel_display_power_put(dev_priv, power_domain);
9718
9719         return active;
9720 }
9721
9722 static u32 intel_cursor_base(const struct intel_plane_state *plane_state)
9723 {
9724         struct drm_i915_private *dev_priv =
9725                 to_i915(plane_state->base.plane->dev);
9726         const struct drm_framebuffer *fb = plane_state->base.fb;
9727         const struct drm_i915_gem_object *obj = intel_fb_obj(fb);
9728         u32 base;
9729
9730         if (INTEL_INFO(dev_priv)->display.cursor_needs_physical)
9731                 base = obj->phys_handle->busaddr;
9732         else
9733                 base = intel_plane_ggtt_offset(plane_state);
9734
9735         base += plane_state->color_plane[0].offset;
9736
9737         /* ILK+ do this automagically */
9738         if (HAS_GMCH_DISPLAY(dev_priv) &&
9739             plane_state->base.rotation & DRM_MODE_ROTATE_180)
9740                 base += (plane_state->base.crtc_h *
9741                          plane_state->base.crtc_w - 1) * fb->format->cpp[0];
9742
9743         return base;
9744 }
9745
9746 static u32 intel_cursor_position(const struct intel_plane_state *plane_state)
9747 {
9748         int x = plane_state->base.crtc_x;
9749         int y = plane_state->base.crtc_y;
9750         u32 pos = 0;
9751
9752         if (x < 0) {
9753                 pos |= CURSOR_POS_SIGN << CURSOR_X_SHIFT;
9754                 x = -x;
9755         }
9756         pos |= x << CURSOR_X_SHIFT;
9757
9758         if (y < 0) {
9759                 pos |= CURSOR_POS_SIGN << CURSOR_Y_SHIFT;
9760                 y = -y;
9761         }
9762         pos |= y << CURSOR_Y_SHIFT;
9763
9764         return pos;
9765 }
9766
9767 static bool intel_cursor_size_ok(const struct intel_plane_state *plane_state)
9768 {
9769         const struct drm_mode_config *config =
9770                 &plane_state->base.plane->dev->mode_config;
9771         int width = plane_state->base.crtc_w;
9772         int height = plane_state->base.crtc_h;
9773
9774         return width > 0 && width <= config->cursor_width &&
9775                 height > 0 && height <= config->cursor_height;
9776 }
9777
9778 static int intel_cursor_check_surface(struct intel_plane_state *plane_state)
9779 {
9780         const struct drm_framebuffer *fb = plane_state->base.fb;
9781         unsigned int rotation = plane_state->base.rotation;
9782         int src_x, src_y;
9783         u32 offset;
9784         int ret;
9785
9786         intel_fill_fb_ggtt_view(&plane_state->view, fb, rotation);
9787         plane_state->color_plane[0].stride = intel_fb_pitch(fb, 0, rotation);
9788
9789         ret = intel_plane_check_stride(plane_state);
9790         if (ret)
9791                 return ret;
9792
9793         src_x = plane_state->base.src_x >> 16;
9794         src_y = plane_state->base.src_y >> 16;
9795
9796         intel_add_fb_offsets(&src_x, &src_y, plane_state, 0);
9797         offset = intel_plane_compute_aligned_offset(&src_x, &src_y,
9798                                                     plane_state, 0);
9799
9800         if (src_x != 0 || src_y != 0) {
9801                 DRM_DEBUG_KMS("Arbitrary cursor panning not supported\n");
9802                 return -EINVAL;
9803         }
9804
9805         plane_state->color_plane[0].offset = offset;
9806
9807         return 0;
9808 }
9809
9810 static int intel_check_cursor(struct intel_crtc_state *crtc_state,
9811                               struct intel_plane_state *plane_state)
9812 {
9813         const struct drm_framebuffer *fb = plane_state->base.fb;
9814         int ret;
9815
9816         if (fb && fb->modifier != DRM_FORMAT_MOD_LINEAR) {
9817                 DRM_DEBUG_KMS("cursor cannot be tiled\n");
9818                 return -EINVAL;
9819         }
9820
9821         ret = drm_atomic_helper_check_plane_state(&plane_state->base,
9822                                                   &crtc_state->base,
9823                                                   DRM_PLANE_HELPER_NO_SCALING,
9824                                                   DRM_PLANE_HELPER_NO_SCALING,
9825                                                   true, true);
9826         if (ret)
9827                 return ret;
9828
9829         if (!plane_state->base.visible)
9830                 return 0;
9831
9832         ret = intel_plane_check_src_coordinates(plane_state);
9833         if (ret)
9834                 return ret;
9835
9836         ret = intel_cursor_check_surface(plane_state);
9837         if (ret)
9838                 return ret;
9839
9840         return 0;
9841 }
9842
9843 static unsigned int
9844 i845_cursor_max_stride(struct intel_plane *plane,
9845                        u32 pixel_format, u64 modifier,
9846                        unsigned int rotation)
9847 {
9848         return 2048;
9849 }
9850
9851 static u32 i845_cursor_ctl(const struct intel_crtc_state *crtc_state,
9852                            const struct intel_plane_state *plane_state)
9853 {
9854         return CURSOR_ENABLE |
9855                 CURSOR_GAMMA_ENABLE |
9856                 CURSOR_FORMAT_ARGB |
9857                 CURSOR_STRIDE(plane_state->color_plane[0].stride);
9858 }
9859
9860 static bool i845_cursor_size_ok(const struct intel_plane_state *plane_state)
9861 {
9862         int width = plane_state->base.crtc_w;
9863
9864         /*
9865          * 845g/865g are only limited by the width of their cursors,
9866          * the height is arbitrary up to the precision of the register.
9867          */
9868         return intel_cursor_size_ok(plane_state) && IS_ALIGNED(width, 64);
9869 }
9870
9871 static int i845_check_cursor(struct intel_crtc_state *crtc_state,
9872                              struct intel_plane_state *plane_state)
9873 {
9874         const struct drm_framebuffer *fb = plane_state->base.fb;
9875         int ret;
9876
9877         ret = intel_check_cursor(crtc_state, plane_state);
9878         if (ret)
9879                 return ret;
9880
9881         /* if we want to turn off the cursor ignore width and height */
9882         if (!fb)
9883                 return 0;
9884
9885         /* Check for which cursor types we support */
9886         if (!i845_cursor_size_ok(plane_state)) {
9887                 DRM_DEBUG("Cursor dimension %dx%d not supported\n",
9888                           plane_state->base.crtc_w,
9889                           plane_state->base.crtc_h);
9890                 return -EINVAL;
9891         }
9892
9893         WARN_ON(plane_state->base.visible &&
9894                 plane_state->color_plane[0].stride != fb->pitches[0]);
9895
9896         switch (fb->pitches[0]) {
9897         case 256:
9898         case 512:
9899         case 1024:
9900         case 2048:
9901                 break;
9902         default:
9903                 DRM_DEBUG_KMS("Invalid cursor stride (%u)\n",
9904                               fb->pitches[0]);
9905                 return -EINVAL;
9906         }
9907
9908         plane_state->ctl = i845_cursor_ctl(crtc_state, plane_state);
9909
9910         return 0;
9911 }
9912
9913 static void i845_update_cursor(struct intel_plane *plane,
9914                                const struct intel_crtc_state *crtc_state,
9915                                const struct intel_plane_state *plane_state)
9916 {
9917         struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
9918         u32 cntl = 0, base = 0, pos = 0, size = 0;
9919         unsigned long irqflags;
9920
9921         if (plane_state && plane_state->base.visible) {
9922                 unsigned int width = plane_state->base.crtc_w;
9923                 unsigned int height = plane_state->base.crtc_h;
9924
9925                 cntl = plane_state->ctl;
9926                 size = (height << 12) | width;
9927
9928                 base = intel_cursor_base(plane_state);
9929                 pos = intel_cursor_position(plane_state);
9930         }
9931
9932         spin_lock_irqsave(&dev_priv->uncore.lock, irqflags);
9933
9934         /* On these chipsets we can only modify the base/size/stride
9935          * whilst the cursor is disabled.
9936          */
9937         if (plane->cursor.base != base ||
9938             plane->cursor.size != size ||
9939             plane->cursor.cntl != cntl) {
9940                 I915_WRITE_FW(CURCNTR(PIPE_A), 0);
9941                 I915_WRITE_FW(CURBASE(PIPE_A), base);
9942                 I915_WRITE_FW(CURSIZE, size);
9943                 I915_WRITE_FW(CURPOS(PIPE_A), pos);
9944                 I915_WRITE_FW(CURCNTR(PIPE_A), cntl);
9945
9946                 plane->cursor.base = base;
9947                 plane->cursor.size = size;
9948                 plane->cursor.cntl = cntl;
9949         } else {
9950                 I915_WRITE_FW(CURPOS(PIPE_A), pos);
9951         }
9952
9953         spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags);
9954 }
9955
9956 static void i845_disable_cursor(struct intel_plane *plane,
9957                                 const struct intel_crtc_state *crtc_state)
9958 {
9959         i845_update_cursor(plane, crtc_state, NULL);
9960 }
9961
9962 static bool i845_cursor_get_hw_state(struct intel_plane *plane,
9963                                      enum pipe *pipe)
9964 {
9965         struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
9966         enum intel_display_power_domain power_domain;
9967         bool ret;
9968
9969         power_domain = POWER_DOMAIN_PIPE(PIPE_A);
9970         if (!intel_display_power_get_if_enabled(dev_priv, power_domain))
9971                 return false;
9972
9973         ret = I915_READ(CURCNTR(PIPE_A)) & CURSOR_ENABLE;
9974
9975         *pipe = PIPE_A;
9976
9977         intel_display_power_put(dev_priv, power_domain);
9978
9979         return ret;
9980 }
9981
9982 static unsigned int
9983 i9xx_cursor_max_stride(struct intel_plane *plane,
9984                        u32 pixel_format, u64 modifier,
9985                        unsigned int rotation)
9986 {
9987         return plane->base.dev->mode_config.cursor_width * 4;
9988 }
9989
9990 static u32 i9xx_cursor_ctl(const struct intel_crtc_state *crtc_state,
9991                            const struct intel_plane_state *plane_state)
9992 {
9993         struct drm_i915_private *dev_priv =
9994                 to_i915(plane_state->base.plane->dev);
9995         struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
9996         u32 cntl = 0;
9997
9998         if (IS_GEN(dev_priv, 6) || IS_IVYBRIDGE(dev_priv))
9999                 cntl |= MCURSOR_TRICKLE_FEED_DISABLE;
10000
10001         if (INTEL_GEN(dev_priv) <= 10) {
10002                 cntl |= MCURSOR_GAMMA_ENABLE;
10003
10004                 if (HAS_DDI(dev_priv))
10005                         cntl |= MCURSOR_PIPE_CSC_ENABLE;
10006         }
10007
10008         if (INTEL_GEN(dev_priv) < 5 && !IS_G4X(dev_priv))
10009                 cntl |= MCURSOR_PIPE_SELECT(crtc->pipe);
10010
10011         switch (plane_state->base.crtc_w) {
10012         case 64:
10013                 cntl |= MCURSOR_MODE_64_ARGB_AX;
10014                 break;
10015         case 128:
10016                 cntl |= MCURSOR_MODE_128_ARGB_AX;
10017                 break;
10018         case 256:
10019                 cntl |= MCURSOR_MODE_256_ARGB_AX;
10020                 break;
10021         default:
10022                 MISSING_CASE(plane_state->base.crtc_w);
10023                 return 0;
10024         }
10025
10026         if (plane_state->base.rotation & DRM_MODE_ROTATE_180)
10027                 cntl |= MCURSOR_ROTATE_180;
10028
10029         return cntl;
10030 }
10031
10032 static bool i9xx_cursor_size_ok(const struct intel_plane_state *plane_state)
10033 {
10034         struct drm_i915_private *dev_priv =
10035                 to_i915(plane_state->base.plane->dev);
10036         int width = plane_state->base.crtc_w;
10037         int height = plane_state->base.crtc_h;
10038
10039         if (!intel_cursor_size_ok(plane_state))
10040                 return false;
10041
10042         /* Cursor width is limited to a few power-of-two sizes */
10043         switch (width) {
10044         case 256:
10045         case 128:
10046         case 64:
10047                 break;
10048         default:
10049                 return false;
10050         }
10051
10052         /*
10053          * IVB+ have CUR_FBC_CTL which allows an arbitrary cursor
10054          * height from 8 lines up to the cursor width, when the
10055          * cursor is not rotated. Everything else requires square
10056          * cursors.
10057          */
10058         if (HAS_CUR_FBC(dev_priv) &&
10059             plane_state->base.rotation & DRM_MODE_ROTATE_0) {
10060                 if (height < 8 || height > width)
10061                         return false;
10062         } else {
10063                 if (height != width)
10064                         return false;
10065         }
10066
10067         return true;
10068 }
10069
10070 static int i9xx_check_cursor(struct intel_crtc_state *crtc_state,
10071                              struct intel_plane_state *plane_state)
10072 {
10073         struct intel_plane *plane = to_intel_plane(plane_state->base.plane);
10074         struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
10075         const struct drm_framebuffer *fb = plane_state->base.fb;
10076         enum pipe pipe = plane->pipe;
10077         int ret;
10078
10079         ret = intel_check_cursor(crtc_state, plane_state);
10080         if (ret)
10081                 return ret;
10082
10083         /* if we want to turn off the cursor ignore width and height */
10084         if (!fb)
10085                 return 0;
10086
10087         /* Check for which cursor types we support */
10088         if (!i9xx_cursor_size_ok(plane_state)) {
10089                 DRM_DEBUG("Cursor dimension %dx%d not supported\n",
10090                           plane_state->base.crtc_w,
10091                           plane_state->base.crtc_h);
10092                 return -EINVAL;
10093         }
10094
10095         WARN_ON(plane_state->base.visible &&
10096                 plane_state->color_plane[0].stride != fb->pitches[0]);
10097
10098         if (fb->pitches[0] != plane_state->base.crtc_w * fb->format->cpp[0]) {
10099                 DRM_DEBUG_KMS("Invalid cursor stride (%u) (cursor width %d)\n",
10100                               fb->pitches[0], plane_state->base.crtc_w);
10101                 return -EINVAL;
10102         }
10103
10104         /*
10105          * There's something wrong with the cursor on CHV pipe C.
10106          * If it straddles the left edge of the screen then
10107          * moving it away from the edge or disabling it often
10108          * results in a pipe underrun, and often that can lead to
10109          * dead pipe (constant underrun reported, and it scans
10110          * out just a solid color). To recover from that, the
10111          * display power well must be turned off and on again.
10112          * Refuse the put the cursor into that compromised position.
10113          */
10114         if (IS_CHERRYVIEW(dev_priv) && pipe == PIPE_C &&
10115             plane_state->base.visible && plane_state->base.crtc_x < 0) {
10116                 DRM_DEBUG_KMS("CHV cursor C not allowed to straddle the left screen edge\n");
10117                 return -EINVAL;
10118         }
10119
10120         plane_state->ctl = i9xx_cursor_ctl(crtc_state, plane_state);
10121
10122         return 0;
10123 }
10124
10125 static void i9xx_update_cursor(struct intel_plane *plane,
10126                                const struct intel_crtc_state *crtc_state,
10127                                const struct intel_plane_state *plane_state)
10128 {
10129         struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
10130         enum pipe pipe = plane->pipe;
10131         u32 cntl = 0, base = 0, pos = 0, fbc_ctl = 0;
10132         unsigned long irqflags;
10133
10134         if (plane_state && plane_state->base.visible) {
10135                 cntl = plane_state->ctl;
10136
10137                 if (plane_state->base.crtc_h != plane_state->base.crtc_w)
10138                         fbc_ctl = CUR_FBC_CTL_EN | (plane_state->base.crtc_h - 1);
10139
10140                 base = intel_cursor_base(plane_state);
10141                 pos = intel_cursor_position(plane_state);
10142         }
10143
10144         spin_lock_irqsave(&dev_priv->uncore.lock, irqflags);
10145
10146         /*
10147          * On some platforms writing CURCNTR first will also
10148          * cause CURPOS to be armed by the CURBASE write.
10149          * Without the CURCNTR write the CURPOS write would
10150          * arm itself. Thus we always update CURCNTR before
10151          * CURPOS.
10152          *
10153          * On other platforms CURPOS always requires the
10154          * CURBASE write to arm the update. Additonally
10155          * a write to any of the cursor register will cancel
10156          * an already armed cursor update. Thus leaving out
10157          * the CURBASE write after CURPOS could lead to a
10158          * cursor that doesn't appear to move, or even change
10159          * shape. Thus we always write CURBASE.
10160          *
10161          * The other registers are armed by by the CURBASE write
10162          * except when the plane is getting enabled at which time
10163          * the CURCNTR write arms the update.
10164          */
10165
10166         if (INTEL_GEN(dev_priv) >= 9)
10167                 skl_write_cursor_wm(plane, crtc_state);
10168
10169         if (plane->cursor.base != base ||
10170             plane->cursor.size != fbc_ctl ||
10171             plane->cursor.cntl != cntl) {
10172                 if (HAS_CUR_FBC(dev_priv))
10173                         I915_WRITE_FW(CUR_FBC_CTL(pipe), fbc_ctl);
10174                 I915_WRITE_FW(CURCNTR(pipe), cntl);
10175                 I915_WRITE_FW(CURPOS(pipe), pos);
10176                 I915_WRITE_FW(CURBASE(pipe), base);
10177
10178                 plane->cursor.base = base;
10179                 plane->cursor.size = fbc_ctl;
10180                 plane->cursor.cntl = cntl;
10181         } else {
10182                 I915_WRITE_FW(CURPOS(pipe), pos);
10183                 I915_WRITE_FW(CURBASE(pipe), base);
10184         }
10185
10186         spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags);
10187 }
10188
10189 static void i9xx_disable_cursor(struct intel_plane *plane,
10190                                 const struct intel_crtc_state *crtc_state)
10191 {
10192         i9xx_update_cursor(plane, crtc_state, NULL);
10193 }
10194
10195 static bool i9xx_cursor_get_hw_state(struct intel_plane *plane,
10196                                      enum pipe *pipe)
10197 {
10198         struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
10199         enum intel_display_power_domain power_domain;
10200         bool ret;
10201         u32 val;
10202
10203         /*
10204          * Not 100% correct for planes that can move between pipes,
10205          * but that's only the case for gen2-3 which don't have any
10206          * display power wells.
10207          */
10208         power_domain = POWER_DOMAIN_PIPE(plane->pipe);
10209         if (!intel_display_power_get_if_enabled(dev_priv, power_domain))
10210                 return false;
10211
10212         val = I915_READ(CURCNTR(plane->pipe));
10213
10214         ret = val & MCURSOR_MODE;
10215
10216         if (INTEL_GEN(dev_priv) >= 5 || IS_G4X(dev_priv))
10217                 *pipe = plane->pipe;
10218         else
10219                 *pipe = (val & MCURSOR_PIPE_SELECT_MASK) >>
10220                         MCURSOR_PIPE_SELECT_SHIFT;
10221
10222         intel_display_power_put(dev_priv, power_domain);
10223
10224         return ret;
10225 }
10226
10227 /* VESA 640x480x72Hz mode to set on the pipe */
10228 static const struct drm_display_mode load_detect_mode = {
10229         DRM_MODE("640x480", DRM_MODE_TYPE_DEFAULT, 31500, 640, 664,
10230                  704, 832, 0, 480, 489, 491, 520, 0, DRM_MODE_FLAG_NHSYNC | DRM_MODE_FLAG_NVSYNC),
10231 };
10232
10233 struct drm_framebuffer *
10234 intel_framebuffer_create(struct drm_i915_gem_object *obj,
10235                          struct drm_mode_fb_cmd2 *mode_cmd)
10236 {
10237         struct intel_framebuffer *intel_fb;
10238         int ret;
10239
10240         intel_fb = kzalloc(sizeof(*intel_fb), GFP_KERNEL);
10241         if (!intel_fb)
10242                 return ERR_PTR(-ENOMEM);
10243
10244         ret = intel_framebuffer_init(intel_fb, obj, mode_cmd);
10245         if (ret)
10246                 goto err;
10247
10248         return &intel_fb->base;
10249
10250 err:
10251         kfree(intel_fb);
10252         return ERR_PTR(ret);
10253 }
10254
10255 static int intel_modeset_disable_planes(struct drm_atomic_state *state,
10256                                         struct drm_crtc *crtc)
10257 {
10258         struct drm_plane *plane;
10259         struct drm_plane_state *plane_state;
10260         int ret, i;
10261
10262         ret = drm_atomic_add_affected_planes(state, crtc);
10263         if (ret)
10264                 return ret;
10265
10266         for_each_new_plane_in_state(state, plane, plane_state, i) {
10267                 if (plane_state->crtc != crtc)
10268                         continue;
10269
10270                 ret = drm_atomic_set_crtc_for_plane(plane_state, NULL);
10271                 if (ret)
10272                         return ret;
10273
10274                 drm_atomic_set_fb_for_plane(plane_state, NULL);
10275         }
10276
10277         return 0;
10278 }
10279
10280 int intel_get_load_detect_pipe(struct drm_connector *connector,
10281                                const struct drm_display_mode *mode,
10282                                struct intel_load_detect_pipe *old,
10283                                struct drm_modeset_acquire_ctx *ctx)
10284 {
10285         struct intel_crtc *intel_crtc;
10286         struct intel_encoder *intel_encoder =
10287                 intel_attached_encoder(connector);
10288         struct drm_crtc *possible_crtc;
10289         struct drm_encoder *encoder = &intel_encoder->base;
10290         struct drm_crtc *crtc = NULL;
10291         struct drm_device *dev = encoder->dev;
10292         struct drm_i915_private *dev_priv = to_i915(dev);
10293         struct drm_mode_config *config = &dev->mode_config;
10294         struct drm_atomic_state *state = NULL, *restore_state = NULL;
10295         struct drm_connector_state *connector_state;
10296         struct intel_crtc_state *crtc_state;
10297         int ret, i = -1;
10298
10299         DRM_DEBUG_KMS("[CONNECTOR:%d:%s], [ENCODER:%d:%s]\n",
10300                       connector->base.id, connector->name,
10301                       encoder->base.id, encoder->name);
10302
10303         old->restore_state = NULL;
10304
10305         WARN_ON(!drm_modeset_is_locked(&config->connection_mutex));
10306
10307         /*
10308          * Algorithm gets a little messy:
10309          *
10310          *   - if the connector already has an assigned crtc, use it (but make
10311          *     sure it's on first)
10312          *
10313          *   - try to find the first unused crtc that can drive this connector,
10314          *     and use that if we find one
10315          */
10316
10317         /* See if we already have a CRTC for this connector */
10318         if (connector->state->crtc) {
10319                 crtc = connector->state->crtc;
10320
10321                 ret = drm_modeset_lock(&crtc->mutex, ctx);
10322                 if (ret)
10323                         goto fail;
10324
10325                 /* Make sure the crtc and connector are running */
10326                 goto found;
10327         }
10328
10329         /* Find an unused one (if possible) */
10330         for_each_crtc(dev, possible_crtc) {
10331                 i++;
10332                 if (!(encoder->possible_crtcs & (1 << i)))
10333                         continue;
10334
10335                 ret = drm_modeset_lock(&possible_crtc->mutex, ctx);
10336                 if (ret)
10337                         goto fail;
10338
10339                 if (possible_crtc->state->enable) {
10340                         drm_modeset_unlock(&possible_crtc->mutex);
10341                         continue;
10342                 }
10343
10344                 crtc = possible_crtc;
10345                 break;
10346         }
10347
10348         /*
10349          * If we didn't find an unused CRTC, don't use any.
10350          */
10351         if (!crtc) {
10352                 DRM_DEBUG_KMS("no pipe available for load-detect\n");
10353                 ret = -ENODEV;
10354                 goto fail;
10355         }
10356
10357 found:
10358         intel_crtc = to_intel_crtc(crtc);
10359
10360         state = drm_atomic_state_alloc(dev);
10361         restore_state = drm_atomic_state_alloc(dev);
10362         if (!state || !restore_state) {
10363                 ret = -ENOMEM;
10364                 goto fail;
10365         }
10366
10367         state->acquire_ctx = ctx;
10368         restore_state->acquire_ctx = ctx;
10369
10370         connector_state = drm_atomic_get_connector_state(state, connector);
10371         if (IS_ERR(connector_state)) {
10372                 ret = PTR_ERR(connector_state);
10373                 goto fail;
10374         }
10375
10376         ret = drm_atomic_set_crtc_for_connector(connector_state, crtc);
10377         if (ret)
10378                 goto fail;
10379
10380         crtc_state = intel_atomic_get_crtc_state(state, intel_crtc);
10381         if (IS_ERR(crtc_state)) {
10382                 ret = PTR_ERR(crtc_state);
10383                 goto fail;
10384         }
10385
10386         crtc_state->base.active = crtc_state->base.enable = true;
10387
10388         if (!mode)
10389                 mode = &load_detect_mode;
10390
10391         ret = drm_atomic_set_mode_for_crtc(&crtc_state->base, mode);
10392         if (ret)
10393                 goto fail;
10394
10395         ret = intel_modeset_disable_planes(state, crtc);
10396         if (ret)
10397                 goto fail;
10398
10399         ret = PTR_ERR_OR_ZERO(drm_atomic_get_connector_state(restore_state, connector));
10400         if (!ret)
10401                 ret = PTR_ERR_OR_ZERO(drm_atomic_get_crtc_state(restore_state, crtc));
10402         if (!ret)
10403                 ret = drm_atomic_add_affected_planes(restore_state, crtc);
10404         if (ret) {
10405                 DRM_DEBUG_KMS("Failed to create a copy of old state to restore: %i\n", ret);
10406                 goto fail;
10407         }
10408
10409         ret = drm_atomic_commit(state);
10410         if (ret) {
10411                 DRM_DEBUG_KMS("failed to set mode on load-detect pipe\n");
10412                 goto fail;
10413         }
10414
10415         old->restore_state = restore_state;
10416         drm_atomic_state_put(state);
10417
10418         /* let the connector get through one full cycle before testing */
10419         intel_wait_for_vblank(dev_priv, intel_crtc->pipe);
10420         return true;
10421
10422 fail:
10423         if (state) {
10424                 drm_atomic_state_put(state);
10425                 state = NULL;
10426         }
10427         if (restore_state) {
10428                 drm_atomic_state_put(restore_state);
10429                 restore_state = NULL;
10430         }
10431
10432         if (ret == -EDEADLK)
10433                 return ret;
10434
10435         return false;
10436 }
10437
10438 void intel_release_load_detect_pipe(struct drm_connector *connector,
10439                                     struct intel_load_detect_pipe *old,
10440                                     struct drm_modeset_acquire_ctx *ctx)
10441 {
10442         struct intel_encoder *intel_encoder =
10443                 intel_attached_encoder(connector);
10444         struct drm_encoder *encoder = &intel_encoder->base;
10445         struct drm_atomic_state *state = old->restore_state;
10446         int ret;
10447
10448         DRM_DEBUG_KMS("[CONNECTOR:%d:%s], [ENCODER:%d:%s]\n",
10449                       connector->base.id, connector->name,
10450                       encoder->base.id, encoder->name);
10451
10452         if (!state)
10453                 return;
10454
10455         ret = drm_atomic_helper_commit_duplicated_state(state, ctx);
10456         if (ret)
10457                 DRM_DEBUG_KMS("Couldn't release load detect pipe: %i\n", ret);
10458         drm_atomic_state_put(state);
10459 }
10460
10461 static int i9xx_pll_refclk(struct drm_device *dev,
10462                            const struct intel_crtc_state *pipe_config)
10463 {
10464         struct drm_i915_private *dev_priv = to_i915(dev);
10465         u32 dpll = pipe_config->dpll_hw_state.dpll;
10466
10467         if ((dpll & PLL_REF_INPUT_MASK) == PLLB_REF_INPUT_SPREADSPECTRUMIN)
10468                 return dev_priv->vbt.lvds_ssc_freq;
10469         else if (HAS_PCH_SPLIT(dev_priv))
10470                 return 120000;
10471         else if (!IS_GEN(dev_priv, 2))
10472                 return 96000;
10473         else
10474                 return 48000;
10475 }
10476
10477 /* Returns the clock of the currently programmed mode of the given pipe. */
10478 static void i9xx_crtc_clock_get(struct intel_crtc *crtc,
10479                                 struct intel_crtc_state *pipe_config)
10480 {
10481         struct drm_device *dev = crtc->base.dev;
10482         struct drm_i915_private *dev_priv = to_i915(dev);
10483         int pipe = pipe_config->cpu_transcoder;
10484         u32 dpll = pipe_config->dpll_hw_state.dpll;
10485         u32 fp;
10486         struct dpll clock;
10487         int port_clock;
10488         int refclk = i9xx_pll_refclk(dev, pipe_config);
10489
10490         if ((dpll & DISPLAY_RATE_SELECT_FPA1) == 0)
10491                 fp = pipe_config->dpll_hw_state.fp0;
10492         else
10493                 fp = pipe_config->dpll_hw_state.fp1;
10494
10495         clock.m1 = (fp & FP_M1_DIV_MASK) >> FP_M1_DIV_SHIFT;
10496         if (IS_PINEVIEW(dev_priv)) {
10497                 clock.n = ffs((fp & FP_N_PINEVIEW_DIV_MASK) >> FP_N_DIV_SHIFT) - 1;
10498                 clock.m2 = (fp & FP_M2_PINEVIEW_DIV_MASK) >> FP_M2_DIV_SHIFT;
10499         } else {
10500                 clock.n = (fp & FP_N_DIV_MASK) >> FP_N_DIV_SHIFT;
10501                 clock.m2 = (fp & FP_M2_DIV_MASK) >> FP_M2_DIV_SHIFT;
10502         }
10503
10504         if (!IS_GEN(dev_priv, 2)) {
10505                 if (IS_PINEVIEW(dev_priv))
10506                         clock.p1 = ffs((dpll & DPLL_FPA01_P1_POST_DIV_MASK_PINEVIEW) >>
10507                                 DPLL_FPA01_P1_POST_DIV_SHIFT_PINEVIEW);
10508                 else
10509                         clock.p1 = ffs((dpll & DPLL_FPA01_P1_POST_DIV_MASK) >>
10510                                DPLL_FPA01_P1_POST_DIV_SHIFT);
10511
10512                 switch (dpll & DPLL_MODE_MASK) {
10513                 case DPLLB_MODE_DAC_SERIAL:
10514                         clock.p2 = dpll & DPLL_DAC_SERIAL_P2_CLOCK_DIV_5 ?
10515                                 5 : 10;
10516                         break;
10517                 case DPLLB_MODE_LVDS:
10518                         clock.p2 = dpll & DPLLB_LVDS_P2_CLOCK_DIV_7 ?
10519                                 7 : 14;
10520                         break;
10521                 default:
10522                         DRM_DEBUG_KMS("Unknown DPLL mode %08x in programmed "
10523                                   "mode\n", (int)(dpll & DPLL_MODE_MASK));
10524                         return;
10525                 }
10526
10527                 if (IS_PINEVIEW(dev_priv))
10528                         port_clock = pnv_calc_dpll_params(refclk, &clock);
10529                 else
10530                         port_clock = i9xx_calc_dpll_params(refclk, &clock);
10531         } else {
10532                 u32 lvds = IS_I830(dev_priv) ? 0 : I915_READ(LVDS);
10533                 bool is_lvds = (pipe == 1) && (lvds & LVDS_PORT_EN);
10534
10535                 if (is_lvds) {
10536                         clock.p1 = ffs((dpll & DPLL_FPA01_P1_POST_DIV_MASK_I830_LVDS) >>
10537                                        DPLL_FPA01_P1_POST_DIV_SHIFT);
10538
10539                         if (lvds & LVDS_CLKB_POWER_UP)
10540                                 clock.p2 = 7;
10541                         else
10542                                 clock.p2 = 14;
10543                 } else {
10544                         if (dpll & PLL_P1_DIVIDE_BY_TWO)
10545                                 clock.p1 = 2;
10546                         else {
10547                                 clock.p1 = ((dpll & DPLL_FPA01_P1_POST_DIV_MASK_I830) >>
10548                                             DPLL_FPA01_P1_POST_DIV_SHIFT) + 2;
10549                         }
10550                         if (dpll & PLL_P2_DIVIDE_BY_4)
10551                                 clock.p2 = 4;
10552                         else
10553                                 clock.p2 = 2;
10554                 }
10555
10556                 port_clock = i9xx_calc_dpll_params(refclk, &clock);
10557         }
10558
10559         /*
10560          * This value includes pixel_multiplier. We will use
10561          * port_clock to compute adjusted_mode.crtc_clock in the
10562          * encoder's get_config() function.
10563          */
10564         pipe_config->port_clock = port_clock;
10565 }
10566
10567 int intel_dotclock_calculate(int link_freq,
10568                              const struct intel_link_m_n *m_n)
10569 {
10570         /*
10571          * The calculation for the data clock is:
10572          * pixel_clock = ((m/n)*(link_clock * nr_lanes))/bpp
10573          * But we want to avoid losing precison if possible, so:
10574          * pixel_clock = ((m * link_clock * nr_lanes)/(n*bpp))
10575          *
10576          * and the link clock is simpler:
10577          * link_clock = (m * link_clock) / n
10578          */
10579
10580         if (!m_n->link_n)
10581                 return 0;
10582
10583         return div_u64(mul_u32_u32(m_n->link_m, link_freq), m_n->link_n);
10584 }
10585
10586 static void ironlake_pch_clock_get(struct intel_crtc *crtc,
10587                                    struct intel_crtc_state *pipe_config)
10588 {
10589         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
10590
10591         /* read out port_clock from the DPLL */
10592         i9xx_crtc_clock_get(crtc, pipe_config);
10593
10594         /*
10595          * In case there is an active pipe without active ports,
10596          * we may need some idea for the dotclock anyway.
10597          * Calculate one based on the FDI configuration.
10598          */
10599         pipe_config->base.adjusted_mode.crtc_clock =
10600                 intel_dotclock_calculate(intel_fdi_link_freq(dev_priv, pipe_config),
10601                                          &pipe_config->fdi_m_n);
10602 }
10603
10604 /* Returns the currently programmed mode of the given encoder. */
10605 struct drm_display_mode *
10606 intel_encoder_current_mode(struct intel_encoder *encoder)
10607 {
10608         struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
10609         struct intel_crtc_state *crtc_state;
10610         struct drm_display_mode *mode;
10611         struct intel_crtc *crtc;
10612         enum pipe pipe;
10613
10614         if (!encoder->get_hw_state(encoder, &pipe))
10615                 return NULL;
10616
10617         crtc = intel_get_crtc_for_pipe(dev_priv, pipe);
10618
10619         mode = kzalloc(sizeof(*mode), GFP_KERNEL);
10620         if (!mode)
10621                 return NULL;
10622
10623         crtc_state = kzalloc(sizeof(*crtc_state), GFP_KERNEL);
10624         if (!crtc_state) {
10625                 kfree(mode);
10626                 return NULL;
10627         }
10628
10629         crtc_state->base.crtc = &crtc->base;
10630
10631         if (!dev_priv->display.get_pipe_config(crtc, crtc_state)) {
10632                 kfree(crtc_state);
10633                 kfree(mode);
10634                 return NULL;
10635         }
10636
10637         encoder->get_config(encoder, crtc_state);
10638
10639         intel_mode_from_pipe_config(mode, crtc_state);
10640
10641         kfree(crtc_state);
10642
10643         return mode;
10644 }
10645
10646 static void intel_crtc_destroy(struct drm_crtc *crtc)
10647 {
10648         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
10649
10650         drm_crtc_cleanup(crtc);
10651         kfree(intel_crtc);
10652 }
10653
10654 /**
10655  * intel_wm_need_update - Check whether watermarks need updating
10656  * @plane: drm plane
10657  * @state: new plane state
10658  *
10659  * Check current plane state versus the new one to determine whether
10660  * watermarks need to be recalculated.
10661  *
10662  * Returns true or false.
10663  */
10664 static bool intel_wm_need_update(struct intel_plane_state *cur,
10665                                  struct intel_plane_state *new)
10666 {
10667         /* Update watermarks on tiling or size changes. */
10668         if (new->base.visible != cur->base.visible)
10669                 return true;
10670
10671         if (!cur->base.fb || !new->base.fb)
10672                 return false;
10673
10674         if (cur->base.fb->modifier != new->base.fb->modifier ||
10675             cur->base.rotation != new->base.rotation ||
10676             drm_rect_width(&new->base.src) != drm_rect_width(&cur->base.src) ||
10677             drm_rect_height(&new->base.src) != drm_rect_height(&cur->base.src) ||
10678             drm_rect_width(&new->base.dst) != drm_rect_width(&cur->base.dst) ||
10679             drm_rect_height(&new->base.dst) != drm_rect_height(&cur->base.dst))
10680                 return true;
10681
10682         return false;
10683 }
10684
10685 static bool needs_scaling(const struct intel_plane_state *state)
10686 {
10687         int src_w = drm_rect_width(&state->base.src) >> 16;
10688         int src_h = drm_rect_height(&state->base.src) >> 16;
10689         int dst_w = drm_rect_width(&state->base.dst);
10690         int dst_h = drm_rect_height(&state->base.dst);
10691
10692         return (src_w != dst_w || src_h != dst_h);
10693 }
10694
10695 int intel_plane_atomic_calc_changes(const struct intel_crtc_state *old_crtc_state,
10696                                     struct drm_crtc_state *crtc_state,
10697                                     const struct intel_plane_state *old_plane_state,
10698                                     struct drm_plane_state *plane_state)
10699 {
10700         struct intel_crtc_state *pipe_config = to_intel_crtc_state(crtc_state);
10701         struct drm_crtc *crtc = crtc_state->crtc;
10702         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
10703         struct intel_plane *plane = to_intel_plane(plane_state->plane);
10704         struct drm_device *dev = crtc->dev;
10705         struct drm_i915_private *dev_priv = to_i915(dev);
10706         bool mode_changed = needs_modeset(crtc_state);
10707         bool was_crtc_enabled = old_crtc_state->base.active;
10708         bool is_crtc_enabled = crtc_state->active;
10709         bool turn_off, turn_on, visible, was_visible;
10710         struct drm_framebuffer *fb = plane_state->fb;
10711         int ret;
10712
10713         if (INTEL_GEN(dev_priv) >= 9 && plane->id != PLANE_CURSOR) {
10714                 ret = skl_update_scaler_plane(
10715                         to_intel_crtc_state(crtc_state),
10716                         to_intel_plane_state(plane_state));
10717                 if (ret)
10718                         return ret;
10719         }
10720
10721         was_visible = old_plane_state->base.visible;
10722         visible = plane_state->visible;
10723
10724         if (!was_crtc_enabled && WARN_ON(was_visible))
10725                 was_visible = false;
10726
10727         /*
10728          * Visibility is calculated as if the crtc was on, but
10729          * after scaler setup everything depends on it being off
10730          * when the crtc isn't active.
10731          *
10732          * FIXME this is wrong for watermarks. Watermarks should also
10733          * be computed as if the pipe would be active. Perhaps move
10734          * per-plane wm computation to the .check_plane() hook, and
10735          * only combine the results from all planes in the current place?
10736          */
10737         if (!is_crtc_enabled) {
10738                 plane_state->visible = visible = false;
10739                 to_intel_crtc_state(crtc_state)->active_planes &= ~BIT(plane->id);
10740         }
10741
10742         if (!was_visible && !visible)
10743                 return 0;
10744
10745         if (fb != old_plane_state->base.fb)
10746                 pipe_config->fb_changed = true;
10747
10748         turn_off = was_visible && (!visible || mode_changed);
10749         turn_on = visible && (!was_visible || mode_changed);
10750
10751         DRM_DEBUG_ATOMIC("[CRTC:%d:%s] has [PLANE:%d:%s] with fb %i\n",
10752                          intel_crtc->base.base.id, intel_crtc->base.name,
10753                          plane->base.base.id, plane->base.name,
10754                          fb ? fb->base.id : -1);
10755
10756         DRM_DEBUG_ATOMIC("[PLANE:%d:%s] visible %i -> %i, off %i, on %i, ms %i\n",
10757                          plane->base.base.id, plane->base.name,
10758                          was_visible, visible,
10759                          turn_off, turn_on, mode_changed);
10760
10761         if (turn_on) {
10762                 if (INTEL_GEN(dev_priv) < 5 && !IS_G4X(dev_priv))
10763                         pipe_config->update_wm_pre = true;
10764
10765                 /* must disable cxsr around plane enable/disable */
10766                 if (plane->id != PLANE_CURSOR)
10767                         pipe_config->disable_cxsr = true;
10768         } else if (turn_off) {
10769                 if (INTEL_GEN(dev_priv) < 5 && !IS_G4X(dev_priv))
10770                         pipe_config->update_wm_post = true;
10771
10772                 /* must disable cxsr around plane enable/disable */
10773                 if (plane->id != PLANE_CURSOR)
10774                         pipe_config->disable_cxsr = true;
10775         } else if (intel_wm_need_update(to_intel_plane_state(plane->base.state),
10776                                         to_intel_plane_state(plane_state))) {
10777                 if (INTEL_GEN(dev_priv) < 5 && !IS_G4X(dev_priv)) {
10778                         /* FIXME bollocks */
10779                         pipe_config->update_wm_pre = true;
10780                         pipe_config->update_wm_post = true;
10781                 }
10782         }
10783
10784         if (visible || was_visible)
10785                 pipe_config->fb_bits |= plane->frontbuffer_bit;
10786
10787         /*
10788          * ILK/SNB DVSACNTR/Sprite Enable
10789          * IVB SPR_CTL/Sprite Enable
10790          * "When in Self Refresh Big FIFO mode, a write to enable the
10791          *  plane will be internally buffered and delayed while Big FIFO
10792          *  mode is exiting."
10793          *
10794          * Which means that enabling the sprite can take an extra frame
10795          * when we start in big FIFO mode (LP1+). Thus we need to drop
10796          * down to LP0 and wait for vblank in order to make sure the
10797          * sprite gets enabled on the next vblank after the register write.
10798          * Doing otherwise would risk enabling the sprite one frame after
10799          * we've already signalled flip completion. We can resume LP1+
10800          * once the sprite has been enabled.
10801          *
10802          *
10803          * WaCxSRDisabledForSpriteScaling:ivb
10804          * IVB SPR_SCALE/Scaling Enable
10805          * "Low Power watermarks must be disabled for at least one
10806          *  frame before enabling sprite scaling, and kept disabled
10807          *  until sprite scaling is disabled."
10808          *
10809          * ILK/SNB DVSASCALE/Scaling Enable
10810          * "When in Self Refresh Big FIFO mode, scaling enable will be
10811          *  masked off while Big FIFO mode is exiting."
10812          *
10813          * Despite the w/a only being listed for IVB we assume that
10814          * the ILK/SNB note has similar ramifications, hence we apply
10815          * the w/a on all three platforms.
10816          */
10817         if (plane->id == PLANE_SPRITE0 &&
10818             (IS_GEN_RANGE(dev_priv, 5, 6) ||
10819              IS_IVYBRIDGE(dev_priv)) &&
10820             (turn_on || (!needs_scaling(old_plane_state) &&
10821                          needs_scaling(to_intel_plane_state(plane_state)))))
10822                 pipe_config->disable_lp_wm = true;
10823
10824         return 0;
10825 }
10826
10827 static bool encoders_cloneable(const struct intel_encoder *a,
10828                                const struct intel_encoder *b)
10829 {
10830         /* masks could be asymmetric, so check both ways */
10831         return a == b || (a->cloneable & (1 << b->type) &&
10832                           b->cloneable & (1 << a->type));
10833 }
10834
10835 static bool check_single_encoder_cloning(struct drm_atomic_state *state,
10836                                          struct intel_crtc *crtc,
10837                                          struct intel_encoder *encoder)
10838 {
10839         struct intel_encoder *source_encoder;
10840         struct drm_connector *connector;
10841         struct drm_connector_state *connector_state;
10842         int i;
10843
10844         for_each_new_connector_in_state(state, connector, connector_state, i) {
10845                 if (connector_state->crtc != &crtc->base)
10846                         continue;
10847
10848                 source_encoder =
10849                         to_intel_encoder(connector_state->best_encoder);
10850                 if (!encoders_cloneable(encoder, source_encoder))
10851                         return false;
10852         }
10853
10854         return true;
10855 }
10856
10857 static int icl_add_linked_planes(struct intel_atomic_state *state)
10858 {
10859         struct intel_plane *plane, *linked;
10860         struct intel_plane_state *plane_state, *linked_plane_state;
10861         int i;
10862
10863         for_each_new_intel_plane_in_state(state, plane, plane_state, i) {
10864                 linked = plane_state->linked_plane;
10865
10866                 if (!linked)
10867                         continue;
10868
10869                 linked_plane_state = intel_atomic_get_plane_state(state, linked);
10870                 if (IS_ERR(linked_plane_state))
10871                         return PTR_ERR(linked_plane_state);
10872
10873                 WARN_ON(linked_plane_state->linked_plane != plane);
10874                 WARN_ON(linked_plane_state->slave == plane_state->slave);
10875         }
10876
10877         return 0;
10878 }
10879
10880 static int icl_check_nv12_planes(struct intel_crtc_state *crtc_state)
10881 {
10882         struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
10883         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
10884         struct intel_atomic_state *state = to_intel_atomic_state(crtc_state->base.state);
10885         struct intel_plane *plane, *linked;
10886         struct intel_plane_state *plane_state;
10887         int i;
10888
10889         if (INTEL_GEN(dev_priv) < 11)
10890                 return 0;
10891
10892         /*
10893          * Destroy all old plane links and make the slave plane invisible
10894          * in the crtc_state->active_planes mask.
10895          */
10896         for_each_new_intel_plane_in_state(state, plane, plane_state, i) {
10897                 if (plane->pipe != crtc->pipe || !plane_state->linked_plane)
10898                         continue;
10899
10900                 plane_state->linked_plane = NULL;
10901                 if (plane_state->slave && !plane_state->base.visible) {
10902                         crtc_state->active_planes &= ~BIT(plane->id);
10903                         crtc_state->update_planes |= BIT(plane->id);
10904                 }
10905
10906                 plane_state->slave = false;
10907         }
10908
10909         if (!crtc_state->nv12_planes)
10910                 return 0;
10911
10912         for_each_new_intel_plane_in_state(state, plane, plane_state, i) {
10913                 struct intel_plane_state *linked_state = NULL;
10914
10915                 if (plane->pipe != crtc->pipe ||
10916                     !(crtc_state->nv12_planes & BIT(plane->id)))
10917                         continue;
10918
10919                 for_each_intel_plane_on_crtc(&dev_priv->drm, crtc, linked) {
10920                         if (!icl_is_nv12_y_plane(linked->id))
10921                                 continue;
10922
10923                         if (crtc_state->active_planes & BIT(linked->id))
10924                                 continue;
10925
10926                         linked_state = intel_atomic_get_plane_state(state, linked);
10927                         if (IS_ERR(linked_state))
10928                                 return PTR_ERR(linked_state);
10929
10930                         break;
10931                 }
10932
10933                 if (!linked_state) {
10934                         DRM_DEBUG_KMS("Need %d free Y planes for NV12\n",
10935                                       hweight8(crtc_state->nv12_planes));
10936
10937                         return -EINVAL;
10938                 }
10939
10940                 plane_state->linked_plane = linked;
10941
10942                 linked_state->slave = true;
10943                 linked_state->linked_plane = plane;
10944                 crtc_state->active_planes |= BIT(linked->id);
10945                 crtc_state->update_planes |= BIT(linked->id);
10946                 DRM_DEBUG_KMS("Using %s as Y plane for %s\n", linked->base.name, plane->base.name);
10947         }
10948
10949         return 0;
10950 }
10951
10952 static int intel_crtc_atomic_check(struct drm_crtc *crtc,
10953                                    struct drm_crtc_state *crtc_state)
10954 {
10955         struct drm_i915_private *dev_priv = to_i915(crtc->dev);
10956         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
10957         struct intel_crtc_state *pipe_config =
10958                 to_intel_crtc_state(crtc_state);
10959         int ret;
10960         bool mode_changed = needs_modeset(crtc_state);
10961
10962         if (mode_changed && !crtc_state->active)
10963                 pipe_config->update_wm_post = true;
10964
10965         if (mode_changed && crtc_state->enable &&
10966             dev_priv->display.crtc_compute_clock &&
10967             !WARN_ON(pipe_config->shared_dpll)) {
10968                 ret = dev_priv->display.crtc_compute_clock(intel_crtc,
10969                                                            pipe_config);
10970                 if (ret)
10971                         return ret;
10972         }
10973
10974         if (crtc_state->color_mgmt_changed) {
10975                 ret = intel_color_check(pipe_config);
10976                 if (ret)
10977                         return ret;
10978
10979                 /*
10980                  * Changing color management on Intel hardware is
10981                  * handled as part of planes update.
10982                  */
10983                 crtc_state->planes_changed = true;
10984         }
10985
10986         ret = 0;
10987         if (dev_priv->display.compute_pipe_wm) {
10988                 ret = dev_priv->display.compute_pipe_wm(pipe_config);
10989                 if (ret) {
10990                         DRM_DEBUG_KMS("Target pipe watermarks are invalid\n");
10991                         return ret;
10992                 }
10993         }
10994
10995         if (dev_priv->display.compute_intermediate_wm) {
10996                 if (WARN_ON(!dev_priv->display.compute_pipe_wm))
10997                         return 0;
10998
10999                 /*
11000                  * Calculate 'intermediate' watermarks that satisfy both the
11001                  * old state and the new state.  We can program these
11002                  * immediately.
11003                  */
11004                 ret = dev_priv->display.compute_intermediate_wm(pipe_config);
11005                 if (ret) {
11006                         DRM_DEBUG_KMS("No valid intermediate pipe watermarks are possible\n");
11007                         return ret;
11008                 }
11009         }
11010
11011         if (INTEL_GEN(dev_priv) >= 9) {
11012                 if (mode_changed || pipe_config->update_pipe)
11013                         ret = skl_update_scaler_crtc(pipe_config);
11014
11015                 if (!ret)
11016                         ret = icl_check_nv12_planes(pipe_config);
11017                 if (!ret)
11018                         ret = skl_check_pipe_max_pixel_rate(intel_crtc,
11019                                                             pipe_config);
11020                 if (!ret)
11021                         ret = intel_atomic_setup_scalers(dev_priv, intel_crtc,
11022                                                          pipe_config);
11023         }
11024
11025         if (HAS_IPS(dev_priv))
11026                 pipe_config->ips_enabled = hsw_compute_ips_config(pipe_config);
11027
11028         return ret;
11029 }
11030
11031 static const struct drm_crtc_helper_funcs intel_helper_funcs = {
11032         .atomic_check = intel_crtc_atomic_check,
11033 };
11034
11035 static void intel_modeset_update_connector_atomic_state(struct drm_device *dev)
11036 {
11037         struct intel_connector *connector;
11038         struct drm_connector_list_iter conn_iter;
11039
11040         drm_connector_list_iter_begin(dev, &conn_iter);
11041         for_each_intel_connector_iter(connector, &conn_iter) {
11042                 if (connector->base.state->crtc)
11043                         drm_connector_put(&connector->base);
11044
11045                 if (connector->base.encoder) {
11046                         connector->base.state->best_encoder =
11047                                 connector->base.encoder;
11048                         connector->base.state->crtc =
11049                                 connector->base.encoder->crtc;
11050
11051                         drm_connector_get(&connector->base);
11052                 } else {
11053                         connector->base.state->best_encoder = NULL;
11054                         connector->base.state->crtc = NULL;
11055                 }
11056         }
11057         drm_connector_list_iter_end(&conn_iter);
11058 }
11059
11060 static int
11061 compute_sink_pipe_bpp(const struct drm_connector_state *conn_state,
11062                       struct intel_crtc_state *pipe_config)
11063 {
11064         struct drm_connector *connector = conn_state->connector;
11065         const struct drm_display_info *info = &connector->display_info;
11066         int bpp;
11067
11068         switch (conn_state->max_bpc) {
11069         case 6 ... 7:
11070                 bpp = 6 * 3;
11071                 break;
11072         case 8 ... 9:
11073                 bpp = 8 * 3;
11074                 break;
11075         case 10 ... 11:
11076                 bpp = 10 * 3;
11077                 break;
11078         case 12:
11079                 bpp = 12 * 3;
11080                 break;
11081         default:
11082                 return -EINVAL;
11083         }
11084
11085         if (bpp < pipe_config->pipe_bpp) {
11086                 DRM_DEBUG_KMS("[CONNECTOR:%d:%s] Limiting display bpp to %d instead of "
11087                               "EDID bpp %d, requested bpp %d, max platform bpp %d\n",
11088                               connector->base.id, connector->name,
11089                               bpp, 3 * info->bpc, 3 * conn_state->max_requested_bpc,
11090                               pipe_config->pipe_bpp);
11091
11092                 pipe_config->pipe_bpp = bpp;
11093         }
11094
11095         return 0;
11096 }
11097
11098 static int
11099 compute_baseline_pipe_bpp(struct intel_crtc *crtc,
11100                           struct intel_crtc_state *pipe_config)
11101 {
11102         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
11103         struct drm_atomic_state *state = pipe_config->base.state;
11104         struct drm_connector *connector;
11105         struct drm_connector_state *connector_state;
11106         int bpp, i;
11107
11108         if ((IS_G4X(dev_priv) || IS_VALLEYVIEW(dev_priv) ||
11109             IS_CHERRYVIEW(dev_priv)))
11110                 bpp = 10*3;
11111         else if (INTEL_GEN(dev_priv) >= 5)
11112                 bpp = 12*3;
11113         else
11114                 bpp = 8*3;
11115
11116         pipe_config->pipe_bpp = bpp;
11117
11118         /* Clamp display bpp to connector max bpp */
11119         for_each_new_connector_in_state(state, connector, connector_state, i) {
11120                 int ret;
11121
11122                 if (connector_state->crtc != &crtc->base)
11123                         continue;
11124
11125                 ret = compute_sink_pipe_bpp(connector_state, pipe_config);
11126                 if (ret)
11127                         return ret;
11128         }
11129
11130         return 0;
11131 }
11132
11133 static void intel_dump_crtc_timings(const struct drm_display_mode *mode)
11134 {
11135         DRM_DEBUG_KMS("crtc timings: %d %d %d %d %d %d %d %d %d, "
11136                         "type: 0x%x flags: 0x%x\n",
11137                 mode->crtc_clock,
11138                 mode->crtc_hdisplay, mode->crtc_hsync_start,
11139                 mode->crtc_hsync_end, mode->crtc_htotal,
11140                 mode->crtc_vdisplay, mode->crtc_vsync_start,
11141                 mode->crtc_vsync_end, mode->crtc_vtotal, mode->type, mode->flags);
11142 }
11143
11144 static inline void
11145 intel_dump_m_n_config(struct intel_crtc_state *pipe_config, char *id,
11146                       unsigned int lane_count, struct intel_link_m_n *m_n)
11147 {
11148         DRM_DEBUG_KMS("%s: lanes: %i; gmch_m: %u, gmch_n: %u, link_m: %u, link_n: %u, tu: %u\n",
11149                       id, lane_count,
11150                       m_n->gmch_m, m_n->gmch_n,
11151                       m_n->link_m, m_n->link_n, m_n->tu);
11152 }
11153
11154 #define OUTPUT_TYPE(x) [INTEL_OUTPUT_ ## x] = #x
11155
11156 static const char * const output_type_str[] = {
11157         OUTPUT_TYPE(UNUSED),
11158         OUTPUT_TYPE(ANALOG),
11159         OUTPUT_TYPE(DVO),
11160         OUTPUT_TYPE(SDVO),
11161         OUTPUT_TYPE(LVDS),
11162         OUTPUT_TYPE(TVOUT),
11163         OUTPUT_TYPE(HDMI),
11164         OUTPUT_TYPE(DP),
11165         OUTPUT_TYPE(EDP),
11166         OUTPUT_TYPE(DSI),
11167         OUTPUT_TYPE(DDI),
11168         OUTPUT_TYPE(DP_MST),
11169 };
11170
11171 #undef OUTPUT_TYPE
11172
11173 static void snprintf_output_types(char *buf, size_t len,
11174                                   unsigned int output_types)
11175 {
11176         char *str = buf;
11177         int i;
11178
11179         str[0] = '\0';
11180
11181         for (i = 0; i < ARRAY_SIZE(output_type_str); i++) {
11182                 int r;
11183
11184                 if ((output_types & BIT(i)) == 0)
11185                         continue;
11186
11187                 r = snprintf(str, len, "%s%s",
11188                              str != buf ? "," : "", output_type_str[i]);
11189                 if (r >= len)
11190                         break;
11191                 str += r;
11192                 len -= r;
11193
11194                 output_types &= ~BIT(i);
11195         }
11196
11197         WARN_ON_ONCE(output_types != 0);
11198 }
11199
11200 static const char * const output_format_str[] = {
11201         [INTEL_OUTPUT_FORMAT_INVALID] = "Invalid",
11202         [INTEL_OUTPUT_FORMAT_RGB] = "RGB",
11203         [INTEL_OUTPUT_FORMAT_YCBCR420] = "YCBCR4:2:0",
11204         [INTEL_OUTPUT_FORMAT_YCBCR444] = "YCBCR4:4:4",
11205 };
11206
11207 static const char *output_formats(enum intel_output_format format)
11208 {
11209         if (format >= ARRAY_SIZE(output_format_str))
11210                 format = INTEL_OUTPUT_FORMAT_INVALID;
11211         return output_format_str[format];
11212 }
11213
11214 static void intel_dump_pipe_config(struct intel_crtc *crtc,
11215                                    struct intel_crtc_state *pipe_config,
11216                                    const char *context)
11217 {
11218         struct drm_device *dev = crtc->base.dev;
11219         struct drm_i915_private *dev_priv = to_i915(dev);
11220         struct drm_plane *plane;
11221         struct intel_plane *intel_plane;
11222         struct intel_plane_state *state;
11223         struct drm_framebuffer *fb;
11224         char buf[64];
11225
11226         DRM_DEBUG_KMS("[CRTC:%d:%s]%s\n",
11227                       crtc->base.base.id, crtc->base.name, context);
11228
11229         snprintf_output_types(buf, sizeof(buf), pipe_config->output_types);
11230         DRM_DEBUG_KMS("output_types: %s (0x%x)\n",
11231                       buf, pipe_config->output_types);
11232
11233         DRM_DEBUG_KMS("output format: %s\n",
11234                       output_formats(pipe_config->output_format));
11235
11236         DRM_DEBUG_KMS("cpu_transcoder: %s, pipe bpp: %i, dithering: %i\n",
11237                       transcoder_name(pipe_config->cpu_transcoder),
11238                       pipe_config->pipe_bpp, pipe_config->dither);
11239
11240         if (pipe_config->has_pch_encoder)
11241                 intel_dump_m_n_config(pipe_config, "fdi",
11242                                       pipe_config->fdi_lanes,
11243                                       &pipe_config->fdi_m_n);
11244
11245         if (intel_crtc_has_dp_encoder(pipe_config)) {
11246                 intel_dump_m_n_config(pipe_config, "dp m_n",
11247                                 pipe_config->lane_count, &pipe_config->dp_m_n);
11248                 if (pipe_config->has_drrs)
11249                         intel_dump_m_n_config(pipe_config, "dp m2_n2",
11250                                               pipe_config->lane_count,
11251                                               &pipe_config->dp_m2_n2);
11252         }
11253
11254         DRM_DEBUG_KMS("audio: %i, infoframes: %i\n",
11255                       pipe_config->has_audio, pipe_config->has_infoframe);
11256
11257         DRM_DEBUG_KMS("requested mode:\n");
11258         drm_mode_debug_printmodeline(&pipe_config->base.mode);
11259         DRM_DEBUG_KMS("adjusted mode:\n");
11260         drm_mode_debug_printmodeline(&pipe_config->base.adjusted_mode);
11261         intel_dump_crtc_timings(&pipe_config->base.adjusted_mode);
11262         DRM_DEBUG_KMS("port clock: %d, pipe src size: %dx%d, pixel rate %d\n",
11263                       pipe_config->port_clock,
11264                       pipe_config->pipe_src_w, pipe_config->pipe_src_h,
11265                       pipe_config->pixel_rate);
11266
11267         if (INTEL_GEN(dev_priv) >= 9)
11268                 DRM_DEBUG_KMS("num_scalers: %d, scaler_users: 0x%x, scaler_id: %d\n",
11269                               crtc->num_scalers,
11270                               pipe_config->scaler_state.scaler_users,
11271                               pipe_config->scaler_state.scaler_id);
11272
11273         if (HAS_GMCH_DISPLAY(dev_priv))
11274                 DRM_DEBUG_KMS("gmch pfit: control: 0x%08x, ratios: 0x%08x, lvds border: 0x%08x\n",
11275                               pipe_config->gmch_pfit.control,
11276                               pipe_config->gmch_pfit.pgm_ratios,
11277                               pipe_config->gmch_pfit.lvds_border_bits);
11278         else
11279                 DRM_DEBUG_KMS("pch pfit: pos: 0x%08x, size: 0x%08x, %s\n",
11280                               pipe_config->pch_pfit.pos,
11281                               pipe_config->pch_pfit.size,
11282                               enableddisabled(pipe_config->pch_pfit.enabled));
11283
11284         DRM_DEBUG_KMS("ips: %i, double wide: %i\n",
11285                       pipe_config->ips_enabled, pipe_config->double_wide);
11286
11287         intel_dpll_dump_hw_state(dev_priv, &pipe_config->dpll_hw_state);
11288
11289         DRM_DEBUG_KMS("planes on this crtc\n");
11290         list_for_each_entry(plane, &dev->mode_config.plane_list, head) {
11291                 struct drm_format_name_buf format_name;
11292                 intel_plane = to_intel_plane(plane);
11293                 if (intel_plane->pipe != crtc->pipe)
11294                         continue;
11295
11296                 state = to_intel_plane_state(plane->state);
11297                 fb = state->base.fb;
11298                 if (!fb) {
11299                         DRM_DEBUG_KMS("[PLANE:%d:%s] disabled, scaler_id = %d\n",
11300                                       plane->base.id, plane->name, state->scaler_id);
11301                         continue;
11302                 }
11303
11304                 DRM_DEBUG_KMS("[PLANE:%d:%s] FB:%d, fb = %ux%u format = %s\n",
11305                               plane->base.id, plane->name,
11306                               fb->base.id, fb->width, fb->height,
11307                               drm_get_format_name(fb->format->format, &format_name));
11308                 if (INTEL_GEN(dev_priv) >= 9)
11309                         DRM_DEBUG_KMS("\tscaler:%d src %dx%d+%d+%d dst %dx%d+%d+%d\n",
11310                                       state->scaler_id,
11311                                       state->base.src.x1 >> 16,
11312                                       state->base.src.y1 >> 16,
11313                                       drm_rect_width(&state->base.src) >> 16,
11314                                       drm_rect_height(&state->base.src) >> 16,
11315                                       state->base.dst.x1, state->base.dst.y1,
11316                                       drm_rect_width(&state->base.dst),
11317                                       drm_rect_height(&state->base.dst));
11318         }
11319 }
11320
11321 static bool check_digital_port_conflicts(struct drm_atomic_state *state)
11322 {
11323         struct drm_device *dev = state->dev;
11324         struct drm_connector *connector;
11325         struct drm_connector_list_iter conn_iter;
11326         unsigned int used_ports = 0;
11327         unsigned int used_mst_ports = 0;
11328         bool ret = true;
11329
11330         /*
11331          * Walk the connector list instead of the encoder
11332          * list to detect the problem on ddi platforms
11333          * where there's just one encoder per digital port.
11334          */
11335         drm_connector_list_iter_begin(dev, &conn_iter);
11336         drm_for_each_connector_iter(connector, &conn_iter) {
11337                 struct drm_connector_state *connector_state;
11338                 struct intel_encoder *encoder;
11339
11340                 connector_state = drm_atomic_get_new_connector_state(state, connector);
11341                 if (!connector_state)
11342                         connector_state = connector->state;
11343
11344                 if (!connector_state->best_encoder)
11345                         continue;
11346
11347                 encoder = to_intel_encoder(connector_state->best_encoder);
11348
11349                 WARN_ON(!connector_state->crtc);
11350
11351                 switch (encoder->type) {
11352                         unsigned int port_mask;
11353                 case INTEL_OUTPUT_DDI:
11354                         if (WARN_ON(!HAS_DDI(to_i915(dev))))
11355                                 break;
11356                         /* else: fall through */
11357                 case INTEL_OUTPUT_DP:
11358                 case INTEL_OUTPUT_HDMI:
11359                 case INTEL_OUTPUT_EDP:
11360                         port_mask = 1 << encoder->port;
11361
11362                         /* the same port mustn't appear more than once */
11363                         if (used_ports & port_mask)
11364                                 ret = false;
11365
11366                         used_ports |= port_mask;
11367                         break;
11368                 case INTEL_OUTPUT_DP_MST:
11369                         used_mst_ports |=
11370                                 1 << encoder->port;
11371                         break;
11372                 default:
11373                         break;
11374                 }
11375         }
11376         drm_connector_list_iter_end(&conn_iter);
11377
11378         /* can't mix MST and SST/HDMI on the same port */
11379         if (used_ports & used_mst_ports)
11380                 return false;
11381
11382         return ret;
11383 }
11384
11385 static void
11386 clear_intel_crtc_state(struct intel_crtc_state *crtc_state)
11387 {
11388         struct drm_i915_private *dev_priv =
11389                 to_i915(crtc_state->base.crtc->dev);
11390         struct intel_crtc_scaler_state scaler_state;
11391         struct intel_dpll_hw_state dpll_hw_state;
11392         struct intel_shared_dpll *shared_dpll;
11393         struct intel_crtc_wm_state wm_state;
11394         bool force_thru, ips_force_disable;
11395
11396         /* FIXME: before the switch to atomic started, a new pipe_config was
11397          * kzalloc'd. Code that depends on any field being zero should be
11398          * fixed, so that the crtc_state can be safely duplicated. For now,
11399          * only fields that are know to not cause problems are preserved. */
11400
11401         scaler_state = crtc_state->scaler_state;
11402         shared_dpll = crtc_state->shared_dpll;
11403         dpll_hw_state = crtc_state->dpll_hw_state;
11404         force_thru = crtc_state->pch_pfit.force_thru;
11405         ips_force_disable = crtc_state->ips_force_disable;
11406         if (IS_G4X(dev_priv) ||
11407             IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
11408                 wm_state = crtc_state->wm;
11409
11410         /* Keep base drm_crtc_state intact, only clear our extended struct */
11411         BUILD_BUG_ON(offsetof(struct intel_crtc_state, base));
11412         memset(&crtc_state->base + 1, 0,
11413                sizeof(*crtc_state) - sizeof(crtc_state->base));
11414
11415         crtc_state->scaler_state = scaler_state;
11416         crtc_state->shared_dpll = shared_dpll;
11417         crtc_state->dpll_hw_state = dpll_hw_state;
11418         crtc_state->pch_pfit.force_thru = force_thru;
11419         crtc_state->ips_force_disable = ips_force_disable;
11420         if (IS_G4X(dev_priv) ||
11421             IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
11422                 crtc_state->wm = wm_state;
11423 }
11424
11425 static int
11426 intel_modeset_pipe_config(struct drm_crtc *crtc,
11427                           struct intel_crtc_state *pipe_config)
11428 {
11429         struct drm_atomic_state *state = pipe_config->base.state;
11430         struct intel_encoder *encoder;
11431         struct drm_connector *connector;
11432         struct drm_connector_state *connector_state;
11433         int base_bpp, ret;
11434         int i;
11435         bool retry = true;
11436
11437         clear_intel_crtc_state(pipe_config);
11438
11439         pipe_config->cpu_transcoder =
11440                 (enum transcoder) to_intel_crtc(crtc)->pipe;
11441
11442         /*
11443          * Sanitize sync polarity flags based on requested ones. If neither
11444          * positive or negative polarity is requested, treat this as meaning
11445          * negative polarity.
11446          */
11447         if (!(pipe_config->base.adjusted_mode.flags &
11448               (DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_NHSYNC)))
11449                 pipe_config->base.adjusted_mode.flags |= DRM_MODE_FLAG_NHSYNC;
11450
11451         if (!(pipe_config->base.adjusted_mode.flags &
11452               (DRM_MODE_FLAG_PVSYNC | DRM_MODE_FLAG_NVSYNC)))
11453                 pipe_config->base.adjusted_mode.flags |= DRM_MODE_FLAG_NVSYNC;
11454
11455         ret = compute_baseline_pipe_bpp(to_intel_crtc(crtc),
11456                                         pipe_config);
11457         if (ret)
11458                 return ret;
11459
11460         base_bpp = pipe_config->pipe_bpp;
11461
11462         /*
11463          * Determine the real pipe dimensions. Note that stereo modes can
11464          * increase the actual pipe size due to the frame doubling and
11465          * insertion of additional space for blanks between the frame. This
11466          * is stored in the crtc timings. We use the requested mode to do this
11467          * computation to clearly distinguish it from the adjusted mode, which
11468          * can be changed by the connectors in the below retry loop.
11469          */
11470         drm_mode_get_hv_timing(&pipe_config->base.mode,
11471                                &pipe_config->pipe_src_w,
11472                                &pipe_config->pipe_src_h);
11473
11474         for_each_new_connector_in_state(state, connector, connector_state, i) {
11475                 if (connector_state->crtc != crtc)
11476                         continue;
11477
11478                 encoder = to_intel_encoder(connector_state->best_encoder);
11479
11480                 if (!check_single_encoder_cloning(state, to_intel_crtc(crtc), encoder)) {
11481                         DRM_DEBUG_KMS("rejecting invalid cloning configuration\n");
11482                         return -EINVAL;
11483                 }
11484
11485                 /*
11486                  * Determine output_types before calling the .compute_config()
11487                  * hooks so that the hooks can use this information safely.
11488                  */
11489                 if (encoder->compute_output_type)
11490                         pipe_config->output_types |=
11491                                 BIT(encoder->compute_output_type(encoder, pipe_config,
11492                                                                  connector_state));
11493                 else
11494                         pipe_config->output_types |= BIT(encoder->type);
11495         }
11496
11497 encoder_retry:
11498         /* Ensure the port clock defaults are reset when retrying. */
11499         pipe_config->port_clock = 0;
11500         pipe_config->pixel_multiplier = 1;
11501
11502         /* Fill in default crtc timings, allow encoders to overwrite them. */
11503         drm_mode_set_crtcinfo(&pipe_config->base.adjusted_mode,
11504                               CRTC_STEREO_DOUBLE);
11505
11506         /* Pass our mode to the connectors and the CRTC to give them a chance to
11507          * adjust it according to limitations or connector properties, and also
11508          * a chance to reject the mode entirely.
11509          */
11510         for_each_new_connector_in_state(state, connector, connector_state, i) {
11511                 if (connector_state->crtc != crtc)
11512                         continue;
11513
11514                 encoder = to_intel_encoder(connector_state->best_encoder);
11515
11516                 if (!(encoder->compute_config(encoder, pipe_config, connector_state))) {
11517                         DRM_DEBUG_KMS("Encoder config failure\n");
11518                         return -EINVAL;
11519                 }
11520         }
11521
11522         /* Set default port clock if not overwritten by the encoder. Needs to be
11523          * done afterwards in case the encoder adjusts the mode. */
11524         if (!pipe_config->port_clock)
11525                 pipe_config->port_clock = pipe_config->base.adjusted_mode.crtc_clock
11526                         * pipe_config->pixel_multiplier;
11527
11528         ret = intel_crtc_compute_config(to_intel_crtc(crtc), pipe_config);
11529         if (ret == -EDEADLK)
11530                 return ret;
11531         if (ret < 0) {
11532                 DRM_DEBUG_KMS("CRTC fixup failed\n");
11533                 return ret;
11534         }
11535
11536         if (ret == RETRY) {
11537                 if (WARN(!retry, "loop in pipe configuration computation\n"))
11538                         return -EINVAL;
11539
11540                 DRM_DEBUG_KMS("CRTC bw constrained, retrying\n");
11541                 retry = false;
11542                 goto encoder_retry;
11543         }
11544
11545         /* Dithering seems to not pass-through bits correctly when it should, so
11546          * only enable it on 6bpc panels and when its not a compliance
11547          * test requesting 6bpc video pattern.
11548          */
11549         pipe_config->dither = (pipe_config->pipe_bpp == 6*3) &&
11550                 !pipe_config->dither_force_disable;
11551         DRM_DEBUG_KMS("hw max bpp: %i, pipe bpp: %i, dithering: %i\n",
11552                       base_bpp, pipe_config->pipe_bpp, pipe_config->dither);
11553
11554         return 0;
11555 }
11556
11557 static bool intel_fuzzy_clock_check(int clock1, int clock2)
11558 {
11559         int diff;
11560
11561         if (clock1 == clock2)
11562                 return true;
11563
11564         if (!clock1 || !clock2)
11565                 return false;
11566
11567         diff = abs(clock1 - clock2);
11568
11569         if (((((diff + clock1 + clock2) * 100)) / (clock1 + clock2)) < 105)
11570                 return true;
11571
11572         return false;
11573 }
11574
11575 static bool
11576 intel_compare_m_n(unsigned int m, unsigned int n,
11577                   unsigned int m2, unsigned int n2,
11578                   bool exact)
11579 {
11580         if (m == m2 && n == n2)
11581                 return true;
11582
11583         if (exact || !m || !n || !m2 || !n2)
11584                 return false;
11585
11586         BUILD_BUG_ON(DATA_LINK_M_N_MASK > INT_MAX);
11587
11588         if (n > n2) {
11589                 while (n > n2) {
11590                         m2 <<= 1;
11591                         n2 <<= 1;
11592                 }
11593         } else if (n < n2) {
11594                 while (n < n2) {
11595                         m <<= 1;
11596                         n <<= 1;
11597                 }
11598         }
11599
11600         if (n != n2)
11601                 return false;
11602
11603         return intel_fuzzy_clock_check(m, m2);
11604 }
11605
11606 static bool
11607 intel_compare_link_m_n(const struct intel_link_m_n *m_n,
11608                        struct intel_link_m_n *m2_n2,
11609                        bool adjust)
11610 {
11611         if (m_n->tu == m2_n2->tu &&
11612             intel_compare_m_n(m_n->gmch_m, m_n->gmch_n,
11613                               m2_n2->gmch_m, m2_n2->gmch_n, !adjust) &&
11614             intel_compare_m_n(m_n->link_m, m_n->link_n,
11615                               m2_n2->link_m, m2_n2->link_n, !adjust)) {
11616                 if (adjust)
11617                         *m2_n2 = *m_n;
11618
11619                 return true;
11620         }
11621
11622         return false;
11623 }
11624
11625 static void __printf(3, 4)
11626 pipe_config_err(bool adjust, const char *name, const char *format, ...)
11627 {
11628         struct va_format vaf;
11629         va_list args;
11630
11631         va_start(args, format);
11632         vaf.fmt = format;
11633         vaf.va = &args;
11634
11635         if (adjust)
11636                 drm_dbg(DRM_UT_KMS, "mismatch in %s %pV", name, &vaf);
11637         else
11638                 drm_err("mismatch in %s %pV", name, &vaf);
11639
11640         va_end(args);
11641 }
11642
11643 static bool
11644 intel_pipe_config_compare(struct drm_i915_private *dev_priv,
11645                           struct intel_crtc_state *current_config,
11646                           struct intel_crtc_state *pipe_config,
11647                           bool adjust)
11648 {
11649         bool ret = true;
11650         bool fixup_inherited = adjust &&
11651                 (current_config->base.mode.private_flags & I915_MODE_FLAG_INHERITED) &&
11652                 !(pipe_config->base.mode.private_flags & I915_MODE_FLAG_INHERITED);
11653
11654 #define PIPE_CONF_CHECK_X(name) do { \
11655         if (current_config->name != pipe_config->name) { \
11656                 pipe_config_err(adjust, __stringify(name), \
11657                           "(expected 0x%08x, found 0x%08x)\n", \
11658                           current_config->name, \
11659                           pipe_config->name); \
11660                 ret = false; \
11661         } \
11662 } while (0)
11663
11664 #define PIPE_CONF_CHECK_I(name) do { \
11665         if (current_config->name != pipe_config->name) { \
11666                 pipe_config_err(adjust, __stringify(name), \
11667                           "(expected %i, found %i)\n", \
11668                           current_config->name, \
11669                           pipe_config->name); \
11670                 ret = false; \
11671         } \
11672 } while (0)
11673
11674 #define PIPE_CONF_CHECK_BOOL(name) do { \
11675         if (current_config->name != pipe_config->name) { \
11676                 pipe_config_err(adjust, __stringify(name), \
11677                           "(expected %s, found %s)\n", \
11678                           yesno(current_config->name), \
11679                           yesno(pipe_config->name)); \
11680                 ret = false; \
11681         } \
11682 } while (0)
11683
11684 /*
11685  * Checks state where we only read out the enabling, but not the entire
11686  * state itself (like full infoframes or ELD for audio). These states
11687  * require a full modeset on bootup to fix up.
11688  */
11689 #define PIPE_CONF_CHECK_BOOL_INCOMPLETE(name) do { \
11690         if (!fixup_inherited || (!current_config->name && !pipe_config->name)) { \
11691                 PIPE_CONF_CHECK_BOOL(name); \
11692         } else { \
11693                 pipe_config_err(adjust, __stringify(name), \
11694                           "unable to verify whether state matches exactly, forcing modeset (expected %s, found %s)\n", \
11695                           yesno(current_config->name), \
11696                           yesno(pipe_config->name)); \
11697                 ret = false; \
11698         } \
11699 } while (0)
11700
11701 #define PIPE_CONF_CHECK_P(name) do { \
11702         if (current_config->name != pipe_config->name) { \
11703                 pipe_config_err(adjust, __stringify(name), \
11704                           "(expected %p, found %p)\n", \
11705                           current_config->name, \
11706                           pipe_config->name); \
11707                 ret = false; \
11708         } \
11709 } while (0)
11710
11711 #define PIPE_CONF_CHECK_M_N(name) do { \
11712         if (!intel_compare_link_m_n(&current_config->name, \
11713                                     &pipe_config->name,\
11714                                     adjust)) { \
11715                 pipe_config_err(adjust, __stringify(name), \
11716                           "(expected tu %i gmch %i/%i link %i/%i, " \
11717                           "found tu %i, gmch %i/%i link %i/%i)\n", \
11718                           current_config->name.tu, \
11719                           current_config->name.gmch_m, \
11720                           current_config->name.gmch_n, \
11721                           current_config->name.link_m, \
11722                           current_config->name.link_n, \
11723                           pipe_config->name.tu, \
11724                           pipe_config->name.gmch_m, \
11725                           pipe_config->name.gmch_n, \
11726                           pipe_config->name.link_m, \
11727                           pipe_config->name.link_n); \
11728                 ret = false; \
11729         } \
11730 } while (0)
11731
11732 /* This is required for BDW+ where there is only one set of registers for
11733  * switching between high and low RR.
11734  * This macro can be used whenever a comparison has to be made between one
11735  * hw state and multiple sw state variables.
11736  */
11737 #define PIPE_CONF_CHECK_M_N_ALT(name, alt_name) do { \
11738         if (!intel_compare_link_m_n(&current_config->name, \
11739                                     &pipe_config->name, adjust) && \
11740             !intel_compare_link_m_n(&current_config->alt_name, \
11741                                     &pipe_config->name, adjust)) { \
11742                 pipe_config_err(adjust, __stringify(name), \
11743                           "(expected tu %i gmch %i/%i link %i/%i, " \
11744                           "or tu %i gmch %i/%i link %i/%i, " \
11745                           "found tu %i, gmch %i/%i link %i/%i)\n", \
11746                           current_config->name.tu, \
11747                           current_config->name.gmch_m, \
11748                           current_config->name.gmch_n, \
11749                           current_config->name.link_m, \
11750                           current_config->name.link_n, \
11751                           current_config->alt_name.tu, \
11752                           current_config->alt_name.gmch_m, \
11753                           current_config->alt_name.gmch_n, \
11754                           current_config->alt_name.link_m, \
11755                           current_config->alt_name.link_n, \
11756                           pipe_config->name.tu, \
11757                           pipe_config->name.gmch_m, \
11758                           pipe_config->name.gmch_n, \
11759                           pipe_config->name.link_m, \
11760                           pipe_config->name.link_n); \
11761                 ret = false; \
11762         } \
11763 } while (0)
11764
11765 #define PIPE_CONF_CHECK_FLAGS(name, mask) do { \
11766         if ((current_config->name ^ pipe_config->name) & (mask)) { \
11767                 pipe_config_err(adjust, __stringify(name), \
11768                           "(%x) (expected %i, found %i)\n", \
11769                           (mask), \
11770                           current_config->name & (mask), \
11771                           pipe_config->name & (mask)); \
11772                 ret = false; \
11773         } \
11774 } while (0)
11775
11776 #define PIPE_CONF_CHECK_CLOCK_FUZZY(name) do { \
11777         if (!intel_fuzzy_clock_check(current_config->name, pipe_config->name)) { \
11778                 pipe_config_err(adjust, __stringify(name), \
11779                           "(expected %i, found %i)\n", \
11780                           current_config->name, \
11781                           pipe_config->name); \
11782                 ret = false; \
11783         } \
11784 } while (0)
11785
11786 #define PIPE_CONF_QUIRK(quirk)  \
11787         ((current_config->quirks | pipe_config->quirks) & (quirk))
11788
11789         PIPE_CONF_CHECK_I(cpu_transcoder);
11790
11791         PIPE_CONF_CHECK_BOOL(has_pch_encoder);
11792         PIPE_CONF_CHECK_I(fdi_lanes);
11793         PIPE_CONF_CHECK_M_N(fdi_m_n);
11794
11795         PIPE_CONF_CHECK_I(lane_count);
11796         PIPE_CONF_CHECK_X(lane_lat_optim_mask);
11797
11798         if (INTEL_GEN(dev_priv) < 8) {
11799                 PIPE_CONF_CHECK_M_N(dp_m_n);
11800
11801                 if (current_config->has_drrs)
11802                         PIPE_CONF_CHECK_M_N(dp_m2_n2);
11803         } else
11804                 PIPE_CONF_CHECK_M_N_ALT(dp_m_n, dp_m2_n2);
11805
11806         PIPE_CONF_CHECK_X(output_types);
11807
11808         PIPE_CONF_CHECK_I(base.adjusted_mode.crtc_hdisplay);
11809         PIPE_CONF_CHECK_I(base.adjusted_mode.crtc_htotal);
11810         PIPE_CONF_CHECK_I(base.adjusted_mode.crtc_hblank_start);
11811         PIPE_CONF_CHECK_I(base.adjusted_mode.crtc_hblank_end);
11812         PIPE_CONF_CHECK_I(base.adjusted_mode.crtc_hsync_start);
11813         PIPE_CONF_CHECK_I(base.adjusted_mode.crtc_hsync_end);
11814
11815         PIPE_CONF_CHECK_I(base.adjusted_mode.crtc_vdisplay);
11816         PIPE_CONF_CHECK_I(base.adjusted_mode.crtc_vtotal);
11817         PIPE_CONF_CHECK_I(base.adjusted_mode.crtc_vblank_start);
11818         PIPE_CONF_CHECK_I(base.adjusted_mode.crtc_vblank_end);
11819         PIPE_CONF_CHECK_I(base.adjusted_mode.crtc_vsync_start);
11820         PIPE_CONF_CHECK_I(base.adjusted_mode.crtc_vsync_end);
11821
11822         PIPE_CONF_CHECK_I(pixel_multiplier);
11823         PIPE_CONF_CHECK_I(output_format);
11824         PIPE_CONF_CHECK_BOOL(has_hdmi_sink);
11825         if ((INTEL_GEN(dev_priv) < 8 && !IS_HASWELL(dev_priv)) ||
11826             IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
11827                 PIPE_CONF_CHECK_BOOL(limited_color_range);
11828
11829         PIPE_CONF_CHECK_BOOL(hdmi_scrambling);
11830         PIPE_CONF_CHECK_BOOL(hdmi_high_tmds_clock_ratio);
11831         PIPE_CONF_CHECK_BOOL_INCOMPLETE(has_infoframe);
11832
11833         PIPE_CONF_CHECK_BOOL_INCOMPLETE(has_audio);
11834
11835         PIPE_CONF_CHECK_FLAGS(base.adjusted_mode.flags,
11836                               DRM_MODE_FLAG_INTERLACE);
11837
11838         if (!PIPE_CONF_QUIRK(PIPE_CONFIG_QUIRK_MODE_SYNC_FLAGS)) {
11839                 PIPE_CONF_CHECK_FLAGS(base.adjusted_mode.flags,
11840                                       DRM_MODE_FLAG_PHSYNC);
11841                 PIPE_CONF_CHECK_FLAGS(base.adjusted_mode.flags,
11842                                       DRM_MODE_FLAG_NHSYNC);
11843                 PIPE_CONF_CHECK_FLAGS(base.adjusted_mode.flags,
11844                                       DRM_MODE_FLAG_PVSYNC);
11845                 PIPE_CONF_CHECK_FLAGS(base.adjusted_mode.flags,
11846                                       DRM_MODE_FLAG_NVSYNC);
11847         }
11848
11849         PIPE_CONF_CHECK_X(gmch_pfit.control);
11850         /* pfit ratios are autocomputed by the hw on gen4+ */
11851         if (INTEL_GEN(dev_priv) < 4)
11852                 PIPE_CONF_CHECK_X(gmch_pfit.pgm_ratios);
11853         PIPE_CONF_CHECK_X(gmch_pfit.lvds_border_bits);
11854
11855         if (!adjust) {
11856                 PIPE_CONF_CHECK_I(pipe_src_w);
11857                 PIPE_CONF_CHECK_I(pipe_src_h);
11858
11859                 PIPE_CONF_CHECK_BOOL(pch_pfit.enabled);
11860                 if (current_config->pch_pfit.enabled) {
11861                         PIPE_CONF_CHECK_X(pch_pfit.pos);
11862                         PIPE_CONF_CHECK_X(pch_pfit.size);
11863                 }
11864
11865                 PIPE_CONF_CHECK_I(scaler_state.scaler_id);
11866                 PIPE_CONF_CHECK_CLOCK_FUZZY(pixel_rate);
11867         }
11868
11869         PIPE_CONF_CHECK_BOOL(double_wide);
11870
11871         PIPE_CONF_CHECK_P(shared_dpll);
11872         PIPE_CONF_CHECK_X(dpll_hw_state.dpll);
11873         PIPE_CONF_CHECK_X(dpll_hw_state.dpll_md);
11874         PIPE_CONF_CHECK_X(dpll_hw_state.fp0);
11875         PIPE_CONF_CHECK_X(dpll_hw_state.fp1);
11876         PIPE_CONF_CHECK_X(dpll_hw_state.wrpll);
11877         PIPE_CONF_CHECK_X(dpll_hw_state.spll);
11878         PIPE_CONF_CHECK_X(dpll_hw_state.ctrl1);
11879         PIPE_CONF_CHECK_X(dpll_hw_state.cfgcr1);
11880         PIPE_CONF_CHECK_X(dpll_hw_state.cfgcr2);
11881         PIPE_CONF_CHECK_X(dpll_hw_state.cfgcr0);
11882         PIPE_CONF_CHECK_X(dpll_hw_state.ebb0);
11883         PIPE_CONF_CHECK_X(dpll_hw_state.ebb4);
11884         PIPE_CONF_CHECK_X(dpll_hw_state.pll0);
11885         PIPE_CONF_CHECK_X(dpll_hw_state.pll1);
11886         PIPE_CONF_CHECK_X(dpll_hw_state.pll2);
11887         PIPE_CONF_CHECK_X(dpll_hw_state.pll3);
11888         PIPE_CONF_CHECK_X(dpll_hw_state.pll6);
11889         PIPE_CONF_CHECK_X(dpll_hw_state.pll8);
11890         PIPE_CONF_CHECK_X(dpll_hw_state.pll9);
11891         PIPE_CONF_CHECK_X(dpll_hw_state.pll10);
11892         PIPE_CONF_CHECK_X(dpll_hw_state.pcsdw12);
11893         PIPE_CONF_CHECK_X(dpll_hw_state.mg_refclkin_ctl);
11894         PIPE_CONF_CHECK_X(dpll_hw_state.mg_clktop2_coreclkctl1);
11895         PIPE_CONF_CHECK_X(dpll_hw_state.mg_clktop2_hsclkctl);
11896         PIPE_CONF_CHECK_X(dpll_hw_state.mg_pll_div0);
11897         PIPE_CONF_CHECK_X(dpll_hw_state.mg_pll_div1);
11898         PIPE_CONF_CHECK_X(dpll_hw_state.mg_pll_lf);
11899         PIPE_CONF_CHECK_X(dpll_hw_state.mg_pll_frac_lock);
11900         PIPE_CONF_CHECK_X(dpll_hw_state.mg_pll_ssc);
11901         PIPE_CONF_CHECK_X(dpll_hw_state.mg_pll_bias);
11902         PIPE_CONF_CHECK_X(dpll_hw_state.mg_pll_tdc_coldst_bias);
11903
11904         PIPE_CONF_CHECK_X(dsi_pll.ctrl);
11905         PIPE_CONF_CHECK_X(dsi_pll.div);
11906
11907         if (IS_G4X(dev_priv) || INTEL_GEN(dev_priv) >= 5)
11908                 PIPE_CONF_CHECK_I(pipe_bpp);
11909
11910         PIPE_CONF_CHECK_CLOCK_FUZZY(base.adjusted_mode.crtc_clock);
11911         PIPE_CONF_CHECK_CLOCK_FUZZY(port_clock);
11912
11913         PIPE_CONF_CHECK_I(min_voltage_level);
11914
11915 #undef PIPE_CONF_CHECK_X
11916 #undef PIPE_CONF_CHECK_I
11917 #undef PIPE_CONF_CHECK_BOOL
11918 #undef PIPE_CONF_CHECK_BOOL_INCOMPLETE
11919 #undef PIPE_CONF_CHECK_P
11920 #undef PIPE_CONF_CHECK_FLAGS
11921 #undef PIPE_CONF_CHECK_CLOCK_FUZZY
11922 #undef PIPE_CONF_QUIRK
11923
11924         return ret;
11925 }
11926
11927 static void intel_pipe_config_sanity_check(struct drm_i915_private *dev_priv,
11928                                            const struct intel_crtc_state *pipe_config)
11929 {
11930         if (pipe_config->has_pch_encoder) {
11931                 int fdi_dotclock = intel_dotclock_calculate(intel_fdi_link_freq(dev_priv, pipe_config),
11932                                                             &pipe_config->fdi_m_n);
11933                 int dotclock = pipe_config->base.adjusted_mode.crtc_clock;
11934
11935                 /*
11936                  * FDI already provided one idea for the dotclock.
11937                  * Yell if the encoder disagrees.
11938                  */
11939                 WARN(!intel_fuzzy_clock_check(fdi_dotclock, dotclock),
11940                      "FDI dotclock and encoder dotclock mismatch, fdi: %i, encoder: %i\n",
11941                      fdi_dotclock, dotclock);
11942         }
11943 }
11944
11945 static void verify_wm_state(struct drm_crtc *crtc,
11946                             struct drm_crtc_state *new_state)
11947 {
11948         struct drm_i915_private *dev_priv = to_i915(crtc->dev);
11949         struct skl_ddb_allocation hw_ddb, *sw_ddb;
11950         struct skl_pipe_wm hw_wm, *sw_wm;
11951         struct skl_plane_wm *hw_plane_wm, *sw_plane_wm;
11952         struct skl_ddb_entry *hw_ddb_entry, *sw_ddb_entry;
11953         struct skl_ddb_entry hw_ddb_y[I915_MAX_PLANES];
11954         struct skl_ddb_entry hw_ddb_uv[I915_MAX_PLANES];
11955         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
11956         const enum pipe pipe = intel_crtc->pipe;
11957         int plane, level, max_level = ilk_wm_max_level(dev_priv);
11958
11959         if (INTEL_GEN(dev_priv) < 9 || !new_state->active)
11960                 return;
11961
11962         skl_pipe_wm_get_hw_state(intel_crtc, &hw_wm);
11963         sw_wm = &to_intel_crtc_state(new_state)->wm.skl.optimal;
11964
11965         skl_pipe_ddb_get_hw_state(intel_crtc, hw_ddb_y, hw_ddb_uv);
11966
11967         skl_ddb_get_hw_state(dev_priv, &hw_ddb);
11968         sw_ddb = &dev_priv->wm.skl_hw.ddb;
11969
11970         if (INTEL_GEN(dev_priv) >= 11)
11971                 if (hw_ddb.enabled_slices != sw_ddb->enabled_slices)
11972                         DRM_ERROR("mismatch in DBUF Slices (expected %u, got %u)\n",
11973                                   sw_ddb->enabled_slices,
11974                                   hw_ddb.enabled_slices);
11975         /* planes */
11976         for_each_universal_plane(dev_priv, pipe, plane) {
11977                 hw_plane_wm = &hw_wm.planes[plane];
11978                 sw_plane_wm = &sw_wm->planes[plane];
11979
11980                 /* Watermarks */
11981                 for (level = 0; level <= max_level; level++) {
11982                         if (skl_wm_level_equals(&hw_plane_wm->wm[level],
11983                                                 &sw_plane_wm->wm[level]))
11984                                 continue;
11985
11986                         DRM_ERROR("mismatch in WM pipe %c plane %d level %d (expected e=%d b=%u l=%u, got e=%d b=%u l=%u)\n",
11987                                   pipe_name(pipe), plane + 1, level,
11988                                   sw_plane_wm->wm[level].plane_en,
11989                                   sw_plane_wm->wm[level].plane_res_b,
11990                                   sw_plane_wm->wm[level].plane_res_l,
11991                                   hw_plane_wm->wm[level].plane_en,
11992                                   hw_plane_wm->wm[level].plane_res_b,
11993                                   hw_plane_wm->wm[level].plane_res_l);
11994                 }
11995
11996                 if (!skl_wm_level_equals(&hw_plane_wm->trans_wm,
11997                                          &sw_plane_wm->trans_wm)) {
11998                         DRM_ERROR("mismatch in trans WM pipe %c plane %d (expected e=%d b=%u l=%u, got e=%d b=%u l=%u)\n",
11999                                   pipe_name(pipe), plane + 1,
12000                                   sw_plane_wm->trans_wm.plane_en,
12001                                   sw_plane_wm->trans_wm.plane_res_b,
12002                                   sw_plane_wm->trans_wm.plane_res_l,
12003                                   hw_plane_wm->trans_wm.plane_en,
12004                                   hw_plane_wm->trans_wm.plane_res_b,
12005                                   hw_plane_wm->trans_wm.plane_res_l);
12006                 }
12007
12008                 /* DDB */
12009                 hw_ddb_entry = &hw_ddb_y[plane];
12010                 sw_ddb_entry = &to_intel_crtc_state(new_state)->wm.skl.plane_ddb_y[plane];
12011
12012                 if (!skl_ddb_entry_equal(hw_ddb_entry, sw_ddb_entry)) {
12013                         DRM_ERROR("mismatch in DDB state pipe %c plane %d (expected (%u,%u), found (%u,%u))\n",
12014                                   pipe_name(pipe), plane + 1,
12015                                   sw_ddb_entry->start, sw_ddb_entry->end,
12016                                   hw_ddb_entry->start, hw_ddb_entry->end);
12017                 }
12018         }
12019
12020         /*
12021          * cursor
12022          * If the cursor plane isn't active, we may not have updated it's ddb
12023          * allocation. In that case since the ddb allocation will be updated
12024          * once the plane becomes visible, we can skip this check
12025          */
12026         if (1) {
12027                 hw_plane_wm = &hw_wm.planes[PLANE_CURSOR];
12028                 sw_plane_wm = &sw_wm->planes[PLANE_CURSOR];
12029
12030                 /* Watermarks */
12031                 for (level = 0; level <= max_level; level++) {
12032                         if (skl_wm_level_equals(&hw_plane_wm->wm[level],
12033                                                 &sw_plane_wm->wm[level]))
12034                                 continue;
12035
12036                         DRM_ERROR("mismatch in WM pipe %c cursor level %d (expected e=%d b=%u l=%u, got e=%d b=%u l=%u)\n",
12037                                   pipe_name(pipe), level,
12038                                   sw_plane_wm->wm[level].plane_en,
12039                                   sw_plane_wm->wm[level].plane_res_b,
12040                                   sw_plane_wm->wm[level].plane_res_l,
12041                                   hw_plane_wm->wm[level].plane_en,
12042                                   hw_plane_wm->wm[level].plane_res_b,
12043                                   hw_plane_wm->wm[level].plane_res_l);
12044                 }
12045
12046                 if (!skl_wm_level_equals(&hw_plane_wm->trans_wm,
12047                                          &sw_plane_wm->trans_wm)) {
12048                         DRM_ERROR("mismatch in trans WM pipe %c cursor (expected e=%d b=%u l=%u, got e=%d b=%u l=%u)\n",
12049                                   pipe_name(pipe),
12050                                   sw_plane_wm->trans_wm.plane_en,
12051                                   sw_plane_wm->trans_wm.plane_res_b,
12052                                   sw_plane_wm->trans_wm.plane_res_l,
12053                                   hw_plane_wm->trans_wm.plane_en,
12054                                   hw_plane_wm->trans_wm.plane_res_b,
12055                                   hw_plane_wm->trans_wm.plane_res_l);
12056                 }
12057
12058                 /* DDB */
12059                 hw_ddb_entry = &hw_ddb_y[PLANE_CURSOR];
12060                 sw_ddb_entry = &to_intel_crtc_state(new_state)->wm.skl.plane_ddb_y[PLANE_CURSOR];
12061
12062                 if (!skl_ddb_entry_equal(hw_ddb_entry, sw_ddb_entry)) {
12063                         DRM_ERROR("mismatch in DDB state pipe %c cursor (expected (%u,%u), found (%u,%u))\n",
12064                                   pipe_name(pipe),
12065                                   sw_ddb_entry->start, sw_ddb_entry->end,
12066                                   hw_ddb_entry->start, hw_ddb_entry->end);
12067                 }
12068         }
12069 }
12070
12071 static void
12072 verify_connector_state(struct drm_device *dev,
12073                        struct drm_atomic_state *state,
12074                        struct drm_crtc *crtc)
12075 {
12076         struct drm_connector *connector;
12077         struct drm_connector_state *new_conn_state;
12078         int i;
12079
12080         for_each_new_connector_in_state(state, connector, new_conn_state, i) {
12081                 struct drm_encoder *encoder = connector->encoder;
12082                 struct drm_crtc_state *crtc_state = NULL;
12083
12084                 if (new_conn_state->crtc != crtc)
12085                         continue;
12086
12087                 if (crtc)
12088                         crtc_state = drm_atomic_get_new_crtc_state(state, new_conn_state->crtc);
12089
12090                 intel_connector_verify_state(crtc_state, new_conn_state);
12091
12092                 I915_STATE_WARN(new_conn_state->best_encoder != encoder,
12093                      "connector's atomic encoder doesn't match legacy encoder\n");
12094         }
12095 }
12096
12097 static void
12098 verify_encoder_state(struct drm_device *dev, struct drm_atomic_state *state)
12099 {
12100         struct intel_encoder *encoder;
12101         struct drm_connector *connector;
12102         struct drm_connector_state *old_conn_state, *new_conn_state;
12103         int i;
12104
12105         for_each_intel_encoder(dev, encoder) {
12106                 bool enabled = false, found = false;
12107                 enum pipe pipe;
12108
12109                 DRM_DEBUG_KMS("[ENCODER:%d:%s]\n",
12110                               encoder->base.base.id,
12111                               encoder->base.name);
12112
12113                 for_each_oldnew_connector_in_state(state, connector, old_conn_state,
12114                                                    new_conn_state, i) {
12115                         if (old_conn_state->best_encoder == &encoder->base)
12116                                 found = true;
12117
12118                         if (new_conn_state->best_encoder != &encoder->base)
12119                                 continue;
12120                         found = enabled = true;
12121
12122                         I915_STATE_WARN(new_conn_state->crtc !=
12123                                         encoder->base.crtc,
12124                              "connector's crtc doesn't match encoder crtc\n");
12125                 }
12126
12127                 if (!found)
12128                         continue;
12129
12130                 I915_STATE_WARN(!!encoder->base.crtc != enabled,
12131                      "encoder's enabled state mismatch "
12132                      "(expected %i, found %i)\n",
12133                      !!encoder->base.crtc, enabled);
12134
12135                 if (!encoder->base.crtc) {
12136                         bool active;
12137
12138                         active = encoder->get_hw_state(encoder, &pipe);
12139                         I915_STATE_WARN(active,
12140                              "encoder detached but still enabled on pipe %c.\n",
12141                              pipe_name(pipe));
12142                 }
12143         }
12144 }
12145
12146 static void
12147 verify_crtc_state(struct drm_crtc *crtc,
12148                   struct drm_crtc_state *old_crtc_state,
12149                   struct drm_crtc_state *new_crtc_state)
12150 {
12151         struct drm_device *dev = crtc->dev;
12152         struct drm_i915_private *dev_priv = to_i915(dev);
12153         struct intel_encoder *encoder;
12154         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
12155         struct intel_crtc_state *pipe_config, *sw_config;
12156         struct drm_atomic_state *old_state;
12157         bool active;
12158
12159         old_state = old_crtc_state->state;
12160         __drm_atomic_helper_crtc_destroy_state(old_crtc_state);
12161         pipe_config = to_intel_crtc_state(old_crtc_state);
12162         memset(pipe_config, 0, sizeof(*pipe_config));
12163         pipe_config->base.crtc = crtc;
12164         pipe_config->base.state = old_state;
12165
12166         DRM_DEBUG_KMS("[CRTC:%d:%s]\n", crtc->base.id, crtc->name);
12167
12168         active = dev_priv->display.get_pipe_config(intel_crtc, pipe_config);
12169
12170         /* we keep both pipes enabled on 830 */
12171         if (IS_I830(dev_priv))
12172                 active = new_crtc_state->active;
12173
12174         I915_STATE_WARN(new_crtc_state->active != active,
12175              "crtc active state doesn't match with hw state "
12176              "(expected %i, found %i)\n", new_crtc_state->active, active);
12177
12178         I915_STATE_WARN(intel_crtc->active != new_crtc_state->active,
12179              "transitional active state does not match atomic hw state "
12180              "(expected %i, found %i)\n", new_crtc_state->active, intel_crtc->active);
12181
12182         for_each_encoder_on_crtc(dev, crtc, encoder) {
12183                 enum pipe pipe;
12184
12185                 active = encoder->get_hw_state(encoder, &pipe);
12186                 I915_STATE_WARN(active != new_crtc_state->active,
12187                         "[ENCODER:%i] active %i with crtc active %i\n",
12188                         encoder->base.base.id, active, new_crtc_state->active);
12189
12190                 I915_STATE_WARN(active && intel_crtc->pipe != pipe,
12191                                 "Encoder connected to wrong pipe %c\n",
12192                                 pipe_name(pipe));
12193
12194                 if (active)
12195                         encoder->get_config(encoder, pipe_config);
12196         }
12197
12198         intel_crtc_compute_pixel_rate(pipe_config);
12199
12200         if (!new_crtc_state->active)
12201                 return;
12202
12203         intel_pipe_config_sanity_check(dev_priv, pipe_config);
12204
12205         sw_config = to_intel_crtc_state(new_crtc_state);
12206         if (!intel_pipe_config_compare(dev_priv, sw_config,
12207                                        pipe_config, false)) {
12208                 I915_STATE_WARN(1, "pipe state doesn't match!\n");
12209                 intel_dump_pipe_config(intel_crtc, pipe_config,
12210                                        "[hw state]");
12211                 intel_dump_pipe_config(intel_crtc, sw_config,
12212                                        "[sw state]");
12213         }
12214 }
12215
12216 static void
12217 intel_verify_planes(struct intel_atomic_state *state)
12218 {
12219         struct intel_plane *plane;
12220         const struct intel_plane_state *plane_state;
12221         int i;
12222
12223         for_each_new_intel_plane_in_state(state, plane,
12224                                           plane_state, i)
12225                 assert_plane(plane, plane_state->base.visible);
12226 }
12227
12228 static void
12229 verify_single_dpll_state(struct drm_i915_private *dev_priv,
12230                          struct intel_shared_dpll *pll,
12231                          struct drm_crtc *crtc,
12232                          struct drm_crtc_state *new_state)
12233 {
12234         struct intel_dpll_hw_state dpll_hw_state;
12235         unsigned int crtc_mask;
12236         bool active;
12237
12238         memset(&dpll_hw_state, 0, sizeof(dpll_hw_state));
12239
12240         DRM_DEBUG_KMS("%s\n", pll->info->name);
12241
12242         active = pll->info->funcs->get_hw_state(dev_priv, pll, &dpll_hw_state);
12243
12244         if (!(pll->info->flags & INTEL_DPLL_ALWAYS_ON)) {
12245                 I915_STATE_WARN(!pll->on && pll->active_mask,
12246                      "pll in active use but not on in sw tracking\n");
12247                 I915_STATE_WARN(pll->on && !pll->active_mask,
12248                      "pll is on but not used by any active crtc\n");
12249                 I915_STATE_WARN(pll->on != active,
12250                      "pll on state mismatch (expected %i, found %i)\n",
12251                      pll->on, active);
12252         }
12253
12254         if (!crtc) {
12255                 I915_STATE_WARN(pll->active_mask & ~pll->state.crtc_mask,
12256                                 "more active pll users than references: %x vs %x\n",
12257                                 pll->active_mask, pll->state.crtc_mask);
12258
12259                 return;
12260         }
12261
12262         crtc_mask = drm_crtc_mask(crtc);
12263
12264         if (new_state->active)
12265                 I915_STATE_WARN(!(pll->active_mask & crtc_mask),
12266                                 "pll active mismatch (expected pipe %c in active mask 0x%02x)\n",
12267                                 pipe_name(drm_crtc_index(crtc)), pll->active_mask);
12268         else
12269                 I915_STATE_WARN(pll->active_mask & crtc_mask,
12270                                 "pll active mismatch (didn't expect pipe %c in active mask 0x%02x)\n",
12271                                 pipe_name(drm_crtc_index(crtc)), pll->active_mask);
12272
12273         I915_STATE_WARN(!(pll->state.crtc_mask & crtc_mask),
12274                         "pll enabled crtcs mismatch (expected 0x%x in 0x%02x)\n",
12275                         crtc_mask, pll->state.crtc_mask);
12276
12277         I915_STATE_WARN(pll->on && memcmp(&pll->state.hw_state,
12278                                           &dpll_hw_state,
12279                                           sizeof(dpll_hw_state)),
12280                         "pll hw state mismatch\n");
12281 }
12282
12283 static void
12284 verify_shared_dpll_state(struct drm_device *dev, struct drm_crtc *crtc,
12285                          struct drm_crtc_state *old_crtc_state,
12286                          struct drm_crtc_state *new_crtc_state)
12287 {
12288         struct drm_i915_private *dev_priv = to_i915(dev);
12289         struct intel_crtc_state *old_state = to_intel_crtc_state(old_crtc_state);
12290         struct intel_crtc_state *new_state = to_intel_crtc_state(new_crtc_state);
12291
12292         if (new_state->shared_dpll)
12293                 verify_single_dpll_state(dev_priv, new_state->shared_dpll, crtc, new_crtc_state);
12294
12295         if (old_state->shared_dpll &&
12296             old_state->shared_dpll != new_state->shared_dpll) {
12297                 unsigned int crtc_mask = drm_crtc_mask(crtc);
12298                 struct intel_shared_dpll *pll = old_state->shared_dpll;
12299
12300                 I915_STATE_WARN(pll->active_mask & crtc_mask,
12301                                 "pll active mismatch (didn't expect pipe %c in active mask)\n",
12302                                 pipe_name(drm_crtc_index(crtc)));
12303                 I915_STATE_WARN(pll->state.crtc_mask & crtc_mask,
12304                                 "pll enabled crtcs mismatch (found %x in enabled mask)\n",
12305                                 pipe_name(drm_crtc_index(crtc)));
12306         }
12307 }
12308
12309 static void
12310 intel_modeset_verify_crtc(struct drm_crtc *crtc,
12311                           struct drm_atomic_state *state,
12312                           struct drm_crtc_state *old_state,
12313                           struct drm_crtc_state *new_state)
12314 {
12315         if (!needs_modeset(new_state) &&
12316             !to_intel_crtc_state(new_state)->update_pipe)
12317                 return;
12318
12319         verify_wm_state(crtc, new_state);
12320         verify_connector_state(crtc->dev, state, crtc);
12321         verify_crtc_state(crtc, old_state, new_state);
12322         verify_shared_dpll_state(crtc->dev, crtc, old_state, new_state);
12323 }
12324
12325 static void
12326 verify_disabled_dpll_state(struct drm_device *dev)
12327 {
12328         struct drm_i915_private *dev_priv = to_i915(dev);
12329         int i;
12330
12331         for (i = 0; i < dev_priv->num_shared_dpll; i++)
12332                 verify_single_dpll_state(dev_priv, &dev_priv->shared_dplls[i], NULL, NULL);
12333 }
12334
12335 static void
12336 intel_modeset_verify_disabled(struct drm_device *dev,
12337                               struct drm_atomic_state *state)
12338 {
12339         verify_encoder_state(dev, state);
12340         verify_connector_state(dev, state, NULL);
12341         verify_disabled_dpll_state(dev);
12342 }
12343
12344 static void update_scanline_offset(const struct intel_crtc_state *crtc_state)
12345 {
12346         struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
12347         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
12348
12349         /*
12350          * The scanline counter increments at the leading edge of hsync.
12351          *
12352          * On most platforms it starts counting from vtotal-1 on the
12353          * first active line. That means the scanline counter value is
12354          * always one less than what we would expect. Ie. just after
12355          * start of vblank, which also occurs at start of hsync (on the
12356          * last active line), the scanline counter will read vblank_start-1.
12357          *
12358          * On gen2 the scanline counter starts counting from 1 instead
12359          * of vtotal-1, so we have to subtract one (or rather add vtotal-1
12360          * to keep the value positive), instead of adding one.
12361          *
12362          * On HSW+ the behaviour of the scanline counter depends on the output
12363          * type. For DP ports it behaves like most other platforms, but on HDMI
12364          * there's an extra 1 line difference. So we need to add two instead of
12365          * one to the value.
12366          *
12367          * On VLV/CHV DSI the scanline counter would appear to increment
12368          * approx. 1/3 of a scanline before start of vblank. Unfortunately
12369          * that means we can't tell whether we're in vblank or not while
12370          * we're on that particular line. We must still set scanline_offset
12371          * to 1 so that the vblank timestamps come out correct when we query
12372          * the scanline counter from within the vblank interrupt handler.
12373          * However if queried just before the start of vblank we'll get an
12374          * answer that's slightly in the future.
12375          */
12376         if (IS_GEN(dev_priv, 2)) {
12377                 const struct drm_display_mode *adjusted_mode = &crtc_state->base.adjusted_mode;
12378                 int vtotal;
12379
12380                 vtotal = adjusted_mode->crtc_vtotal;
12381                 if (adjusted_mode->flags & DRM_MODE_FLAG_INTERLACE)
12382                         vtotal /= 2;
12383
12384                 crtc->scanline_offset = vtotal - 1;
12385         } else if (HAS_DDI(dev_priv) &&
12386                    intel_crtc_has_type(crtc_state, INTEL_OUTPUT_HDMI)) {
12387                 crtc->scanline_offset = 2;
12388         } else
12389                 crtc->scanline_offset = 1;
12390 }
12391
12392 static void intel_modeset_clear_plls(struct drm_atomic_state *state)
12393 {
12394         struct drm_device *dev = state->dev;
12395         struct drm_i915_private *dev_priv = to_i915(dev);
12396         struct drm_crtc *crtc;
12397         struct drm_crtc_state *old_crtc_state, *new_crtc_state;
12398         int i;
12399
12400         if (!dev_priv->display.crtc_compute_clock)
12401                 return;
12402
12403         for_each_oldnew_crtc_in_state(state, crtc, old_crtc_state, new_crtc_state, i) {
12404                 struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
12405                 struct intel_shared_dpll *old_dpll =
12406                         to_intel_crtc_state(old_crtc_state)->shared_dpll;
12407
12408                 if (!needs_modeset(new_crtc_state))
12409                         continue;
12410
12411                 to_intel_crtc_state(new_crtc_state)->shared_dpll = NULL;
12412
12413                 if (!old_dpll)
12414                         continue;
12415
12416                 intel_release_shared_dpll(old_dpll, intel_crtc, state);
12417         }
12418 }
12419
12420 /*
12421  * This implements the workaround described in the "notes" section of the mode
12422  * set sequence documentation. When going from no pipes or single pipe to
12423  * multiple pipes, and planes are enabled after the pipe, we need to wait at
12424  * least 2 vblanks on the first pipe before enabling planes on the second pipe.
12425  */
12426 static int haswell_mode_set_planes_workaround(struct drm_atomic_state *state)
12427 {
12428         struct drm_crtc_state *crtc_state;
12429         struct intel_crtc *intel_crtc;
12430         struct drm_crtc *crtc;
12431         struct intel_crtc_state *first_crtc_state = NULL;
12432         struct intel_crtc_state *other_crtc_state = NULL;
12433         enum pipe first_pipe = INVALID_PIPE, enabled_pipe = INVALID_PIPE;
12434         int i;
12435
12436         /* look at all crtc's that are going to be enabled in during modeset */
12437         for_each_new_crtc_in_state(state, crtc, crtc_state, i) {
12438                 intel_crtc = to_intel_crtc(crtc);
12439
12440                 if (!crtc_state->active || !needs_modeset(crtc_state))
12441                         continue;
12442
12443                 if (first_crtc_state) {
12444                         other_crtc_state = to_intel_crtc_state(crtc_state);
12445                         break;
12446                 } else {
12447                         first_crtc_state = to_intel_crtc_state(crtc_state);
12448                         first_pipe = intel_crtc->pipe;
12449                 }
12450         }
12451
12452         /* No workaround needed? */
12453         if (!first_crtc_state)
12454                 return 0;
12455
12456         /* w/a possibly needed, check how many crtc's are already enabled. */
12457         for_each_intel_crtc(state->dev, intel_crtc) {
12458                 struct intel_crtc_state *pipe_config;
12459
12460                 pipe_config = intel_atomic_get_crtc_state(state, intel_crtc);
12461                 if (IS_ERR(pipe_config))
12462                         return PTR_ERR(pipe_config);
12463
12464                 pipe_config->hsw_workaround_pipe = INVALID_PIPE;
12465
12466                 if (!pipe_config->base.active ||
12467                     needs_modeset(&pipe_config->base))
12468                         continue;
12469
12470                 /* 2 or more enabled crtcs means no need for w/a */
12471                 if (enabled_pipe != INVALID_PIPE)
12472                         return 0;
12473
12474                 enabled_pipe = intel_crtc->pipe;
12475         }
12476
12477         if (enabled_pipe != INVALID_PIPE)
12478                 first_crtc_state->hsw_workaround_pipe = enabled_pipe;
12479         else if (other_crtc_state)
12480                 other_crtc_state->hsw_workaround_pipe = first_pipe;
12481
12482         return 0;
12483 }
12484
12485 static int intel_lock_all_pipes(struct drm_atomic_state *state)
12486 {
12487         struct drm_crtc *crtc;
12488
12489         /* Add all pipes to the state */
12490         for_each_crtc(state->dev, crtc) {
12491                 struct drm_crtc_state *crtc_state;
12492
12493                 crtc_state = drm_atomic_get_crtc_state(state, crtc);
12494                 if (IS_ERR(crtc_state))
12495                         return PTR_ERR(crtc_state);
12496         }
12497
12498         return 0;
12499 }
12500
12501 static int intel_modeset_all_pipes(struct drm_atomic_state *state)
12502 {
12503         struct drm_crtc *crtc;
12504
12505         /*
12506          * Add all pipes to the state, and force
12507          * a modeset on all the active ones.
12508          */
12509         for_each_crtc(state->dev, crtc) {
12510                 struct drm_crtc_state *crtc_state;
12511                 int ret;
12512
12513                 crtc_state = drm_atomic_get_crtc_state(state, crtc);
12514                 if (IS_ERR(crtc_state))
12515                         return PTR_ERR(crtc_state);
12516
12517                 if (!crtc_state->active || needs_modeset(crtc_state))
12518                         continue;
12519
12520                 crtc_state->mode_changed = true;
12521
12522                 ret = drm_atomic_add_affected_connectors(state, crtc);
12523                 if (ret)
12524                         return ret;
12525
12526                 ret = drm_atomic_add_affected_planes(state, crtc);
12527                 if (ret)
12528                         return ret;
12529         }
12530
12531         return 0;
12532 }
12533
12534 static int intel_modeset_checks(struct drm_atomic_state *state)
12535 {
12536         struct intel_atomic_state *intel_state = to_intel_atomic_state(state);
12537         struct drm_i915_private *dev_priv = to_i915(state->dev);
12538         struct drm_crtc *crtc;
12539         struct drm_crtc_state *old_crtc_state, *new_crtc_state;
12540         int ret = 0, i;
12541
12542         if (!check_digital_port_conflicts(state)) {
12543                 DRM_DEBUG_KMS("rejecting conflicting digital port configuration\n");
12544                 return -EINVAL;
12545         }
12546
12547         intel_state->modeset = true;
12548         intel_state->active_crtcs = dev_priv->active_crtcs;
12549         intel_state->cdclk.logical = dev_priv->cdclk.logical;
12550         intel_state->cdclk.actual = dev_priv->cdclk.actual;
12551
12552         for_each_oldnew_crtc_in_state(state, crtc, old_crtc_state, new_crtc_state, i) {
12553                 if (new_crtc_state->active)
12554                         intel_state->active_crtcs |= 1 << i;
12555                 else
12556                         intel_state->active_crtcs &= ~(1 << i);
12557
12558                 if (old_crtc_state->active != new_crtc_state->active)
12559                         intel_state->active_pipe_changes |= drm_crtc_mask(crtc);
12560         }
12561
12562         /*
12563          * See if the config requires any additional preparation, e.g.
12564          * to adjust global state with pipes off.  We need to do this
12565          * here so we can get the modeset_pipe updated config for the new
12566          * mode set on this crtc.  For other crtcs we need to use the
12567          * adjusted_mode bits in the crtc directly.
12568          */
12569         if (dev_priv->display.modeset_calc_cdclk) {
12570                 ret = dev_priv->display.modeset_calc_cdclk(state);
12571                 if (ret < 0)
12572                         return ret;
12573
12574                 /*
12575                  * Writes to dev_priv->cdclk.logical must protected by
12576                  * holding all the crtc locks, even if we don't end up
12577                  * touching the hardware
12578                  */
12579                 if (intel_cdclk_changed(&dev_priv->cdclk.logical,
12580                                         &intel_state->cdclk.logical)) {
12581                         ret = intel_lock_all_pipes(state);
12582                         if (ret < 0)
12583                                 return ret;
12584                 }
12585
12586                 /* All pipes must be switched off while we change the cdclk. */
12587                 if (intel_cdclk_needs_modeset(&dev_priv->cdclk.actual,
12588                                               &intel_state->cdclk.actual)) {
12589                         ret = intel_modeset_all_pipes(state);
12590                         if (ret < 0)
12591                                 return ret;
12592                 }
12593
12594                 DRM_DEBUG_KMS("New cdclk calculated to be logical %u kHz, actual %u kHz\n",
12595                               intel_state->cdclk.logical.cdclk,
12596                               intel_state->cdclk.actual.cdclk);
12597                 DRM_DEBUG_KMS("New voltage level calculated to be logical %u, actual %u\n",
12598                               intel_state->cdclk.logical.voltage_level,
12599                               intel_state->cdclk.actual.voltage_level);
12600         } else {
12601                 to_intel_atomic_state(state)->cdclk.logical = dev_priv->cdclk.logical;
12602         }
12603
12604         intel_modeset_clear_plls(state);
12605
12606         if (IS_HASWELL(dev_priv))
12607                 return haswell_mode_set_planes_workaround(state);
12608
12609         return 0;
12610 }
12611
12612 /*
12613  * Handle calculation of various watermark data at the end of the atomic check
12614  * phase.  The code here should be run after the per-crtc and per-plane 'check'
12615  * handlers to ensure that all derived state has been updated.
12616  */
12617 static int calc_watermark_data(struct intel_atomic_state *state)
12618 {
12619         struct drm_device *dev = state->base.dev;
12620         struct drm_i915_private *dev_priv = to_i915(dev);
12621
12622         /* Is there platform-specific watermark information to calculate? */
12623         if (dev_priv->display.compute_global_watermarks)
12624                 return dev_priv->display.compute_global_watermarks(state);
12625
12626         return 0;
12627 }
12628
12629 /**
12630  * intel_atomic_check - validate state object
12631  * @dev: drm device
12632  * @state: state to validate
12633  */
12634 static int intel_atomic_check(struct drm_device *dev,
12635                               struct drm_atomic_state *state)
12636 {
12637         struct drm_i915_private *dev_priv = to_i915(dev);
12638         struct intel_atomic_state *intel_state = to_intel_atomic_state(state);
12639         struct drm_crtc *crtc;
12640         struct drm_crtc_state *old_crtc_state, *crtc_state;
12641         int ret, i;
12642         bool any_ms = false;
12643
12644         /* Catch I915_MODE_FLAG_INHERITED */
12645         for_each_oldnew_crtc_in_state(state, crtc, old_crtc_state,
12646                                       crtc_state, i) {
12647                 if (crtc_state->mode.private_flags !=
12648                     old_crtc_state->mode.private_flags)
12649                         crtc_state->mode_changed = true;
12650         }
12651
12652         ret = drm_atomic_helper_check_modeset(dev, state);
12653         if (ret)
12654                 return ret;
12655
12656         for_each_oldnew_crtc_in_state(state, crtc, old_crtc_state, crtc_state, i) {
12657                 struct intel_crtc_state *pipe_config =
12658                         to_intel_crtc_state(crtc_state);
12659
12660                 if (!needs_modeset(crtc_state))
12661                         continue;
12662
12663                 if (!crtc_state->enable) {
12664                         any_ms = true;
12665                         continue;
12666                 }
12667
12668                 ret = intel_modeset_pipe_config(crtc, pipe_config);
12669                 if (ret == -EDEADLK)
12670                         return ret;
12671                 if (ret) {
12672                         intel_dump_pipe_config(to_intel_crtc(crtc),
12673                                                pipe_config, "[failed]");
12674                         return ret;
12675                 }
12676
12677                 if (i915_modparams.fastboot &&
12678                     intel_pipe_config_compare(dev_priv,
12679                                         to_intel_crtc_state(old_crtc_state),
12680                                         pipe_config, true)) {
12681                         crtc_state->mode_changed = false;
12682                         pipe_config->update_pipe = true;
12683                 }
12684
12685                 if (needs_modeset(crtc_state))
12686                         any_ms = true;
12687
12688                 intel_dump_pipe_config(to_intel_crtc(crtc), pipe_config,
12689                                        needs_modeset(crtc_state) ?
12690                                        "[modeset]" : "[fastset]");
12691         }
12692
12693         if (any_ms) {
12694                 ret = intel_modeset_checks(state);
12695
12696                 if (ret)
12697                         return ret;
12698         } else {
12699                 intel_state->cdclk.logical = dev_priv->cdclk.logical;
12700         }
12701
12702         ret = icl_add_linked_planes(intel_state);
12703         if (ret)
12704                 return ret;
12705
12706         ret = drm_atomic_helper_check_planes(dev, state);
12707         if (ret)
12708                 return ret;
12709
12710         intel_fbc_choose_crtc(dev_priv, intel_state);
12711         return calc_watermark_data(intel_state);
12712 }
12713
12714 static int intel_atomic_prepare_commit(struct drm_device *dev,
12715                                        struct drm_atomic_state *state)
12716 {
12717         return drm_atomic_helper_prepare_planes(dev, state);
12718 }
12719
12720 u32 intel_crtc_get_vblank_counter(struct intel_crtc *crtc)
12721 {
12722         struct drm_device *dev = crtc->base.dev;
12723
12724         if (!dev->max_vblank_count)
12725                 return (u32)drm_crtc_accurate_vblank_count(&crtc->base);
12726
12727         return dev->driver->get_vblank_counter(dev, crtc->pipe);
12728 }
12729
12730 static void intel_update_crtc(struct drm_crtc *crtc,
12731                               struct drm_atomic_state *state,
12732                               struct drm_crtc_state *old_crtc_state,
12733                               struct drm_crtc_state *new_crtc_state)
12734 {
12735         struct drm_device *dev = crtc->dev;
12736         struct drm_i915_private *dev_priv = to_i915(dev);
12737         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
12738         struct intel_crtc_state *pipe_config = to_intel_crtc_state(new_crtc_state);
12739         bool modeset = needs_modeset(new_crtc_state);
12740         struct intel_plane_state *new_plane_state =
12741                 intel_atomic_get_new_plane_state(to_intel_atomic_state(state),
12742                                                  to_intel_plane(crtc->primary));
12743
12744         if (modeset) {
12745                 update_scanline_offset(pipe_config);
12746                 dev_priv->display.crtc_enable(pipe_config, state);
12747
12748                 /* vblanks work again, re-enable pipe CRC. */
12749                 intel_crtc_enable_pipe_crc(intel_crtc);
12750         } else {
12751                 intel_pre_plane_update(to_intel_crtc_state(old_crtc_state),
12752                                        pipe_config);
12753         }
12754
12755         if (pipe_config->update_pipe && !pipe_config->enable_fbc)
12756                 intel_fbc_disable(intel_crtc);
12757         else if (new_plane_state)
12758                 intel_fbc_enable(intel_crtc, pipe_config, new_plane_state);
12759
12760         intel_begin_crtc_commit(crtc, old_crtc_state);
12761
12762         if (INTEL_GEN(dev_priv) >= 9)
12763                 skl_update_planes_on_crtc(to_intel_atomic_state(state), intel_crtc);
12764         else
12765                 i9xx_update_planes_on_crtc(to_intel_atomic_state(state), intel_crtc);
12766
12767         intel_finish_crtc_commit(crtc, old_crtc_state);
12768 }
12769
12770 static void intel_update_crtcs(struct drm_atomic_state *state)
12771 {
12772         struct drm_crtc *crtc;
12773         struct drm_crtc_state *old_crtc_state, *new_crtc_state;
12774         int i;
12775
12776         for_each_oldnew_crtc_in_state(state, crtc, old_crtc_state, new_crtc_state, i) {
12777                 if (!new_crtc_state->active)
12778                         continue;
12779
12780                 intel_update_crtc(crtc, state, old_crtc_state,
12781                                   new_crtc_state);
12782         }
12783 }
12784
12785 static void skl_update_crtcs(struct drm_atomic_state *state)
12786 {
12787         struct drm_i915_private *dev_priv = to_i915(state->dev);
12788         struct intel_atomic_state *intel_state = to_intel_atomic_state(state);
12789         struct drm_crtc *crtc;
12790         struct intel_crtc *intel_crtc;
12791         struct drm_crtc_state *old_crtc_state, *new_crtc_state;
12792         struct intel_crtc_state *cstate;
12793         unsigned int updated = 0;
12794         bool progress;
12795         enum pipe pipe;
12796         int i;
12797         u8 hw_enabled_slices = dev_priv->wm.skl_hw.ddb.enabled_slices;
12798         u8 required_slices = intel_state->wm_results.ddb.enabled_slices;
12799         struct skl_ddb_entry entries[I915_MAX_PIPES] = {};
12800
12801         for_each_oldnew_crtc_in_state(state, crtc, old_crtc_state, new_crtc_state, i)
12802                 /* ignore allocations for crtc's that have been turned off. */
12803                 if (new_crtc_state->active)
12804                         entries[i] = to_intel_crtc_state(old_crtc_state)->wm.skl.ddb;
12805
12806         /* If 2nd DBuf slice required, enable it here */
12807         if (INTEL_GEN(dev_priv) >= 11 && required_slices > hw_enabled_slices)
12808                 icl_dbuf_slices_update(dev_priv, required_slices);
12809
12810         /*
12811          * Whenever the number of active pipes changes, we need to make sure we
12812          * update the pipes in the right order so that their ddb allocations
12813          * never overlap with eachother inbetween CRTC updates. Otherwise we'll
12814          * cause pipe underruns and other bad stuff.
12815          */
12816         do {
12817                 progress = false;
12818
12819                 for_each_oldnew_crtc_in_state(state, crtc, old_crtc_state, new_crtc_state, i) {
12820                         bool vbl_wait = false;
12821                         unsigned int cmask = drm_crtc_mask(crtc);
12822
12823                         intel_crtc = to_intel_crtc(crtc);
12824                         cstate = to_intel_crtc_state(new_crtc_state);
12825                         pipe = intel_crtc->pipe;
12826
12827                         if (updated & cmask || !cstate->base.active)
12828                                 continue;
12829
12830                         if (skl_ddb_allocation_overlaps(&cstate->wm.skl.ddb,
12831                                                         entries,
12832                                                         INTEL_INFO(dev_priv)->num_pipes, i))
12833                                 continue;
12834
12835                         updated |= cmask;
12836                         entries[i] = cstate->wm.skl.ddb;
12837
12838                         /*
12839                          * If this is an already active pipe, it's DDB changed,
12840                          * and this isn't the last pipe that needs updating
12841                          * then we need to wait for a vblank to pass for the
12842                          * new ddb allocation to take effect.
12843                          */
12844                         if (!skl_ddb_entry_equal(&cstate->wm.skl.ddb,
12845                                                  &to_intel_crtc_state(old_crtc_state)->wm.skl.ddb) &&
12846                             !new_crtc_state->active_changed &&
12847                             intel_state->wm_results.dirty_pipes != updated)
12848                                 vbl_wait = true;
12849
12850                         intel_update_crtc(crtc, state, old_crtc_state,
12851                                           new_crtc_state);
12852
12853                         if (vbl_wait)
12854                                 intel_wait_for_vblank(dev_priv, pipe);
12855
12856                         progress = true;
12857                 }
12858         } while (progress);
12859
12860         /* If 2nd DBuf slice is no more required disable it */
12861         if (INTEL_GEN(dev_priv) >= 11 && required_slices < hw_enabled_slices)
12862                 icl_dbuf_slices_update(dev_priv, required_slices);
12863 }
12864
12865 static void intel_atomic_helper_free_state(struct drm_i915_private *dev_priv)
12866 {
12867         struct intel_atomic_state *state, *next;
12868         struct llist_node *freed;
12869
12870         freed = llist_del_all(&dev_priv->atomic_helper.free_list);
12871         llist_for_each_entry_safe(state, next, freed, freed)
12872                 drm_atomic_state_put(&state->base);
12873 }
12874
12875 static void intel_atomic_helper_free_state_worker(struct work_struct *work)
12876 {
12877         struct drm_i915_private *dev_priv =
12878                 container_of(work, typeof(*dev_priv), atomic_helper.free_work);
12879
12880         intel_atomic_helper_free_state(dev_priv);
12881 }
12882
12883 static void intel_atomic_commit_fence_wait(struct intel_atomic_state *intel_state)
12884 {
12885         struct wait_queue_entry wait_fence, wait_reset;
12886         struct drm_i915_private *dev_priv = to_i915(intel_state->base.dev);
12887
12888         init_wait_entry(&wait_fence, 0);
12889         init_wait_entry(&wait_reset, 0);
12890         for (;;) {
12891                 prepare_to_wait(&intel_state->commit_ready.wait,
12892                                 &wait_fence, TASK_UNINTERRUPTIBLE);
12893                 prepare_to_wait(&dev_priv->gpu_error.wait_queue,
12894                                 &wait_reset, TASK_UNINTERRUPTIBLE);
12895
12896
12897                 if (i915_sw_fence_done(&intel_state->commit_ready)
12898                     || test_bit(I915_RESET_MODESET, &dev_priv->gpu_error.flags))
12899                         break;
12900
12901                 schedule();
12902         }
12903         finish_wait(&intel_state->commit_ready.wait, &wait_fence);
12904         finish_wait(&dev_priv->gpu_error.wait_queue, &wait_reset);
12905 }
12906
12907 static void intel_atomic_cleanup_work(struct work_struct *work)
12908 {
12909         struct drm_atomic_state *state =
12910                 container_of(work, struct drm_atomic_state, commit_work);
12911         struct drm_i915_private *i915 = to_i915(state->dev);
12912
12913         drm_atomic_helper_cleanup_planes(&i915->drm, state);
12914         drm_atomic_helper_commit_cleanup_done(state);
12915         drm_atomic_state_put(state);
12916
12917         intel_atomic_helper_free_state(i915);
12918 }
12919
12920 static void intel_atomic_commit_tail(struct drm_atomic_state *state)
12921 {
12922         struct drm_device *dev = state->dev;
12923         struct intel_atomic_state *intel_state = to_intel_atomic_state(state);
12924         struct drm_i915_private *dev_priv = to_i915(dev);
12925         struct drm_crtc_state *old_crtc_state, *new_crtc_state;
12926         struct intel_crtc_state *new_intel_crtc_state, *old_intel_crtc_state;
12927         struct drm_crtc *crtc;
12928         struct intel_crtc *intel_crtc;
12929         u64 put_domains[I915_MAX_PIPES] = {};
12930         int i;
12931
12932         intel_atomic_commit_fence_wait(intel_state);
12933
12934         drm_atomic_helper_wait_for_dependencies(state);
12935
12936         if (intel_state->modeset)
12937                 intel_display_power_get(dev_priv, POWER_DOMAIN_MODESET);
12938
12939         for_each_oldnew_crtc_in_state(state, crtc, old_crtc_state, new_crtc_state, i) {
12940                 old_intel_crtc_state = to_intel_crtc_state(old_crtc_state);
12941                 new_intel_crtc_state = to_intel_crtc_state(new_crtc_state);
12942                 intel_crtc = to_intel_crtc(crtc);
12943
12944                 if (needs_modeset(new_crtc_state) ||
12945                     to_intel_crtc_state(new_crtc_state)->update_pipe) {
12946
12947                         put_domains[intel_crtc->pipe] =
12948                                 modeset_get_crtc_power_domains(crtc,
12949                                         new_intel_crtc_state);
12950                 }
12951
12952                 if (!needs_modeset(new_crtc_state))
12953                         continue;
12954
12955                 intel_pre_plane_update(old_intel_crtc_state, new_intel_crtc_state);
12956
12957                 if (old_crtc_state->active) {
12958                         intel_crtc_disable_planes(intel_state, intel_crtc);
12959
12960                         /*
12961                          * We need to disable pipe CRC before disabling the pipe,
12962                          * or we race against vblank off.
12963                          */
12964                         intel_crtc_disable_pipe_crc(intel_crtc);
12965
12966                         dev_priv->display.crtc_disable(old_intel_crtc_state, state);
12967                         intel_crtc->active = false;
12968                         intel_fbc_disable(intel_crtc);
12969                         intel_disable_shared_dpll(old_intel_crtc_state);
12970
12971                         /*
12972                          * Underruns don't always raise
12973                          * interrupts, so check manually.
12974                          */
12975                         intel_check_cpu_fifo_underruns(dev_priv);
12976                         intel_check_pch_fifo_underruns(dev_priv);
12977
12978                         /* FIXME unify this for all platforms */
12979                         if (!new_crtc_state->active &&
12980                             !HAS_GMCH_DISPLAY(dev_priv) &&
12981                             dev_priv->display.initial_watermarks)
12982                                 dev_priv->display.initial_watermarks(intel_state,
12983                                                                      new_intel_crtc_state);
12984                 }
12985         }
12986
12987         /* FIXME: Eventually get rid of our intel_crtc->config pointer */
12988         for_each_new_crtc_in_state(state, crtc, new_crtc_state, i)
12989                 to_intel_crtc(crtc)->config = to_intel_crtc_state(new_crtc_state);
12990
12991         if (intel_state->modeset) {
12992                 drm_atomic_helper_update_legacy_modeset_state(state->dev, state);
12993
12994                 intel_set_cdclk(dev_priv, &dev_priv->cdclk.actual);
12995
12996                 /*
12997                  * SKL workaround: bspec recommends we disable the SAGV when we
12998                  * have more then one pipe enabled
12999                  */
13000                 if (!intel_can_enable_sagv(state))
13001                         intel_disable_sagv(dev_priv);
13002
13003                 intel_modeset_verify_disabled(dev, state);
13004         }
13005
13006         /* Complete the events for pipes that have now been disabled */
13007         for_each_new_crtc_in_state(state, crtc, new_crtc_state, i) {
13008                 bool modeset = needs_modeset(new_crtc_state);
13009
13010                 /* Complete events for now disable pipes here. */
13011                 if (modeset && !new_crtc_state->active && new_crtc_state->event) {
13012                         spin_lock_irq(&dev->event_lock);
13013                         drm_crtc_send_vblank_event(crtc, new_crtc_state->event);
13014                         spin_unlock_irq(&dev->event_lock);
13015
13016                         new_crtc_state->event = NULL;
13017                 }
13018         }
13019
13020         /* Now enable the clocks, plane, pipe, and connectors that we set up. */
13021         dev_priv->display.update_crtcs(state);
13022
13023         /* FIXME: We should call drm_atomic_helper_commit_hw_done() here
13024          * already, but still need the state for the delayed optimization. To
13025          * fix this:
13026          * - wrap the optimization/post_plane_update stuff into a per-crtc work.
13027          * - schedule that vblank worker _before_ calling hw_done
13028          * - at the start of commit_tail, cancel it _synchrously
13029          * - switch over to the vblank wait helper in the core after that since
13030          *   we don't need out special handling any more.
13031          */
13032         drm_atomic_helper_wait_for_flip_done(dev, state);
13033
13034         /*
13035          * Now that the vblank has passed, we can go ahead and program the
13036          * optimal watermarks on platforms that need two-step watermark
13037          * programming.
13038          *
13039          * TODO: Move this (and other cleanup) to an async worker eventually.
13040          */
13041         for_each_new_crtc_in_state(state, crtc, new_crtc_state, i) {
13042                 new_intel_crtc_state = to_intel_crtc_state(new_crtc_state);
13043
13044                 if (dev_priv->display.optimize_watermarks)
13045                         dev_priv->display.optimize_watermarks(intel_state,
13046                                                               new_intel_crtc_state);
13047         }
13048
13049         for_each_oldnew_crtc_in_state(state, crtc, old_crtc_state, new_crtc_state, i) {
13050                 intel_post_plane_update(to_intel_crtc_state(old_crtc_state));
13051
13052                 if (put_domains[i])
13053                         modeset_put_power_domains(dev_priv, put_domains[i]);
13054
13055                 intel_modeset_verify_crtc(crtc, state, old_crtc_state, new_crtc_state);
13056         }
13057
13058         if (intel_state->modeset)
13059                 intel_verify_planes(intel_state);
13060
13061         if (intel_state->modeset && intel_can_enable_sagv(state))
13062                 intel_enable_sagv(dev_priv);
13063
13064         drm_atomic_helper_commit_hw_done(state);
13065
13066         if (intel_state->modeset) {
13067                 /* As one of the primary mmio accessors, KMS has a high
13068                  * likelihood of triggering bugs in unclaimed access. After we
13069                  * finish modesetting, see if an error has been flagged, and if
13070                  * so enable debugging for the next modeset - and hope we catch
13071                  * the culprit.
13072                  */
13073                 intel_uncore_arm_unclaimed_mmio_detection(dev_priv);
13074                 intel_display_power_put(dev_priv, POWER_DOMAIN_MODESET);
13075         }
13076
13077         /*
13078          * Defer the cleanup of the old state to a separate worker to not
13079          * impede the current task (userspace for blocking modesets) that
13080          * are executed inline. For out-of-line asynchronous modesets/flips,
13081          * deferring to a new worker seems overkill, but we would place a
13082          * schedule point (cond_resched()) here anyway to keep latencies
13083          * down.
13084          */
13085         INIT_WORK(&state->commit_work, intel_atomic_cleanup_work);
13086         queue_work(system_highpri_wq, &state->commit_work);
13087 }
13088
13089 static void intel_atomic_commit_work(struct work_struct *work)
13090 {
13091         struct drm_atomic_state *state =
13092                 container_of(work, struct drm_atomic_state, commit_work);
13093
13094         intel_atomic_commit_tail(state);
13095 }
13096
13097 static int __i915_sw_fence_call
13098 intel_atomic_commit_ready(struct i915_sw_fence *fence,
13099                           enum i915_sw_fence_notify notify)
13100 {
13101         struct intel_atomic_state *state =
13102                 container_of(fence, struct intel_atomic_state, commit_ready);
13103
13104         switch (notify) {
13105         case FENCE_COMPLETE:
13106                 /* we do blocking waits in the worker, nothing to do here */
13107                 break;
13108         case FENCE_FREE:
13109                 {
13110                         struct intel_atomic_helper *helper =
13111                                 &to_i915(state->base.dev)->atomic_helper;
13112
13113                         if (llist_add(&state->freed, &helper->free_list))
13114                                 schedule_work(&helper->free_work);
13115                         break;
13116                 }
13117         }
13118
13119         return NOTIFY_DONE;
13120 }
13121
13122 static void intel_atomic_track_fbs(struct drm_atomic_state *state)
13123 {
13124         struct drm_plane_state *old_plane_state, *new_plane_state;
13125         struct drm_plane *plane;
13126         int i;
13127
13128         for_each_oldnew_plane_in_state(state, plane, old_plane_state, new_plane_state, i)
13129                 i915_gem_track_fb(intel_fb_obj(old_plane_state->fb),
13130                                   intel_fb_obj(new_plane_state->fb),
13131                                   to_intel_plane(plane)->frontbuffer_bit);
13132 }
13133
13134 /**
13135  * intel_atomic_commit - commit validated state object
13136  * @dev: DRM device
13137  * @state: the top-level driver state object
13138  * @nonblock: nonblocking commit
13139  *
13140  * This function commits a top-level state object that has been validated
13141  * with drm_atomic_helper_check().
13142  *
13143  * RETURNS
13144  * Zero for success or -errno.
13145  */
13146 static int intel_atomic_commit(struct drm_device *dev,
13147                                struct drm_atomic_state *state,
13148                                bool nonblock)
13149 {
13150         struct intel_atomic_state *intel_state = to_intel_atomic_state(state);
13151         struct drm_i915_private *dev_priv = to_i915(dev);
13152         int ret = 0;
13153
13154         drm_atomic_state_get(state);
13155         i915_sw_fence_init(&intel_state->commit_ready,
13156                            intel_atomic_commit_ready);
13157
13158         /*
13159          * The intel_legacy_cursor_update() fast path takes care
13160          * of avoiding the vblank waits for simple cursor
13161          * movement and flips. For cursor on/off and size changes,
13162          * we want to perform the vblank waits so that watermark
13163          * updates happen during the correct frames. Gen9+ have
13164          * double buffered watermarks and so shouldn't need this.
13165          *
13166          * Unset state->legacy_cursor_update before the call to
13167          * drm_atomic_helper_setup_commit() because otherwise
13168          * drm_atomic_helper_wait_for_flip_done() is a noop and
13169          * we get FIFO underruns because we didn't wait
13170          * for vblank.
13171          *
13172          * FIXME doing watermarks and fb cleanup from a vblank worker
13173          * (assuming we had any) would solve these problems.
13174          */
13175         if (INTEL_GEN(dev_priv) < 9 && state->legacy_cursor_update) {
13176                 struct intel_crtc_state *new_crtc_state;
13177                 struct intel_crtc *crtc;
13178                 int i;
13179
13180                 for_each_new_intel_crtc_in_state(intel_state, crtc, new_crtc_state, i)
13181                         if (new_crtc_state->wm.need_postvbl_update ||
13182                             new_crtc_state->update_wm_post)
13183                                 state->legacy_cursor_update = false;
13184         }
13185
13186         ret = intel_atomic_prepare_commit(dev, state);
13187         if (ret) {
13188                 DRM_DEBUG_ATOMIC("Preparing state failed with %i\n", ret);
13189                 i915_sw_fence_commit(&intel_state->commit_ready);
13190                 return ret;
13191         }
13192
13193         ret = drm_atomic_helper_setup_commit(state, nonblock);
13194         if (!ret)
13195                 ret = drm_atomic_helper_swap_state(state, true);
13196
13197         if (ret) {
13198                 i915_sw_fence_commit(&intel_state->commit_ready);
13199
13200                 drm_atomic_helper_cleanup_planes(dev, state);
13201                 return ret;
13202         }
13203         dev_priv->wm.distrust_bios_wm = false;
13204         intel_shared_dpll_swap_state(state);
13205         intel_atomic_track_fbs(state);
13206
13207         if (intel_state->modeset) {
13208                 memcpy(dev_priv->min_cdclk, intel_state->min_cdclk,
13209                        sizeof(intel_state->min_cdclk));
13210                 memcpy(dev_priv->min_voltage_level,
13211                        intel_state->min_voltage_level,
13212                        sizeof(intel_state->min_voltage_level));
13213                 dev_priv->active_crtcs = intel_state->active_crtcs;
13214                 dev_priv->cdclk.logical = intel_state->cdclk.logical;
13215                 dev_priv->cdclk.actual = intel_state->cdclk.actual;
13216         }
13217
13218         drm_atomic_state_get(state);
13219         INIT_WORK(&state->commit_work, intel_atomic_commit_work);
13220
13221         i915_sw_fence_commit(&intel_state->commit_ready);
13222         if (nonblock && intel_state->modeset) {
13223                 queue_work(dev_priv->modeset_wq, &state->commit_work);
13224         } else if (nonblock) {
13225                 queue_work(system_unbound_wq, &state->commit_work);
13226         } else {
13227                 if (intel_state->modeset)
13228                         flush_workqueue(dev_priv->modeset_wq);
13229                 intel_atomic_commit_tail(state);
13230         }
13231
13232         return 0;
13233 }
13234
13235 static const struct drm_crtc_funcs intel_crtc_funcs = {
13236         .gamma_set = drm_atomic_helper_legacy_gamma_set,
13237         .set_config = drm_atomic_helper_set_config,
13238         .destroy = intel_crtc_destroy,
13239         .page_flip = drm_atomic_helper_page_flip,
13240         .atomic_duplicate_state = intel_crtc_duplicate_state,
13241         .atomic_destroy_state = intel_crtc_destroy_state,
13242         .set_crc_source = intel_crtc_set_crc_source,
13243         .verify_crc_source = intel_crtc_verify_crc_source,
13244         .get_crc_sources = intel_crtc_get_crc_sources,
13245 };
13246
13247 struct wait_rps_boost {
13248         struct wait_queue_entry wait;
13249
13250         struct drm_crtc *crtc;
13251         struct i915_request *request;
13252 };
13253
13254 static int do_rps_boost(struct wait_queue_entry *_wait,
13255                         unsigned mode, int sync, void *key)
13256 {
13257         struct wait_rps_boost *wait = container_of(_wait, typeof(*wait), wait);
13258         struct i915_request *rq = wait->request;
13259
13260         /*
13261          * If we missed the vblank, but the request is already running it
13262          * is reasonable to assume that it will complete before the next
13263          * vblank without our intervention, so leave RPS alone.
13264          */
13265         if (!i915_request_started(rq))
13266                 gen6_rps_boost(rq, NULL);
13267         i915_request_put(rq);
13268
13269         drm_crtc_vblank_put(wait->crtc);
13270
13271         list_del(&wait->wait.entry);
13272         kfree(wait);
13273         return 1;
13274 }
13275
13276 static void add_rps_boost_after_vblank(struct drm_crtc *crtc,
13277                                        struct dma_fence *fence)
13278 {
13279         struct wait_rps_boost *wait;
13280
13281         if (!dma_fence_is_i915(fence))
13282                 return;
13283
13284         if (INTEL_GEN(to_i915(crtc->dev)) < 6)
13285                 return;
13286
13287         if (drm_crtc_vblank_get(crtc))
13288                 return;
13289
13290         wait = kmalloc(sizeof(*wait), GFP_KERNEL);
13291         if (!wait) {
13292                 drm_crtc_vblank_put(crtc);
13293                 return;
13294         }
13295
13296         wait->request = to_request(dma_fence_get(fence));
13297         wait->crtc = crtc;
13298
13299         wait->wait.func = do_rps_boost;
13300         wait->wait.flags = 0;
13301
13302         add_wait_queue(drm_crtc_vblank_waitqueue(crtc), &wait->wait);
13303 }
13304
13305 static int intel_plane_pin_fb(struct intel_plane_state *plane_state)
13306 {
13307         struct intel_plane *plane = to_intel_plane(plane_state->base.plane);
13308         struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
13309         struct drm_framebuffer *fb = plane_state->base.fb;
13310         struct i915_vma *vma;
13311
13312         if (plane->id == PLANE_CURSOR &&
13313             INTEL_INFO(dev_priv)->display.cursor_needs_physical) {
13314                 struct drm_i915_gem_object *obj = intel_fb_obj(fb);
13315                 const int align = intel_cursor_alignment(dev_priv);
13316                 int err;
13317
13318                 err = i915_gem_object_attach_phys(obj, align);
13319                 if (err)
13320                         return err;
13321         }
13322
13323         vma = intel_pin_and_fence_fb_obj(fb,
13324                                          &plane_state->view,
13325                                          intel_plane_uses_fence(plane_state),
13326                                          &plane_state->flags);
13327         if (IS_ERR(vma))
13328                 return PTR_ERR(vma);
13329
13330         plane_state->vma = vma;
13331
13332         return 0;
13333 }
13334
13335 static void intel_plane_unpin_fb(struct intel_plane_state *old_plane_state)
13336 {
13337         struct i915_vma *vma;
13338
13339         vma = fetch_and_zero(&old_plane_state->vma);
13340         if (vma)
13341                 intel_unpin_fb_vma(vma, old_plane_state->flags);
13342 }
13343
13344 static void fb_obj_bump_render_priority(struct drm_i915_gem_object *obj)
13345 {
13346         struct i915_sched_attr attr = {
13347                 .priority = I915_PRIORITY_DISPLAY,
13348         };
13349
13350         i915_gem_object_wait_priority(obj, 0, &attr);
13351 }
13352
13353 /**
13354  * intel_prepare_plane_fb - Prepare fb for usage on plane
13355  * @plane: drm plane to prepare for
13356  * @new_state: the plane state being prepared
13357  *
13358  * Prepares a framebuffer for usage on a display plane.  Generally this
13359  * involves pinning the underlying object and updating the frontbuffer tracking
13360  * bits.  Some older platforms need special physical address handling for
13361  * cursor planes.
13362  *
13363  * Must be called with struct_mutex held.
13364  *
13365  * Returns 0 on success, negative error code on failure.
13366  */
13367 int
13368 intel_prepare_plane_fb(struct drm_plane *plane,
13369                        struct drm_plane_state *new_state)
13370 {
13371         struct intel_atomic_state *intel_state =
13372                 to_intel_atomic_state(new_state->state);
13373         struct drm_i915_private *dev_priv = to_i915(plane->dev);
13374         struct drm_framebuffer *fb = new_state->fb;
13375         struct drm_i915_gem_object *obj = intel_fb_obj(fb);
13376         struct drm_i915_gem_object *old_obj = intel_fb_obj(plane->state->fb);
13377         int ret;
13378
13379         if (old_obj) {
13380                 struct drm_crtc_state *crtc_state =
13381                         drm_atomic_get_new_crtc_state(new_state->state,
13382                                                       plane->state->crtc);
13383
13384                 /* Big Hammer, we also need to ensure that any pending
13385                  * MI_WAIT_FOR_EVENT inside a user batch buffer on the
13386                  * current scanout is retired before unpinning the old
13387                  * framebuffer. Note that we rely on userspace rendering
13388                  * into the buffer attached to the pipe they are waiting
13389                  * on. If not, userspace generates a GPU hang with IPEHR
13390                  * point to the MI_WAIT_FOR_EVENT.
13391                  *
13392                  * This should only fail upon a hung GPU, in which case we
13393                  * can safely continue.
13394                  */
13395                 if (needs_modeset(crtc_state)) {
13396                         ret = i915_sw_fence_await_reservation(&intel_state->commit_ready,
13397                                                               old_obj->resv, NULL,
13398                                                               false, 0,
13399                                                               GFP_KERNEL);
13400                         if (ret < 0)
13401                                 return ret;
13402                 }
13403         }
13404
13405         if (new_state->fence) { /* explicit fencing */
13406                 ret = i915_sw_fence_await_dma_fence(&intel_state->commit_ready,
13407                                                     new_state->fence,
13408                                                     I915_FENCE_TIMEOUT,
13409                                                     GFP_KERNEL);
13410                 if (ret < 0)
13411                         return ret;
13412         }
13413
13414         if (!obj)
13415                 return 0;
13416
13417         ret = i915_gem_object_pin_pages(obj);
13418         if (ret)
13419                 return ret;
13420
13421         ret = mutex_lock_interruptible(&dev_priv->drm.struct_mutex);
13422         if (ret) {
13423                 i915_gem_object_unpin_pages(obj);
13424                 return ret;
13425         }
13426
13427         ret = intel_plane_pin_fb(to_intel_plane_state(new_state));
13428
13429         mutex_unlock(&dev_priv->drm.struct_mutex);
13430         i915_gem_object_unpin_pages(obj);
13431         if (ret)
13432                 return ret;
13433
13434         fb_obj_bump_render_priority(obj);
13435         intel_fb_obj_flush(obj, ORIGIN_DIRTYFB);
13436
13437         if (!new_state->fence) { /* implicit fencing */
13438                 struct dma_fence *fence;
13439
13440                 ret = i915_sw_fence_await_reservation(&intel_state->commit_ready,
13441                                                       obj->resv, NULL,
13442                                                       false, I915_FENCE_TIMEOUT,
13443                                                       GFP_KERNEL);
13444                 if (ret < 0)
13445                         return ret;
13446
13447                 fence = reservation_object_get_excl_rcu(obj->resv);
13448                 if (fence) {
13449                         add_rps_boost_after_vblank(new_state->crtc, fence);
13450                         dma_fence_put(fence);
13451                 }
13452         } else {
13453                 add_rps_boost_after_vblank(new_state->crtc, new_state->fence);
13454         }
13455
13456         /*
13457          * We declare pageflips to be interactive and so merit a small bias
13458          * towards upclocking to deliver the frame on time. By only changing
13459          * the RPS thresholds to sample more regularly and aim for higher
13460          * clocks we can hopefully deliver low power workloads (like kodi)
13461          * that are not quite steady state without resorting to forcing
13462          * maximum clocks following a vblank miss (see do_rps_boost()).
13463          */
13464         if (!intel_state->rps_interactive) {
13465                 intel_rps_mark_interactive(dev_priv, true);
13466                 intel_state->rps_interactive = true;
13467         }
13468
13469         return 0;
13470 }
13471
13472 /**
13473  * intel_cleanup_plane_fb - Cleans up an fb after plane use
13474  * @plane: drm plane to clean up for
13475  * @old_state: the state from the previous modeset
13476  *
13477  * Cleans up a framebuffer that has just been removed from a plane.
13478  *
13479  * Must be called with struct_mutex held.
13480  */
13481 void
13482 intel_cleanup_plane_fb(struct drm_plane *plane,
13483                        struct drm_plane_state *old_state)
13484 {
13485         struct intel_atomic_state *intel_state =
13486                 to_intel_atomic_state(old_state->state);
13487         struct drm_i915_private *dev_priv = to_i915(plane->dev);
13488
13489         if (intel_state->rps_interactive) {
13490                 intel_rps_mark_interactive(dev_priv, false);
13491                 intel_state->rps_interactive = false;
13492         }
13493
13494         /* Should only be called after a successful intel_prepare_plane_fb()! */
13495         mutex_lock(&dev_priv->drm.struct_mutex);
13496         intel_plane_unpin_fb(to_intel_plane_state(old_state));
13497         mutex_unlock(&dev_priv->drm.struct_mutex);
13498 }
13499
13500 int
13501 skl_max_scale(const struct intel_crtc_state *crtc_state,
13502               u32 pixel_format)
13503 {
13504         struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
13505         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
13506         int max_scale, mult;
13507         int crtc_clock, max_dotclk, tmpclk1, tmpclk2;
13508
13509         if (!crtc_state->base.enable)
13510                 return DRM_PLANE_HELPER_NO_SCALING;
13511
13512         crtc_clock = crtc_state->base.adjusted_mode.crtc_clock;
13513         max_dotclk = to_intel_atomic_state(crtc_state->base.state)->cdclk.logical.cdclk;
13514
13515         if (IS_GEMINILAKE(dev_priv) || INTEL_GEN(dev_priv) >= 10)
13516                 max_dotclk *= 2;
13517
13518         if (WARN_ON_ONCE(!crtc_clock || max_dotclk < crtc_clock))
13519                 return DRM_PLANE_HELPER_NO_SCALING;
13520
13521         /*
13522          * skl max scale is lower of:
13523          *    close to 3 but not 3, -1 is for that purpose
13524          *            or
13525          *    cdclk/crtc_clock
13526          */
13527         mult = pixel_format == DRM_FORMAT_NV12 ? 2 : 3;
13528         tmpclk1 = (1 << 16) * mult - 1;
13529         tmpclk2 = (1 << 8) * ((max_dotclk << 8) / crtc_clock);
13530         max_scale = min(tmpclk1, tmpclk2);
13531
13532         return max_scale;
13533 }
13534
13535 static void intel_begin_crtc_commit(struct drm_crtc *crtc,
13536                                     struct drm_crtc_state *old_crtc_state)
13537 {
13538         struct drm_device *dev = crtc->dev;
13539         struct drm_i915_private *dev_priv = to_i915(dev);
13540         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
13541         struct intel_crtc_state *old_intel_cstate =
13542                 to_intel_crtc_state(old_crtc_state);
13543         struct intel_atomic_state *old_intel_state =
13544                 to_intel_atomic_state(old_crtc_state->state);
13545         struct intel_crtc_state *intel_cstate =
13546                 intel_atomic_get_new_crtc_state(old_intel_state, intel_crtc);
13547         bool modeset = needs_modeset(&intel_cstate->base);
13548
13549         if (!modeset &&
13550             (intel_cstate->base.color_mgmt_changed ||
13551              intel_cstate->update_pipe)) {
13552                 intel_color_set_csc(intel_cstate);
13553                 intel_color_load_luts(intel_cstate);
13554         }
13555
13556         /* Perform vblank evasion around commit operation */
13557         intel_pipe_update_start(intel_cstate);
13558
13559         if (modeset)
13560                 goto out;
13561
13562         if (intel_cstate->update_pipe)
13563                 intel_update_pipe_config(old_intel_cstate, intel_cstate);
13564         else if (INTEL_GEN(dev_priv) >= 9)
13565                 skl_detach_scalers(intel_cstate);
13566
13567 out:
13568         if (dev_priv->display.atomic_update_watermarks)
13569                 dev_priv->display.atomic_update_watermarks(old_intel_state,
13570                                                            intel_cstate);
13571 }
13572
13573 void intel_crtc_arm_fifo_underrun(struct intel_crtc *crtc,
13574                                   struct intel_crtc_state *crtc_state)
13575 {
13576         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
13577
13578         if (!IS_GEN(dev_priv, 2))
13579                 intel_set_cpu_fifo_underrun_reporting(dev_priv, crtc->pipe, true);
13580
13581         if (crtc_state->has_pch_encoder) {
13582                 enum pipe pch_transcoder =
13583                         intel_crtc_pch_transcoder(crtc);
13584
13585                 intel_set_pch_fifo_underrun_reporting(dev_priv, pch_transcoder, true);
13586         }
13587 }
13588
13589 static void intel_finish_crtc_commit(struct drm_crtc *crtc,
13590                                      struct drm_crtc_state *old_crtc_state)
13591 {
13592         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
13593         struct intel_atomic_state *old_intel_state =
13594                 to_intel_atomic_state(old_crtc_state->state);
13595         struct intel_crtc_state *new_crtc_state =
13596                 intel_atomic_get_new_crtc_state(old_intel_state, intel_crtc);
13597
13598         intel_pipe_update_end(new_crtc_state);
13599
13600         if (new_crtc_state->update_pipe &&
13601             !needs_modeset(&new_crtc_state->base) &&
13602             old_crtc_state->mode.private_flags & I915_MODE_FLAG_INHERITED)
13603                 intel_crtc_arm_fifo_underrun(intel_crtc, new_crtc_state);
13604 }
13605
13606 /**
13607  * intel_plane_destroy - destroy a plane
13608  * @plane: plane to destroy
13609  *
13610  * Common destruction function for all types of planes (primary, cursor,
13611  * sprite).
13612  */
13613 void intel_plane_destroy(struct drm_plane *plane)
13614 {
13615         drm_plane_cleanup(plane);
13616         kfree(to_intel_plane(plane));
13617 }
13618
13619 static bool i8xx_plane_format_mod_supported(struct drm_plane *_plane,
13620                                             u32 format, u64 modifier)
13621 {
13622         switch (modifier) {
13623         case DRM_FORMAT_MOD_LINEAR:
13624         case I915_FORMAT_MOD_X_TILED:
13625                 break;
13626         default:
13627                 return false;
13628         }
13629
13630         switch (format) {
13631         case DRM_FORMAT_C8:
13632         case DRM_FORMAT_RGB565:
13633         case DRM_FORMAT_XRGB1555:
13634         case DRM_FORMAT_XRGB8888:
13635                 return modifier == DRM_FORMAT_MOD_LINEAR ||
13636                         modifier == I915_FORMAT_MOD_X_TILED;
13637         default:
13638                 return false;
13639         }
13640 }
13641
13642 static bool i965_plane_format_mod_supported(struct drm_plane *_plane,
13643                                             u32 format, u64 modifier)
13644 {
13645         switch (modifier) {
13646         case DRM_FORMAT_MOD_LINEAR:
13647         case I915_FORMAT_MOD_X_TILED:
13648                 break;
13649         default:
13650                 return false;
13651         }
13652
13653         switch (format) {
13654         case DRM_FORMAT_C8:
13655         case DRM_FORMAT_RGB565:
13656         case DRM_FORMAT_XRGB8888:
13657         case DRM_FORMAT_XBGR8888:
13658         case DRM_FORMAT_XRGB2101010:
13659         case DRM_FORMAT_XBGR2101010:
13660                 return modifier == DRM_FORMAT_MOD_LINEAR ||
13661                         modifier == I915_FORMAT_MOD_X_TILED;
13662         default:
13663                 return false;
13664         }
13665 }
13666
13667 static bool intel_cursor_format_mod_supported(struct drm_plane *_plane,
13668                                               u32 format, u64 modifier)
13669 {
13670         return modifier == DRM_FORMAT_MOD_LINEAR &&
13671                 format == DRM_FORMAT_ARGB8888;
13672 }
13673
13674 static const struct drm_plane_funcs i965_plane_funcs = {
13675         .update_plane = drm_atomic_helper_update_plane,
13676         .disable_plane = drm_atomic_helper_disable_plane,
13677         .destroy = intel_plane_destroy,
13678         .atomic_get_property = intel_plane_atomic_get_property,
13679         .atomic_set_property = intel_plane_atomic_set_property,
13680         .atomic_duplicate_state = intel_plane_duplicate_state,
13681         .atomic_destroy_state = intel_plane_destroy_state,
13682         .format_mod_supported = i965_plane_format_mod_supported,
13683 };
13684
13685 static const struct drm_plane_funcs i8xx_plane_funcs = {
13686         .update_plane = drm_atomic_helper_update_plane,
13687         .disable_plane = drm_atomic_helper_disable_plane,
13688         .destroy = intel_plane_destroy,
13689         .atomic_get_property = intel_plane_atomic_get_property,
13690         .atomic_set_property = intel_plane_atomic_set_property,
13691         .atomic_duplicate_state = intel_plane_duplicate_state,
13692         .atomic_destroy_state = intel_plane_destroy_state,
13693         .format_mod_supported = i8xx_plane_format_mod_supported,
13694 };
13695
13696 static int
13697 intel_legacy_cursor_update(struct drm_plane *plane,
13698                            struct drm_crtc *crtc,
13699                            struct drm_framebuffer *fb,
13700                            int crtc_x, int crtc_y,
13701                            unsigned int crtc_w, unsigned int crtc_h,
13702                            uint32_t src_x, uint32_t src_y,
13703                            uint32_t src_w, uint32_t src_h,
13704                            struct drm_modeset_acquire_ctx *ctx)
13705 {
13706         struct drm_i915_private *dev_priv = to_i915(crtc->dev);
13707         int ret;
13708         struct drm_plane_state *old_plane_state, *new_plane_state;
13709         struct intel_plane *intel_plane = to_intel_plane(plane);
13710         struct drm_framebuffer *old_fb;
13711         struct intel_crtc_state *crtc_state =
13712                 to_intel_crtc_state(crtc->state);
13713         struct intel_crtc_state *new_crtc_state;
13714
13715         /*
13716          * When crtc is inactive or there is a modeset pending,
13717          * wait for it to complete in the slowpath
13718          */
13719         if (!crtc_state->base.active || needs_modeset(&crtc_state->base) ||
13720             crtc_state->update_pipe)
13721                 goto slow;
13722
13723         old_plane_state = plane->state;
13724         /*
13725          * Don't do an async update if there is an outstanding commit modifying
13726          * the plane.  This prevents our async update's changes from getting
13727          * overridden by a previous synchronous update's state.
13728          */
13729         if (old_plane_state->commit &&
13730             !try_wait_for_completion(&old_plane_state->commit->hw_done))
13731                 goto slow;
13732
13733         /*
13734          * If any parameters change that may affect watermarks,
13735          * take the slowpath. Only changing fb or position should be
13736          * in the fastpath.
13737          */
13738         if (old_plane_state->crtc != crtc ||
13739             old_plane_state->src_w != src_w ||
13740             old_plane_state->src_h != src_h ||
13741             old_plane_state->crtc_w != crtc_w ||
13742             old_plane_state->crtc_h != crtc_h ||
13743             !old_plane_state->fb != !fb)
13744                 goto slow;
13745
13746         new_plane_state = intel_plane_duplicate_state(plane);
13747         if (!new_plane_state)
13748                 return -ENOMEM;
13749
13750         new_crtc_state = to_intel_crtc_state(intel_crtc_duplicate_state(crtc));
13751         if (!new_crtc_state) {
13752                 ret = -ENOMEM;
13753                 goto out_free;
13754         }
13755
13756         drm_atomic_set_fb_for_plane(new_plane_state, fb);
13757
13758         new_plane_state->src_x = src_x;
13759         new_plane_state->src_y = src_y;
13760         new_plane_state->src_w = src_w;
13761         new_plane_state->src_h = src_h;
13762         new_plane_state->crtc_x = crtc_x;
13763         new_plane_state->crtc_y = crtc_y;
13764         new_plane_state->crtc_w = crtc_w;
13765         new_plane_state->crtc_h = crtc_h;
13766
13767         ret = intel_plane_atomic_check_with_state(crtc_state, new_crtc_state,
13768                                                   to_intel_plane_state(old_plane_state),
13769                                                   to_intel_plane_state(new_plane_state));
13770         if (ret)
13771                 goto out_free;
13772
13773         ret = mutex_lock_interruptible(&dev_priv->drm.struct_mutex);
13774         if (ret)
13775                 goto out_free;
13776
13777         ret = intel_plane_pin_fb(to_intel_plane_state(new_plane_state));
13778         if (ret)
13779                 goto out_unlock;
13780
13781         intel_fb_obj_flush(intel_fb_obj(fb), ORIGIN_FLIP);
13782
13783         old_fb = old_plane_state->fb;
13784         i915_gem_track_fb(intel_fb_obj(old_fb), intel_fb_obj(fb),
13785                           intel_plane->frontbuffer_bit);
13786
13787         /* Swap plane state */
13788         plane->state = new_plane_state;
13789
13790         /*
13791          * We cannot swap crtc_state as it may be in use by an atomic commit or
13792          * page flip that's running simultaneously. If we swap crtc_state and
13793          * destroy the old state, we will cause a use-after-free there.
13794          *
13795          * Only update active_planes, which is needed for our internal
13796          * bookkeeping. Either value will do the right thing when updating
13797          * planes atomically. If the cursor was part of the atomic update then
13798          * we would have taken the slowpath.
13799          */
13800         crtc_state->active_planes = new_crtc_state->active_planes;
13801
13802         if (plane->state->visible) {
13803                 trace_intel_update_plane(plane, to_intel_crtc(crtc));
13804                 intel_plane->update_plane(intel_plane, crtc_state,
13805                                           to_intel_plane_state(plane->state));
13806         } else {
13807                 trace_intel_disable_plane(plane, to_intel_crtc(crtc));
13808                 intel_plane->disable_plane(intel_plane, crtc_state);
13809         }
13810
13811         intel_plane_unpin_fb(to_intel_plane_state(old_plane_state));
13812
13813 out_unlock:
13814         mutex_unlock(&dev_priv->drm.struct_mutex);
13815 out_free:
13816         if (new_crtc_state)
13817                 intel_crtc_destroy_state(crtc, &new_crtc_state->base);
13818         if (ret)
13819                 intel_plane_destroy_state(plane, new_plane_state);
13820         else
13821                 intel_plane_destroy_state(plane, old_plane_state);
13822         return ret;
13823
13824 slow:
13825         return drm_atomic_helper_update_plane(plane, crtc, fb,
13826                                               crtc_x, crtc_y, crtc_w, crtc_h,
13827                                               src_x, src_y, src_w, src_h, ctx);
13828 }
13829
13830 static const struct drm_plane_funcs intel_cursor_plane_funcs = {
13831         .update_plane = intel_legacy_cursor_update,
13832         .disable_plane = drm_atomic_helper_disable_plane,
13833         .destroy = intel_plane_destroy,
13834         .atomic_get_property = intel_plane_atomic_get_property,
13835         .atomic_set_property = intel_plane_atomic_set_property,
13836         .atomic_duplicate_state = intel_plane_duplicate_state,
13837         .atomic_destroy_state = intel_plane_destroy_state,
13838         .format_mod_supported = intel_cursor_format_mod_supported,
13839 };
13840
13841 static bool i9xx_plane_has_fbc(struct drm_i915_private *dev_priv,
13842                                enum i9xx_plane_id i9xx_plane)
13843 {
13844         if (!HAS_FBC(dev_priv))
13845                 return false;
13846
13847         if (IS_BROADWELL(dev_priv) || IS_HASWELL(dev_priv))
13848                 return i9xx_plane == PLANE_A; /* tied to pipe A */
13849         else if (IS_IVYBRIDGE(dev_priv))
13850                 return i9xx_plane == PLANE_A || i9xx_plane == PLANE_B ||
13851                         i9xx_plane == PLANE_C;
13852         else if (INTEL_GEN(dev_priv) >= 4)
13853                 return i9xx_plane == PLANE_A || i9xx_plane == PLANE_B;
13854         else
13855                 return i9xx_plane == PLANE_A;
13856 }
13857
13858 static struct intel_plane *
13859 intel_primary_plane_create(struct drm_i915_private *dev_priv, enum pipe pipe)
13860 {
13861         struct intel_plane *plane;
13862         const struct drm_plane_funcs *plane_funcs;
13863         unsigned int supported_rotations;
13864         unsigned int possible_crtcs;
13865         const u64 *modifiers;
13866         const u32 *formats;
13867         int num_formats;
13868         int ret;
13869
13870         if (INTEL_GEN(dev_priv) >= 9)
13871                 return skl_universal_plane_create(dev_priv, pipe,
13872                                                   PLANE_PRIMARY);
13873
13874         plane = intel_plane_alloc();
13875         if (IS_ERR(plane))
13876                 return plane;
13877
13878         plane->pipe = pipe;
13879         /*
13880          * On gen2/3 only plane A can do FBC, but the panel fitter and LVDS
13881          * port is hooked to pipe B. Hence we want plane A feeding pipe B.
13882          */
13883         if (HAS_FBC(dev_priv) && INTEL_GEN(dev_priv) < 4)
13884                 plane->i9xx_plane = (enum i9xx_plane_id) !pipe;
13885         else
13886                 plane->i9xx_plane = (enum i9xx_plane_id) pipe;
13887         plane->id = PLANE_PRIMARY;
13888         plane->frontbuffer_bit = INTEL_FRONTBUFFER(pipe, plane->id);
13889
13890         plane->has_fbc = i9xx_plane_has_fbc(dev_priv, plane->i9xx_plane);
13891         if (plane->has_fbc) {
13892                 struct intel_fbc *fbc = &dev_priv->fbc;
13893
13894                 fbc->possible_framebuffer_bits |= plane->frontbuffer_bit;
13895         }
13896
13897         if (INTEL_GEN(dev_priv) >= 4) {
13898                 formats = i965_primary_formats;
13899                 num_formats = ARRAY_SIZE(i965_primary_formats);
13900                 modifiers = i9xx_format_modifiers;
13901
13902                 plane->max_stride = i9xx_plane_max_stride;
13903                 plane->update_plane = i9xx_update_plane;
13904                 plane->disable_plane = i9xx_disable_plane;
13905                 plane->get_hw_state = i9xx_plane_get_hw_state;
13906                 plane->check_plane = i9xx_plane_check;
13907
13908                 plane_funcs = &i965_plane_funcs;
13909         } else {
13910                 formats = i8xx_primary_formats;
13911                 num_formats = ARRAY_SIZE(i8xx_primary_formats);
13912                 modifiers = i9xx_format_modifiers;
13913
13914                 plane->max_stride = i9xx_plane_max_stride;
13915                 plane->update_plane = i9xx_update_plane;
13916                 plane->disable_plane = i9xx_disable_plane;
13917                 plane->get_hw_state = i9xx_plane_get_hw_state;
13918                 plane->check_plane = i9xx_plane_check;
13919
13920                 plane_funcs = &i8xx_plane_funcs;
13921         }
13922
13923         possible_crtcs = BIT(pipe);
13924
13925         if (INTEL_GEN(dev_priv) >= 5 || IS_G4X(dev_priv))
13926                 ret = drm_universal_plane_init(&dev_priv->drm, &plane->base,
13927                                                possible_crtcs, plane_funcs,
13928                                                formats, num_formats, modifiers,
13929                                                DRM_PLANE_TYPE_PRIMARY,
13930                                                "primary %c", pipe_name(pipe));
13931         else
13932                 ret = drm_universal_plane_init(&dev_priv->drm, &plane->base,
13933                                                possible_crtcs, plane_funcs,
13934                                                formats, num_formats, modifiers,
13935                                                DRM_PLANE_TYPE_PRIMARY,
13936                                                "plane %c",
13937                                                plane_name(plane->i9xx_plane));
13938         if (ret)
13939                 goto fail;
13940
13941         if (IS_CHERRYVIEW(dev_priv) && pipe == PIPE_B) {
13942                 supported_rotations =
13943                         DRM_MODE_ROTATE_0 | DRM_MODE_ROTATE_180 |
13944                         DRM_MODE_REFLECT_X;
13945         } else if (INTEL_GEN(dev_priv) >= 4) {
13946                 supported_rotations =
13947                         DRM_MODE_ROTATE_0 | DRM_MODE_ROTATE_180;
13948         } else {
13949                 supported_rotations = DRM_MODE_ROTATE_0;
13950         }
13951
13952         if (INTEL_GEN(dev_priv) >= 4)
13953                 drm_plane_create_rotation_property(&plane->base,
13954                                                    DRM_MODE_ROTATE_0,
13955                                                    supported_rotations);
13956
13957         drm_plane_helper_add(&plane->base, &intel_plane_helper_funcs);
13958
13959         return plane;
13960
13961 fail:
13962         intel_plane_free(plane);
13963
13964         return ERR_PTR(ret);
13965 }
13966
13967 static struct intel_plane *
13968 intel_cursor_plane_create(struct drm_i915_private *dev_priv,
13969                           enum pipe pipe)
13970 {
13971         unsigned int possible_crtcs;
13972         struct intel_plane *cursor;
13973         int ret;
13974
13975         cursor = intel_plane_alloc();
13976         if (IS_ERR(cursor))
13977                 return cursor;
13978
13979         cursor->pipe = pipe;
13980         cursor->i9xx_plane = (enum i9xx_plane_id) pipe;
13981         cursor->id = PLANE_CURSOR;
13982         cursor->frontbuffer_bit = INTEL_FRONTBUFFER(pipe, cursor->id);
13983
13984         if (IS_I845G(dev_priv) || IS_I865G(dev_priv)) {
13985                 cursor->max_stride = i845_cursor_max_stride;
13986                 cursor->update_plane = i845_update_cursor;
13987                 cursor->disable_plane = i845_disable_cursor;
13988                 cursor->get_hw_state = i845_cursor_get_hw_state;
13989                 cursor->check_plane = i845_check_cursor;
13990         } else {
13991                 cursor->max_stride = i9xx_cursor_max_stride;
13992                 cursor->update_plane = i9xx_update_cursor;
13993                 cursor->disable_plane = i9xx_disable_cursor;
13994                 cursor->get_hw_state = i9xx_cursor_get_hw_state;
13995                 cursor->check_plane = i9xx_check_cursor;
13996         }
13997
13998         cursor->cursor.base = ~0;
13999         cursor->cursor.cntl = ~0;
14000
14001         if (IS_I845G(dev_priv) || IS_I865G(dev_priv) || HAS_CUR_FBC(dev_priv))
14002                 cursor->cursor.size = ~0;
14003
14004         possible_crtcs = BIT(pipe);
14005
14006         ret = drm_universal_plane_init(&dev_priv->drm, &cursor->base,
14007                                        possible_crtcs, &intel_cursor_plane_funcs,
14008                                        intel_cursor_formats,
14009                                        ARRAY_SIZE(intel_cursor_formats),
14010                                        cursor_format_modifiers,
14011                                        DRM_PLANE_TYPE_CURSOR,
14012                                        "cursor %c", pipe_name(pipe));
14013         if (ret)
14014                 goto fail;
14015
14016         if (INTEL_GEN(dev_priv) >= 4)
14017                 drm_plane_create_rotation_property(&cursor->base,
14018                                                    DRM_MODE_ROTATE_0,
14019                                                    DRM_MODE_ROTATE_0 |
14020                                                    DRM_MODE_ROTATE_180);
14021
14022         drm_plane_helper_add(&cursor->base, &intel_plane_helper_funcs);
14023
14024         return cursor;
14025
14026 fail:
14027         intel_plane_free(cursor);
14028
14029         return ERR_PTR(ret);
14030 }
14031
14032 static void intel_crtc_init_scalers(struct intel_crtc *crtc,
14033                                     struct intel_crtc_state *crtc_state)
14034 {
14035         struct intel_crtc_scaler_state *scaler_state =
14036                 &crtc_state->scaler_state;
14037         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
14038         int i;
14039
14040         crtc->num_scalers = dev_priv->info.num_scalers[crtc->pipe];
14041         if (!crtc->num_scalers)
14042                 return;
14043
14044         for (i = 0; i < crtc->num_scalers; i++) {
14045                 struct intel_scaler *scaler = &scaler_state->scalers[i];
14046
14047                 scaler->in_use = 0;
14048                 scaler->mode = 0;
14049         }
14050
14051         scaler_state->scaler_id = -1;
14052 }
14053
14054 static int intel_crtc_init(struct drm_i915_private *dev_priv, enum pipe pipe)
14055 {
14056         struct intel_crtc *intel_crtc;
14057         struct intel_crtc_state *crtc_state = NULL;
14058         struct intel_plane *primary = NULL;
14059         struct intel_plane *cursor = NULL;
14060         int sprite, ret;
14061
14062         intel_crtc = kzalloc(sizeof(*intel_crtc), GFP_KERNEL);
14063         if (!intel_crtc)
14064                 return -ENOMEM;
14065
14066         crtc_state = kzalloc(sizeof(*crtc_state), GFP_KERNEL);
14067         if (!crtc_state) {
14068                 ret = -ENOMEM;
14069                 goto fail;
14070         }
14071         intel_crtc->config = crtc_state;
14072         intel_crtc->base.state = &crtc_state->base;
14073         crtc_state->base.crtc = &intel_crtc->base;
14074
14075         primary = intel_primary_plane_create(dev_priv, pipe);
14076         if (IS_ERR(primary)) {
14077                 ret = PTR_ERR(primary);
14078                 goto fail;
14079         }
14080         intel_crtc->plane_ids_mask |= BIT(primary->id);
14081
14082         for_each_sprite(dev_priv, pipe, sprite) {
14083                 struct intel_plane *plane;
14084
14085                 plane = intel_sprite_plane_create(dev_priv, pipe, sprite);
14086                 if (IS_ERR(plane)) {
14087                         ret = PTR_ERR(plane);
14088                         goto fail;
14089                 }
14090                 intel_crtc->plane_ids_mask |= BIT(plane->id);
14091         }
14092
14093         cursor = intel_cursor_plane_create(dev_priv, pipe);
14094         if (IS_ERR(cursor)) {
14095                 ret = PTR_ERR(cursor);
14096                 goto fail;
14097         }
14098         intel_crtc->plane_ids_mask |= BIT(cursor->id);
14099
14100         ret = drm_crtc_init_with_planes(&dev_priv->drm, &intel_crtc->base,
14101                                         &primary->base, &cursor->base,
14102                                         &intel_crtc_funcs,
14103                                         "pipe %c", pipe_name(pipe));
14104         if (ret)
14105                 goto fail;
14106
14107         intel_crtc->pipe = pipe;
14108
14109         /* initialize shared scalers */
14110         intel_crtc_init_scalers(intel_crtc, crtc_state);
14111
14112         BUG_ON(pipe >= ARRAY_SIZE(dev_priv->pipe_to_crtc_mapping) ||
14113                dev_priv->pipe_to_crtc_mapping[pipe] != NULL);
14114         dev_priv->pipe_to_crtc_mapping[pipe] = intel_crtc;
14115
14116         if (INTEL_GEN(dev_priv) < 9) {
14117                 enum i9xx_plane_id i9xx_plane = primary->i9xx_plane;
14118
14119                 BUG_ON(i9xx_plane >= ARRAY_SIZE(dev_priv->plane_to_crtc_mapping) ||
14120                        dev_priv->plane_to_crtc_mapping[i9xx_plane] != NULL);
14121                 dev_priv->plane_to_crtc_mapping[i9xx_plane] = intel_crtc;
14122         }
14123
14124         drm_crtc_helper_add(&intel_crtc->base, &intel_helper_funcs);
14125
14126         intel_color_init(intel_crtc);
14127
14128         WARN_ON(drm_crtc_index(&intel_crtc->base) != intel_crtc->pipe);
14129
14130         return 0;
14131
14132 fail:
14133         /*
14134          * drm_mode_config_cleanup() will free up any
14135          * crtcs/planes already initialized.
14136          */
14137         kfree(crtc_state);
14138         kfree(intel_crtc);
14139
14140         return ret;
14141 }
14142
14143 int intel_get_pipe_from_crtc_id_ioctl(struct drm_device *dev, void *data,
14144                                       struct drm_file *file)
14145 {
14146         struct drm_i915_get_pipe_from_crtc_id *pipe_from_crtc_id = data;
14147         struct drm_crtc *drmmode_crtc;
14148         struct intel_crtc *crtc;
14149
14150         drmmode_crtc = drm_crtc_find(dev, file, pipe_from_crtc_id->crtc_id);
14151         if (!drmmode_crtc)
14152                 return -ENOENT;
14153
14154         crtc = to_intel_crtc(drmmode_crtc);
14155         pipe_from_crtc_id->pipe = crtc->pipe;
14156
14157         return 0;
14158 }
14159
14160 static int intel_encoder_clones(struct intel_encoder *encoder)
14161 {
14162         struct drm_device *dev = encoder->base.dev;
14163         struct intel_encoder *source_encoder;
14164         int index_mask = 0;
14165         int entry = 0;
14166
14167         for_each_intel_encoder(dev, source_encoder) {
14168                 if (encoders_cloneable(encoder, source_encoder))
14169                         index_mask |= (1 << entry);
14170
14171                 entry++;
14172         }
14173
14174         return index_mask;
14175 }
14176
14177 static bool has_edp_a(struct drm_i915_private *dev_priv)
14178 {
14179         if (!IS_MOBILE(dev_priv))
14180                 return false;
14181
14182         if ((I915_READ(DP_A) & DP_DETECTED) == 0)
14183                 return false;
14184
14185         if (IS_GEN(dev_priv, 5) && (I915_READ(FUSE_STRAP) & ILK_eDP_A_DISABLE))
14186                 return false;
14187
14188         return true;
14189 }
14190
14191 static bool intel_crt_present(struct drm_i915_private *dev_priv)
14192 {
14193         if (INTEL_GEN(dev_priv) >= 9)
14194                 return false;
14195
14196         if (IS_HSW_ULT(dev_priv) || IS_BDW_ULT(dev_priv))
14197                 return false;
14198
14199         if (IS_CHERRYVIEW(dev_priv))
14200                 return false;
14201
14202         if (HAS_PCH_LPT_H(dev_priv) &&
14203             I915_READ(SFUSE_STRAP) & SFUSE_STRAP_CRT_DISABLED)
14204                 return false;
14205
14206         /* DDI E can't be used if DDI A requires 4 lanes */
14207         if (HAS_DDI(dev_priv) && I915_READ(DDI_BUF_CTL(PORT_A)) & DDI_A_4_LANES)
14208                 return false;
14209
14210         if (!dev_priv->vbt.int_crt_support)
14211                 return false;
14212
14213         return true;
14214 }
14215
14216 void intel_pps_unlock_regs_wa(struct drm_i915_private *dev_priv)
14217 {
14218         int pps_num;
14219         int pps_idx;
14220
14221         if (HAS_DDI(dev_priv))
14222                 return;
14223         /*
14224          * This w/a is needed at least on CPT/PPT, but to be sure apply it
14225          * everywhere where registers can be write protected.
14226          */
14227         if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
14228                 pps_num = 2;
14229         else
14230                 pps_num = 1;
14231
14232         for (pps_idx = 0; pps_idx < pps_num; pps_idx++) {
14233                 u32 val = I915_READ(PP_CONTROL(pps_idx));
14234
14235                 val = (val & ~PANEL_UNLOCK_MASK) | PANEL_UNLOCK_REGS;
14236                 I915_WRITE(PP_CONTROL(pps_idx), val);
14237         }
14238 }
14239
14240 static void intel_pps_init(struct drm_i915_private *dev_priv)
14241 {
14242         if (HAS_PCH_SPLIT(dev_priv) || IS_GEN9_LP(dev_priv))
14243                 dev_priv->pps_mmio_base = PCH_PPS_BASE;
14244         else if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
14245                 dev_priv->pps_mmio_base = VLV_PPS_BASE;
14246         else
14247                 dev_priv->pps_mmio_base = PPS_BASE;
14248
14249         intel_pps_unlock_regs_wa(dev_priv);
14250 }
14251
14252 static void intel_setup_outputs(struct drm_i915_private *dev_priv)
14253 {
14254         struct intel_encoder *encoder;
14255         bool dpd_is_edp = false;
14256
14257         intel_pps_init(dev_priv);
14258
14259         if (!HAS_DISPLAY(dev_priv))
14260                 return;
14261
14262         /*
14263          * intel_edp_init_connector() depends on this completing first, to
14264          * prevent the registeration of both eDP and LVDS and the incorrect
14265          * sharing of the PPS.
14266          */
14267         intel_lvds_init(dev_priv);
14268
14269         if (intel_crt_present(dev_priv))
14270                 intel_crt_init(dev_priv);
14271
14272         if (IS_ICELAKE(dev_priv)) {
14273                 intel_ddi_init(dev_priv, PORT_A);
14274                 intel_ddi_init(dev_priv, PORT_B);
14275                 intel_ddi_init(dev_priv, PORT_C);
14276                 intel_ddi_init(dev_priv, PORT_D);
14277                 intel_ddi_init(dev_priv, PORT_E);
14278                 intel_ddi_init(dev_priv, PORT_F);
14279                 icl_dsi_init(dev_priv);
14280         } else if (IS_GEN9_LP(dev_priv)) {
14281                 /*
14282                  * FIXME: Broxton doesn't support port detection via the
14283                  * DDI_BUF_CTL_A or SFUSE_STRAP registers, find another way to
14284                  * detect the ports.
14285                  */
14286                 intel_ddi_init(dev_priv, PORT_A);
14287                 intel_ddi_init(dev_priv, PORT_B);
14288                 intel_ddi_init(dev_priv, PORT_C);
14289
14290                 vlv_dsi_init(dev_priv);
14291         } else if (HAS_DDI(dev_priv)) {
14292                 int found;
14293
14294                 /*
14295                  * Haswell uses DDI functions to detect digital outputs.
14296                  * On SKL pre-D0 the strap isn't connected, so we assume
14297                  * it's there.
14298                  */
14299                 found = I915_READ(DDI_BUF_CTL(PORT_A)) & DDI_INIT_DISPLAY_DETECTED;
14300                 /* WaIgnoreDDIAStrap: skl */
14301                 if (found || IS_GEN9_BC(dev_priv))
14302                         intel_ddi_init(dev_priv, PORT_A);
14303
14304                 /* DDI B, C, D, and F detection is indicated by the SFUSE_STRAP
14305                  * register */
14306                 found = I915_READ(SFUSE_STRAP);
14307
14308                 if (found & SFUSE_STRAP_DDIB_DETECTED)
14309                         intel_ddi_init(dev_priv, PORT_B);
14310                 if (found & SFUSE_STRAP_DDIC_DETECTED)
14311                         intel_ddi_init(dev_priv, PORT_C);
14312                 if (found & SFUSE_STRAP_DDID_DETECTED)
14313                         intel_ddi_init(dev_priv, PORT_D);
14314                 if (found & SFUSE_STRAP_DDIF_DETECTED)
14315                         intel_ddi_init(dev_priv, PORT_F);
14316                 /*
14317                  * On SKL we don't have a way to detect DDI-E so we rely on VBT.
14318                  */
14319                 if (IS_GEN9_BC(dev_priv) &&
14320                     (dev_priv->vbt.ddi_port_info[PORT_E].supports_dp ||
14321                      dev_priv->vbt.ddi_port_info[PORT_E].supports_dvi ||
14322                      dev_priv->vbt.ddi_port_info[PORT_E].supports_hdmi))
14323                         intel_ddi_init(dev_priv, PORT_E);
14324
14325         } else if (HAS_PCH_SPLIT(dev_priv)) {
14326                 int found;
14327                 dpd_is_edp = intel_dp_is_port_edp(dev_priv, PORT_D);
14328
14329                 if (has_edp_a(dev_priv))
14330                         intel_dp_init(dev_priv, DP_A, PORT_A);
14331
14332                 if (I915_READ(PCH_HDMIB) & SDVO_DETECTED) {
14333                         /* PCH SDVOB multiplex with HDMIB */
14334                         found = intel_sdvo_init(dev_priv, PCH_SDVOB, PORT_B);
14335                         if (!found)
14336                                 intel_hdmi_init(dev_priv, PCH_HDMIB, PORT_B);
14337                         if (!found && (I915_READ(PCH_DP_B) & DP_DETECTED))
14338                                 intel_dp_init(dev_priv, PCH_DP_B, PORT_B);
14339                 }
14340
14341                 if (I915_READ(PCH_HDMIC) & SDVO_DETECTED)
14342                         intel_hdmi_init(dev_priv, PCH_HDMIC, PORT_C);
14343
14344                 if (!dpd_is_edp && I915_READ(PCH_HDMID) & SDVO_DETECTED)
14345                         intel_hdmi_init(dev_priv, PCH_HDMID, PORT_D);
14346
14347                 if (I915_READ(PCH_DP_C) & DP_DETECTED)
14348                         intel_dp_init(dev_priv, PCH_DP_C, PORT_C);
14349
14350                 if (I915_READ(PCH_DP_D) & DP_DETECTED)
14351                         intel_dp_init(dev_priv, PCH_DP_D, PORT_D);
14352         } else if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) {
14353                 bool has_edp, has_port;
14354
14355                 /*
14356                  * The DP_DETECTED bit is the latched state of the DDC
14357                  * SDA pin at boot. However since eDP doesn't require DDC
14358                  * (no way to plug in a DP->HDMI dongle) the DDC pins for
14359                  * eDP ports may have been muxed to an alternate function.
14360                  * Thus we can't rely on the DP_DETECTED bit alone to detect
14361                  * eDP ports. Consult the VBT as well as DP_DETECTED to
14362                  * detect eDP ports.
14363                  *
14364                  * Sadly the straps seem to be missing sometimes even for HDMI
14365                  * ports (eg. on Voyo V3 - CHT x7-Z8700), so check both strap
14366                  * and VBT for the presence of the port. Additionally we can't
14367                  * trust the port type the VBT declares as we've seen at least
14368                  * HDMI ports that the VBT claim are DP or eDP.
14369                  */
14370                 has_edp = intel_dp_is_port_edp(dev_priv, PORT_B);
14371                 has_port = intel_bios_is_port_present(dev_priv, PORT_B);
14372                 if (I915_READ(VLV_DP_B) & DP_DETECTED || has_port)
14373                         has_edp &= intel_dp_init(dev_priv, VLV_DP_B, PORT_B);
14374                 if ((I915_READ(VLV_HDMIB) & SDVO_DETECTED || has_port) && !has_edp)
14375                         intel_hdmi_init(dev_priv, VLV_HDMIB, PORT_B);
14376
14377                 has_edp = intel_dp_is_port_edp(dev_priv, PORT_C);
14378                 has_port = intel_bios_is_port_present(dev_priv, PORT_C);
14379                 if (I915_READ(VLV_DP_C) & DP_DETECTED || has_port)
14380                         has_edp &= intel_dp_init(dev_priv, VLV_DP_C, PORT_C);
14381                 if ((I915_READ(VLV_HDMIC) & SDVO_DETECTED || has_port) && !has_edp)
14382                         intel_hdmi_init(dev_priv, VLV_HDMIC, PORT_C);
14383
14384                 if (IS_CHERRYVIEW(dev_priv)) {
14385                         /*
14386                          * eDP not supported on port D,
14387                          * so no need to worry about it
14388                          */
14389                         has_port = intel_bios_is_port_present(dev_priv, PORT_D);
14390                         if (I915_READ(CHV_DP_D) & DP_DETECTED || has_port)
14391                                 intel_dp_init(dev_priv, CHV_DP_D, PORT_D);
14392                         if (I915_READ(CHV_HDMID) & SDVO_DETECTED || has_port)
14393                                 intel_hdmi_init(dev_priv, CHV_HDMID, PORT_D);
14394                 }
14395
14396                 vlv_dsi_init(dev_priv);
14397         } else if (!IS_GEN(dev_priv, 2) && !IS_PINEVIEW(dev_priv)) {
14398                 bool found = false;
14399
14400                 if (I915_READ(GEN3_SDVOB) & SDVO_DETECTED) {
14401                         DRM_DEBUG_KMS("probing SDVOB\n");
14402                         found = intel_sdvo_init(dev_priv, GEN3_SDVOB, PORT_B);
14403                         if (!found && IS_G4X(dev_priv)) {
14404                                 DRM_DEBUG_KMS("probing HDMI on SDVOB\n");
14405                                 intel_hdmi_init(dev_priv, GEN4_HDMIB, PORT_B);
14406                         }
14407
14408                         if (!found && IS_G4X(dev_priv))
14409                                 intel_dp_init(dev_priv, DP_B, PORT_B);
14410                 }
14411
14412                 /* Before G4X SDVOC doesn't have its own detect register */
14413
14414                 if (I915_READ(GEN3_SDVOB) & SDVO_DETECTED) {
14415                         DRM_DEBUG_KMS("probing SDVOC\n");
14416                         found = intel_sdvo_init(dev_priv, GEN3_SDVOC, PORT_C);
14417                 }
14418
14419                 if (!found && (I915_READ(GEN3_SDVOC) & SDVO_DETECTED)) {
14420
14421                         if (IS_G4X(dev_priv)) {
14422                                 DRM_DEBUG_KMS("probing HDMI on SDVOC\n");
14423                                 intel_hdmi_init(dev_priv, GEN4_HDMIC, PORT_C);
14424                         }
14425                         if (IS_G4X(dev_priv))
14426                                 intel_dp_init(dev_priv, DP_C, PORT_C);
14427                 }
14428
14429                 if (IS_G4X(dev_priv) && (I915_READ(DP_D) & DP_DETECTED))
14430                         intel_dp_init(dev_priv, DP_D, PORT_D);
14431         } else if (IS_GEN(dev_priv, 2))
14432                 intel_dvo_init(dev_priv);
14433
14434         if (SUPPORTS_TV(dev_priv))
14435                 intel_tv_init(dev_priv);
14436
14437         intel_psr_init(dev_priv);
14438
14439         for_each_intel_encoder(&dev_priv->drm, encoder) {
14440                 encoder->base.possible_crtcs = encoder->crtc_mask;
14441                 encoder->base.possible_clones =
14442                         intel_encoder_clones(encoder);
14443         }
14444
14445         intel_init_pch_refclk(dev_priv);
14446
14447         drm_helper_move_panel_connectors_to_head(&dev_priv->drm);
14448 }
14449
14450 static void intel_user_framebuffer_destroy(struct drm_framebuffer *fb)
14451 {
14452         struct intel_framebuffer *intel_fb = to_intel_framebuffer(fb);
14453         struct drm_i915_gem_object *obj = intel_fb_obj(fb);
14454
14455         drm_framebuffer_cleanup(fb);
14456
14457         i915_gem_object_lock(obj);
14458         WARN_ON(!obj->framebuffer_references--);
14459         i915_gem_object_unlock(obj);
14460
14461         i915_gem_object_put(obj);
14462
14463         kfree(intel_fb);
14464 }
14465
14466 static int intel_user_framebuffer_create_handle(struct drm_framebuffer *fb,
14467                                                 struct drm_file *file,
14468                                                 unsigned int *handle)
14469 {
14470         struct drm_i915_gem_object *obj = intel_fb_obj(fb);
14471
14472         if (obj->userptr.mm) {
14473                 DRM_DEBUG("attempting to use a userptr for a framebuffer, denied\n");
14474                 return -EINVAL;
14475         }
14476
14477         return drm_gem_handle_create(file, &obj->base, handle);
14478 }
14479
14480 static int intel_user_framebuffer_dirty(struct drm_framebuffer *fb,
14481                                         struct drm_file *file,
14482                                         unsigned flags, unsigned color,
14483                                         struct drm_clip_rect *clips,
14484                                         unsigned num_clips)
14485 {
14486         struct drm_i915_gem_object *obj = intel_fb_obj(fb);
14487
14488         i915_gem_object_flush_if_display(obj);
14489         intel_fb_obj_flush(obj, ORIGIN_DIRTYFB);
14490
14491         return 0;
14492 }
14493
14494 static const struct drm_framebuffer_funcs intel_fb_funcs = {
14495         .destroy = intel_user_framebuffer_destroy,
14496         .create_handle = intel_user_framebuffer_create_handle,
14497         .dirty = intel_user_framebuffer_dirty,
14498 };
14499
14500 static
14501 u32 intel_fb_pitch_limit(struct drm_i915_private *dev_priv,
14502                          u32 pixel_format, u64 fb_modifier)
14503 {
14504         struct intel_crtc *crtc;
14505         struct intel_plane *plane;
14506
14507         /*
14508          * We assume the primary plane for pipe A has
14509          * the highest stride limits of them all.
14510          */
14511         crtc = intel_get_crtc_for_pipe(dev_priv, PIPE_A);
14512         plane = to_intel_plane(crtc->base.primary);
14513
14514         return plane->max_stride(plane, pixel_format, fb_modifier,
14515                                  DRM_MODE_ROTATE_0);
14516 }
14517
14518 static int intel_framebuffer_init(struct intel_framebuffer *intel_fb,
14519                                   struct drm_i915_gem_object *obj,
14520                                   struct drm_mode_fb_cmd2 *mode_cmd)
14521 {
14522         struct drm_i915_private *dev_priv = to_i915(obj->base.dev);
14523         struct drm_framebuffer *fb = &intel_fb->base;
14524         u32 pitch_limit;
14525         unsigned int tiling, stride;
14526         int ret = -EINVAL;
14527         int i;
14528
14529         i915_gem_object_lock(obj);
14530         obj->framebuffer_references++;
14531         tiling = i915_gem_object_get_tiling(obj);
14532         stride = i915_gem_object_get_stride(obj);
14533         i915_gem_object_unlock(obj);
14534
14535         if (mode_cmd->flags & DRM_MODE_FB_MODIFIERS) {
14536                 /*
14537                  * If there's a fence, enforce that
14538                  * the fb modifier and tiling mode match.
14539                  */
14540                 if (tiling != I915_TILING_NONE &&
14541                     tiling != intel_fb_modifier_to_tiling(mode_cmd->modifier[0])) {
14542                         DRM_DEBUG_KMS("tiling_mode doesn't match fb modifier\n");
14543                         goto err;
14544                 }
14545         } else {
14546                 if (tiling == I915_TILING_X) {
14547                         mode_cmd->modifier[0] = I915_FORMAT_MOD_X_TILED;
14548                 } else if (tiling == I915_TILING_Y) {
14549                         DRM_DEBUG_KMS("No Y tiling for legacy addfb\n");
14550                         goto err;
14551                 }
14552         }
14553
14554         if (!drm_any_plane_has_format(&dev_priv->drm,
14555                                       mode_cmd->pixel_format,
14556                                       mode_cmd->modifier[0])) {
14557                 struct drm_format_name_buf format_name;
14558
14559                 DRM_DEBUG_KMS("unsupported pixel format %s / modifier 0x%llx\n",
14560                               drm_get_format_name(mode_cmd->pixel_format,
14561                                                   &format_name),
14562                               mode_cmd->modifier[0]);
14563                 goto err;
14564         }
14565
14566         /*
14567          * gen2/3 display engine uses the fence if present,
14568          * so the tiling mode must match the fb modifier exactly.
14569          */
14570         if (INTEL_GEN(dev_priv) < 4 &&
14571             tiling != intel_fb_modifier_to_tiling(mode_cmd->modifier[0])) {
14572                 DRM_DEBUG_KMS("tiling_mode must match fb modifier exactly on gen2/3\n");
14573                 goto err;
14574         }
14575
14576         pitch_limit = intel_fb_pitch_limit(dev_priv, mode_cmd->pixel_format,
14577                                            mode_cmd->modifier[0]);
14578         if (mode_cmd->pitches[0] > pitch_limit) {
14579                 DRM_DEBUG_KMS("%s pitch (%u) must be at most %d\n",
14580                               mode_cmd->modifier[0] != DRM_FORMAT_MOD_LINEAR ?
14581                               "tiled" : "linear",
14582                               mode_cmd->pitches[0], pitch_limit);
14583                 goto err;
14584         }
14585
14586         /*
14587          * If there's a fence, enforce that
14588          * the fb pitch and fence stride match.
14589          */
14590         if (tiling != I915_TILING_NONE && mode_cmd->pitches[0] != stride) {
14591                 DRM_DEBUG_KMS("pitch (%d) must match tiling stride (%d)\n",
14592                               mode_cmd->pitches[0], stride);
14593                 goto err;
14594         }
14595
14596         /* FIXME need to adjust LINOFF/TILEOFF accordingly. */
14597         if (mode_cmd->offsets[0] != 0)
14598                 goto err;
14599
14600         drm_helper_mode_fill_fb_struct(&dev_priv->drm, fb, mode_cmd);
14601
14602         if (fb->format->format == DRM_FORMAT_NV12 &&
14603             (fb->width < SKL_MIN_YUV_420_SRC_W ||
14604              fb->height < SKL_MIN_YUV_420_SRC_H ||
14605              (fb->width % 4) != 0 || (fb->height % 4) != 0)) {
14606                 DRM_DEBUG_KMS("src dimensions not correct for NV12\n");
14607                 goto err;
14608         }
14609
14610         for (i = 0; i < fb->format->num_planes; i++) {
14611                 u32 stride_alignment;
14612
14613                 if (mode_cmd->handles[i] != mode_cmd->handles[0]) {
14614                         DRM_DEBUG_KMS("bad plane %d handle\n", i);
14615                         goto err;
14616                 }
14617
14618                 stride_alignment = intel_fb_stride_alignment(fb, i);
14619
14620                 /*
14621                  * Display WA #0531: skl,bxt,kbl,glk
14622                  *
14623                  * Render decompression and plane width > 3840
14624                  * combined with horizontal panning requires the
14625                  * plane stride to be a multiple of 4. We'll just
14626                  * require the entire fb to accommodate that to avoid
14627                  * potential runtime errors at plane configuration time.
14628                  */
14629                 if (IS_GEN(dev_priv, 9) && i == 0 && fb->width > 3840 &&
14630                     is_ccs_modifier(fb->modifier))
14631                         stride_alignment *= 4;
14632
14633                 if (fb->pitches[i] & (stride_alignment - 1)) {
14634                         DRM_DEBUG_KMS("plane %d pitch (%d) must be at least %u byte aligned\n",
14635                                       i, fb->pitches[i], stride_alignment);
14636                         goto err;
14637                 }
14638
14639                 fb->obj[i] = &obj->base;
14640         }
14641
14642         ret = intel_fill_fb_info(dev_priv, fb);
14643         if (ret)
14644                 goto err;
14645
14646         ret = drm_framebuffer_init(&dev_priv->drm, fb, &intel_fb_funcs);
14647         if (ret) {
14648                 DRM_ERROR("framebuffer init failed %d\n", ret);
14649                 goto err;
14650         }
14651
14652         return 0;
14653
14654 err:
14655         i915_gem_object_lock(obj);
14656         obj->framebuffer_references--;
14657         i915_gem_object_unlock(obj);
14658         return ret;
14659 }
14660
14661 static struct drm_framebuffer *
14662 intel_user_framebuffer_create(struct drm_device *dev,
14663                               struct drm_file *filp,
14664                               const struct drm_mode_fb_cmd2 *user_mode_cmd)
14665 {
14666         struct drm_framebuffer *fb;
14667         struct drm_i915_gem_object *obj;
14668         struct drm_mode_fb_cmd2 mode_cmd = *user_mode_cmd;
14669
14670         obj = i915_gem_object_lookup(filp, mode_cmd.handles[0]);
14671         if (!obj)
14672                 return ERR_PTR(-ENOENT);
14673
14674         fb = intel_framebuffer_create(obj, &mode_cmd);
14675         if (IS_ERR(fb))
14676                 i915_gem_object_put(obj);
14677
14678         return fb;
14679 }
14680
14681 static void intel_atomic_state_free(struct drm_atomic_state *state)
14682 {
14683         struct intel_atomic_state *intel_state = to_intel_atomic_state(state);
14684
14685         drm_atomic_state_default_release(state);
14686
14687         i915_sw_fence_fini(&intel_state->commit_ready);
14688
14689         kfree(state);
14690 }
14691
14692 static enum drm_mode_status
14693 intel_mode_valid(struct drm_device *dev,
14694                  const struct drm_display_mode *mode)
14695 {
14696         struct drm_i915_private *dev_priv = to_i915(dev);
14697         int hdisplay_max, htotal_max;
14698         int vdisplay_max, vtotal_max;
14699
14700         /*
14701          * Can't reject DBLSCAN here because Xorg ddxen can add piles
14702          * of DBLSCAN modes to the output's mode list when they detect
14703          * the scaling mode property on the connector. And they don't
14704          * ask the kernel to validate those modes in any way until
14705          * modeset time at which point the client gets a protocol error.
14706          * So in order to not upset those clients we silently ignore the
14707          * DBLSCAN flag on such connectors. For other connectors we will
14708          * reject modes with the DBLSCAN flag in encoder->compute_config().
14709          * And we always reject DBLSCAN modes in connector->mode_valid()
14710          * as we never want such modes on the connector's mode list.
14711          */
14712
14713         if (mode->vscan > 1)
14714                 return MODE_NO_VSCAN;
14715
14716         if (mode->flags & DRM_MODE_FLAG_HSKEW)
14717                 return MODE_H_ILLEGAL;
14718
14719         if (mode->flags & (DRM_MODE_FLAG_CSYNC |
14720                            DRM_MODE_FLAG_NCSYNC |
14721                            DRM_MODE_FLAG_PCSYNC))
14722                 return MODE_HSYNC;
14723
14724         if (mode->flags & (DRM_MODE_FLAG_BCAST |
14725                            DRM_MODE_FLAG_PIXMUX |
14726                            DRM_MODE_FLAG_CLKDIV2))
14727                 return MODE_BAD;
14728
14729         if (INTEL_GEN(dev_priv) >= 9 ||
14730             IS_BROADWELL(dev_priv) || IS_HASWELL(dev_priv)) {
14731                 hdisplay_max = 8192; /* FDI max 4096 handled elsewhere */
14732                 vdisplay_max = 4096;
14733                 htotal_max = 8192;
14734                 vtotal_max = 8192;
14735         } else if (INTEL_GEN(dev_priv) >= 3) {
14736                 hdisplay_max = 4096;
14737                 vdisplay_max = 4096;
14738                 htotal_max = 8192;
14739                 vtotal_max = 8192;
14740         } else {
14741                 hdisplay_max = 2048;
14742                 vdisplay_max = 2048;
14743                 htotal_max = 4096;
14744                 vtotal_max = 4096;
14745         }
14746
14747         if (mode->hdisplay > hdisplay_max ||
14748             mode->hsync_start > htotal_max ||
14749             mode->hsync_end > htotal_max ||
14750             mode->htotal > htotal_max)
14751                 return MODE_H_ILLEGAL;
14752
14753         if (mode->vdisplay > vdisplay_max ||
14754             mode->vsync_start > vtotal_max ||
14755             mode->vsync_end > vtotal_max ||
14756             mode->vtotal > vtotal_max)
14757                 return MODE_V_ILLEGAL;
14758
14759         return MODE_OK;
14760 }
14761
14762 static const struct drm_mode_config_funcs intel_mode_funcs = {
14763         .fb_create = intel_user_framebuffer_create,
14764         .get_format_info = intel_get_format_info,
14765         .output_poll_changed = intel_fbdev_output_poll_changed,
14766         .mode_valid = intel_mode_valid,
14767         .atomic_check = intel_atomic_check,
14768         .atomic_commit = intel_atomic_commit,
14769         .atomic_state_alloc = intel_atomic_state_alloc,
14770         .atomic_state_clear = intel_atomic_state_clear,
14771         .atomic_state_free = intel_atomic_state_free,
14772 };
14773
14774 /**
14775  * intel_init_display_hooks - initialize the display modesetting hooks
14776  * @dev_priv: device private
14777  */
14778 void intel_init_display_hooks(struct drm_i915_private *dev_priv)
14779 {
14780         intel_init_cdclk_hooks(dev_priv);
14781
14782         if (INTEL_GEN(dev_priv) >= 9) {
14783                 dev_priv->display.get_pipe_config = haswell_get_pipe_config;
14784                 dev_priv->display.get_initial_plane_config =
14785                         skylake_get_initial_plane_config;
14786                 dev_priv->display.crtc_compute_clock =
14787                         haswell_crtc_compute_clock;
14788                 dev_priv->display.crtc_enable = haswell_crtc_enable;
14789                 dev_priv->display.crtc_disable = haswell_crtc_disable;
14790         } else if (HAS_DDI(dev_priv)) {
14791                 dev_priv->display.get_pipe_config = haswell_get_pipe_config;
14792                 dev_priv->display.get_initial_plane_config =
14793                         i9xx_get_initial_plane_config;
14794                 dev_priv->display.crtc_compute_clock =
14795                         haswell_crtc_compute_clock;
14796                 dev_priv->display.crtc_enable = haswell_crtc_enable;
14797                 dev_priv->display.crtc_disable = haswell_crtc_disable;
14798         } else if (HAS_PCH_SPLIT(dev_priv)) {
14799                 dev_priv->display.get_pipe_config = ironlake_get_pipe_config;
14800                 dev_priv->display.get_initial_plane_config =
14801                         i9xx_get_initial_plane_config;
14802                 dev_priv->display.crtc_compute_clock =
14803                         ironlake_crtc_compute_clock;
14804                 dev_priv->display.crtc_enable = ironlake_crtc_enable;
14805                 dev_priv->display.crtc_disable = ironlake_crtc_disable;
14806         } else if (IS_CHERRYVIEW(dev_priv)) {
14807                 dev_priv->display.get_pipe_config = i9xx_get_pipe_config;
14808                 dev_priv->display.get_initial_plane_config =
14809                         i9xx_get_initial_plane_config;
14810                 dev_priv->display.crtc_compute_clock = chv_crtc_compute_clock;
14811                 dev_priv->display.crtc_enable = valleyview_crtc_enable;
14812                 dev_priv->display.crtc_disable = i9xx_crtc_disable;
14813         } else if (IS_VALLEYVIEW(dev_priv)) {
14814                 dev_priv->display.get_pipe_config = i9xx_get_pipe_config;
14815                 dev_priv->display.get_initial_plane_config =
14816                         i9xx_get_initial_plane_config;
14817                 dev_priv->display.crtc_compute_clock = vlv_crtc_compute_clock;
14818                 dev_priv->display.crtc_enable = valleyview_crtc_enable;
14819                 dev_priv->display.crtc_disable = i9xx_crtc_disable;
14820         } else if (IS_G4X(dev_priv)) {
14821                 dev_priv->display.get_pipe_config = i9xx_get_pipe_config;
14822                 dev_priv->display.get_initial_plane_config =
14823                         i9xx_get_initial_plane_config;
14824                 dev_priv->display.crtc_compute_clock = g4x_crtc_compute_clock;
14825                 dev_priv->display.crtc_enable = i9xx_crtc_enable;
14826                 dev_priv->display.crtc_disable = i9xx_crtc_disable;
14827         } else if (IS_PINEVIEW(dev_priv)) {
14828                 dev_priv->display.get_pipe_config = i9xx_get_pipe_config;
14829                 dev_priv->display.get_initial_plane_config =
14830                         i9xx_get_initial_plane_config;
14831                 dev_priv->display.crtc_compute_clock = pnv_crtc_compute_clock;
14832                 dev_priv->display.crtc_enable = i9xx_crtc_enable;
14833                 dev_priv->display.crtc_disable = i9xx_crtc_disable;
14834         } else if (!IS_GEN(dev_priv, 2)) {
14835                 dev_priv->display.get_pipe_config = i9xx_get_pipe_config;
14836                 dev_priv->display.get_initial_plane_config =
14837                         i9xx_get_initial_plane_config;
14838                 dev_priv->display.crtc_compute_clock = i9xx_crtc_compute_clock;
14839                 dev_priv->display.crtc_enable = i9xx_crtc_enable;
14840                 dev_priv->display.crtc_disable = i9xx_crtc_disable;
14841         } else {
14842                 dev_priv->display.get_pipe_config = i9xx_get_pipe_config;
14843                 dev_priv->display.get_initial_plane_config =
14844                         i9xx_get_initial_plane_config;
14845                 dev_priv->display.crtc_compute_clock = i8xx_crtc_compute_clock;
14846                 dev_priv->display.crtc_enable = i9xx_crtc_enable;
14847                 dev_priv->display.crtc_disable = i9xx_crtc_disable;
14848         }
14849
14850         if (IS_GEN(dev_priv, 5)) {
14851                 dev_priv->display.fdi_link_train = ironlake_fdi_link_train;
14852         } else if (IS_GEN(dev_priv, 6)) {
14853                 dev_priv->display.fdi_link_train = gen6_fdi_link_train;
14854         } else if (IS_IVYBRIDGE(dev_priv)) {
14855                 /* FIXME: detect B0+ stepping and use auto training */
14856                 dev_priv->display.fdi_link_train = ivb_manual_fdi_link_train;
14857         } else if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv)) {
14858                 dev_priv->display.fdi_link_train = hsw_fdi_link_train;
14859         }
14860
14861         if (INTEL_GEN(dev_priv) >= 9)
14862                 dev_priv->display.update_crtcs = skl_update_crtcs;
14863         else
14864                 dev_priv->display.update_crtcs = intel_update_crtcs;
14865 }
14866
14867 /* Disable the VGA plane that we never use */
14868 static void i915_disable_vga(struct drm_i915_private *dev_priv)
14869 {
14870         struct pci_dev *pdev = dev_priv->drm.pdev;
14871         u8 sr1;
14872         i915_reg_t vga_reg = i915_vgacntrl_reg(dev_priv);
14873
14874         /* WaEnableVGAAccessThroughIOPort:ctg,elk,ilk,snb,ivb,vlv,hsw */
14875         vga_get_uninterruptible(pdev, VGA_RSRC_LEGACY_IO);
14876         outb(SR01, VGA_SR_INDEX);
14877         sr1 = inb(VGA_SR_DATA);
14878         outb(sr1 | 1<<5, VGA_SR_DATA);
14879         vga_put(pdev, VGA_RSRC_LEGACY_IO);
14880         udelay(300);
14881
14882         I915_WRITE(vga_reg, VGA_DISP_DISABLE);
14883         POSTING_READ(vga_reg);
14884 }
14885
14886 void intel_modeset_init_hw(struct drm_device *dev)
14887 {
14888         struct drm_i915_private *dev_priv = to_i915(dev);
14889
14890         intel_update_cdclk(dev_priv);
14891         intel_dump_cdclk_state(&dev_priv->cdclk.hw, "Current CDCLK");
14892         dev_priv->cdclk.logical = dev_priv->cdclk.actual = dev_priv->cdclk.hw;
14893 }
14894
14895 /*
14896  * Calculate what we think the watermarks should be for the state we've read
14897  * out of the hardware and then immediately program those watermarks so that
14898  * we ensure the hardware settings match our internal state.
14899  *
14900  * We can calculate what we think WM's should be by creating a duplicate of the
14901  * current state (which was constructed during hardware readout) and running it
14902  * through the atomic check code to calculate new watermark values in the
14903  * state object.
14904  */
14905 static void sanitize_watermarks(struct drm_device *dev)
14906 {
14907         struct drm_i915_private *dev_priv = to_i915(dev);
14908         struct drm_atomic_state *state;
14909         struct intel_atomic_state *intel_state;
14910         struct drm_crtc *crtc;
14911         struct drm_crtc_state *cstate;
14912         struct drm_modeset_acquire_ctx ctx;
14913         int ret;
14914         int i;
14915
14916         /* Only supported on platforms that use atomic watermark design */
14917         if (!dev_priv->display.optimize_watermarks)
14918                 return;
14919
14920         /*
14921          * We need to hold connection_mutex before calling duplicate_state so
14922          * that the connector loop is protected.
14923          */
14924         drm_modeset_acquire_init(&ctx, 0);
14925 retry:
14926         ret = drm_modeset_lock_all_ctx(dev, &ctx);
14927         if (ret == -EDEADLK) {
14928                 drm_modeset_backoff(&ctx);
14929                 goto retry;
14930         } else if (WARN_ON(ret)) {
14931                 goto fail;
14932         }
14933
14934         state = drm_atomic_helper_duplicate_state(dev, &ctx);
14935         if (WARN_ON(IS_ERR(state)))
14936                 goto fail;
14937
14938         intel_state = to_intel_atomic_state(state);
14939
14940         /*
14941          * Hardware readout is the only time we don't want to calculate
14942          * intermediate watermarks (since we don't trust the current
14943          * watermarks).
14944          */
14945         if (!HAS_GMCH_DISPLAY(dev_priv))
14946                 intel_state->skip_intermediate_wm = true;
14947
14948         ret = intel_atomic_check(dev, state);
14949         if (ret) {
14950                 /*
14951                  * If we fail here, it means that the hardware appears to be
14952                  * programmed in a way that shouldn't be possible, given our
14953                  * understanding of watermark requirements.  This might mean a
14954                  * mistake in the hardware readout code or a mistake in the
14955                  * watermark calculations for a given platform.  Raise a WARN
14956                  * so that this is noticeable.
14957                  *
14958                  * If this actually happens, we'll have to just leave the
14959                  * BIOS-programmed watermarks untouched and hope for the best.
14960                  */
14961                 WARN(true, "Could not determine valid watermarks for inherited state\n");
14962                 goto put_state;
14963         }
14964
14965         /* Write calculated watermark values back */
14966         for_each_new_crtc_in_state(state, crtc, cstate, i) {
14967                 struct intel_crtc_state *cs = to_intel_crtc_state(cstate);
14968
14969                 cs->wm.need_postvbl_update = true;
14970                 dev_priv->display.optimize_watermarks(intel_state, cs);
14971
14972                 to_intel_crtc_state(crtc->state)->wm = cs->wm;
14973         }
14974
14975 put_state:
14976         drm_atomic_state_put(state);
14977 fail:
14978         drm_modeset_drop_locks(&ctx);
14979         drm_modeset_acquire_fini(&ctx);
14980 }
14981
14982 static void intel_update_fdi_pll_freq(struct drm_i915_private *dev_priv)
14983 {
14984         if (IS_GEN(dev_priv, 5)) {
14985                 u32 fdi_pll_clk =
14986                         I915_READ(FDI_PLL_BIOS_0) & FDI_PLL_FB_CLOCK_MASK;
14987
14988                 dev_priv->fdi_pll_freq = (fdi_pll_clk + 2) * 10000;
14989         } else if (IS_GEN(dev_priv, 6) || IS_IVYBRIDGE(dev_priv)) {
14990                 dev_priv->fdi_pll_freq = 270000;
14991         } else {
14992                 return;
14993         }
14994
14995         DRM_DEBUG_DRIVER("FDI PLL freq=%d\n", dev_priv->fdi_pll_freq);
14996 }
14997
14998 static int intel_initial_commit(struct drm_device *dev)
14999 {
15000         struct drm_atomic_state *state = NULL;
15001         struct drm_modeset_acquire_ctx ctx;
15002         struct drm_crtc *crtc;
15003         struct drm_crtc_state *crtc_state;
15004         int ret = 0;
15005
15006         state = drm_atomic_state_alloc(dev);
15007         if (!state)
15008                 return -ENOMEM;
15009
15010         drm_modeset_acquire_init(&ctx, 0);
15011
15012 retry:
15013         state->acquire_ctx = &ctx;
15014
15015         drm_for_each_crtc(crtc, dev) {
15016                 crtc_state = drm_atomic_get_crtc_state(state, crtc);
15017                 if (IS_ERR(crtc_state)) {
15018                         ret = PTR_ERR(crtc_state);
15019                         goto out;
15020                 }
15021
15022                 if (crtc_state->active) {
15023                         ret = drm_atomic_add_affected_planes(state, crtc);
15024                         if (ret)
15025                                 goto out;
15026
15027                         /*
15028                          * FIXME hack to force a LUT update to avoid the
15029                          * plane update forcing the pipe gamma on without
15030                          * having a proper LUT loaded. Remove once we
15031                          * have readout for pipe gamma enable.
15032                          */
15033                         crtc_state->color_mgmt_changed = true;
15034                 }
15035         }
15036
15037         ret = drm_atomic_commit(state);
15038
15039 out:
15040         if (ret == -EDEADLK) {
15041                 drm_atomic_state_clear(state);
15042                 drm_modeset_backoff(&ctx);
15043                 goto retry;
15044         }
15045
15046         drm_atomic_state_put(state);
15047
15048         drm_modeset_drop_locks(&ctx);
15049         drm_modeset_acquire_fini(&ctx);
15050
15051         return ret;
15052 }
15053
15054 int intel_modeset_init(struct drm_device *dev)
15055 {
15056         struct drm_i915_private *dev_priv = to_i915(dev);
15057         struct i915_ggtt *ggtt = &dev_priv->ggtt;
15058         enum pipe pipe;
15059         struct intel_crtc *crtc;
15060         int ret;
15061
15062         dev_priv->modeset_wq = alloc_ordered_workqueue("i915_modeset", 0);
15063
15064         drm_mode_config_init(dev);
15065
15066         dev->mode_config.min_width = 0;
15067         dev->mode_config.min_height = 0;
15068
15069         dev->mode_config.preferred_depth = 24;
15070         dev->mode_config.prefer_shadow = 1;
15071
15072         dev->mode_config.allow_fb_modifiers = true;
15073
15074         dev->mode_config.funcs = &intel_mode_funcs;
15075
15076         init_llist_head(&dev_priv->atomic_helper.free_list);
15077         INIT_WORK(&dev_priv->atomic_helper.free_work,
15078                   intel_atomic_helper_free_state_worker);
15079
15080         intel_init_quirks(dev_priv);
15081
15082         intel_fbc_init(dev_priv);
15083
15084         intel_init_pm(dev_priv);
15085
15086         /*
15087          * There may be no VBT; and if the BIOS enabled SSC we can
15088          * just keep using it to avoid unnecessary flicker.  Whereas if the
15089          * BIOS isn't using it, don't assume it will work even if the VBT
15090          * indicates as much.
15091          */
15092         if (HAS_PCH_IBX(dev_priv) || HAS_PCH_CPT(dev_priv)) {
15093                 bool bios_lvds_use_ssc = !!(I915_READ(PCH_DREF_CONTROL) &
15094                                             DREF_SSC1_ENABLE);
15095
15096                 if (dev_priv->vbt.lvds_use_ssc != bios_lvds_use_ssc) {
15097                         DRM_DEBUG_KMS("SSC %sabled by BIOS, overriding VBT which says %sabled\n",
15098                                      bios_lvds_use_ssc ? "en" : "dis",
15099                                      dev_priv->vbt.lvds_use_ssc ? "en" : "dis");
15100                         dev_priv->vbt.lvds_use_ssc = bios_lvds_use_ssc;
15101                 }
15102         }
15103
15104         /* maximum framebuffer dimensions */
15105         if (IS_GEN(dev_priv, 2)) {
15106                 dev->mode_config.max_width = 2048;
15107                 dev->mode_config.max_height = 2048;
15108         } else if (IS_GEN(dev_priv, 3)) {
15109                 dev->mode_config.max_width = 4096;
15110                 dev->mode_config.max_height = 4096;
15111         } else {
15112                 dev->mode_config.max_width = 8192;
15113                 dev->mode_config.max_height = 8192;
15114         }
15115
15116         if (IS_I845G(dev_priv) || IS_I865G(dev_priv)) {
15117                 dev->mode_config.cursor_width = IS_I845G(dev_priv) ? 64 : 512;
15118                 dev->mode_config.cursor_height = 1023;
15119         } else if (IS_GEN(dev_priv, 2)) {
15120                 dev->mode_config.cursor_width = 64;
15121                 dev->mode_config.cursor_height = 64;
15122         } else {
15123                 dev->mode_config.cursor_width = 256;
15124                 dev->mode_config.cursor_height = 256;
15125         }
15126
15127         dev->mode_config.fb_base = ggtt->gmadr.start;
15128
15129         DRM_DEBUG_KMS("%d display pipe%s available.\n",
15130                       INTEL_INFO(dev_priv)->num_pipes,
15131                       INTEL_INFO(dev_priv)->num_pipes > 1 ? "s" : "");
15132
15133         for_each_pipe(dev_priv, pipe) {
15134                 ret = intel_crtc_init(dev_priv, pipe);
15135                 if (ret) {
15136                         drm_mode_config_cleanup(dev);
15137                         return ret;
15138                 }
15139         }
15140
15141         intel_shared_dpll_init(dev);
15142         intel_update_fdi_pll_freq(dev_priv);
15143
15144         intel_update_czclk(dev_priv);
15145         intel_modeset_init_hw(dev);
15146
15147         if (dev_priv->max_cdclk_freq == 0)
15148                 intel_update_max_cdclk(dev_priv);
15149
15150         /* Just disable it once at startup */
15151         i915_disable_vga(dev_priv);
15152         intel_setup_outputs(dev_priv);
15153
15154         drm_modeset_lock_all(dev);
15155         intel_modeset_setup_hw_state(dev, dev->mode_config.acquire_ctx);
15156         drm_modeset_unlock_all(dev);
15157
15158         for_each_intel_crtc(dev, crtc) {
15159                 struct intel_initial_plane_config plane_config = {};
15160
15161                 if (!crtc->active)
15162                         continue;
15163
15164                 /*
15165                  * Note that reserving the BIOS fb up front prevents us
15166                  * from stuffing other stolen allocations like the ring
15167                  * on top.  This prevents some ugliness at boot time, and
15168                  * can even allow for smooth boot transitions if the BIOS
15169                  * fb is large enough for the active pipe configuration.
15170                  */
15171                 dev_priv->display.get_initial_plane_config(crtc,
15172                                                            &plane_config);
15173
15174                 /*
15175                  * If the fb is shared between multiple heads, we'll
15176                  * just get the first one.
15177                  */
15178                 intel_find_initial_plane_obj(crtc, &plane_config);
15179         }
15180
15181         /*
15182          * Make sure hardware watermarks really match the state we read out.
15183          * Note that we need to do this after reconstructing the BIOS fb's
15184          * since the watermark calculation done here will use pstate->fb.
15185          */
15186         if (!HAS_GMCH_DISPLAY(dev_priv))
15187                 sanitize_watermarks(dev);
15188
15189         /*
15190          * Force all active planes to recompute their states. So that on
15191          * mode_setcrtc after probe, all the intel_plane_state variables
15192          * are already calculated and there is no assert_plane warnings
15193          * during bootup.
15194          */
15195         ret = intel_initial_commit(dev);
15196         if (ret)
15197                 DRM_DEBUG_KMS("Initial commit in probe failed.\n");
15198
15199         return 0;
15200 }
15201
15202 void i830_enable_pipe(struct drm_i915_private *dev_priv, enum pipe pipe)
15203 {
15204         struct intel_crtc *crtc = intel_get_crtc_for_pipe(dev_priv, pipe);
15205         /* 640x480@60Hz, ~25175 kHz */
15206         struct dpll clock = {
15207                 .m1 = 18,
15208                 .m2 = 7,
15209                 .p1 = 13,
15210                 .p2 = 4,
15211                 .n = 2,
15212         };
15213         u32 dpll, fp;
15214         int i;
15215
15216         WARN_ON(i9xx_calc_dpll_params(48000, &clock) != 25154);
15217
15218         DRM_DEBUG_KMS("enabling pipe %c due to force quirk (vco=%d dot=%d)\n",
15219                       pipe_name(pipe), clock.vco, clock.dot);
15220
15221         fp = i9xx_dpll_compute_fp(&clock);
15222         dpll = (I915_READ(DPLL(pipe)) & DPLL_DVO_2X_MODE) |
15223                 DPLL_VGA_MODE_DIS |
15224                 ((clock.p1 - 2) << DPLL_FPA01_P1_POST_DIV_SHIFT) |
15225                 PLL_P2_DIVIDE_BY_4 |
15226                 PLL_REF_INPUT_DREFCLK |
15227                 DPLL_VCO_ENABLE;
15228
15229         I915_WRITE(FP0(pipe), fp);
15230         I915_WRITE(FP1(pipe), fp);
15231
15232         I915_WRITE(HTOTAL(pipe), (640 - 1) | ((800 - 1) << 16));
15233         I915_WRITE(HBLANK(pipe), (640 - 1) | ((800 - 1) << 16));
15234         I915_WRITE(HSYNC(pipe), (656 - 1) | ((752 - 1) << 16));
15235         I915_WRITE(VTOTAL(pipe), (480 - 1) | ((525 - 1) << 16));
15236         I915_WRITE(VBLANK(pipe), (480 - 1) | ((525 - 1) << 16));
15237         I915_WRITE(VSYNC(pipe), (490 - 1) | ((492 - 1) << 16));
15238         I915_WRITE(PIPESRC(pipe), ((640 - 1) << 16) | (480 - 1));
15239
15240         /*
15241          * Apparently we need to have VGA mode enabled prior to changing
15242          * the P1/P2 dividers. Otherwise the DPLL will keep using the old
15243          * dividers, even though the register value does change.
15244          */
15245         I915_WRITE(DPLL(pipe), dpll & ~DPLL_VGA_MODE_DIS);
15246         I915_WRITE(DPLL(pipe), dpll);
15247
15248         /* Wait for the clocks to stabilize. */
15249         POSTING_READ(DPLL(pipe));
15250         udelay(150);
15251
15252         /* The pixel multiplier can only be updated once the
15253          * DPLL is enabled and the clocks are stable.
15254          *
15255          * So write it again.
15256          */
15257         I915_WRITE(DPLL(pipe), dpll);
15258
15259         /* We do this three times for luck */
15260         for (i = 0; i < 3 ; i++) {
15261                 I915_WRITE(DPLL(pipe), dpll);
15262                 POSTING_READ(DPLL(pipe));
15263                 udelay(150); /* wait for warmup */
15264         }
15265
15266         I915_WRITE(PIPECONF(pipe), PIPECONF_ENABLE | PIPECONF_PROGRESSIVE);
15267         POSTING_READ(PIPECONF(pipe));
15268
15269         intel_wait_for_pipe_scanline_moving(crtc);
15270 }
15271
15272 void i830_disable_pipe(struct drm_i915_private *dev_priv, enum pipe pipe)
15273 {
15274         struct intel_crtc *crtc = intel_get_crtc_for_pipe(dev_priv, pipe);
15275
15276         DRM_DEBUG_KMS("disabling pipe %c due to force quirk\n",
15277                       pipe_name(pipe));
15278
15279         WARN_ON(I915_READ(DSPCNTR(PLANE_A)) & DISPLAY_PLANE_ENABLE);
15280         WARN_ON(I915_READ(DSPCNTR(PLANE_B)) & DISPLAY_PLANE_ENABLE);
15281         WARN_ON(I915_READ(DSPCNTR(PLANE_C)) & DISPLAY_PLANE_ENABLE);
15282         WARN_ON(I915_READ(CURCNTR(PIPE_A)) & MCURSOR_MODE);
15283         WARN_ON(I915_READ(CURCNTR(PIPE_B)) & MCURSOR_MODE);
15284
15285         I915_WRITE(PIPECONF(pipe), 0);
15286         POSTING_READ(PIPECONF(pipe));
15287
15288         intel_wait_for_pipe_scanline_stopped(crtc);
15289
15290         I915_WRITE(DPLL(pipe), DPLL_VGA_MODE_DIS);
15291         POSTING_READ(DPLL(pipe));
15292 }
15293
15294 static void
15295 intel_sanitize_plane_mapping(struct drm_i915_private *dev_priv)
15296 {
15297         struct intel_crtc *crtc;
15298
15299         if (INTEL_GEN(dev_priv) >= 4)
15300                 return;
15301
15302         for_each_intel_crtc(&dev_priv->drm, crtc) {
15303                 struct intel_plane *plane =
15304                         to_intel_plane(crtc->base.primary);
15305                 struct intel_crtc *plane_crtc;
15306                 enum pipe pipe;
15307
15308                 if (!plane->get_hw_state(plane, &pipe))
15309                         continue;
15310
15311                 if (pipe == crtc->pipe)
15312                         continue;
15313
15314                 DRM_DEBUG_KMS("[PLANE:%d:%s] attached to the wrong pipe, disabling plane\n",
15315                               plane->base.base.id, plane->base.name);
15316
15317                 plane_crtc = intel_get_crtc_for_pipe(dev_priv, pipe);
15318                 intel_plane_disable_noatomic(plane_crtc, plane);
15319         }
15320 }
15321
15322 static bool intel_crtc_has_encoders(struct intel_crtc *crtc)
15323 {
15324         struct drm_device *dev = crtc->base.dev;
15325         struct intel_encoder *encoder;
15326
15327         for_each_encoder_on_crtc(dev, &crtc->base, encoder)
15328                 return true;
15329
15330         return false;
15331 }
15332
15333 static struct intel_connector *intel_encoder_find_connector(struct intel_encoder *encoder)
15334 {
15335         struct drm_device *dev = encoder->base.dev;
15336         struct intel_connector *connector;
15337
15338         for_each_connector_on_encoder(dev, &encoder->base, connector)
15339                 return connector;
15340
15341         return NULL;
15342 }
15343
15344 static bool has_pch_trancoder(struct drm_i915_private *dev_priv,
15345                               enum pipe pch_transcoder)
15346 {
15347         return HAS_PCH_IBX(dev_priv) || HAS_PCH_CPT(dev_priv) ||
15348                 (HAS_PCH_LPT_H(dev_priv) && pch_transcoder == PIPE_A);
15349 }
15350
15351 static void intel_sanitize_crtc(struct intel_crtc *crtc,
15352                                 struct drm_modeset_acquire_ctx *ctx)
15353 {
15354         struct drm_device *dev = crtc->base.dev;
15355         struct drm_i915_private *dev_priv = to_i915(dev);
15356         struct intel_crtc_state *crtc_state = to_intel_crtc_state(crtc->base.state);
15357         enum transcoder cpu_transcoder = crtc_state->cpu_transcoder;
15358
15359         /* Clear any frame start delays used for debugging left by the BIOS */
15360         if (crtc->active && !transcoder_is_dsi(cpu_transcoder)) {
15361                 i915_reg_t reg = PIPECONF(cpu_transcoder);
15362
15363                 I915_WRITE(reg,
15364                            I915_READ(reg) & ~PIPECONF_FRAME_START_DELAY_MASK);
15365         }
15366
15367         if (crtc_state->base.active) {
15368                 struct intel_plane *plane;
15369
15370                 /* Disable everything but the primary plane */
15371                 for_each_intel_plane_on_crtc(dev, crtc, plane) {
15372                         const struct intel_plane_state *plane_state =
15373                                 to_intel_plane_state(plane->base.state);
15374
15375                         if (plane_state->base.visible &&
15376                             plane->base.type != DRM_PLANE_TYPE_PRIMARY)
15377                                 intel_plane_disable_noatomic(crtc, plane);
15378                 }
15379         }
15380
15381         /* Adjust the state of the output pipe according to whether we
15382          * have active connectors/encoders. */
15383         if (crtc_state->base.active && !intel_crtc_has_encoders(crtc))
15384                 intel_crtc_disable_noatomic(&crtc->base, ctx);
15385
15386         if (crtc_state->base.active || HAS_GMCH_DISPLAY(dev_priv)) {
15387                 /*
15388                  * We start out with underrun reporting disabled to avoid races.
15389                  * For correct bookkeeping mark this on active crtcs.
15390                  *
15391                  * Also on gmch platforms we dont have any hardware bits to
15392                  * disable the underrun reporting. Which means we need to start
15393                  * out with underrun reporting disabled also on inactive pipes,
15394                  * since otherwise we'll complain about the garbage we read when
15395                  * e.g. coming up after runtime pm.
15396                  *
15397                  * No protection against concurrent access is required - at
15398                  * worst a fifo underrun happens which also sets this to false.
15399                  */
15400                 crtc->cpu_fifo_underrun_disabled = true;
15401                 /*
15402                  * We track the PCH trancoder underrun reporting state
15403                  * within the crtc. With crtc for pipe A housing the underrun
15404                  * reporting state for PCH transcoder A, crtc for pipe B housing
15405                  * it for PCH transcoder B, etc. LPT-H has only PCH transcoder A,
15406                  * and marking underrun reporting as disabled for the non-existing
15407                  * PCH transcoders B and C would prevent enabling the south
15408                  * error interrupt (see cpt_can_enable_serr_int()).
15409                  */
15410                 if (has_pch_trancoder(dev_priv, crtc->pipe))
15411                         crtc->pch_fifo_underrun_disabled = true;
15412         }
15413 }
15414
15415 static void intel_sanitize_encoder(struct intel_encoder *encoder)
15416 {
15417         struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
15418         struct intel_connector *connector;
15419
15420         /* We need to check both for a crtc link (meaning that the
15421          * encoder is active and trying to read from a pipe) and the
15422          * pipe itself being active. */
15423         bool has_active_crtc = encoder->base.crtc &&
15424                 to_intel_crtc(encoder->base.crtc)->active;
15425
15426         connector = intel_encoder_find_connector(encoder);
15427         if (connector && !has_active_crtc) {
15428                 DRM_DEBUG_KMS("[ENCODER:%d:%s] has active connectors but no active pipe!\n",
15429                               encoder->base.base.id,
15430                               encoder->base.name);
15431
15432                 /* Connector is active, but has no active pipe. This is
15433                  * fallout from our resume register restoring. Disable
15434                  * the encoder manually again. */
15435                 if (encoder->base.crtc) {
15436                         struct drm_crtc_state *crtc_state = encoder->base.crtc->state;
15437
15438                         DRM_DEBUG_KMS("[ENCODER:%d:%s] manually disabled\n",
15439                                       encoder->base.base.id,
15440                                       encoder->base.name);
15441                         if (encoder->disable)
15442                                 encoder->disable(encoder, to_intel_crtc_state(crtc_state), connector->base.state);
15443                         if (encoder->post_disable)
15444                                 encoder->post_disable(encoder, to_intel_crtc_state(crtc_state), connector->base.state);
15445                 }
15446                 encoder->base.crtc = NULL;
15447
15448                 /* Inconsistent output/port/pipe state happens presumably due to
15449                  * a bug in one of the get_hw_state functions. Or someplace else
15450                  * in our code, like the register restore mess on resume. Clamp
15451                  * things to off as a safer default. */
15452
15453                 connector->base.dpms = DRM_MODE_DPMS_OFF;
15454                 connector->base.encoder = NULL;
15455         }
15456
15457         /* notify opregion of the sanitized encoder state */
15458         intel_opregion_notify_encoder(encoder, connector && has_active_crtc);
15459
15460         if (INTEL_GEN(dev_priv) >= 11)
15461                 icl_sanitize_encoder_pll_mapping(encoder);
15462 }
15463
15464 void i915_redisable_vga_power_on(struct drm_i915_private *dev_priv)
15465 {
15466         i915_reg_t vga_reg = i915_vgacntrl_reg(dev_priv);
15467
15468         if (!(I915_READ(vga_reg) & VGA_DISP_DISABLE)) {
15469                 DRM_DEBUG_KMS("Something enabled VGA plane, disabling it\n");
15470                 i915_disable_vga(dev_priv);
15471         }
15472 }
15473
15474 void i915_redisable_vga(struct drm_i915_private *dev_priv)
15475 {
15476         /* This function can be called both from intel_modeset_setup_hw_state or
15477          * at a very early point in our resume sequence, where the power well
15478          * structures are not yet restored. Since this function is at a very
15479          * paranoid "someone might have enabled VGA while we were not looking"
15480          * level, just check if the power well is enabled instead of trying to
15481          * follow the "don't touch the power well if we don't need it" policy
15482          * the rest of the driver uses. */
15483         if (!intel_display_power_get_if_enabled(dev_priv, POWER_DOMAIN_VGA))
15484                 return;
15485
15486         i915_redisable_vga_power_on(dev_priv);
15487
15488         intel_display_power_put(dev_priv, POWER_DOMAIN_VGA);
15489 }
15490
15491 /* FIXME read out full plane state for all planes */
15492 static void readout_plane_state(struct drm_i915_private *dev_priv)
15493 {
15494         struct intel_plane *plane;
15495         struct intel_crtc *crtc;
15496
15497         for_each_intel_plane(&dev_priv->drm, plane) {
15498                 struct intel_plane_state *plane_state =
15499                         to_intel_plane_state(plane->base.state);
15500                 struct intel_crtc_state *crtc_state;
15501                 enum pipe pipe = PIPE_A;
15502                 bool visible;
15503
15504                 visible = plane->get_hw_state(plane, &pipe);
15505
15506                 crtc = intel_get_crtc_for_pipe(dev_priv, pipe);
15507                 crtc_state = to_intel_crtc_state(crtc->base.state);
15508
15509                 intel_set_plane_visible(crtc_state, plane_state, visible);
15510
15511                 DRM_DEBUG_KMS("[PLANE:%d:%s] hw state readout: %s, pipe %c\n",
15512                               plane->base.base.id, plane->base.name,
15513                               enableddisabled(visible), pipe_name(pipe));
15514         }
15515
15516         for_each_intel_crtc(&dev_priv->drm, crtc) {
15517                 struct intel_crtc_state *crtc_state =
15518                         to_intel_crtc_state(crtc->base.state);
15519
15520                 fixup_active_planes(crtc_state);
15521         }
15522 }
15523
15524 static void intel_modeset_readout_hw_state(struct drm_device *dev)
15525 {
15526         struct drm_i915_private *dev_priv = to_i915(dev);
15527         enum pipe pipe;
15528         struct intel_crtc *crtc;
15529         struct intel_encoder *encoder;
15530         struct intel_connector *connector;
15531         struct drm_connector_list_iter conn_iter;
15532         int i;
15533
15534         dev_priv->active_crtcs = 0;
15535
15536         for_each_intel_crtc(dev, crtc) {
15537                 struct intel_crtc_state *crtc_state =
15538                         to_intel_crtc_state(crtc->base.state);
15539
15540                 __drm_atomic_helper_crtc_destroy_state(&crtc_state->base);
15541                 memset(crtc_state, 0, sizeof(*crtc_state));
15542                 crtc_state->base.crtc = &crtc->base;
15543
15544                 crtc_state->base.active = crtc_state->base.enable =
15545                         dev_priv->display.get_pipe_config(crtc, crtc_state);
15546
15547                 crtc->base.enabled = crtc_state->base.enable;
15548                 crtc->active = crtc_state->base.active;
15549
15550                 if (crtc_state->base.active)
15551                         dev_priv->active_crtcs |= 1 << crtc->pipe;
15552
15553                 DRM_DEBUG_KMS("[CRTC:%d:%s] hw state readout: %s\n",
15554                               crtc->base.base.id, crtc->base.name,
15555                               enableddisabled(crtc_state->base.active));
15556         }
15557
15558         readout_plane_state(dev_priv);
15559
15560         for (i = 0; i < dev_priv->num_shared_dpll; i++) {
15561                 struct intel_shared_dpll *pll = &dev_priv->shared_dplls[i];
15562
15563                 pll->on = pll->info->funcs->get_hw_state(dev_priv, pll,
15564                                                         &pll->state.hw_state);
15565                 pll->state.crtc_mask = 0;
15566                 for_each_intel_crtc(dev, crtc) {
15567                         struct intel_crtc_state *crtc_state =
15568                                 to_intel_crtc_state(crtc->base.state);
15569
15570                         if (crtc_state->base.active &&
15571                             crtc_state->shared_dpll == pll)
15572                                 pll->state.crtc_mask |= 1 << crtc->pipe;
15573                 }
15574                 pll->active_mask = pll->state.crtc_mask;
15575
15576                 DRM_DEBUG_KMS("%s hw state readout: crtc_mask 0x%08x, on %i\n",
15577                               pll->info->name, pll->state.crtc_mask, pll->on);
15578         }
15579
15580         for_each_intel_encoder(dev, encoder) {
15581                 pipe = 0;
15582
15583                 if (encoder->get_hw_state(encoder, &pipe)) {
15584                         struct intel_crtc_state *crtc_state;
15585
15586                         crtc = intel_get_crtc_for_pipe(dev_priv, pipe);
15587                         crtc_state = to_intel_crtc_state(crtc->base.state);
15588
15589                         encoder->base.crtc = &crtc->base;
15590                         encoder->get_config(encoder, crtc_state);
15591                 } else {
15592                         encoder->base.crtc = NULL;
15593                 }
15594
15595                 DRM_DEBUG_KMS("[ENCODER:%d:%s] hw state readout: %s, pipe %c\n",
15596                               encoder->base.base.id, encoder->base.name,
15597                               enableddisabled(encoder->base.crtc),
15598                               pipe_name(pipe));
15599         }
15600
15601         drm_connector_list_iter_begin(dev, &conn_iter);
15602         for_each_intel_connector_iter(connector, &conn_iter) {
15603                 if (connector->get_hw_state(connector)) {
15604                         connector->base.dpms = DRM_MODE_DPMS_ON;
15605
15606                         encoder = connector->encoder;
15607                         connector->base.encoder = &encoder->base;
15608
15609                         if (encoder->base.crtc &&
15610                             encoder->base.crtc->state->active) {
15611                                 /*
15612                                  * This has to be done during hardware readout
15613                                  * because anything calling .crtc_disable may
15614                                  * rely on the connector_mask being accurate.
15615                                  */
15616                                 encoder->base.crtc->state->connector_mask |=
15617                                         drm_connector_mask(&connector->base);
15618                                 encoder->base.crtc->state->encoder_mask |=
15619                                         drm_encoder_mask(&encoder->base);
15620                         }
15621
15622                 } else {
15623                         connector->base.dpms = DRM_MODE_DPMS_OFF;
15624                         connector->base.encoder = NULL;
15625                 }
15626                 DRM_DEBUG_KMS("[CONNECTOR:%d:%s] hw state readout: %s\n",
15627                               connector->base.base.id, connector->base.name,
15628                               enableddisabled(connector->base.encoder));
15629         }
15630         drm_connector_list_iter_end(&conn_iter);
15631
15632         for_each_intel_crtc(dev, crtc) {
15633                 struct intel_crtc_state *crtc_state =
15634                         to_intel_crtc_state(crtc->base.state);
15635                 int min_cdclk = 0;
15636
15637                 memset(&crtc->base.mode, 0, sizeof(crtc->base.mode));
15638                 if (crtc_state->base.active) {
15639                         intel_mode_from_pipe_config(&crtc->base.mode, crtc_state);
15640                         crtc->base.mode.hdisplay = crtc_state->pipe_src_w;
15641                         crtc->base.mode.vdisplay = crtc_state->pipe_src_h;
15642                         intel_mode_from_pipe_config(&crtc_state->base.adjusted_mode, crtc_state);
15643                         WARN_ON(drm_atomic_set_mode_for_crtc(crtc->base.state, &crtc->base.mode));
15644
15645                         /*
15646                          * The initial mode needs to be set in order to keep
15647                          * the atomic core happy. It wants a valid mode if the
15648                          * crtc's enabled, so we do the above call.
15649                          *
15650                          * But we don't set all the derived state fully, hence
15651                          * set a flag to indicate that a full recalculation is
15652                          * needed on the next commit.
15653                          */
15654                         crtc_state->base.mode.private_flags = I915_MODE_FLAG_INHERITED;
15655
15656                         intel_crtc_compute_pixel_rate(crtc_state);
15657
15658                         if (dev_priv->display.modeset_calc_cdclk) {
15659                                 min_cdclk = intel_crtc_compute_min_cdclk(crtc_state);
15660                                 if (WARN_ON(min_cdclk < 0))
15661                                         min_cdclk = 0;
15662                         }
15663
15664                         drm_calc_timestamping_constants(&crtc->base,
15665                                                         &crtc_state->base.adjusted_mode);
15666                         update_scanline_offset(crtc_state);
15667                 }
15668
15669                 dev_priv->min_cdclk[crtc->pipe] = min_cdclk;
15670                 dev_priv->min_voltage_level[crtc->pipe] =
15671                         crtc_state->min_voltage_level;
15672
15673                 intel_pipe_config_sanity_check(dev_priv, crtc_state);
15674         }
15675 }
15676
15677 static void
15678 get_encoder_power_domains(struct drm_i915_private *dev_priv)
15679 {
15680         struct intel_encoder *encoder;
15681
15682         for_each_intel_encoder(&dev_priv->drm, encoder) {
15683                 u64 get_domains;
15684                 enum intel_display_power_domain domain;
15685                 struct intel_crtc_state *crtc_state;
15686
15687                 if (!encoder->get_power_domains)
15688                         continue;
15689
15690                 /*
15691                  * MST-primary and inactive encoders don't have a crtc state
15692                  * and neither of these require any power domain references.
15693                  */
15694                 if (!encoder->base.crtc)
15695                         continue;
15696
15697                 crtc_state = to_intel_crtc_state(encoder->base.crtc->state);
15698                 get_domains = encoder->get_power_domains(encoder, crtc_state);
15699                 for_each_power_domain(domain, get_domains)
15700                         intel_display_power_get(dev_priv, domain);
15701         }
15702 }
15703
15704 static void intel_early_display_was(struct drm_i915_private *dev_priv)
15705 {
15706         /* Display WA #1185 WaDisableDARBFClkGating:cnl,glk */
15707         if (IS_CANNONLAKE(dev_priv) || IS_GEMINILAKE(dev_priv))
15708                 I915_WRITE(GEN9_CLKGATE_DIS_0, I915_READ(GEN9_CLKGATE_DIS_0) |
15709                            DARBF_GATING_DIS);
15710
15711         if (IS_HASWELL(dev_priv)) {
15712                 /*
15713                  * WaRsPkgCStateDisplayPMReq:hsw
15714                  * System hang if this isn't done before disabling all planes!
15715                  */
15716                 I915_WRITE(CHICKEN_PAR1_1,
15717                            I915_READ(CHICKEN_PAR1_1) | FORCE_ARB_IDLE_PLANES);
15718         }
15719 }
15720
15721 static void ibx_sanitize_pch_hdmi_port(struct drm_i915_private *dev_priv,
15722                                        enum port port, i915_reg_t hdmi_reg)
15723 {
15724         u32 val = I915_READ(hdmi_reg);
15725
15726         if (val & SDVO_ENABLE ||
15727             (val & SDVO_PIPE_SEL_MASK) == SDVO_PIPE_SEL(PIPE_A))
15728                 return;
15729
15730         DRM_DEBUG_KMS("Sanitizing transcoder select for HDMI %c\n",
15731                       port_name(port));
15732
15733         val &= ~SDVO_PIPE_SEL_MASK;
15734         val |= SDVO_PIPE_SEL(PIPE_A);
15735
15736         I915_WRITE(hdmi_reg, val);
15737 }
15738
15739 static void ibx_sanitize_pch_dp_port(struct drm_i915_private *dev_priv,
15740                                      enum port port, i915_reg_t dp_reg)
15741 {
15742         u32 val = I915_READ(dp_reg);
15743
15744         if (val & DP_PORT_EN ||
15745             (val & DP_PIPE_SEL_MASK) == DP_PIPE_SEL(PIPE_A))
15746                 return;
15747
15748         DRM_DEBUG_KMS("Sanitizing transcoder select for DP %c\n",
15749                       port_name(port));
15750
15751         val &= ~DP_PIPE_SEL_MASK;
15752         val |= DP_PIPE_SEL(PIPE_A);
15753
15754         I915_WRITE(dp_reg, val);
15755 }
15756
15757 static void ibx_sanitize_pch_ports(struct drm_i915_private *dev_priv)
15758 {
15759         /*
15760          * The BIOS may select transcoder B on some of the PCH
15761          * ports even it doesn't enable the port. This would trip
15762          * assert_pch_dp_disabled() and assert_pch_hdmi_disabled().
15763          * Sanitize the transcoder select bits to prevent that. We
15764          * assume that the BIOS never actually enabled the port,
15765          * because if it did we'd actually have to toggle the port
15766          * on and back off to make the transcoder A select stick
15767          * (see. intel_dp_link_down(), intel_disable_hdmi(),
15768          * intel_disable_sdvo()).
15769          */
15770         ibx_sanitize_pch_dp_port(dev_priv, PORT_B, PCH_DP_B);
15771         ibx_sanitize_pch_dp_port(dev_priv, PORT_C, PCH_DP_C);
15772         ibx_sanitize_pch_dp_port(dev_priv, PORT_D, PCH_DP_D);
15773
15774         /* PCH SDVOB multiplex with HDMIB */
15775         ibx_sanitize_pch_hdmi_port(dev_priv, PORT_B, PCH_HDMIB);
15776         ibx_sanitize_pch_hdmi_port(dev_priv, PORT_C, PCH_HDMIC);
15777         ibx_sanitize_pch_hdmi_port(dev_priv, PORT_D, PCH_HDMID);
15778 }
15779
15780 /* Scan out the current hw modeset state,
15781  * and sanitizes it to the current state
15782  */
15783 static void
15784 intel_modeset_setup_hw_state(struct drm_device *dev,
15785                              struct drm_modeset_acquire_ctx *ctx)
15786 {
15787         struct drm_i915_private *dev_priv = to_i915(dev);
15788         struct intel_crtc *crtc;
15789         struct intel_crtc_state *crtc_state;
15790         struct intel_encoder *encoder;
15791         int i;
15792
15793         intel_display_power_get(dev_priv, POWER_DOMAIN_INIT);
15794
15795         intel_early_display_was(dev_priv);
15796         intel_modeset_readout_hw_state(dev);
15797
15798         /* HW state is read out, now we need to sanitize this mess. */
15799         get_encoder_power_domains(dev_priv);
15800
15801         if (HAS_PCH_IBX(dev_priv))
15802                 ibx_sanitize_pch_ports(dev_priv);
15803
15804         /*
15805          * intel_sanitize_plane_mapping() may need to do vblank
15806          * waits, so we need vblank interrupts restored beforehand.
15807          */
15808         for_each_intel_crtc(&dev_priv->drm, crtc) {
15809                 drm_crtc_vblank_reset(&crtc->base);
15810
15811                 if (crtc->base.state->active)
15812                         drm_crtc_vblank_on(&crtc->base);
15813         }
15814
15815         intel_sanitize_plane_mapping(dev_priv);
15816
15817         for_each_intel_encoder(dev, encoder)
15818                 intel_sanitize_encoder(encoder);
15819
15820         for_each_intel_crtc(&dev_priv->drm, crtc) {
15821                 crtc_state = to_intel_crtc_state(crtc->base.state);
15822                 intel_sanitize_crtc(crtc, ctx);
15823                 intel_dump_pipe_config(crtc, crtc_state,
15824                                        "[setup_hw_state]");
15825         }
15826
15827         intel_modeset_update_connector_atomic_state(dev);
15828
15829         for (i = 0; i < dev_priv->num_shared_dpll; i++) {
15830                 struct intel_shared_dpll *pll = &dev_priv->shared_dplls[i];
15831
15832                 if (!pll->on || pll->active_mask)
15833                         continue;
15834
15835                 DRM_DEBUG_KMS("%s enabled but not in use, disabling\n",
15836                               pll->info->name);
15837
15838                 pll->info->funcs->disable(dev_priv, pll);
15839                 pll->on = false;
15840         }
15841
15842         if (IS_G4X(dev_priv)) {
15843                 g4x_wm_get_hw_state(dev_priv);
15844                 g4x_wm_sanitize(dev_priv);
15845         } else if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) {
15846                 vlv_wm_get_hw_state(dev_priv);
15847                 vlv_wm_sanitize(dev_priv);
15848         } else if (INTEL_GEN(dev_priv) >= 9) {
15849                 skl_wm_get_hw_state(dev_priv);
15850         } else if (HAS_PCH_SPLIT(dev_priv)) {
15851                 ilk_wm_get_hw_state(dev_priv);
15852         }
15853
15854         for_each_intel_crtc(dev, crtc) {
15855                 u64 put_domains;
15856
15857                 crtc_state = to_intel_crtc_state(crtc->base.state);
15858                 put_domains = modeset_get_crtc_power_domains(&crtc->base, crtc_state);
15859                 if (WARN_ON(put_domains))
15860                         modeset_put_power_domains(dev_priv, put_domains);
15861         }
15862
15863         intel_display_power_put(dev_priv, POWER_DOMAIN_INIT);
15864
15865         intel_fbc_init_pipe_state(dev_priv);
15866 }
15867
15868 void intel_display_resume(struct drm_device *dev)
15869 {
15870         struct drm_i915_private *dev_priv = to_i915(dev);
15871         struct drm_atomic_state *state = dev_priv->modeset_restore_state;
15872         struct drm_modeset_acquire_ctx ctx;
15873         int ret;
15874
15875         dev_priv->modeset_restore_state = NULL;
15876         if (state)
15877                 state->acquire_ctx = &ctx;
15878
15879         drm_modeset_acquire_init(&ctx, 0);
15880
15881         while (1) {
15882                 ret = drm_modeset_lock_all_ctx(dev, &ctx);
15883                 if (ret != -EDEADLK)
15884                         break;
15885
15886                 drm_modeset_backoff(&ctx);
15887         }
15888
15889         if (!ret)
15890                 ret = __intel_display_resume(dev, state, &ctx);
15891
15892         intel_enable_ipc(dev_priv);
15893         drm_modeset_drop_locks(&ctx);
15894         drm_modeset_acquire_fini(&ctx);
15895
15896         if (ret)
15897                 DRM_ERROR("Restoring old state failed with %i\n", ret);
15898         if (state)
15899                 drm_atomic_state_put(state);
15900 }
15901
15902 static void intel_hpd_poll_fini(struct drm_device *dev)
15903 {
15904         struct intel_connector *connector;
15905         struct drm_connector_list_iter conn_iter;
15906
15907         /* Kill all the work that may have been queued by hpd. */
15908         drm_connector_list_iter_begin(dev, &conn_iter);
15909         for_each_intel_connector_iter(connector, &conn_iter) {
15910                 if (connector->modeset_retry_work.func)
15911                         cancel_work_sync(&connector->modeset_retry_work);
15912                 if (connector->hdcp.shim) {
15913                         cancel_delayed_work_sync(&connector->hdcp.check_work);
15914                         cancel_work_sync(&connector->hdcp.prop_work);
15915                 }
15916         }
15917         drm_connector_list_iter_end(&conn_iter);
15918 }
15919
15920 void intel_modeset_cleanup(struct drm_device *dev)
15921 {
15922         struct drm_i915_private *dev_priv = to_i915(dev);
15923
15924         flush_workqueue(dev_priv->modeset_wq);
15925
15926         flush_work(&dev_priv->atomic_helper.free_work);
15927         WARN_ON(!llist_empty(&dev_priv->atomic_helper.free_list));
15928
15929         /*
15930          * Interrupts and polling as the first thing to avoid creating havoc.
15931          * Too much stuff here (turning of connectors, ...) would
15932          * experience fancy races otherwise.
15933          */
15934         intel_irq_uninstall(dev_priv);
15935
15936         /*
15937          * Due to the hpd irq storm handling the hotplug work can re-arm the
15938          * poll handlers. Hence disable polling after hpd handling is shut down.
15939          */
15940         intel_hpd_poll_fini(dev);
15941
15942         /* poll work can call into fbdev, hence clean that up afterwards */
15943         intel_fbdev_fini(dev_priv);
15944
15945         intel_unregister_dsm_handler();
15946
15947         intel_fbc_global_disable(dev_priv);
15948
15949         /* flush any delayed tasks or pending work */
15950         flush_scheduled_work();
15951
15952         drm_mode_config_cleanup(dev);
15953
15954         intel_overlay_cleanup(dev_priv);
15955
15956         intel_teardown_gmbus(dev_priv);
15957
15958         destroy_workqueue(dev_priv->modeset_wq);
15959
15960         intel_fbc_cleanup_cfb(dev_priv);
15961 }
15962
15963 /*
15964  * set vga decode state - true == enable VGA decode
15965  */
15966 int intel_modeset_vga_set_state(struct drm_i915_private *dev_priv, bool state)
15967 {
15968         unsigned reg = INTEL_GEN(dev_priv) >= 6 ? SNB_GMCH_CTRL : INTEL_GMCH_CTRL;
15969         u16 gmch_ctrl;
15970
15971         if (pci_read_config_word(dev_priv->bridge_dev, reg, &gmch_ctrl)) {
15972                 DRM_ERROR("failed to read control word\n");
15973                 return -EIO;
15974         }
15975
15976         if (!!(gmch_ctrl & INTEL_GMCH_VGA_DISABLE) == !state)
15977                 return 0;
15978
15979         if (state)
15980                 gmch_ctrl &= ~INTEL_GMCH_VGA_DISABLE;
15981         else
15982                 gmch_ctrl |= INTEL_GMCH_VGA_DISABLE;
15983
15984         if (pci_write_config_word(dev_priv->bridge_dev, reg, gmch_ctrl)) {
15985                 DRM_ERROR("failed to write control word\n");
15986                 return -EIO;
15987         }
15988
15989         return 0;
15990 }
15991
15992 #if IS_ENABLED(CONFIG_DRM_I915_CAPTURE_ERROR)
15993
15994 struct intel_display_error_state {
15995
15996         u32 power_well_driver;
15997
15998         int num_transcoders;
15999
16000         struct intel_cursor_error_state {
16001                 u32 control;
16002                 u32 position;
16003                 u32 base;
16004                 u32 size;
16005         } cursor[I915_MAX_PIPES];
16006
16007         struct intel_pipe_error_state {
16008                 bool power_domain_on;
16009                 u32 source;
16010                 u32 stat;
16011         } pipe[I915_MAX_PIPES];
16012
16013         struct intel_plane_error_state {
16014                 u32 control;
16015                 u32 stride;
16016                 u32 size;
16017                 u32 pos;
16018                 u32 addr;
16019                 u32 surface;
16020                 u32 tile_offset;
16021         } plane[I915_MAX_PIPES];
16022
16023         struct intel_transcoder_error_state {
16024                 bool power_domain_on;
16025                 enum transcoder cpu_transcoder;
16026
16027                 u32 conf;
16028
16029                 u32 htotal;
16030                 u32 hblank;
16031                 u32 hsync;
16032                 u32 vtotal;
16033                 u32 vblank;
16034                 u32 vsync;
16035         } transcoder[4];
16036 };
16037
16038 struct intel_display_error_state *
16039 intel_display_capture_error_state(struct drm_i915_private *dev_priv)
16040 {
16041         struct intel_display_error_state *error;
16042         int transcoders[] = {
16043                 TRANSCODER_A,
16044                 TRANSCODER_B,
16045                 TRANSCODER_C,
16046                 TRANSCODER_EDP,
16047         };
16048         int i;
16049
16050         if (!HAS_DISPLAY(dev_priv))
16051                 return NULL;
16052
16053         error = kzalloc(sizeof(*error), GFP_ATOMIC);
16054         if (error == NULL)
16055                 return NULL;
16056
16057         if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
16058                 error->power_well_driver = I915_READ(HSW_PWR_WELL_CTL2);
16059
16060         for_each_pipe(dev_priv, i) {
16061                 error->pipe[i].power_domain_on =
16062                         __intel_display_power_is_enabled(dev_priv,
16063                                                          POWER_DOMAIN_PIPE(i));
16064                 if (!error->pipe[i].power_domain_on)
16065                         continue;
16066
16067                 error->cursor[i].control = I915_READ(CURCNTR(i));
16068                 error->cursor[i].position = I915_READ(CURPOS(i));
16069                 error->cursor[i].base = I915_READ(CURBASE(i));
16070
16071                 error->plane[i].control = I915_READ(DSPCNTR(i));
16072                 error->plane[i].stride = I915_READ(DSPSTRIDE(i));
16073                 if (INTEL_GEN(dev_priv) <= 3) {
16074                         error->plane[i].size = I915_READ(DSPSIZE(i));
16075                         error->plane[i].pos = I915_READ(DSPPOS(i));
16076                 }
16077                 if (INTEL_GEN(dev_priv) <= 7 && !IS_HASWELL(dev_priv))
16078                         error->plane[i].addr = I915_READ(DSPADDR(i));
16079                 if (INTEL_GEN(dev_priv) >= 4) {
16080                         error->plane[i].surface = I915_READ(DSPSURF(i));
16081                         error->plane[i].tile_offset = I915_READ(DSPTILEOFF(i));
16082                 }
16083
16084                 error->pipe[i].source = I915_READ(PIPESRC(i));
16085
16086                 if (HAS_GMCH_DISPLAY(dev_priv))
16087                         error->pipe[i].stat = I915_READ(PIPESTAT(i));
16088         }
16089
16090         /* Note: this does not include DSI transcoders. */
16091         error->num_transcoders = INTEL_INFO(dev_priv)->num_pipes;
16092         if (HAS_DDI(dev_priv))
16093                 error->num_transcoders++; /* Account for eDP. */
16094
16095         for (i = 0; i < error->num_transcoders; i++) {
16096                 enum transcoder cpu_transcoder = transcoders[i];
16097
16098                 error->transcoder[i].power_domain_on =
16099                         __intel_display_power_is_enabled(dev_priv,
16100                                 POWER_DOMAIN_TRANSCODER(cpu_transcoder));
16101                 if (!error->transcoder[i].power_domain_on)
16102                         continue;
16103
16104                 error->transcoder[i].cpu_transcoder = cpu_transcoder;
16105
16106                 error->transcoder[i].conf = I915_READ(PIPECONF(cpu_transcoder));
16107                 error->transcoder[i].htotal = I915_READ(HTOTAL(cpu_transcoder));
16108                 error->transcoder[i].hblank = I915_READ(HBLANK(cpu_transcoder));
16109                 error->transcoder[i].hsync = I915_READ(HSYNC(cpu_transcoder));
16110                 error->transcoder[i].vtotal = I915_READ(VTOTAL(cpu_transcoder));
16111                 error->transcoder[i].vblank = I915_READ(VBLANK(cpu_transcoder));
16112                 error->transcoder[i].vsync = I915_READ(VSYNC(cpu_transcoder));
16113         }
16114
16115         return error;
16116 }
16117
16118 #define err_printf(e, ...) i915_error_printf(e, __VA_ARGS__)
16119
16120 void
16121 intel_display_print_error_state(struct drm_i915_error_state_buf *m,
16122                                 struct intel_display_error_state *error)
16123 {
16124         struct drm_i915_private *dev_priv = m->i915;
16125         int i;
16126
16127         if (!error)
16128                 return;
16129
16130         err_printf(m, "Num Pipes: %d\n", INTEL_INFO(dev_priv)->num_pipes);
16131         if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
16132                 err_printf(m, "PWR_WELL_CTL2: %08x\n",
16133                            error->power_well_driver);
16134         for_each_pipe(dev_priv, i) {
16135                 err_printf(m, "Pipe [%d]:\n", i);
16136                 err_printf(m, "  Power: %s\n",
16137                            onoff(error->pipe[i].power_domain_on));
16138                 err_printf(m, "  SRC: %08x\n", error->pipe[i].source);
16139                 err_printf(m, "  STAT: %08x\n", error->pipe[i].stat);
16140
16141                 err_printf(m, "Plane [%d]:\n", i);
16142                 err_printf(m, "  CNTR: %08x\n", error->plane[i].control);
16143                 err_printf(m, "  STRIDE: %08x\n", error->plane[i].stride);
16144                 if (INTEL_GEN(dev_priv) <= 3) {
16145                         err_printf(m, "  SIZE: %08x\n", error->plane[i].size);
16146                         err_printf(m, "  POS: %08x\n", error->plane[i].pos);
16147                 }
16148                 if (INTEL_GEN(dev_priv) <= 7 && !IS_HASWELL(dev_priv))
16149                         err_printf(m, "  ADDR: %08x\n", error->plane[i].addr);
16150                 if (INTEL_GEN(dev_priv) >= 4) {
16151                         err_printf(m, "  SURF: %08x\n", error->plane[i].surface);
16152                         err_printf(m, "  TILEOFF: %08x\n", error->plane[i].tile_offset);
16153                 }
16154
16155                 err_printf(m, "Cursor [%d]:\n", i);
16156                 err_printf(m, "  CNTR: %08x\n", error->cursor[i].control);
16157                 err_printf(m, "  POS: %08x\n", error->cursor[i].position);
16158                 err_printf(m, "  BASE: %08x\n", error->cursor[i].base);
16159         }
16160
16161         for (i = 0; i < error->num_transcoders; i++) {
16162                 err_printf(m, "CPU transcoder: %s\n",
16163                            transcoder_name(error->transcoder[i].cpu_transcoder));
16164                 err_printf(m, "  Power: %s\n",
16165                            onoff(error->transcoder[i].power_domain_on));
16166                 err_printf(m, "  CONF: %08x\n", error->transcoder[i].conf);
16167                 err_printf(m, "  HTOTAL: %08x\n", error->transcoder[i].htotal);
16168                 err_printf(m, "  HBLANK: %08x\n", error->transcoder[i].hblank);
16169                 err_printf(m, "  HSYNC: %08x\n", error->transcoder[i].hsync);
16170                 err_printf(m, "  VTOTAL: %08x\n", error->transcoder[i].vtotal);
16171                 err_printf(m, "  VBLANK: %08x\n", error->transcoder[i].vblank);
16172                 err_printf(m, "  VSYNC: %08x\n", error->transcoder[i].vsync);
16173         }
16174 }
16175
16176 #endif