drivers/gpu/drm/i915/display/intel_display.c

   1 /*
   2  * Copyright © 2006-2007 Intel Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
  21  * DEALINGS IN THE SOFTWARE.
  22  *
  23  * Authors:
  24  *      Eric Anholt <eric@anholt.net>
  25  */
  26
  27 #include <linux/i2c.h>
  28 #include <linux/input.h>
  29 #include <linux/intel-iommu.h>
  30 #include <linux/kernel.h>
  31 #include <linux/module.h>
  32 #include <linux/reservation.h>
  33 #include <linux/slab.h>
  34 #include <linux/vgaarb.h>
  35
  36 #include <drm/drm_atomic.h>
  37 #include <drm/drm_atomic_helper.h>
  38 #include <drm/drm_atomic_uapi.h>
  39 #include <drm/drm_dp_helper.h>
  40 #include <drm/drm_edid.h>
  41 #include <drm/drm_fourcc.h>
  42 #include <drm/drm_plane_helper.h>
  43 #include <drm/drm_probe_helper.h>
  44 #include <drm/drm_rect.h>
  45 #include <drm/i915_drm.h>
  46
  47 #include "display/intel_crt.h"
  48 #include "display/intel_ddi.h"
  49 #include "display/intel_dp.h"
  50 #include "display/intel_dsi.h"
  51 #include "display/intel_dvo.h"
  52 #include "display/intel_gmbus.h"
  53 #include "display/intel_hdmi.h"
  54 #include "display/intel_lvds.h"
  55 #include "display/intel_sdvo.h"
  56 #include "display/intel_tv.h"
  57 #include "display/intel_vdsc.h"
  58
  59 #include "i915_drv.h"
  60 #include "i915_trace.h"
  61 #include "intel_acpi.h"
  62 #include "intel_atomic.h"
  63 #include "intel_atomic_plane.h"
  64 #include "intel_bw.h"
  65 #include "intel_color.h"
  66 #include "intel_cdclk.h"
  67 #include "intel_drv.h"
  68 #include "intel_fbc.h"
  69 #include "intel_fbdev.h"
  70 #include "intel_fifo_underrun.h"
  71 #include "intel_frontbuffer.h"
  72 #include "intel_hdcp.h"
  73 #include "intel_hotplug.h"
  74 #include "intel_overlay.h"
  75 #include "intel_pipe_crc.h"
  76 #include "intel_pm.h"
  77 #include "intel_psr.h"
  78 #include "intel_quirks.h"
  79 #include "intel_sideband.h"
  80 #include "intel_sprite.h"
  81 #include "intel_tc.h"
  82
  83 /* Primary plane formats for gen <= 3 */
  84 static const u32 i8xx_primary_formats[] = {
  85         DRM_FORMAT_C8,
  86         DRM_FORMAT_RGB565,
  87         DRM_FORMAT_XRGB1555,
  88         DRM_FORMAT_XRGB8888,
  89 };
  90
  91 /* Primary plane formats for gen >= 4 */
  92 static const u32 i965_primary_formats[] = {
  93         DRM_FORMAT_C8,
  94         DRM_FORMAT_RGB565,
  95         DRM_FORMAT_XRGB8888,
  96         DRM_FORMAT_XBGR8888,
  97         DRM_FORMAT_XRGB2101010,
  98         DRM_FORMAT_XBGR2101010,
  99 };
 100
 101 static const u64 i9xx_format_modifiers[] = {
 102         I915_FORMAT_MOD_X_TILED,
 103         DRM_FORMAT_MOD_LINEAR,
 104         DRM_FORMAT_MOD_INVALID
 105 };
 106
 107 /* Cursor formats */
 108 static const u32 intel_cursor_formats[] = {
 109         DRM_FORMAT_ARGB8888,
 110 };
 111
 112 static const u64 cursor_format_modifiers[] = {
 113         DRM_FORMAT_MOD_LINEAR,
 114         DRM_FORMAT_MOD_INVALID
 115 };
 116
 117 static void i9xx_crtc_clock_get(struct intel_crtc *crtc,
 118                                 struct intel_crtc_state *pipe_config);
 119 static void ironlake_pch_clock_get(struct intel_crtc *crtc,
 120                                    struct intel_crtc_state *pipe_config);
 121
 122 static int intel_framebuffer_init(struct intel_framebuffer *ifb,
 123                                   struct drm_i915_gem_object *obj,
 124                                   struct drm_mode_fb_cmd2 *mode_cmd);
 125 static void intel_set_pipe_timings(const struct intel_crtc_state *crtc_state);
 126 static void intel_set_pipe_src_size(const struct intel_crtc_state *crtc_state);
 127 static void intel_cpu_transcoder_set_m_n(const struct intel_crtc_state *crtc_state,
 128                                          const struct intel_link_m_n *m_n,
 129                                          const struct intel_link_m_n *m2_n2);
 130 static void i9xx_set_pipeconf(const struct intel_crtc_state *crtc_state);
 131 static void ironlake_set_pipeconf(const struct intel_crtc_state *crtc_state);
 132 static void haswell_set_pipeconf(const struct intel_crtc_state *crtc_state);
 133 static void bdw_set_pipemisc(const struct intel_crtc_state *crtc_state);
 134 static void vlv_prepare_pll(struct intel_crtc *crtc,
 135                             const struct intel_crtc_state *pipe_config);
 136 static void chv_prepare_pll(struct intel_crtc *crtc,
 137                             const struct intel_crtc_state *pipe_config);
 138 static void intel_begin_crtc_commit(struct intel_atomic_state *, struct intel_crtc *);
 139 static void intel_finish_crtc_commit(struct intel_atomic_state *, struct intel_crtc *);
 140 static void intel_crtc_init_scalers(struct intel_crtc *crtc,
 141                                     struct intel_crtc_state *crtc_state);
 142 static void skylake_pfit_enable(const struct intel_crtc_state *crtc_state);
 143 static void ironlake_pfit_disable(const struct intel_crtc_state *old_crtc_state);
 144 static void ironlake_pfit_enable(const struct intel_crtc_state *crtc_state);
 145 static void intel_modeset_setup_hw_state(struct drm_device *dev,
 146                                          struct drm_modeset_acquire_ctx *ctx);
 147 static void intel_pre_disable_primary_noatomic(struct drm_crtc *crtc);
 148
 149 struct intel_limit {
 150         struct {
 151                 int min, max;
 152         } dot, vco, n, m, m1, m2, p, p1;
 153
 154         struct {
 155                 int dot_limit;
 156                 int p2_slow, p2_fast;
 157         } p2;
 158 };
 159
 160 /* returns HPLL frequency in kHz */
 161 int vlv_get_hpll_vco(struct drm_i915_private *dev_priv)
 162 {
 163         int hpll_freq, vco_freq[] = { 800, 1600, 2000, 2400 };
 164
 165         /* Obtain SKU information */
 166         hpll_freq = vlv_cck_read(dev_priv, CCK_FUSE_REG) &
 167                 CCK_FUSE_HPLL_FREQ_MASK;
 168
 169         return vco_freq[hpll_freq] * 1000;
 170 }
 171
 172 int vlv_get_cck_clock(struct drm_i915_private *dev_priv,
 173                       const char *name, u32 reg, int ref_freq)
 174 {
 175         u32 val;
 176         int divider;
 177
 178         val = vlv_cck_read(dev_priv, reg);
 179         divider = val & CCK_FREQUENCY_VALUES;
 180
 181         WARN((val & CCK_FREQUENCY_STATUS) !=
 182              (divider << CCK_FREQUENCY_STATUS_SHIFT),
 183              "%s change in progress\n", name);
 184
 185         return DIV_ROUND_CLOSEST(ref_freq << 1, divider + 1);
 186 }
 187
 188 int vlv_get_cck_clock_hpll(struct drm_i915_private *dev_priv,
 189                            const char *name, u32 reg)
 190 {
 191         int hpll;
 192
 193         vlv_cck_get(dev_priv);
 194
 195         if (dev_priv->hpll_freq == 0)
 196                 dev_priv->hpll_freq = vlv_get_hpll_vco(dev_priv);
 197
 198         hpll = vlv_get_cck_clock(dev_priv, name, reg, dev_priv->hpll_freq);
 199
 200         vlv_cck_put(dev_priv);
 201
 202         return hpll;
 203 }
 204
 205 static void intel_update_czclk(struct drm_i915_private *dev_priv)
 206 {
 207         if (!(IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)))
 208                 return;
 209
 210         dev_priv->czclk_freq = vlv_get_cck_clock_hpll(dev_priv, "czclk",
 211                                                       CCK_CZ_CLOCK_CONTROL);
 212
 213         DRM_DEBUG_DRIVER("CZ clock rate: %d kHz\n", dev_priv->czclk_freq);
 214 }
 215
 216 static inline u32 /* units of 100MHz */
 217 intel_fdi_link_freq(struct drm_i915_private *dev_priv,
 218                     const struct intel_crtc_state *pipe_config)
 219 {
 220         if (HAS_DDI(dev_priv))
 221                 return pipe_config->port_clock; /* SPLL */
 222         else
 223                 return dev_priv->fdi_pll_freq;
 224 }
 225
 226 static const struct intel_limit intel_limits_i8xx_dac = {
 227         .dot = { .min = 25000, .max = 350000 },
 228         .vco = { .min = 908000, .max = 1512000 },
 229         .n = { .min = 2, .max = 16 },
 230         .m = { .min = 96, .max = 140 },
 231         .m1 = { .min = 18, .max = 26 },
 232         .m2 = { .min = 6, .max = 16 },
 233         .p = { .min = 4, .max = 128 },
 234         .p1 = { .min = 2, .max = 33 },
 235         .p2 = { .dot_limit = 165000,
 236                 .p2_slow = 4, .p2_fast = 2 },
 237 };
 238
 239 static const struct intel_limit intel_limits_i8xx_dvo = {
 240         .dot = { .min = 25000, .max = 350000 },
 241         .vco = { .min = 908000, .max = 1512000 },
 242         .n = { .min = 2, .max = 16 },
 243         .m = { .min = 96, .max = 140 },
 244         .m1 = { .min = 18, .max = 26 },
 245         .m2 = { .min = 6, .max = 16 },
 246         .p = { .min = 4, .max = 128 },
 247         .p1 = { .min = 2, .max = 33 },
 248         .p2 = { .dot_limit = 165000,
 249                 .p2_slow = 4, .p2_fast = 4 },
 250 };
 251
 252 static const struct intel_limit intel_limits_i8xx_lvds = {
 253         .dot = { .min = 25000, .max = 350000 },
 254         .vco = { .min = 908000, .max = 1512000 },
 255         .n = { .min = 2, .max = 16 },
 256         .m = { .min = 96, .max = 140 },
 257         .m1 = { .min = 18, .max = 26 },
 258         .m2 = { .min = 6, .max = 16 },
 259         .p = { .min = 4, .max = 128 },
 260         .p1 = { .min = 1, .max = 6 },
 261         .p2 = { .dot_limit = 165000,
 262                 .p2_slow = 14, .p2_fast = 7 },
 263 };
 264
 265 static const struct intel_limit intel_limits_i9xx_sdvo = {
 266         .dot = { .min = 20000, .max = 400000 },
 267         .vco = { .min = 1400000, .max = 2800000 },
 268         .n = { .min = 1, .max = 6 },
 269         .m = { .min = 70, .max = 120 },
 270         .m1 = { .min = 8, .max = 18 },
 271         .m2 = { .min = 3, .max = 7 },
 272         .p = { .min = 5, .max = 80 },
 273         .p1 = { .min = 1, .max = 8 },
 274         .p2 = { .dot_limit = 200000,
 275                 .p2_slow = 10, .p2_fast = 5 },
 276 };
 277
 278 static const struct intel_limit intel_limits_i9xx_lvds = {
 279         .dot = { .min = 20000, .max = 400000 },
 280         .vco = { .min = 1400000, .max = 2800000 },
 281         .n = { .min = 1, .max = 6 },
 282         .m = { .min = 70, .max = 120 },
 283         .m1 = { .min = 8, .max = 18 },
 284         .m2 = { .min = 3, .max = 7 },
 285         .p = { .min = 7, .max = 98 },
 286         .p1 = { .min = 1, .max = 8 },
 287         .p2 = { .dot_limit = 112000,
 288                 .p2_slow = 14, .p2_fast = 7 },
 289 };
 290
 291
 292 static const struct intel_limit intel_limits_g4x_sdvo = {
 293         .dot = { .min = 25000, .max = 270000 },
 294         .vco = { .min = 1750000, .max = 3500000},
 295         .n = { .min = 1, .max = 4 },
 296         .m = { .min = 104, .max = 138 },
 297         .m1 = { .min = 17, .max = 23 },
 298         .m2 = { .min = 5, .max = 11 },
 299         .p = { .min = 10, .max = 30 },
 300         .p1 = { .min = 1, .max = 3},
 301         .p2 = { .dot_limit = 270000,
 302                 .p2_slow = 10,
 303                 .p2_fast = 10
 304         },
 305 };
 306
 307 static const struct intel_limit intel_limits_g4x_hdmi = {
 308         .dot = { .min = 22000, .max = 400000 },
 309         .vco = { .min = 1750000, .max = 3500000},
 310         .n = { .min = 1, .max = 4 },
 311         .m = { .min = 104, .max = 138 },
 312         .m1 = { .min = 16, .max = 23 },
 313         .m2 = { .min = 5, .max = 11 },
 314         .p = { .min = 5, .max = 80 },
 315         .p1 = { .min = 1, .max = 8},
 316         .p2 = { .dot_limit = 165000,
 317                 .p2_slow = 10, .p2_fast = 5 },
 318 };
 319
 320 static const struct intel_limit intel_limits_g4x_single_channel_lvds = {
 321         .dot = { .min = 20000, .max = 115000 },
 322         .vco = { .min = 1750000, .max = 3500000 },
 323         .n = { .min = 1, .max = 3 },
 324         .m = { .min = 104, .max = 138 },
 325         .m1 = { .min = 17, .max = 23 },
 326         .m2 = { .min = 5, .max = 11 },
 327         .p = { .min = 28, .max = 112 },
 328         .p1 = { .min = 2, .max = 8 },
 329         .p2 = { .dot_limit = 0,
 330                 .p2_slow = 14, .p2_fast = 14
 331         },
 332 };
 333
 334 static const struct intel_limit intel_limits_g4x_dual_channel_lvds = {
 335         .dot = { .min = 80000, .max = 224000 },
 336         .vco = { .min = 1750000, .max = 3500000 },
 337         .n = { .min = 1, .max = 3 },
 338         .m = { .min = 104, .max = 138 },
 339         .m1 = { .min = 17, .max = 23 },
 340         .m2 = { .min = 5, .max = 11 },
 341         .p = { .min = 14, .max = 42 },
 342         .p1 = { .min = 2, .max = 6 },
 343         .p2 = { .dot_limit = 0,
 344                 .p2_slow = 7, .p2_fast = 7
 345         },
 346 };
 347
 348 static const struct intel_limit intel_limits_pineview_sdvo = {
 349         .dot = { .min = 20000, .max = 400000},
 350         .vco = { .min = 1700000, .max = 3500000 },
 351         /* Pineview's Ncounter is a ring counter */
 352         .n = { .min = 3, .max = 6 },
 353         .m = { .min = 2, .max = 256 },
 354         /* Pineview only has one combined m divider, which we treat as m2. */
 355         .m1 = { .min = 0, .max = 0 },
 356         .m2 = { .min = 0, .max = 254 },
 357         .p = { .min = 5, .max = 80 },
 358         .p1 = { .min = 1, .max = 8 },
 359         .p2 = { .dot_limit = 200000,
 360                 .p2_slow = 10, .p2_fast = 5 },
 361 };
 362
 363 static const struct intel_limit intel_limits_pineview_lvds = {
 364         .dot = { .min = 20000, .max = 400000 },
 365         .vco = { .min = 1700000, .max = 3500000 },
 366         .n = { .min = 3, .max = 6 },
 367         .m = { .min = 2, .max = 256 },
 368         .m1 = { .min = 0, .max = 0 },
 369         .m2 = { .min = 0, .max = 254 },
 370         .p = { .min = 7, .max = 112 },
 371         .p1 = { .min = 1, .max = 8 },
 372         .p2 = { .dot_limit = 112000,
 373                 .p2_slow = 14, .p2_fast = 14 },
 374 };
 375
 376 /* Ironlake / Sandybridge
 377  *
 378  * We calculate clock using (register_value + 2) for N/M1/M2, so here
 379  * the range value for them is (actual_value - 2).
 380  */
 381 static const struct intel_limit intel_limits_ironlake_dac = {
 382         .dot = { .min = 25000, .max = 350000 },
 383         .vco = { .min = 1760000, .max = 3510000 },
 384         .n = { .min = 1, .max = 5 },
 385         .m = { .min = 79, .max = 127 },
 386         .m1 = { .min = 12, .max = 22 },
 387         .m2 = { .min = 5, .max = 9 },
 388         .p = { .min = 5, .max = 80 },
 389         .p1 = { .min = 1, .max = 8 },
 390         .p2 = { .dot_limit = 225000,
 391                 .p2_slow = 10, .p2_fast = 5 },
 392 };
 393
 394 static const struct intel_limit intel_limits_ironlake_single_lvds = {
 395         .dot = { .min = 25000, .max = 350000 },
 396         .vco = { .min = 1760000, .max = 3510000 },
 397         .n = { .min = 1, .max = 3 },
 398         .m = { .min = 79, .max = 118 },
 399         .m1 = { .min = 12, .max = 22 },
 400         .m2 = { .min = 5, .max = 9 },
 401         .p = { .min = 28, .max = 112 },
 402         .p1 = { .min = 2, .max = 8 },
 403         .p2 = { .dot_limit = 225000,
 404                 .p2_slow = 14, .p2_fast = 14 },
 405 };
 406
 407 static const struct intel_limit intel_limits_ironlake_dual_lvds = {
 408         .dot = { .min = 25000, .max = 350000 },
 409         .vco = { .min = 1760000, .max = 3510000 },
 410         .n = { .min = 1, .max = 3 },
 411         .m = { .min = 79, .max = 127 },
 412         .m1 = { .min = 12, .max = 22 },
 413         .m2 = { .min = 5, .max = 9 },
 414         .p = { .min = 14, .max = 56 },
 415         .p1 = { .min = 2, .max = 8 },
 416         .p2 = { .dot_limit = 225000,
 417                 .p2_slow = 7, .p2_fast = 7 },
 418 };
 419
 420 /* LVDS 100mhz refclk limits. */
 421 static const struct intel_limit intel_limits_ironlake_single_lvds_100m = {
 422         .dot = { .min = 25000, .max = 350000 },
 423         .vco = { .min = 1760000, .max = 3510000 },
 424         .n = { .min = 1, .max = 2 },
 425         .m = { .min = 79, .max = 126 },
 426         .m1 = { .min = 12, .max = 22 },
 427         .m2 = { .min = 5, .max = 9 },
 428         .p = { .min = 28, .max = 112 },
 429         .p1 = { .min = 2, .max = 8 },
 430         .p2 = { .dot_limit = 225000,
 431                 .p2_slow = 14, .p2_fast = 14 },
 432 };
 433
 434 static const struct intel_limit intel_limits_ironlake_dual_lvds_100m = {
 435         .dot = { .min = 25000, .max = 350000 },
 436         .vco = { .min = 1760000, .max = 3510000 },
 437         .n = { .min = 1, .max = 3 },
 438         .m = { .min = 79, .max = 126 },
 439         .m1 = { .min = 12, .max = 22 },
 440         .m2 = { .min = 5, .max = 9 },
 441         .p = { .min = 14, .max = 42 },
 442         .p1 = { .min = 2, .max = 6 },
 443         .p2 = { .dot_limit = 225000,
 444                 .p2_slow = 7, .p2_fast = 7 },
 445 };
 446
 447 static const struct intel_limit intel_limits_vlv = {
 448          /*
 449           * These are the data rate limits (measured in fast clocks)
 450           * since those are the strictest limits we have. The fast
 451           * clock and actual rate limits are more relaxed, so checking
 452           * them would make no difference.
 453           */
 454         .dot = { .min = 25000 * 5, .max = 270000 * 5 },
 455         .vco = { .min = 4000000, .max = 6000000 },
 456         .n = { .min = 1, .max = 7 },
 457         .m1 = { .min = 2, .max = 3 },
 458         .m2 = { .min = 11, .max = 156 },
 459         .p1 = { .min = 2, .max = 3 },
 460         .p2 = { .p2_slow = 2, .p2_fast = 20 }, /* slow=min, fast=max */
 461 };
 462
 463 static const struct intel_limit intel_limits_chv = {
 464         /*
 465          * These are the data rate limits (measured in fast clocks)
 466          * since those are the strictest limits we have.  The fast
 467          * clock and actual rate limits are more relaxed, so checking
 468          * them would make no difference.
 469          */
 470         .dot = { .min = 25000 * 5, .max = 540000 * 5},
 471         .vco = { .min = 4800000, .max = 6480000 },
 472         .n = { .min = 1, .max = 1 },
 473         .m1 = { .min = 2, .max = 2 },
 474         .m2 = { .min = 24 << 22, .max = 175 << 22 },
 475         .p1 = { .min = 2, .max = 4 },
 476         .p2 = { .p2_slow = 1, .p2_fast = 14 },
 477 };
 478
 479 static const struct intel_limit intel_limits_bxt = {
 480         /* FIXME: find real dot limits */
 481         .dot = { .min = 0, .max = INT_MAX },
 482         .vco = { .min = 4800000, .max = 6700000 },
 483         .n = { .min = 1, .max = 1 },
 484         .m1 = { .min = 2, .max = 2 },
 485         /* FIXME: find real m2 limits */
 486         .m2 = { .min = 2 << 22, .max = 255 << 22 },
 487         .p1 = { .min = 2, .max = 4 },
 488         .p2 = { .p2_slow = 1, .p2_fast = 20 },
 489 };
 490
 491 /* WA Display #0827: Gen9:all */
 492 static void
 493 skl_wa_827(struct drm_i915_private *dev_priv, int pipe, bool enable)
 494 {
 495         if (enable)
 496                 I915_WRITE(CLKGATE_DIS_PSL(pipe),
 497                            I915_READ(CLKGATE_DIS_PSL(pipe)) |
 498                            DUPS1_GATING_DIS | DUPS2_GATING_DIS);
 499         else
 500                 I915_WRITE(CLKGATE_DIS_PSL(pipe),
 501                            I915_READ(CLKGATE_DIS_PSL(pipe)) &
 502                            ~(DUPS1_GATING_DIS | DUPS2_GATING_DIS));
 503 }
 504
 505 /* Wa_2006604312:icl */
 506 static void
 507 icl_wa_scalerclkgating(struct drm_i915_private *dev_priv, enum pipe pipe,
 508                        bool enable)
 509 {
 510         if (enable)
 511                 I915_WRITE(CLKGATE_DIS_PSL(pipe),
 512                            I915_READ(CLKGATE_DIS_PSL(pipe)) | DPFR_GATING_DIS);
 513         else
 514                 I915_WRITE(CLKGATE_DIS_PSL(pipe),
 515                            I915_READ(CLKGATE_DIS_PSL(pipe)) & ~DPFR_GATING_DIS);
 516 }
 517
 518 static bool
 519 needs_modeset(const struct intel_crtc_state *state)
 520 {
 521         return drm_atomic_crtc_needs_modeset(&state->base);
 522 }
 523
 524 /*
 525  * Platform specific helpers to calculate the port PLL loopback- (clock.m),
 526  * and post-divider (clock.p) values, pre- (clock.vco) and post-divided fast
 527  * (clock.dot) clock rates. This fast dot clock is fed to the port's IO logic.
 528  * The helpers' return value is the rate of the clock that is fed to the
 529  * display engine's pipe which can be the above fast dot clock rate or a
 530  * divided-down version of it.
 531  */
 532 /* m1 is reserved as 0 in Pineview, n is a ring counter */
 533 static int pnv_calc_dpll_params(int refclk, struct dpll *clock)
 534 {
 535         clock->m = clock->m2 + 2;
 536         clock->p = clock->p1 * clock->p2;
 537         if (WARN_ON(clock->n == 0 || clock->p == 0))
 538                 return 0;
 539         clock->vco = DIV_ROUND_CLOSEST(refclk * clock->m, clock->n);
 540         clock->dot = DIV_ROUND_CLOSEST(clock->vco, clock->p);
 541
 542         return clock->dot;
 543 }
 544
 545 static u32 i9xx_dpll_compute_m(struct dpll *dpll)
 546 {
 547         return 5 * (dpll->m1 + 2) + (dpll->m2 + 2);
 548 }
 549
 550 static int i9xx_calc_dpll_params(int refclk, struct dpll *clock)
 551 {
 552         clock->m = i9xx_dpll_compute_m(clock);
 553         clock->p = clock->p1 * clock->p2;
 554         if (WARN_ON(clock->n + 2 == 0 || clock->p == 0))
 555                 return 0;
 556         clock->vco = DIV_ROUND_CLOSEST(refclk * clock->m, clock->n + 2);
 557         clock->dot = DIV_ROUND_CLOSEST(clock->vco, clock->p);
 558
 559         return clock->dot;
 560 }
 561
 562 static int vlv_calc_dpll_params(int refclk, struct dpll *clock)
 563 {
 564         clock->m = clock->m1 * clock->m2;
 565         clock->p = clock->p1 * clock->p2;
 566         if (WARN_ON(clock->n == 0 || clock->p == 0))
 567                 return 0;
 568         clock->vco = DIV_ROUND_CLOSEST(refclk * clock->m, clock->n);
 569         clock->dot = DIV_ROUND_CLOSEST(clock->vco, clock->p);
 570
 571         return clock->dot / 5;
 572 }
 573
 574 int chv_calc_dpll_params(int refclk, struct dpll *clock)
 575 {
 576         clock->m = clock->m1 * clock->m2;
 577         clock->p = clock->p1 * clock->p2;
 578         if (WARN_ON(clock->n == 0 || clock->p == 0))
 579                 return 0;
 580         clock->vco = DIV_ROUND_CLOSEST_ULL(mul_u32_u32(refclk, clock->m),
 581                                            clock->n << 22);
 582         clock->dot = DIV_ROUND_CLOSEST(clock->vco, clock->p);
 583
 584         return clock->dot / 5;
 585 }
 586
 587 #define INTELPllInvalid(s)   do { /* DRM_DEBUG(s); */ return false; } while (0)
 588
 589 /*
 590  * Returns whether the given set of divisors are valid for a given refclk with
 591  * the given connectors.
 592  */
 593 static bool intel_PLL_is_valid(struct drm_i915_private *dev_priv,
 594                                const struct intel_limit *limit,
 595                                const struct dpll *clock)
 596 {
 597         if (clock->n   < limit->n.min   || limit->n.max   < clock->n)
 598                 INTELPllInvalid("n out of range\n");
 599         if (clock->p1  < limit->p1.min  || limit->p1.max  < clock->p1)
 600                 INTELPllInvalid("p1 out of range\n");
 601         if (clock->m2  < limit->m2.min  || limit->m2.max  < clock->m2)
 602                 INTELPllInvalid("m2 out of range\n");
 603         if (clock->m1  < limit->m1.min  || limit->m1.max  < clock->m1)
 604                 INTELPllInvalid("m1 out of range\n");
 605
 606         if (!IS_PINEVIEW(dev_priv) && !IS_VALLEYVIEW(dev_priv) &&
 607             !IS_CHERRYVIEW(dev_priv) && !IS_GEN9_LP(dev_priv))
 608                 if (clock->m1 <= clock->m2)
 609                         INTELPllInvalid("m1 <= m2\n");
 610
 611         if (!IS_VALLEYVIEW(dev_priv) && !IS_CHERRYVIEW(dev_priv) &&
 612             !IS_GEN9_LP(dev_priv)) {
 613                 if (clock->p < limit->p.min || limit->p.max < clock->p)
 614                         INTELPllInvalid("p out of range\n");
 615                 if (clock->m < limit->m.min || limit->m.max < clock->m)
 616                         INTELPllInvalid("m out of range\n");
 617         }
 618
 619         if (clock->vco < limit->vco.min || limit->vco.max < clock->vco)
 620                 INTELPllInvalid("vco out of range\n");
 621         /* XXX: We may need to be checking "Dot clock" depending on the multiplier,
 622          * connector, etc., rather than just a single range.
 623          */
 624         if (clock->dot < limit->dot.min || limit->dot.max < clock->dot)
 625                 INTELPllInvalid("dot out of range\n");
 626
 627         return true;
 628 }
 629
 630 static int
 631 i9xx_select_p2_div(const struct intel_limit *limit,
 632                    const struct intel_crtc_state *crtc_state,
 633                    int target)
 634 {
 635         struct drm_i915_private *dev_priv = to_i915(crtc_state->base.crtc->dev);
 636
 637         if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_LVDS)) {
 638                 /*
 639                  * For LVDS just rely on its current settings for dual-channel.
 640                  * We haven't figured out how to reliably set up different
 641                  * single/dual channel state, if we even can.
 642                  */
 643                 if (intel_is_dual_link_lvds(dev_priv))
 644                         return limit->p2.p2_fast;
 645                 else
 646                         return limit->p2.p2_slow;
 647         } else {
 648                 if (target < limit->p2.dot_limit)
 649                         return limit->p2.p2_slow;
 650                 else
 651                         return limit->p2.p2_fast;
 652         }
 653 }
 654
 655 /*
 656  * Returns a set of divisors for the desired target clock with the given
 657  * refclk, or FALSE.  The returned values represent the clock equation:
 658  * reflck * (5 * (m1 + 2) + (m2 + 2)) / (n + 2) / p1 / p2.
 659  *
 660  * Target and reference clocks are specified in kHz.
 661  *
 662  * If match_clock is provided, then best_clock P divider must match the P
 663  * divider from @match_clock used for LVDS downclocking.
 664  */
 665 static bool
 666 i9xx_find_best_dpll(const struct intel_limit *limit,
 667                     struct intel_crtc_state *crtc_state,
 668                     int target, int refclk, struct dpll *match_clock,
 669                     struct dpll *best_clock)
 670 {
 671         struct drm_device *dev = crtc_state->base.crtc->dev;
 672         struct dpll clock;
 673         int err = target;
 674
 675         memset(best_clock, 0, sizeof(*best_clock));
 676
 677         clock.p2 = i9xx_select_p2_div(limit, crtc_state, target);
 678
 679         for (clock.m1 = limit->m1.min; clock.m1 <= limit->m1.max;
 680              clock.m1++) {
 681                 for (clock.m2 = limit->m2.min;
 682                      clock.m2 <= limit->m2.max; clock.m2++) {
 683                         if (clock.m2 >= clock.m1)
 684                                 break;
 685                         for (clock.n = limit->n.min;
 686                              clock.n <= limit->n.max; clock.n++) {
 687                                 for (clock.p1 = limit->p1.min;
 688                                         clock.p1 <= limit->p1.max; clock.p1++) {
 689                                         int this_err;
 690
 691                                         i9xx_calc_dpll_params(refclk, &clock);
 692                                         if (!intel_PLL_is_valid(to_i915(dev),
 693                                                                 limit,
 694                                                                 &clock))
 695                                                 continue;
 696                                         if (match_clock &&
 697                                             clock.p != match_clock->p)
 698                                                 continue;
 699
 700                                         this_err = abs(clock.dot - target);
 701                                         if (this_err < err) {
 702                                                 *best_clock = clock;
 703                                                 err = this_err;
 704                                         }
 705                                 }
 706                         }
 707                 }
 708         }
 709
 710         return (err != target);
 711 }
 712
 713 /*
 714  * Returns a set of divisors for the desired target clock with the given
 715  * refclk, or FALSE.  The returned values represent the clock equation:
 716  * reflck * (5 * (m1 + 2) + (m2 + 2)) / (n + 2) / p1 / p2.
 717  *
 718  * Target and reference clocks are specified in kHz.
 719  *
 720  * If match_clock is provided, then best_clock P divider must match the P
 721  * divider from @match_clock used for LVDS downclocking.
 722  */
 723 static bool
 724 pnv_find_best_dpll(const struct intel_limit *limit,
 725                    struct intel_crtc_state *crtc_state,
 726                    int target, int refclk, struct dpll *match_clock,
 727                    struct dpll *best_clock)
 728 {
 729         struct drm_device *dev = crtc_state->base.crtc->dev;
 730         struct dpll clock;
 731         int err = target;
 732
 733         memset(best_clock, 0, sizeof(*best_clock));
 734
 735         clock.p2 = i9xx_select_p2_div(limit, crtc_state, target);
 736
 737         for (clock.m1 = limit->m1.min; clock.m1 <= limit->m1.max;
 738              clock.m1++) {
 739                 for (clock.m2 = limit->m2.min;
 740                      clock.m2 <= limit->m2.max; clock.m2++) {
 741                         for (clock.n = limit->n.min;
 742                              clock.n <= limit->n.max; clock.n++) {
 743                                 for (clock.p1 = limit->p1.min;
 744                                         clock.p1 <= limit->p1.max; clock.p1++) {
 745                                         int this_err;
 746
 747                                         pnv_calc_dpll_params(refclk, &clock);
 748                                         if (!intel_PLL_is_valid(to_i915(dev),
 749                                                                 limit,
 750                                                                 &clock))
 751                                                 continue;
 752                                         if (match_clock &&
 753                                             clock.p != match_clock->p)
 754                                                 continue;
 755
 756                                         this_err = abs(clock.dot - target);
 757                                         if (this_err < err) {
 758                                                 *best_clock = clock;
 759                                                 err = this_err;
 760                                         }
 761                                 }
 762                         }
 763                 }
 764         }
 765
 766         return (err != target);
 767 }
 768
 769 /*
 770  * Returns a set of divisors for the desired target clock with the given
 771  * refclk, or FALSE.  The returned values represent the clock equation:
 772  * reflck * (5 * (m1 + 2) + (m2 + 2)) / (n + 2) / p1 / p2.
 773  *
 774  * Target and reference clocks are specified in kHz.
 775  *
 776  * If match_clock is provided, then best_clock P divider must match the P
 777  * divider from @match_clock used for LVDS downclocking.
 778  */
 779 static bool
 780 g4x_find_best_dpll(const struct intel_limit *limit,
 781                    struct intel_crtc_state *crtc_state,
 782                    int target, int refclk, struct dpll *match_clock,
 783                    struct dpll *best_clock)
 784 {
 785         struct drm_device *dev = crtc_state->base.crtc->dev;
 786         struct dpll clock;
 787         int max_n;
 788         bool found = false;
 789         /* approximately equals target * 0.00585 */
 790         int err_most = (target >> 8) + (target >> 9);
 791
 792         memset(best_clock, 0, sizeof(*best_clock));
 793
 794         clock.p2 = i9xx_select_p2_div(limit, crtc_state, target);
 795
 796         max_n = limit->n.max;
 797         /* based on hardware requirement, prefer smaller n to precision */
 798         for (clock.n = limit->n.min; clock.n <= max_n; clock.n++) {
 799                 /* based on hardware requirement, prefere larger m1,m2 */
 800                 for (clock.m1 = limit->m1.max;
 801                      clock.m1 >= limit->m1.min; clock.m1--) {
 802                         for (clock.m2 = limit->m2.max;
 803                              clock.m2 >= limit->m2.min; clock.m2--) {
 804                                 for (clock.p1 = limit->p1.max;
 805                                      clock.p1 >= limit->p1.min; clock.p1--) {
 806                                         int this_err;
 807
 808                                         i9xx_calc_dpll_params(refclk, &clock);
 809                                         if (!intel_PLL_is_valid(to_i915(dev),
 810                                                                 limit,
 811                                                                 &clock))
 812                                                 continue;
 813
 814                                         this_err = abs(clock.dot - target);
 815                                         if (this_err < err_most) {
 816                                                 *best_clock = clock;
 817                                                 err_most = this_err;
 818                                                 max_n = clock.n;
 819                                                 found = true;
 820                                         }
 821                                 }
 822                         }
 823                 }
 824         }
 825         return found;
 826 }
 827
 828 /*
 829  * Check if the calculated PLL configuration is more optimal compared to the
 830  * best configuration and error found so far. Return the calculated error.
 831  */
 832 static bool vlv_PLL_is_optimal(struct drm_device *dev, int target_freq,
 833                                const struct dpll *calculated_clock,
 834                                const struct dpll *best_clock,
 835                                unsigned int best_error_ppm,
 836                                unsigned int *error_ppm)
 837 {
 838         /*
 839          * For CHV ignore the error and consider only the P value.
 840          * Prefer a bigger P value based on HW requirements.
 841          */
 842         if (IS_CHERRYVIEW(to_i915(dev))) {
 843                 *error_ppm = 0;
 844
 845                 return calculated_clock->p > best_clock->p;
 846         }
 847
 848         if (WARN_ON_ONCE(!target_freq))
 849                 return false;
 850
 851         *error_ppm = div_u64(1000000ULL *
 852                                 abs(target_freq - calculated_clock->dot),
 853                              target_freq);
 854         /*
 855          * Prefer a better P value over a better (smaller) error if the error
 856          * is small. Ensure this preference for future configurations too by
 857          * setting the error to 0.
 858          */
 859         if (*error_ppm < 100 && calculated_clock->p > best_clock->p) {
 860                 *error_ppm = 0;
 861
 862                 return true;
 863         }
 864
 865         return *error_ppm + 10 < best_error_ppm;
 866 }
 867
 868 /*
 869  * Returns a set of divisors for the desired target clock with the given
 870  * refclk, or FALSE.  The returned values represent the clock equation:
 871  * reflck * (5 * (m1 + 2) + (m2 + 2)) / (n + 2) / p1 / p2.
 872  */
 873 static bool
 874 vlv_find_best_dpll(const struct intel_limit *limit,
 875                    struct intel_crtc_state *crtc_state,
 876                    int target, int refclk, struct dpll *match_clock,
 877                    struct dpll *best_clock)
 878 {
 879         struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
 880         struct drm_device *dev = crtc->base.dev;
 881         struct dpll clock;
 882         unsigned int bestppm = 1000000;
 883         /* min update 19.2 MHz */
 884         int max_n = min(limit->n.max, refclk / 19200);
 885         bool found = false;
 886
 887         target *= 5; /* fast clock */
 888
 889         memset(best_clock, 0, sizeof(*best_clock));
 890
 891         /* based on hardware requirement, prefer smaller n to precision */
 892         for (clock.n = limit->n.min; clock.n <= max_n; clock.n++) {
 893                 for (clock.p1 = limit->p1.max; clock.p1 >= limit->p1.min; clock.p1--) {
 894                         for (clock.p2 = limit->p2.p2_fast; clock.p2 >= limit->p2.p2_slow;
 895                              clock.p2 -= clock.p2 > 10 ? 2 : 1) {
 896                                 clock.p = clock.p1 * clock.p2;
 897                                 /* based on hardware requirement, prefer bigger m1,m2 values */
 898                                 for (clock.m1 = limit->m1.min; clock.m1 <= limit->m1.max; clock.m1++) {
 899                                         unsigned int ppm;
 900
 901                                         clock.m2 = DIV_ROUND_CLOSEST(target * clock.p * clock.n,
 902                                                                      refclk * clock.m1);
 903
 904                                         vlv_calc_dpll_params(refclk, &clock);
 905
 906                                         if (!intel_PLL_is_valid(to_i915(dev),
 907                                                                 limit,
 908                                                                 &clock))
 909                                                 continue;
 910
 911                                         if (!vlv_PLL_is_optimal(dev, target,
 912                                                                 &clock,
 913                                                                 best_clock,
 914                                                                 bestppm, &ppm))
 915                                                 continue;
 916
 917                                         *best_clock = clock;
 918                                         bestppm = ppm;
 919                                         found = true;
 920                                 }
 921                         }
 922                 }
 923         }
 924
 925         return found;
 926 }
 927
 928 /*
 929  * Returns a set of divisors for the desired target clock with the given
 930  * refclk, or FALSE.  The returned values represent the clock equation:
 931  * reflck * (5 * (m1 + 2) + (m2 + 2)) / (n + 2) / p1 / p2.
 932  */
 933 static bool
 934 chv_find_best_dpll(const struct intel_limit *limit,
 935                    struct intel_crtc_state *crtc_state,
 936                    int target, int refclk, struct dpll *match_clock,
 937                    struct dpll *best_clock)
 938 {
 939         struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
 940         struct drm_device *dev = crtc->base.dev;
 941         unsigned int best_error_ppm;
 942         struct dpll clock;
 943         u64 m2;
 944         int found = false;
 945
 946         memset(best_clock, 0, sizeof(*best_clock));
 947         best_error_ppm = 1000000;
 948
 949         /*
 950          * Based on hardware doc, the n always set to 1, and m1 always
 951          * set to 2.  If requires to support 200Mhz refclk, we need to
 952          * revisit this because n may not 1 anymore.
 953          */
 954         clock.n = 1, clock.m1 = 2;
 955         target *= 5;    /* fast clock */
 956
 957         for (clock.p1 = limit->p1.max; clock.p1 >= limit->p1.min; clock.p1--) {
 958                 for (clock.p2 = limit->p2.p2_fast;
 959                                 clock.p2 >= limit->p2.p2_slow;
 960                                 clock.p2 -= clock.p2 > 10 ? 2 : 1) {
 961                         unsigned int error_ppm;
 962
 963                         clock.p = clock.p1 * clock.p2;
 964
 965                         m2 = DIV_ROUND_CLOSEST_ULL(mul_u32_u32(target, clock.p * clock.n) << 22,
 966                                                    refclk * clock.m1);
 967
 968                         if (m2 > INT_MAX/clock.m1)
 969                                 continue;
 970
 971                         clock.m2 = m2;
 972
 973                         chv_calc_dpll_params(refclk, &clock);
 974
 975                         if (!intel_PLL_is_valid(to_i915(dev), limit, &clock))
 976                                 continue;
 977
 978                         if (!vlv_PLL_is_optimal(dev, target, &clock, best_clock,
 979                                                 best_error_ppm, &error_ppm))
 980                                 continue;
 981
 982                         *best_clock = clock;
 983                         best_error_ppm = error_ppm;
 984                         found = true;
 985                 }
 986         }
 987
 988         return found;
 989 }
 990
 991 bool bxt_find_best_dpll(struct intel_crtc_state *crtc_state,
 992                         struct dpll *best_clock)
 993 {
 994         int refclk = 100000;
 995         const struct intel_limit *limit = &intel_limits_bxt;
 996
 997         return chv_find_best_dpll(limit, crtc_state,
 998                                   crtc_state->port_clock, refclk,
 999                                   NULL, best_clock);
1000 }
1001
1002 bool intel_crtc_active(struct intel_crtc *crtc)
1003 {
1004         /* Be paranoid as we can arrive here with only partial
1005          * state retrieved from the hardware during setup.
1006          *
1007          * We can ditch the adjusted_mode.crtc_clock check as soon
1008          * as Haswell has gained clock readout/fastboot support.
1009          *
1010          * We can ditch the crtc->primary->state->fb check as soon as we can
1011          * properly reconstruct framebuffers.
1012          *
1013          * FIXME: The intel_crtc->active here should be switched to
1014          * crtc->state->active once we have proper CRTC states wired up
1015          * for atomic.
1016          */
1017         return crtc->active && crtc->base.primary->state->fb &&
1018                 crtc->config->base.adjusted_mode.crtc_clock;
1019 }
1020
1021 enum transcoder intel_pipe_to_cpu_transcoder(struct drm_i915_private *dev_priv,
1022                                              enum pipe pipe)
1023 {
1024         struct intel_crtc *crtc = intel_get_crtc_for_pipe(dev_priv, pipe);
1025
1026         return crtc->config->cpu_transcoder;
1027 }
1028
1029 static bool pipe_scanline_is_moving(struct drm_i915_private *dev_priv,
1030                                     enum pipe pipe)
1031 {
1032         i915_reg_t reg = PIPEDSL(pipe);
1033         u32 line1, line2;
1034         u32 line_mask;
1035
1036         if (IS_GEN(dev_priv, 2))
1037                 line_mask = DSL_LINEMASK_GEN2;
1038         else
1039                 line_mask = DSL_LINEMASK_GEN3;
1040
1041         line1 = I915_READ(reg) & line_mask;
1042         msleep(5);
1043         line2 = I915_READ(reg) & line_mask;
1044
1045         return line1 != line2;
1046 }
1047
1048 static void wait_for_pipe_scanline_moving(struct intel_crtc *crtc, bool state)
1049 {
1050         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1051         enum pipe pipe = crtc->pipe;
1052
1053         /* Wait for the display line to settle/start moving */
1054         if (wait_for(pipe_scanline_is_moving(dev_priv, pipe) == state, 100))
1055                 DRM_ERROR("pipe %c scanline %s wait timed out\n",
1056                           pipe_name(pipe), onoff(state));
1057 }
1058
1059 static void intel_wait_for_pipe_scanline_stopped(struct intel_crtc *crtc)
1060 {
1061         wait_for_pipe_scanline_moving(crtc, false);
1062 }
1063
1064 static void intel_wait_for_pipe_scanline_moving(struct intel_crtc *crtc)
1065 {
1066         wait_for_pipe_scanline_moving(crtc, true);
1067 }
1068
1069 static void
1070 intel_wait_for_pipe_off(const struct intel_crtc_state *old_crtc_state)
1071 {
1072         struct intel_crtc *crtc = to_intel_crtc(old_crtc_state->base.crtc);
1073         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1074
1075         if (INTEL_GEN(dev_priv) >= 4) {
1076                 enum transcoder cpu_transcoder = old_crtc_state->cpu_transcoder;
1077                 i915_reg_t reg = PIPECONF(cpu_transcoder);
1078
1079                 /* Wait for the Pipe State to go off */
1080                 if (intel_wait_for_register(&dev_priv->uncore,
1081                                             reg, I965_PIPECONF_ACTIVE, 0,
1082                                             100))
1083                         WARN(1, "pipe_off wait timed out\n");
1084         } else {
1085                 intel_wait_for_pipe_scanline_stopped(crtc);
1086         }
1087 }
1088
1089 /* Only for pre-ILK configs */
1090 void assert_pll(struct drm_i915_private *dev_priv,
1091                 enum pipe pipe, bool state)
1092 {
1093         u32 val;
1094         bool cur_state;
1095
1096         val = I915_READ(DPLL(pipe));
1097         cur_state = !!(val & DPLL_VCO_ENABLE);
1098         I915_STATE_WARN(cur_state != state,
1099              "PLL state assertion failure (expected %s, current %s)\n",
1100                         onoff(state), onoff(cur_state));
1101 }
1102
1103 /* XXX: the dsi pll is shared between MIPI DSI ports */
1104 void assert_dsi_pll(struct drm_i915_private *dev_priv, bool state)
1105 {
1106         u32 val;
1107         bool cur_state;
1108
1109         vlv_cck_get(dev_priv);
1110         val = vlv_cck_read(dev_priv, CCK_REG_DSI_PLL_CONTROL);
1111         vlv_cck_put(dev_priv);
1112
1113         cur_state = val & DSI_PLL_VCO_EN;
1114         I915_STATE_WARN(cur_state != state,
1115              "DSI PLL state assertion failure (expected %s, current %s)\n",
1116                         onoff(state), onoff(cur_state));
1117 }
1118
1119 static void assert_fdi_tx(struct drm_i915_private *dev_priv,
1120                           enum pipe pipe, bool state)
1121 {
1122         bool cur_state;
1123         enum transcoder cpu_transcoder = intel_pipe_to_cpu_transcoder(dev_priv,
1124                                                                       pipe);
1125
1126         if (HAS_DDI(dev_priv)) {
1127                 /* DDI does not have a specific FDI_TX register */
1128                 u32 val = I915_READ(TRANS_DDI_FUNC_CTL(cpu_transcoder));
1129                 cur_state = !!(val & TRANS_DDI_FUNC_ENABLE);
1130         } else {
1131                 u32 val = I915_READ(FDI_TX_CTL(pipe));
1132                 cur_state = !!(val & FDI_TX_ENABLE);
1133         }
1134         I915_STATE_WARN(cur_state != state,
1135              "FDI TX state assertion failure (expected %s, current %s)\n",
1136                         onoff(state), onoff(cur_state));
1137 }
1138 #define assert_fdi_tx_enabled(d, p) assert_fdi_tx(d, p, true)
1139 #define assert_fdi_tx_disabled(d, p) assert_fdi_tx(d, p, false)
1140
1141 static void assert_fdi_rx(struct drm_i915_private *dev_priv,
1142                           enum pipe pipe, bool state)
1143 {
1144         u32 val;
1145         bool cur_state;
1146
1147         val = I915_READ(FDI_RX_CTL(pipe));
1148         cur_state = !!(val & FDI_RX_ENABLE);
1149         I915_STATE_WARN(cur_state != state,
1150              "FDI RX state assertion failure (expected %s, current %s)\n",
1151                         onoff(state), onoff(cur_state));
1152 }
1153 #define assert_fdi_rx_enabled(d, p) assert_fdi_rx(d, p, true)
1154 #define assert_fdi_rx_disabled(d, p) assert_fdi_rx(d, p, false)
1155
1156 static void assert_fdi_tx_pll_enabled(struct drm_i915_private *dev_priv,
1157                                       enum pipe pipe)
1158 {
1159         u32 val;
1160
1161         /* ILK FDI PLL is always enabled */
1162         if (IS_GEN(dev_priv, 5))
1163                 return;
1164
1165         /* On Haswell, DDI ports are responsible for the FDI PLL setup */
1166         if (HAS_DDI(dev_priv))
1167                 return;
1168
1169         val = I915_READ(FDI_TX_CTL(pipe));
1170         I915_STATE_WARN(!(val & FDI_TX_PLL_ENABLE), "FDI TX PLL assertion failure, should be active but is disabled\n");
1171 }
1172
1173 void assert_fdi_rx_pll(struct drm_i915_private *dev_priv,
1174                        enum pipe pipe, bool state)
1175 {
1176         u32 val;
1177         bool cur_state;
1178
1179         val = I915_READ(FDI_RX_CTL(pipe));
1180         cur_state = !!(val & FDI_RX_PLL_ENABLE);
1181         I915_STATE_WARN(cur_state != state,
1182              "FDI RX PLL assertion failure (expected %s, current %s)\n",
1183                         onoff(state), onoff(cur_state));
1184 }
1185
1186 void assert_panel_unlocked(struct drm_i915_private *dev_priv, enum pipe pipe)
1187 {
1188         i915_reg_t pp_reg;
1189         u32 val;
1190         enum pipe panel_pipe = INVALID_PIPE;
1191         bool locked = true;
1192
1193         if (WARN_ON(HAS_DDI(dev_priv)))
1194                 return;
1195
1196         if (HAS_PCH_SPLIT(dev_priv)) {
1197                 u32 port_sel;
1198
1199                 pp_reg = PP_CONTROL(0);
1200                 port_sel = I915_READ(PP_ON_DELAYS(0)) & PANEL_PORT_SELECT_MASK;
1201
1202                 switch (port_sel) {
1203                 case PANEL_PORT_SELECT_LVDS:
1204                         intel_lvds_port_enabled(dev_priv, PCH_LVDS, &panel_pipe);
1205                         break;
1206                 case PANEL_PORT_SELECT_DPA:
1207                         intel_dp_port_enabled(dev_priv, DP_A, PORT_A, &panel_pipe);
1208                         break;
1209                 case PANEL_PORT_SELECT_DPC:
1210                         intel_dp_port_enabled(dev_priv, PCH_DP_C, PORT_C, &panel_pipe);
1211                         break;
1212                 case PANEL_PORT_SELECT_DPD:
1213                         intel_dp_port_enabled(dev_priv, PCH_DP_D, PORT_D, &panel_pipe);
1214                         break;
1215                 default:
1216                         MISSING_CASE(port_sel);
1217                         break;
1218                 }
1219         } else if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) {
1220                 /* presumably write lock depends on pipe, not port select */
1221                 pp_reg = PP_CONTROL(pipe);
1222                 panel_pipe = pipe;
1223         } else {
1224                 u32 port_sel;
1225
1226                 pp_reg = PP_CONTROL(0);
1227                 port_sel = I915_READ(PP_ON_DELAYS(0)) & PANEL_PORT_SELECT_MASK;
1228
1229                 WARN_ON(port_sel != PANEL_PORT_SELECT_LVDS);
1230                 intel_lvds_port_enabled(dev_priv, LVDS, &panel_pipe);
1231         }
1232
1233         val = I915_READ(pp_reg);
1234         if (!(val & PANEL_POWER_ON) ||
1235             ((val & PANEL_UNLOCK_MASK) == PANEL_UNLOCK_REGS))
1236                 locked = false;
1237
1238         I915_STATE_WARN(panel_pipe == pipe && locked,
1239              "panel assertion failure, pipe %c regs locked\n",
1240              pipe_name(pipe));
1241 }
1242
1243 void assert_pipe(struct drm_i915_private *dev_priv,
1244                  enum pipe pipe, bool state)
1245 {
1246         bool cur_state;
1247         enum transcoder cpu_transcoder = intel_pipe_to_cpu_transcoder(dev_priv,
1248                                                                       pipe);
1249         enum intel_display_power_domain power_domain;
1250         intel_wakeref_t wakeref;
1251
1252         /* we keep both pipes enabled on 830 */
1253         if (IS_I830(dev_priv))
1254                 state = true;
1255
1256         power_domain = POWER_DOMAIN_TRANSCODER(cpu_transcoder);
1257         wakeref = intel_display_power_get_if_enabled(dev_priv, power_domain);
1258         if (wakeref) {
1259                 u32 val = I915_READ(PIPECONF(cpu_transcoder));
1260                 cur_state = !!(val & PIPECONF_ENABLE);
1261
1262                 intel_display_power_put(dev_priv, power_domain, wakeref);
1263         } else {
1264                 cur_state = false;
1265         }
1266
1267         I915_STATE_WARN(cur_state != state,
1268              "pipe %c assertion failure (expected %s, current %s)\n",
1269                         pipe_name(pipe), onoff(state), onoff(cur_state));
1270 }
1271
1272 static void assert_plane(struct intel_plane *plane, bool state)
1273 {
1274         enum pipe pipe;
1275         bool cur_state;
1276
1277         cur_state = plane->get_hw_state(plane, &pipe);
1278
1279         I915_STATE_WARN(cur_state != state,
1280                         "%s assertion failure (expected %s, current %s)\n",
1281                         plane->base.name, onoff(state), onoff(cur_state));
1282 }
1283
1284 #define assert_plane_enabled(p) assert_plane(p, true)
1285 #define assert_plane_disabled(p) assert_plane(p, false)
1286
1287 static void assert_planes_disabled(struct intel_crtc *crtc)
1288 {
1289         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1290         struct intel_plane *plane;
1291
1292         for_each_intel_plane_on_crtc(&dev_priv->drm, crtc, plane)
1293                 assert_plane_disabled(plane);
1294 }
1295
1296 static void assert_vblank_disabled(struct drm_crtc *crtc)
1297 {
1298         if (I915_STATE_WARN_ON(drm_crtc_vblank_get(crtc) == 0))
1299                 drm_crtc_vblank_put(crtc);
1300 }
1301
1302 void assert_pch_transcoder_disabled(struct drm_i915_private *dev_priv,
1303                                     enum pipe pipe)
1304 {
1305         u32 val;
1306         bool enabled;
1307
1308         val = I915_READ(PCH_TRANSCONF(pipe));
1309         enabled = !!(val & TRANS_ENABLE);
1310         I915_STATE_WARN(enabled,
1311              "transcoder assertion failed, should be off on pipe %c but is still active\n",
1312              pipe_name(pipe));
1313 }
1314
1315 static void assert_pch_dp_disabled(struct drm_i915_private *dev_priv,
1316                                    enum pipe pipe, enum port port,
1317                                    i915_reg_t dp_reg)
1318 {
1319         enum pipe port_pipe;
1320         bool state;
1321
1322         state = intel_dp_port_enabled(dev_priv, dp_reg, port, &port_pipe);
1323
1324         I915_STATE_WARN(state && port_pipe == pipe,
1325                         "PCH DP %c enabled on transcoder %c, should be disabled\n",
1326                         port_name(port), pipe_name(pipe));
1327
1328         I915_STATE_WARN(HAS_PCH_IBX(dev_priv) && !state && port_pipe == PIPE_B,
1329                         "IBX PCH DP %c still using transcoder B\n",
1330                         port_name(port));
1331 }
1332
1333 static void assert_pch_hdmi_disabled(struct drm_i915_private *dev_priv,
1334                                      enum pipe pipe, enum port port,
1335                                      i915_reg_t hdmi_reg)
1336 {
1337         enum pipe port_pipe;
1338         bool state;
1339
1340         state = intel_sdvo_port_enabled(dev_priv, hdmi_reg, &port_pipe);
1341
1342         I915_STATE_WARN(state && port_pipe == pipe,
1343                         "PCH HDMI %c enabled on transcoder %c, should be disabled\n",
1344                         port_name(port), pipe_name(pipe));
1345
1346         I915_STATE_WARN(HAS_PCH_IBX(dev_priv) && !state && port_pipe == PIPE_B,
1347                         "IBX PCH HDMI %c still using transcoder B\n",
1348                         port_name(port));
1349 }
1350
1351 static void assert_pch_ports_disabled(struct drm_i915_private *dev_priv,
1352                                       enum pipe pipe)
1353 {
1354         enum pipe port_pipe;
1355
1356         assert_pch_dp_disabled(dev_priv, pipe, PORT_B, PCH_DP_B);
1357         assert_pch_dp_disabled(dev_priv, pipe, PORT_C, PCH_DP_C);
1358         assert_pch_dp_disabled(dev_priv, pipe, PORT_D, PCH_DP_D);
1359
1360         I915_STATE_WARN(intel_crt_port_enabled(dev_priv, PCH_ADPA, &port_pipe) &&
1361                         port_pipe == pipe,
1362                         "PCH VGA enabled on transcoder %c, should be disabled\n",
1363                         pipe_name(pipe));
1364
1365         I915_STATE_WARN(intel_lvds_port_enabled(dev_priv, PCH_LVDS, &port_pipe) &&
1366                         port_pipe == pipe,
1367                         "PCH LVDS enabled on transcoder %c, should be disabled\n",
1368                         pipe_name(pipe));
1369
1370         /* PCH SDVOB multiplex with HDMIB */
1371         assert_pch_hdmi_disabled(dev_priv, pipe, PORT_B, PCH_HDMIB);
1372         assert_pch_hdmi_disabled(dev_priv, pipe, PORT_C, PCH_HDMIC);
1373         assert_pch_hdmi_disabled(dev_priv, pipe, PORT_D, PCH_HDMID);
1374 }
1375
1376 static void _vlv_enable_pll(struct intel_crtc *crtc,
1377                             const struct intel_crtc_state *pipe_config)
1378 {
1379         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1380         enum pipe pipe = crtc->pipe;
1381
1382         I915_WRITE(DPLL(pipe), pipe_config->dpll_hw_state.dpll);
1383         POSTING_READ(DPLL(pipe));
1384         udelay(150);
1385
1386         if (intel_wait_for_register(&dev_priv->uncore,
1387                                     DPLL(pipe),
1388                                     DPLL_LOCK_VLV,
1389                                     DPLL_LOCK_VLV,
1390                                     1))
1391                 DRM_ERROR("DPLL %d failed to lock\n", pipe);
1392 }
1393
1394 static void vlv_enable_pll(struct intel_crtc *crtc,
1395                            const struct intel_crtc_state *pipe_config)
1396 {
1397         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1398         enum pipe pipe = crtc->pipe;
1399
1400         assert_pipe_disabled(dev_priv, pipe);
1401
1402         /* PLL is protected by panel, make sure we can write it */
1403         assert_panel_unlocked(dev_priv, pipe);
1404
1405         if (pipe_config->dpll_hw_state.dpll & DPLL_VCO_ENABLE)
1406                 _vlv_enable_pll(crtc, pipe_config);
1407
1408         I915_WRITE(DPLL_MD(pipe), pipe_config->dpll_hw_state.dpll_md);
1409         POSTING_READ(DPLL_MD(pipe));
1410 }
1411
1412
1413 static void _chv_enable_pll(struct intel_crtc *crtc,
1414                             const struct intel_crtc_state *pipe_config)
1415 {
1416         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1417         enum pipe pipe = crtc->pipe;
1418         enum dpio_channel port = vlv_pipe_to_channel(pipe);
1419         u32 tmp;
1420
1421         vlv_dpio_get(dev_priv);
1422
1423         /* Enable back the 10bit clock to display controller */
1424         tmp = vlv_dpio_read(dev_priv, pipe, CHV_CMN_DW14(port));
1425         tmp |= DPIO_DCLKP_EN;
1426         vlv_dpio_write(dev_priv, pipe, CHV_CMN_DW14(port), tmp);
1427
1428         vlv_dpio_put(dev_priv);
1429
1430         /*
1431          * Need to wait > 100ns between dclkp clock enable bit and PLL enable.
1432          */
1433         udelay(1);
1434
1435         /* Enable PLL */
1436         I915_WRITE(DPLL(pipe), pipe_config->dpll_hw_state.dpll);
1437
1438         /* Check PLL is locked */
1439         if (intel_wait_for_register(&dev_priv->uncore,
1440                                     DPLL(pipe), DPLL_LOCK_VLV, DPLL_LOCK_VLV,
1441                                     1))
1442                 DRM_ERROR("PLL %d failed to lock\n", pipe);
1443 }
1444
1445 static void chv_enable_pll(struct intel_crtc *crtc,
1446                            const struct intel_crtc_state *pipe_config)
1447 {
1448         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1449         enum pipe pipe = crtc->pipe;
1450
1451         assert_pipe_disabled(dev_priv, pipe);
1452
1453         /* PLL is protected by panel, make sure we can write it */
1454         assert_panel_unlocked(dev_priv, pipe);
1455
1456         if (pipe_config->dpll_hw_state.dpll & DPLL_VCO_ENABLE)
1457                 _chv_enable_pll(crtc, pipe_config);
1458
1459         if (pipe != PIPE_A) {
1460                 /*
1461                  * WaPixelRepeatModeFixForC0:chv
1462                  *
1463                  * DPLLCMD is AWOL. Use chicken bits to propagate
1464                  * the value from DPLLBMD to either pipe B or C.
1465                  */
1466                 I915_WRITE(CBR4_VLV, CBR_DPLLBMD_PIPE(pipe));
1467                 I915_WRITE(DPLL_MD(PIPE_B), pipe_config->dpll_hw_state.dpll_md);
1468                 I915_WRITE(CBR4_VLV, 0);
1469                 dev_priv->chv_dpll_md[pipe] = pipe_config->dpll_hw_state.dpll_md;
1470
1471                 /*
1472                  * DPLLB VGA mode also seems to cause problems.
1473                  * We should always have it disabled.
1474                  */
1475                 WARN_ON((I915_READ(DPLL(PIPE_B)) & DPLL_VGA_MODE_DIS) == 0);
1476         } else {
1477                 I915_WRITE(DPLL_MD(pipe), pipe_config->dpll_hw_state.dpll_md);
1478                 POSTING_READ(DPLL_MD(pipe));
1479         }
1480 }
1481
1482 static bool i9xx_has_pps(struct drm_i915_private *dev_priv)
1483 {
1484         if (IS_I830(dev_priv))
1485                 return false;
1486
1487         return IS_PINEVIEW(dev_priv) || IS_MOBILE(dev_priv);
1488 }
1489
1490 static void i9xx_enable_pll(struct intel_crtc *crtc,
1491                             const struct intel_crtc_state *crtc_state)
1492 {
1493         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1494         i915_reg_t reg = DPLL(crtc->pipe);
1495         u32 dpll = crtc_state->dpll_hw_state.dpll;
1496         int i;
1497
1498         assert_pipe_disabled(dev_priv, crtc->pipe);
1499
1500         /* PLL is protected by panel, make sure we can write it */
1501         if (i9xx_has_pps(dev_priv))
1502                 assert_panel_unlocked(dev_priv, crtc->pipe);
1503
1504         /*
1505          * Apparently we need to have VGA mode enabled prior to changing
1506          * the P1/P2 dividers. Otherwise the DPLL will keep using the old
1507          * dividers, even though the register value does change.
1508          */
1509         I915_WRITE(reg, dpll & ~DPLL_VGA_MODE_DIS);
1510         I915_WRITE(reg, dpll);
1511
1512         /* Wait for the clocks to stabilize. */
1513         POSTING_READ(reg);
1514         udelay(150);
1515
1516         if (INTEL_GEN(dev_priv) >= 4) {
1517                 I915_WRITE(DPLL_MD(crtc->pipe),
1518                            crtc_state->dpll_hw_state.dpll_md);
1519         } else {
1520                 /* The pixel multiplier can only be updated once the
1521                  * DPLL is enabled and the clocks are stable.
1522                  *
1523                  * So write it again.
1524                  */
1525                 I915_WRITE(reg, dpll);
1526         }
1527
1528         /* We do this three times for luck */
1529         for (i = 0; i < 3; i++) {
1530                 I915_WRITE(reg, dpll);
1531                 POSTING_READ(reg);
1532                 udelay(150); /* wait for warmup */
1533         }
1534 }
1535
1536 static void i9xx_disable_pll(const struct intel_crtc_state *crtc_state)
1537 {
1538         struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
1539         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1540         enum pipe pipe = crtc->pipe;
1541
1542         /* Don't disable pipe or pipe PLLs if needed */
1543         if (IS_I830(dev_priv))
1544                 return;
1545
1546         /* Make sure the pipe isn't still relying on us */
1547         assert_pipe_disabled(dev_priv, pipe);
1548
1549         I915_WRITE(DPLL(pipe), DPLL_VGA_MODE_DIS);
1550         POSTING_READ(DPLL(pipe));
1551 }
1552
1553 static void vlv_disable_pll(struct drm_i915_private *dev_priv, enum pipe pipe)
1554 {
1555         u32 val;
1556
1557         /* Make sure the pipe isn't still relying on us */
1558         assert_pipe_disabled(dev_priv, pipe);
1559
1560         val = DPLL_INTEGRATED_REF_CLK_VLV |
1561                 DPLL_REF_CLK_ENABLE_VLV | DPLL_VGA_MODE_DIS;
1562         if (pipe != PIPE_A)
1563                 val |= DPLL_INTEGRATED_CRI_CLK_VLV;
1564
1565         I915_WRITE(DPLL(pipe), val);
1566         POSTING_READ(DPLL(pipe));
1567 }
1568
1569 static void chv_disable_pll(struct drm_i915_private *dev_priv, enum pipe pipe)
1570 {
1571         enum dpio_channel port = vlv_pipe_to_channel(pipe);
1572         u32 val;
1573
1574         /* Make sure the pipe isn't still relying on us */
1575         assert_pipe_disabled(dev_priv, pipe);
1576
1577         val = DPLL_SSC_REF_CLK_CHV |
1578                 DPLL_REF_CLK_ENABLE_VLV | DPLL_VGA_MODE_DIS;
1579         if (pipe != PIPE_A)
1580                 val |= DPLL_INTEGRATED_CRI_CLK_VLV;
1581
1582         I915_WRITE(DPLL(pipe), val);
1583         POSTING_READ(DPLL(pipe));
1584
1585         vlv_dpio_get(dev_priv);
1586
1587         /* Disable 10bit clock to display controller */
1588         val = vlv_dpio_read(dev_priv, pipe, CHV_CMN_DW14(port));
1589         val &= ~DPIO_DCLKP_EN;
1590         vlv_dpio_write(dev_priv, pipe, CHV_CMN_DW14(port), val);
1591
1592         vlv_dpio_put(dev_priv);
1593 }
1594
1595 void vlv_wait_port_ready(struct drm_i915_private *dev_priv,
1596                          struct intel_digital_port *dport,
1597                          unsigned int expected_mask)
1598 {
1599         u32 port_mask;
1600         i915_reg_t dpll_reg;
1601
1602         switch (dport->base.port) {
1603         case PORT_B:
1604                 port_mask = DPLL_PORTB_READY_MASK;
1605                 dpll_reg = DPLL(0);
1606                 break;
1607         case PORT_C:
1608                 port_mask = DPLL_PORTC_READY_MASK;
1609                 dpll_reg = DPLL(0);
1610                 expected_mask <<= 4;
1611                 break;
1612         case PORT_D:
1613                 port_mask = DPLL_PORTD_READY_MASK;
1614                 dpll_reg = DPIO_PHY_STATUS;
1615                 break;
1616         default:
1617                 BUG();
1618         }
1619
1620         if (intel_wait_for_register(&dev_priv->uncore,
1621                                     dpll_reg, port_mask, expected_mask,
1622                                     1000))
1623                 WARN(1, "timed out waiting for port %c ready: got 0x%x, expected 0x%x\n",
1624                      port_name(dport->base.port),
1625                      I915_READ(dpll_reg) & port_mask, expected_mask);
1626 }
1627
1628 static void ironlake_enable_pch_transcoder(const struct intel_crtc_state *crtc_state)
1629 {
1630         struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
1631         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1632         enum pipe pipe = crtc->pipe;
1633         i915_reg_t reg;
1634         u32 val, pipeconf_val;
1635
1636         /* Make sure PCH DPLL is enabled */
1637         assert_shared_dpll_enabled(dev_priv, crtc_state->shared_dpll);
1638
1639         /* FDI must be feeding us bits for PCH ports */
1640         assert_fdi_tx_enabled(dev_priv, pipe);
1641         assert_fdi_rx_enabled(dev_priv, pipe);
1642
1643         if (HAS_PCH_CPT(dev_priv)) {
1644                 /* Workaround: Set the timing override bit before enabling the
1645                  * pch transcoder. */
1646                 reg = TRANS_CHICKEN2(pipe);
1647                 val = I915_READ(reg);
1648                 val |= TRANS_CHICKEN2_TIMING_OVERRIDE;
1649                 I915_WRITE(reg, val);
1650         }
1651
1652         reg = PCH_TRANSCONF(pipe);
1653         val = I915_READ(reg);
1654         pipeconf_val = I915_READ(PIPECONF(pipe));
1655
1656         if (HAS_PCH_IBX(dev_priv)) {
1657                 /*
1658                  * Make the BPC in transcoder be consistent with
1659                  * that in pipeconf reg. For HDMI we must use 8bpc
1660                  * here for both 8bpc and 12bpc.
1661                  */
1662                 val &= ~PIPECONF_BPC_MASK;
1663                 if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_HDMI))
1664                         val |= PIPECONF_8BPC;
1665                 else
1666                         val |= pipeconf_val & PIPECONF_BPC_MASK;
1667         }
1668
1669         val &= ~TRANS_INTERLACE_MASK;
1670         if ((pipeconf_val & PIPECONF_INTERLACE_MASK) == PIPECONF_INTERLACED_ILK) {
1671                 if (HAS_PCH_IBX(dev_priv) &&
1672                     intel_crtc_has_type(crtc_state, INTEL_OUTPUT_SDVO))
1673                         val |= TRANS_LEGACY_INTERLACED_ILK;
1674                 else
1675                         val |= TRANS_INTERLACED;
1676         } else {
1677                 val |= TRANS_PROGRESSIVE;
1678         }
1679
1680         I915_WRITE(reg, val | TRANS_ENABLE);
1681         if (intel_wait_for_register(&dev_priv->uncore,
1682                                     reg, TRANS_STATE_ENABLE, TRANS_STATE_ENABLE,
1683                                     100))
1684                 DRM_ERROR("failed to enable transcoder %c\n", pipe_name(pipe));
1685 }
1686
1687 static void lpt_enable_pch_transcoder(struct drm_i915_private *dev_priv,
1688                                       enum transcoder cpu_transcoder)
1689 {
1690         u32 val, pipeconf_val;
1691
1692         /* FDI must be feeding us bits for PCH ports */
1693         assert_fdi_tx_enabled(dev_priv, (enum pipe) cpu_transcoder);
1694         assert_fdi_rx_enabled(dev_priv, PIPE_A);
1695
1696         /* Workaround: set timing override bit. */
1697         val = I915_READ(TRANS_CHICKEN2(PIPE_A));
1698         val |= TRANS_CHICKEN2_TIMING_OVERRIDE;
1699         I915_WRITE(TRANS_CHICKEN2(PIPE_A), val);
1700
1701         val = TRANS_ENABLE;
1702         pipeconf_val = I915_READ(PIPECONF(cpu_transcoder));
1703
1704         if ((pipeconf_val & PIPECONF_INTERLACE_MASK_HSW) ==
1705             PIPECONF_INTERLACED_ILK)
1706                 val |= TRANS_INTERLACED;
1707         else
1708                 val |= TRANS_PROGRESSIVE;
1709
1710         I915_WRITE(LPT_TRANSCONF, val);
1711         if (intel_wait_for_register(&dev_priv->uncore,
1712                                     LPT_TRANSCONF,
1713                                     TRANS_STATE_ENABLE,
1714                                     TRANS_STATE_ENABLE,
1715                                     100))
1716                 DRM_ERROR("Failed to enable PCH transcoder\n");
1717 }
1718
1719 static void ironlake_disable_pch_transcoder(struct drm_i915_private *dev_priv,
1720                                             enum pipe pipe)
1721 {
1722         i915_reg_t reg;
1723         u32 val;
1724
1725         /* FDI relies on the transcoder */
1726         assert_fdi_tx_disabled(dev_priv, pipe);
1727         assert_fdi_rx_disabled(dev_priv, pipe);
1728
1729         /* Ports must be off as well */
1730         assert_pch_ports_disabled(dev_priv, pipe);
1731
1732         reg = PCH_TRANSCONF(pipe);
1733         val = I915_READ(reg);
1734         val &= ~TRANS_ENABLE;
1735         I915_WRITE(reg, val);
1736         /* wait for PCH transcoder off, transcoder state */
1737         if (intel_wait_for_register(&dev_priv->uncore,
1738                                     reg, TRANS_STATE_ENABLE, 0,
1739                                     50))
1740                 DRM_ERROR("failed to disable transcoder %c\n", pipe_name(pipe));
1741
1742         if (HAS_PCH_CPT(dev_priv)) {
1743                 /* Workaround: Clear the timing override chicken bit again. */
1744                 reg = TRANS_CHICKEN2(pipe);
1745                 val = I915_READ(reg);
1746                 val &= ~TRANS_CHICKEN2_TIMING_OVERRIDE;
1747                 I915_WRITE(reg, val);
1748         }
1749 }
1750
1751 void lpt_disable_pch_transcoder(struct drm_i915_private *dev_priv)
1752 {
1753         u32 val;
1754
1755         val = I915_READ(LPT_TRANSCONF);
1756         val &= ~TRANS_ENABLE;
1757         I915_WRITE(LPT_TRANSCONF, val);
1758         /* wait for PCH transcoder off, transcoder state */
1759         if (intel_wait_for_register(&dev_priv->uncore,
1760                                     LPT_TRANSCONF, TRANS_STATE_ENABLE, 0,
1761                                     50))
1762                 DRM_ERROR("Failed to disable PCH transcoder\n");
1763
1764         /* Workaround: clear timing override bit. */
1765         val = I915_READ(TRANS_CHICKEN2(PIPE_A));
1766         val &= ~TRANS_CHICKEN2_TIMING_OVERRIDE;
1767         I915_WRITE(TRANS_CHICKEN2(PIPE_A), val);
1768 }
1769
1770 enum pipe intel_crtc_pch_transcoder(struct intel_crtc *crtc)
1771 {
1772         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1773
1774         if (HAS_PCH_LPT(dev_priv))
1775                 return PIPE_A;
1776         else
1777                 return crtc->pipe;
1778 }
1779
1780 static u32 intel_crtc_max_vblank_count(const struct intel_crtc_state *crtc_state)
1781 {
1782         struct drm_i915_private *dev_priv = to_i915(crtc_state->base.crtc->dev);
1783
1784         /*
1785          * On i965gm the hardware frame counter reads
1786          * zero when the TV encoder is enabled :(
1787          */
1788         if (IS_I965GM(dev_priv) &&
1789             (crtc_state->output_types & BIT(INTEL_OUTPUT_TVOUT)))
1790                 return 0;
1791
1792         if (INTEL_GEN(dev_priv) >= 5 || IS_G4X(dev_priv))
1793                 return 0xffffffff; /* full 32 bit counter */
1794         else if (INTEL_GEN(dev_priv) >= 3)
1795                 return 0xffffff; /* only 24 bits of frame count */
1796         else
1797                 return 0; /* Gen2 doesn't have a hardware frame counter */
1798 }
1799
1800 static void intel_crtc_vblank_on(const struct intel_crtc_state *crtc_state)
1801 {
1802         struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
1803
1804         drm_crtc_set_max_vblank_count(&crtc->base,
1805                                       intel_crtc_max_vblank_count(crtc_state));
1806         drm_crtc_vblank_on(&crtc->base);
1807 }
1808
1809 static void intel_enable_pipe(const struct intel_crtc_state *new_crtc_state)
1810 {
1811         struct intel_crtc *crtc = to_intel_crtc(new_crtc_state->base.crtc);
1812         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1813         enum transcoder cpu_transcoder = new_crtc_state->cpu_transcoder;
1814         enum pipe pipe = crtc->pipe;
1815         i915_reg_t reg;
1816         u32 val;
1817
1818         DRM_DEBUG_KMS("enabling pipe %c\n", pipe_name(pipe));
1819
1820         assert_planes_disabled(crtc);
1821
1822         /*
1823          * A pipe without a PLL won't actually be able to drive bits from
1824          * a plane.  On ILK+ the pipe PLLs are integrated, so we don't
1825          * need the check.
1826          */
1827         if (HAS_GMCH(dev_priv)) {
1828                 if (intel_crtc_has_type(new_crtc_state, INTEL_OUTPUT_DSI))
1829                         assert_dsi_pll_enabled(dev_priv);
1830                 else
1831                         assert_pll_enabled(dev_priv, pipe);
1832         } else {
1833                 if (new_crtc_state->has_pch_encoder) {
1834                         /* if driving the PCH, we need FDI enabled */
1835                         assert_fdi_rx_pll_enabled(dev_priv,
1836                                                   intel_crtc_pch_transcoder(crtc));
1837                         assert_fdi_tx_pll_enabled(dev_priv,
1838                                                   (enum pipe) cpu_transcoder);
1839                 }
1840                 /* FIXME: assert CPU port conditions for SNB+ */
1841         }
1842
1843         trace_intel_pipe_enable(crtc);
1844
1845         reg = PIPECONF(cpu_transcoder);
1846         val = I915_READ(reg);
1847         if (val & PIPECONF_ENABLE) {
1848                 /* we keep both pipes enabled on 830 */
1849                 WARN_ON(!IS_I830(dev_priv));
1850                 return;
1851         }
1852
1853         I915_WRITE(reg, val | PIPECONF_ENABLE);
1854         POSTING_READ(reg);
1855
1856         /*
1857          * Until the pipe starts PIPEDSL reads will return a stale value,
1858          * which causes an apparent vblank timestamp jump when PIPEDSL
1859          * resets to its proper value. That also messes up the frame count
1860          * when it's derived from the timestamps. So let's wait for the
1861          * pipe to start properly before we call drm_crtc_vblank_on()
1862          */
1863         if (intel_crtc_max_vblank_count(new_crtc_state) == 0)
1864                 intel_wait_for_pipe_scanline_moving(crtc);
1865 }
1866
1867 static void intel_disable_pipe(const struct intel_crtc_state *old_crtc_state)
1868 {
1869         struct intel_crtc *crtc = to_intel_crtc(old_crtc_state->base.crtc);
1870         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1871         enum transcoder cpu_transcoder = old_crtc_state->cpu_transcoder;
1872         enum pipe pipe = crtc->pipe;
1873         i915_reg_t reg;
1874         u32 val;
1875
1876         DRM_DEBUG_KMS("disabling pipe %c\n", pipe_name(pipe));
1877
1878         /*
1879          * Make sure planes won't keep trying to pump pixels to us,
1880          * or we might hang the display.
1881          */
1882         assert_planes_disabled(crtc);
1883
1884         trace_intel_pipe_disable(crtc);
1885
1886         reg = PIPECONF(cpu_transcoder);
1887         val = I915_READ(reg);
1888         if ((val & PIPECONF_ENABLE) == 0)
1889                 return;
1890
1891         /*
1892          * Double wide has implications for planes
1893          * so best keep it disabled when not needed.
1894          */
1895         if (old_crtc_state->double_wide)
1896                 val &= ~PIPECONF_DOUBLE_WIDE;
1897
1898         /* Don't disable pipe or pipe PLLs if needed */
1899         if (!IS_I830(dev_priv))
1900                 val &= ~PIPECONF_ENABLE;
1901
1902         I915_WRITE(reg, val);
1903         if ((val & PIPECONF_ENABLE) == 0)
1904                 intel_wait_for_pipe_off(old_crtc_state);
1905 }
1906
1907 static unsigned int intel_tile_size(const struct drm_i915_private *dev_priv)
1908 {
1909         return IS_GEN(dev_priv, 2) ? 2048 : 4096;
1910 }
1911
1912 static unsigned int
1913 intel_tile_width_bytes(const struct drm_framebuffer *fb, int color_plane)
1914 {
1915         struct drm_i915_private *dev_priv = to_i915(fb->dev);
1916         unsigned int cpp = fb->format->cpp[color_plane];
1917
1918         switch (fb->modifier) {
1919         case DRM_FORMAT_MOD_LINEAR:
1920                 return intel_tile_size(dev_priv);
1921         case I915_FORMAT_MOD_X_TILED:
1922                 if (IS_GEN(dev_priv, 2))
1923                         return 128;
1924                 else
1925                         return 512;
1926         case I915_FORMAT_MOD_Y_TILED_CCS:
1927                 if (color_plane == 1)
1928                         return 128;
1929                 /* fall through */
1930         case I915_FORMAT_MOD_Y_TILED:
1931                 if (IS_GEN(dev_priv, 2) || HAS_128_BYTE_Y_TILING(dev_priv))
1932                         return 128;
1933                 else
1934                         return 512;
1935         case I915_FORMAT_MOD_Yf_TILED_CCS:
1936                 if (color_plane == 1)
1937                         return 128;
1938                 /* fall through */
1939         case I915_FORMAT_MOD_Yf_TILED:
1940                 switch (cpp) {
1941                 case 1:
1942                         return 64;
1943                 case 2:
1944                 case 4:
1945                         return 128;
1946                 case 8:
1947                 case 16:
1948                         return 256;
1949                 default:
1950                         MISSING_CASE(cpp);
1951                         return cpp;
1952                 }
1953                 break;
1954         default:
1955                 MISSING_CASE(fb->modifier);
1956                 return cpp;
1957         }
1958 }
1959
1960 static unsigned int
1961 intel_tile_height(const struct drm_framebuffer *fb, int color_plane)
1962 {
1963         return intel_tile_size(to_i915(fb->dev)) /
1964                 intel_tile_width_bytes(fb, color_plane);
1965 }
1966
1967 /* Return the tile dimensions in pixel units */
1968 static void intel_tile_dims(const struct drm_framebuffer *fb, int color_plane,
1969                             unsigned int *tile_width,
1970                             unsigned int *tile_height)
1971 {
1972         unsigned int tile_width_bytes = intel_tile_width_bytes(fb, color_plane);
1973         unsigned int cpp = fb->format->cpp[color_plane];
1974
1975         *tile_width = tile_width_bytes / cpp;
1976         *tile_height = intel_tile_size(to_i915(fb->dev)) / tile_width_bytes;
1977 }
1978
1979 unsigned int
1980 intel_fb_align_height(const struct drm_framebuffer *fb,
1981                       int color_plane, unsigned int height)
1982 {
1983         unsigned int tile_height = intel_tile_height(fb, color_plane);
1984
1985         return ALIGN(height, tile_height);
1986 }
1987
1988 unsigned int intel_rotation_info_size(const struct intel_rotation_info *rot_info)
1989 {
1990         unsigned int size = 0;
1991         int i;
1992
1993         for (i = 0 ; i < ARRAY_SIZE(rot_info->plane); i++)
1994                 size += rot_info->plane[i].width * rot_info->plane[i].height;
1995
1996         return size;
1997 }
1998
1999 unsigned int intel_remapped_info_size(const struct intel_remapped_info *rem_info)
2000 {
2001         unsigned int size = 0;
2002         int i;
2003
2004         for (i = 0 ; i < ARRAY_SIZE(rem_info->plane); i++)
2005                 size += rem_info->plane[i].width * rem_info->plane[i].height;
2006
2007         return size;
2008 }
2009
2010 static void
2011 intel_fill_fb_ggtt_view(struct i915_ggtt_view *view,
2012                         const struct drm_framebuffer *fb,
2013                         unsigned int rotation)
2014 {
2015         view->type = I915_GGTT_VIEW_NORMAL;
2016         if (drm_rotation_90_or_270(rotation)) {
2017                 view->type = I915_GGTT_VIEW_ROTATED;
2018                 view->rotated = to_intel_framebuffer(fb)->rot_info;
2019         }
2020 }
2021
2022 static unsigned int intel_cursor_alignment(const struct drm_i915_private *dev_priv)
2023 {
2024         if (IS_I830(dev_priv))
2025                 return 16 * 1024;
2026         else if (IS_I85X(dev_priv))
2027                 return 256;
2028         else if (IS_I845G(dev_priv) || IS_I865G(dev_priv))
2029                 return 32;
2030         else
2031                 return 4 * 1024;
2032 }
2033
2034 static unsigned int intel_linear_alignment(const struct drm_i915_private *dev_priv)
2035 {
2036         if (INTEL_GEN(dev_priv) >= 9)
2037                 return 256 * 1024;
2038         else if (IS_I965G(dev_priv) || IS_I965GM(dev_priv) ||
2039                  IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
2040                 return 128 * 1024;
2041         else if (INTEL_GEN(dev_priv) >= 4)
2042                 return 4 * 1024;
2043         else
2044                 return 0;
2045 }
2046
2047 static unsigned int intel_surf_alignment(const struct drm_framebuffer *fb,
2048                                          int color_plane)
2049 {
2050         struct drm_i915_private *dev_priv = to_i915(fb->dev);
2051
2052         /* AUX_DIST needs only 4K alignment */
2053         if (color_plane == 1)
2054                 return 4096;
2055
2056         switch (fb->modifier) {
2057         case DRM_FORMAT_MOD_LINEAR:
2058                 return intel_linear_alignment(dev_priv);
2059         case I915_FORMAT_MOD_X_TILED:
2060                 if (INTEL_GEN(dev_priv) >= 9)
2061                         return 256 * 1024;
2062                 return 0;
2063         case I915_FORMAT_MOD_Y_TILED_CCS:
2064         case I915_FORMAT_MOD_Yf_TILED_CCS:
2065         case I915_FORMAT_MOD_Y_TILED:
2066         case I915_FORMAT_MOD_Yf_TILED:
2067                 return 1 * 1024 * 1024;
2068         default:
2069                 MISSING_CASE(fb->modifier);
2070                 return 0;
2071         }
2072 }
2073
2074 static bool intel_plane_uses_fence(const struct intel_plane_state *plane_state)
2075 {
2076         struct intel_plane *plane = to_intel_plane(plane_state->base.plane);
2077         struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
2078
2079         return INTEL_GEN(dev_priv) < 4 ||
2080                 (plane->has_fbc &&
2081                  plane_state->view.type == I915_GGTT_VIEW_NORMAL);
2082 }
2083
2084 struct i915_vma *
2085 intel_pin_and_fence_fb_obj(struct drm_framebuffer *fb,
2086                            const struct i915_ggtt_view *view,
2087                            bool uses_fence,
2088                            unsigned long *out_flags)
2089 {
2090         struct drm_device *dev = fb->dev;
2091         struct drm_i915_private *dev_priv = to_i915(dev);
2092         struct drm_i915_gem_object *obj = intel_fb_obj(fb);
2093         intel_wakeref_t wakeref;
2094         struct i915_vma *vma;
2095         unsigned int pinctl;
2096         u32 alignment;
2097
2098         WARN_ON(!mutex_is_locked(&dev->struct_mutex));
2099
2100         alignment = intel_surf_alignment(fb, 0);
2101
2102         /* Note that the w/a also requires 64 PTE of padding following the
2103          * bo. We currently fill all unused PTE with the shadow page and so
2104          * we should always have valid PTE following the scanout preventing
2105          * the VT-d warning.
2106          */
2107         if (intel_scanout_needs_vtd_wa(dev_priv) && alignment < 256 * 1024)
2108                 alignment = 256 * 1024;
2109
2110         /*
2111          * Global gtt pte registers are special registers which actually forward
2112          * writes to a chunk of system memory. Which means that there is no risk
2113          * that the register values disappear as soon as we call
2114          * intel_runtime_pm_put(), so it is correct to wrap only the
2115          * pin/unpin/fence and not more.
2116          */
2117         wakeref = intel_runtime_pm_get(&dev_priv->runtime_pm);
2118         i915_gem_object_lock(obj);
2119
2120         atomic_inc(&dev_priv->gpu_error.pending_fb_pin);
2121
2122         pinctl = 0;
2123
2124         /* Valleyview is definitely limited to scanning out the first
2125          * 512MiB. Lets presume this behaviour was inherited from the
2126          * g4x display engine and that all earlier gen are similarly
2127          * limited. Testing suggests that it is a little more
2128          * complicated than this. For example, Cherryview appears quite
2129          * happy to scanout from anywhere within its global aperture.
2130          */
2131         if (HAS_GMCH(dev_priv))
2132                 pinctl |= PIN_MAPPABLE;
2133
2134         vma = i915_gem_object_pin_to_display_plane(obj,
2135                                                    alignment, view, pinctl);
2136         if (IS_ERR(vma))
2137                 goto err;
2138
2139         if (uses_fence && i915_vma_is_map_and_fenceable(vma)) {
2140                 int ret;
2141
2142                 /* Install a fence for tiled scan-out. Pre-i965 always needs a
2143                  * fence, whereas 965+ only requires a fence if using
2144                  * framebuffer compression.  For simplicity, we always, when
2145                  * possible, install a fence as the cost is not that onerous.
2146                  *
2147                  * If we fail to fence the tiled scanout, then either the
2148                  * modeset will reject the change (which is highly unlikely as
2149                  * the affected systems, all but one, do not have unmappable
2150                  * space) or we will not be able to enable full powersaving
2151                  * techniques (also likely not to apply due to various limits
2152                  * FBC and the like impose on the size of the buffer, which
2153                  * presumably we violated anyway with this unmappable buffer).
2154                  * Anyway, it is presumably better to stumble onwards with
2155                  * something and try to run the system in a "less than optimal"
2156                  * mode that matches the user configuration.
2157                  */
2158                 ret = i915_vma_pin_fence(vma);
2159                 if (ret != 0 && INTEL_GEN(dev_priv) < 4) {
2160                         i915_gem_object_unpin_from_display_plane(vma);
2161                         vma = ERR_PTR(ret);
2162                         goto err;
2163                 }
2164
2165                 if (ret == 0 && vma->fence)
2166                         *out_flags |= PLANE_HAS_FENCE;
2167         }
2168
2169         i915_vma_get(vma);
2170 err:
2171         atomic_dec(&dev_priv->gpu_error.pending_fb_pin);
2172
2173         i915_gem_object_unlock(obj);
2174         intel_runtime_pm_put(&dev_priv->runtime_pm, wakeref);
2175         return vma;
2176 }
2177
2178 void intel_unpin_fb_vma(struct i915_vma *vma, unsigned long flags)
2179 {
2180         lockdep_assert_held(&vma->vm->i915->drm.struct_mutex);
2181
2182         i915_gem_object_lock(vma->obj);
2183         if (flags & PLANE_HAS_FENCE)
2184                 i915_vma_unpin_fence(vma);
2185         i915_gem_object_unpin_from_display_plane(vma);
2186         i915_gem_object_unlock(vma->obj);
2187
2188         i915_vma_put(vma);
2189 }
2190
2191 static int intel_fb_pitch(const struct drm_framebuffer *fb, int color_plane,
2192                           unsigned int rotation)
2193 {
2194         if (drm_rotation_90_or_270(rotation))
2195                 return to_intel_framebuffer(fb)->rotated[color_plane].pitch;
2196         else
2197                 return fb->pitches[color_plane];
2198 }
2199
2200 /*
2201  * Convert the x/y offsets into a linear offset.
2202  * Only valid with 0/180 degree rotation, which is fine since linear
2203  * offset is only used with linear buffers on pre-hsw and tiled buffers
2204  * with gen2/3, and 90/270 degree rotations isn't supported on any of them.
2205  */
2206 u32 intel_fb_xy_to_linear(int x, int y,
2207                           const struct intel_plane_state *state,
2208                           int color_plane)
2209 {
2210         const struct drm_framebuffer *fb = state->base.fb;
2211         unsigned int cpp = fb->format->cpp[color_plane];
2212         unsigned int pitch = state->color_plane[color_plane].stride;
2213
2214         return y * pitch + x * cpp;
2215 }
2216
2217 /*
2218  * Add the x/y offsets derived from fb->offsets[] to the user
2219  * specified plane src x/y offsets. The resulting x/y offsets
2220  * specify the start of scanout from the beginning of the gtt mapping.
2221  */
2222 void intel_add_fb_offsets(int *x, int *y,
2223                           const struct intel_plane_state *state,
2224                           int color_plane)
2225
2226 {
2227         *x += state->color_plane[color_plane].x;
2228         *y += state->color_plane[color_plane].y;
2229 }
2230
2231 static u32 intel_adjust_tile_offset(int *x, int *y,
2232                                     unsigned int tile_width,
2233                                     unsigned int tile_height,
2234                                     unsigned int tile_size,
2235                                     unsigned int pitch_tiles,
2236                                     u32 old_offset,
2237                                     u32 new_offset)
2238 {
2239         unsigned int pitch_pixels = pitch_tiles * tile_width;
2240         unsigned int tiles;
2241
2242         WARN_ON(old_offset & (tile_size - 1));
2243         WARN_ON(new_offset & (tile_size - 1));
2244         WARN_ON(new_offset > old_offset);
2245
2246         tiles = (old_offset - new_offset) / tile_size;
2247
2248         *y += tiles / pitch_tiles * tile_height;
2249         *x += tiles % pitch_tiles * tile_width;
2250
2251         /* minimize x in case it got needlessly big */
2252         *y += *x / pitch_pixels * tile_height;
2253         *x %= pitch_pixels;
2254
2255         return new_offset;
2256 }
2257
2258 static bool is_surface_linear(u64 modifier, int color_plane)
2259 {
2260         return modifier == DRM_FORMAT_MOD_LINEAR;
2261 }
2262
2263 static u32 intel_adjust_aligned_offset(int *x, int *y,
2264                                        const struct drm_framebuffer *fb,
2265                                        int color_plane,
2266                                        unsigned int rotation,
2267                                        unsigned int pitch,
2268                                        u32 old_offset, u32 new_offset)
2269 {
2270         struct drm_i915_private *dev_priv = to_i915(fb->dev);
2271         unsigned int cpp = fb->format->cpp[color_plane];
2272
2273         WARN_ON(new_offset > old_offset);
2274
2275         if (!is_surface_linear(fb->modifier, color_plane)) {
2276                 unsigned int tile_size, tile_width, tile_height;
2277                 unsigned int pitch_tiles;
2278
2279                 tile_size = intel_tile_size(dev_priv);
2280                 intel_tile_dims(fb, color_plane, &tile_width, &tile_height);
2281
2282                 if (drm_rotation_90_or_270(rotation)) {
2283                         pitch_tiles = pitch / tile_height;
2284                         swap(tile_width, tile_height);
2285                 } else {
2286                         pitch_tiles = pitch / (tile_width * cpp);
2287                 }
2288
2289                 intel_adjust_tile_offset(x, y, tile_width, tile_height,
2290                                          tile_size, pitch_tiles,
2291                                          old_offset, new_offset);
2292         } else {
2293                 old_offset += *y * pitch + *x * cpp;
2294
2295                 *y = (old_offset - new_offset) / pitch;
2296                 *x = ((old_offset - new_offset) - *y * pitch) / cpp;
2297         }
2298
2299         return new_offset;
2300 }
2301
2302 /*
2303  * Adjust the tile offset by moving the difference into
2304  * the x/y offsets.
2305  */
2306 static u32 intel_plane_adjust_aligned_offset(int *x, int *y,
2307                                              const struct intel_plane_state *state,
2308                                              int color_plane,
2309                                              u32 old_offset, u32 new_offset)
2310 {
2311         return intel_adjust_aligned_offset(x, y, state->base.fb, color_plane,
2312                                            state->base.rotation,
2313                                            state->color_plane[color_plane].stride,
2314                                            old_offset, new_offset);
2315 }
2316
2317 /*
2318  * Computes the aligned offset to the base tile and adjusts
2319  * x, y. bytes per pixel is assumed to be a power-of-two.
2320  *
2321  * In the 90/270 rotated case, x and y are assumed
2322  * to be already rotated to match the rotated GTT view, and
2323  * pitch is the tile_height aligned framebuffer height.
2324  *
2325  * This function is used when computing the derived information
2326  * under intel_framebuffer, so using any of that information
2327  * here is not allowed. Anything under drm_framebuffer can be
2328  * used. This is why the user has to pass in the pitch since it
2329  * is specified in the rotated orientation.
2330  */
2331 static u32 intel_compute_aligned_offset(struct drm_i915_private *dev_priv,
2332                                         int *x, int *y,
2333                                         const struct drm_framebuffer *fb,
2334                                         int color_plane,
2335                                         unsigned int pitch,
2336                                         unsigned int rotation,
2337                                         u32 alignment)
2338 {
2339         unsigned int cpp = fb->format->cpp[color_plane];
2340         u32 offset, offset_aligned;
2341
2342         if (alignment)
2343                 alignment--;
2344
2345         if (!is_surface_linear(fb->modifier, color_plane)) {
2346                 unsigned int tile_size, tile_width, tile_height;
2347                 unsigned int tile_rows, tiles, pitch_tiles;
2348
2349                 tile_size = intel_tile_size(dev_priv);
2350                 intel_tile_dims(fb, color_plane, &tile_width, &tile_height);
2351
2352                 if (drm_rotation_90_or_270(rotation)) {
2353                         pitch_tiles = pitch / tile_height;
2354                         swap(tile_width, tile_height);
2355                 } else {
2356                         pitch_tiles = pitch / (tile_width * cpp);
2357                 }
2358
2359                 tile_rows = *y / tile_height;
2360                 *y %= tile_height;
2361
2362                 tiles = *x / tile_width;
2363                 *x %= tile_width;
2364
2365                 offset = (tile_rows * pitch_tiles + tiles) * tile_size;
2366                 offset_aligned = offset & ~alignment;
2367
2368                 intel_adjust_tile_offset(x, y, tile_width, tile_height,
2369                                          tile_size, pitch_tiles,
2370                                          offset, offset_aligned);
2371         } else {
2372                 offset = *y * pitch + *x * cpp;
2373                 offset_aligned = offset & ~alignment;
2374
2375                 *y = (offset & alignment) / pitch;
2376                 *x = ((offset & alignment) - *y * pitch) / cpp;
2377         }
2378
2379         return offset_aligned;
2380 }
2381
2382 static u32 intel_plane_compute_aligned_offset(int *x, int *y,
2383                                               const struct intel_plane_state *state,
2384                                               int color_plane)
2385 {
2386         struct intel_plane *intel_plane = to_intel_plane(state->base.plane);
2387         struct drm_i915_private *dev_priv = to_i915(intel_plane->base.dev);
2388         const struct drm_framebuffer *fb = state->base.fb;
2389         unsigned int rotation = state->base.rotation;
2390         int pitch = state->color_plane[color_plane].stride;
2391         u32 alignment;
2392
2393         if (intel_plane->id == PLANE_CURSOR)
2394                 alignment = intel_cursor_alignment(dev_priv);
2395         else
2396                 alignment = intel_surf_alignment(fb, color_plane);
2397
2398         return intel_compute_aligned_offset(dev_priv, x, y, fb, color_plane,
2399                                             pitch, rotation, alignment);
2400 }
2401
2402 /* Convert the fb->offset[] into x/y offsets */
2403 static int intel_fb_offset_to_xy(int *x, int *y,
2404                                  const struct drm_framebuffer *fb,
2405                                  int color_plane)
2406 {
2407         struct drm_i915_private *dev_priv = to_i915(fb->dev);
2408         unsigned int height;
2409
2410         if (fb->modifier != DRM_FORMAT_MOD_LINEAR &&
2411             fb->offsets[color_plane] % intel_tile_size(dev_priv)) {
2412                 DRM_DEBUG_KMS("Misaligned offset 0x%08x for color plane %d\n",
2413                               fb->offsets[color_plane], color_plane);
2414                 return -EINVAL;
2415         }
2416
2417         height = drm_framebuffer_plane_height(fb->height, fb, color_plane);
2418         height = ALIGN(height, intel_tile_height(fb, color_plane));
2419
2420         /* Catch potential overflows early */
2421         if (add_overflows_t(u32, mul_u32_u32(height, fb->pitches[color_plane]),
2422                             fb->offsets[color_plane])) {
2423                 DRM_DEBUG_KMS("Bad offset 0x%08x or pitch %d for color plane %d\n",
2424                               fb->offsets[color_plane], fb->pitches[color_plane],
2425                               color_plane);
2426                 return -ERANGE;
2427         }
2428
2429         *x = 0;
2430         *y = 0;
2431
2432         intel_adjust_aligned_offset(x, y,
2433                                     fb, color_plane, DRM_MODE_ROTATE_0,
2434                                     fb->pitches[color_plane],
2435                                     fb->offsets[color_plane], 0);
2436
2437         return 0;
2438 }
2439
2440 static unsigned int intel_fb_modifier_to_tiling(u64 fb_modifier)
2441 {
2442         switch (fb_modifier) {
2443         case I915_FORMAT_MOD_X_TILED:
2444                 return I915_TILING_X;
2445         case I915_FORMAT_MOD_Y_TILED:
2446         case I915_FORMAT_MOD_Y_TILED_CCS:
2447                 return I915_TILING_Y;
2448         default:
2449                 return I915_TILING_NONE;
2450         }
2451 }
2452
2453 /*
2454  * From the Sky Lake PRM:
2455  * "The Color Control Surface (CCS) contains the compression status of
2456  *  the cache-line pairs. The compression state of the cache-line pair
2457  *  is specified by 2 bits in the CCS. Each CCS cache-line represents
2458  *  an area on the main surface of 16 x16 sets of 128 byte Y-tiled
2459  *  cache-line-pairs. CCS is always Y tiled."
2460  *
2461  * Since cache line pairs refers to horizontally adjacent cache lines,
2462  * each cache line in the CCS corresponds to an area of 32x16 cache
2463  * lines on the main surface. Since each pixel is 4 bytes, this gives
2464  * us a ratio of one byte in the CCS for each 8x16 pixels in the
2465  * main surface.
2466  */
2467 static const struct drm_format_info ccs_formats[] = {
2468         { .format = DRM_FORMAT_XRGB8888, .depth = 24, .num_planes = 2,
2469           .cpp = { 4, 1, }, .hsub = 8, .vsub = 16, },
2470         { .format = DRM_FORMAT_XBGR8888, .depth = 24, .num_planes = 2,
2471           .cpp = { 4, 1, }, .hsub = 8, .vsub = 16, },
2472         { .format = DRM_FORMAT_ARGB8888, .depth = 32, .num_planes = 2,
2473           .cpp = { 4, 1, }, .hsub = 8, .vsub = 16, .has_alpha = true, },
2474         { .format = DRM_FORMAT_ABGR8888, .depth = 32, .num_planes = 2,
2475           .cpp = { 4, 1, }, .hsub = 8, .vsub = 16, .has_alpha = true, },
2476 };
2477
2478 static const struct drm_format_info *
2479 lookup_format_info(const struct drm_format_info formats[],
2480                    int num_formats, u32 format)
2481 {
2482         int i;
2483
2484         for (i = 0; i < num_formats; i++) {
2485                 if (formats[i].format == format)
2486                         return &formats[i];
2487         }
2488
2489         return NULL;
2490 }
2491
2492 static const struct drm_format_info *
2493 intel_get_format_info(const struct drm_mode_fb_cmd2 *cmd)
2494 {
2495         switch (cmd->modifier[0]) {
2496         case I915_FORMAT_MOD_Y_TILED_CCS:
2497         case I915_FORMAT_MOD_Yf_TILED_CCS:
2498                 return lookup_format_info(ccs_formats,
2499                                           ARRAY_SIZE(ccs_formats),
2500                                           cmd->pixel_format);
2501         default:
2502                 return NULL;
2503         }
2504 }
2505
2506 bool is_ccs_modifier(u64 modifier)
2507 {
2508         return modifier == I915_FORMAT_MOD_Y_TILED_CCS ||
2509                modifier == I915_FORMAT_MOD_Yf_TILED_CCS;
2510 }
2511
2512 u32 intel_plane_fb_max_stride(struct drm_i915_private *dev_priv,
2513                               u32 pixel_format, u64 modifier)
2514 {
2515         struct intel_crtc *crtc;
2516         struct intel_plane *plane;
2517
2518         /*
2519          * We assume the primary plane for pipe A has
2520          * the highest stride limits of them all.
2521          */
2522         crtc = intel_get_crtc_for_pipe(dev_priv, PIPE_A);
2523         plane = to_intel_plane(crtc->base.primary);
2524
2525         return plane->max_stride(plane, pixel_format, modifier,
2526                                  DRM_MODE_ROTATE_0);
2527 }
2528
2529 static
2530 u32 intel_fb_max_stride(struct drm_i915_private *dev_priv,
2531                         u32 pixel_format, u64 modifier)
2532 {
2533         /*
2534          * Arbitrary limit for gen4+ chosen to match the
2535          * render engine max stride.
2536          *
2537          * The new CCS hash mode makes remapping impossible
2538          */
2539         if (!is_ccs_modifier(modifier)) {
2540                 if (INTEL_GEN(dev_priv) >= 7)
2541                         return 256*1024;
2542                 else if (INTEL_GEN(dev_priv) >= 4)
2543                         return 128*1024;
2544         }
2545
2546         return intel_plane_fb_max_stride(dev_priv, pixel_format, modifier);
2547 }
2548
2549 static u32
2550 intel_fb_stride_alignment(const struct drm_framebuffer *fb, int color_plane)
2551 {
2552         struct drm_i915_private *dev_priv = to_i915(fb->dev);
2553
2554         if (fb->modifier == DRM_FORMAT_MOD_LINEAR) {
2555                 u32 max_stride = intel_plane_fb_max_stride(dev_priv,
2556                                                            fb->format->format,
2557                                                            fb->modifier);
2558
2559                 /*
2560                  * To make remapping with linear generally feasible
2561                  * we need the stride to be page aligned.
2562                  */
2563                 if (fb->pitches[color_plane] > max_stride)
2564                         return intel_tile_size(dev_priv);
2565                 else
2566                         return 64;
2567         } else {
2568                 return intel_tile_width_bytes(fb, color_plane);
2569         }
2570 }
2571
2572 bool intel_plane_can_remap(const struct intel_plane_state *plane_state)
2573 {
2574         struct intel_plane *plane = to_intel_plane(plane_state->base.plane);
2575         struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
2576         const struct drm_framebuffer *fb = plane_state->base.fb;
2577         int i;
2578
2579         /* We don't want to deal with remapping with cursors */
2580         if (plane->id == PLANE_CURSOR)
2581                 return false;
2582
2583         /*
2584          * The display engine limits already match/exceed the
2585          * render engine limits, so not much point in remapping.
2586          * Would also need to deal with the fence POT alignment
2587          * and gen2 2KiB GTT tile size.
2588          */
2589         if (INTEL_GEN(dev_priv) < 4)
2590                 return false;
2591
2592         /*
2593          * The new CCS hash mode isn't compatible with remapping as
2594          * the virtual address of the pages affects the compressed data.
2595          */
2596         if (is_ccs_modifier(fb->modifier))
2597                 return false;
2598
2599         /* Linear needs a page aligned stride for remapping */
2600         if (fb->modifier == DRM_FORMAT_MOD_LINEAR) {
2601                 unsigned int alignment = intel_tile_size(dev_priv) - 1;
2602
2603                 for (i = 0; i < fb->format->num_planes; i++) {
2604                         if (fb->pitches[i] & alignment)
2605                                 return false;
2606                 }
2607         }
2608
2609         return true;
2610 }
2611
2612 static bool intel_plane_needs_remap(const struct intel_plane_state *plane_state)
2613 {
2614         struct intel_plane *plane = to_intel_plane(plane_state->base.plane);
2615         const struct drm_framebuffer *fb = plane_state->base.fb;
2616         unsigned int rotation = plane_state->base.rotation;
2617         u32 stride, max_stride;
2618
2619         /*
2620          * No remapping for invisible planes since we don't have
2621          * an actual source viewport to remap.
2622          */
2623         if (!plane_state->base.visible)
2624                 return false;
2625
2626         if (!intel_plane_can_remap(plane_state))
2627                 return false;
2628
2629         /*
2630          * FIXME: aux plane limits on gen9+ are
2631          * unclear in Bspec, for now no checking.
2632          */
2633         stride = intel_fb_pitch(fb, 0, rotation);
2634         max_stride = plane->max_stride(plane, fb->format->format,
2635                                        fb->modifier, rotation);
2636
2637         return stride > max_stride;
2638 }
2639
2640 static int
2641 intel_fill_fb_info(struct drm_i915_private *dev_priv,
2642                    struct drm_framebuffer *fb)
2643 {
2644         struct intel_framebuffer *intel_fb = to_intel_framebuffer(fb);
2645         struct intel_rotation_info *rot_info = &intel_fb->rot_info;
2646         struct drm_i915_gem_object *obj = intel_fb_obj(fb);
2647         u32 gtt_offset_rotated = 0;
2648         unsigned int max_size = 0;
2649         int i, num_planes = fb->format->num_planes;
2650         unsigned int tile_size = intel_tile_size(dev_priv);
2651
2652         for (i = 0; i < num_planes; i++) {
2653                 unsigned int width, height;
2654                 unsigned int cpp, size;
2655                 u32 offset;
2656                 int x, y;
2657                 int ret;
2658
2659                 cpp = fb->format->cpp[i];
2660                 width = drm_framebuffer_plane_width(fb->width, fb, i);
2661                 height = drm_framebuffer_plane_height(fb->height, fb, i);
2662
2663                 ret = intel_fb_offset_to_xy(&x, &y, fb, i);
2664                 if (ret) {
2665                         DRM_DEBUG_KMS("bad fb plane %d offset: 0x%x\n",
2666                                       i, fb->offsets[i]);
2667                         return ret;
2668                 }
2669
2670                 if (is_ccs_modifier(fb->modifier) && i == 1) {
2671                         int hsub = fb->format->hsub;
2672                         int vsub = fb->format->vsub;
2673                         int tile_width, tile_height;
2674                         int main_x, main_y;
2675                         int ccs_x, ccs_y;
2676
2677                         intel_tile_dims(fb, i, &tile_width, &tile_height);
2678                         tile_width *= hsub;
2679                         tile_height *= vsub;
2680
2681                         ccs_x = (x * hsub) % tile_width;
2682                         ccs_y = (y * vsub) % tile_height;
2683                         main_x = intel_fb->normal[0].x % tile_width;
2684                         main_y = intel_fb->normal[0].y % tile_height;
2685
2686                         /*
2687                          * CCS doesn't have its own x/y offset register, so the intra CCS tile
2688                          * x/y offsets must match between CCS and the main surface.
2689                          */
2690                         if (main_x != ccs_x || main_y != ccs_y) {
2691                                 DRM_DEBUG_KMS("Bad CCS x/y (main %d,%d ccs %d,%d) full (main %d,%d ccs %d,%d)\n",
2692                                               main_x, main_y,
2693                                               ccs_x, ccs_y,
2694                                               intel_fb->normal[0].x,
2695                                               intel_fb->normal[0].y,
2696                                               x, y);
2697                                 return -EINVAL;
2698                         }
2699                 }
2700
2701                 /*
2702                  * The fence (if used) is aligned to the start of the object
2703                  * so having the framebuffer wrap around across the edge of the
2704                  * fenced region doesn't really work. We have no API to configure
2705                  * the fence start offset within the object (nor could we probably
2706                  * on gen2/3). So it's just easier if we just require that the
2707                  * fb layout agrees with the fence layout. We already check that the
2708                  * fb stride matches the fence stride elsewhere.
2709                  */
2710                 if (i == 0 && i915_gem_object_is_tiled(obj) &&
2711                     (x + width) * cpp > fb->pitches[i]) {
2712                         DRM_DEBUG_KMS("bad fb plane %d offset: 0x%x\n",
2713                                       i, fb->offsets[i]);
2714                         return -EINVAL;
2715                 }
2716
2717                 /*
2718                  * First pixel of the framebuffer from
2719                  * the start of the normal gtt mapping.
2720                  */
2721                 intel_fb->normal[i].x = x;
2722                 intel_fb->normal[i].y = y;
2723
2724                 offset = intel_compute_aligned_offset(dev_priv, &x, &y, fb, i,
2725                                                       fb->pitches[i],
2726                                                       DRM_MODE_ROTATE_0,
2727                                                       tile_size);
2728                 offset /= tile_size;
2729
2730                 if (!is_surface_linear(fb->modifier, i)) {
2731                         unsigned int tile_width, tile_height;
2732                         unsigned int pitch_tiles;
2733                         struct drm_rect r;
2734
2735                         intel_tile_dims(fb, i, &tile_width, &tile_height);
2736
2737                         rot_info->plane[i].offset = offset;
2738                         rot_info->plane[i].stride = DIV_ROUND_UP(fb->pitches[i], tile_width * cpp);
2739                         rot_info->plane[i].width = DIV_ROUND_UP(x + width, tile_width);
2740                         rot_info->plane[i].height = DIV_ROUND_UP(y + height, tile_height);
2741
2742                         intel_fb->rotated[i].pitch =
2743                                 rot_info->plane[i].height * tile_height;
2744
2745                         /* how many tiles does this plane need */
2746                         size = rot_info->plane[i].stride * rot_info->plane[i].height;
2747                         /*
2748                          * If the plane isn't horizontally tile aligned,
2749                          * we need one more tile.
2750                          */
2751                         if (x != 0)
2752                                 size++;
2753
2754                         /* rotate the x/y offsets to match the GTT view */
2755                         r.x1 = x;
2756                         r.y1 = y;
2757                         r.x2 = x + width;
2758                         r.y2 = y + height;
2759                         drm_rect_rotate(&r,
2760                                         rot_info->plane[i].width * tile_width,
2761                                         rot_info->plane[i].height * tile_height,
2762                                         DRM_MODE_ROTATE_270);
2763                         x = r.x1;
2764                         y = r.y1;
2765
2766                         /* rotate the tile dimensions to match the GTT view */
2767                         pitch_tiles = intel_fb->rotated[i].pitch / tile_height;
2768                         swap(tile_width, tile_height);
2769
2770                         /*
2771                          * We only keep the x/y offsets, so push all of the
2772                          * gtt offset into the x/y offsets.
2773                          */
2774                         intel_adjust_tile_offset(&x, &y,
2775                                                  tile_width, tile_height,
2776                                                  tile_size, pitch_tiles,
2777                                                  gtt_offset_rotated * tile_size, 0);
2778
2779                         gtt_offset_rotated += rot_info->plane[i].width * rot_info->plane[i].height;
2780
2781                         /*
2782                          * First pixel of the framebuffer from
2783                          * the start of the rotated gtt mapping.
2784                          */
2785                         intel_fb->rotated[i].x = x;
2786                         intel_fb->rotated[i].y = y;
2787                 } else {
2788                         size = DIV_ROUND_UP((y + height) * fb->pitches[i] +
2789                                             x * cpp, tile_size);
2790                 }
2791
2792                 /* how many tiles in total needed in the bo */
2793                 max_size = max(max_size, offset + size);
2794         }
2795
2796         if (mul_u32_u32(max_size, tile_size) > obj->base.size) {
2797                 DRM_DEBUG_KMS("fb too big for bo (need %llu bytes, have %zu bytes)\n",
2798                               mul_u32_u32(max_size, tile_size), obj->base.size);
2799                 return -EINVAL;
2800         }
2801
2802         return 0;
2803 }
2804
2805 static void
2806 intel_plane_remap_gtt(struct intel_plane_state *plane_state)
2807 {
2808         struct drm_i915_private *dev_priv =
2809                 to_i915(plane_state->base.plane->dev);
2810         struct drm_framebuffer *fb = plane_state->base.fb;
2811         struct intel_framebuffer *intel_fb = to_intel_framebuffer(fb);
2812         struct intel_rotation_info *info = &plane_state->view.rotated;
2813         unsigned int rotation = plane_state->base.rotation;
2814         int i, num_planes = fb->format->num_planes;
2815         unsigned int tile_size = intel_tile_size(dev_priv);
2816         unsigned int src_x, src_y;
2817         unsigned int src_w, src_h;
2818         u32 gtt_offset = 0;
2819
2820         memset(&plane_state->view, 0, sizeof(plane_state->view));
2821         plane_state->view.type = drm_rotation_90_or_270(rotation) ?
2822                 I915_GGTT_VIEW_ROTATED : I915_GGTT_VIEW_REMAPPED;
2823
2824         src_x = plane_state->base.src.x1 >> 16;
2825         src_y = plane_state->base.src.y1 >> 16;
2826         src_w = drm_rect_width(&plane_state->base.src) >> 16;
2827         src_h = drm_rect_height(&plane_state->base.src) >> 16;
2828
2829         WARN_ON(is_ccs_modifier(fb->modifier));
2830
2831         /* Make src coordinates relative to the viewport */
2832         drm_rect_translate(&plane_state->base.src,
2833                            -(src_x << 16), -(src_y << 16));
2834
2835         /* Rotate src coordinates to match rotated GTT view */
2836         if (drm_rotation_90_or_270(rotation))
2837                 drm_rect_rotate(&plane_state->base.src,
2838                                 src_w << 16, src_h << 16,
2839                                 DRM_MODE_ROTATE_270);
2840
2841         for (i = 0; i < num_planes; i++) {
2842                 unsigned int hsub = i ? fb->format->hsub : 1;
2843                 unsigned int vsub = i ? fb->format->vsub : 1;
2844                 unsigned int cpp = fb->format->cpp[i];
2845                 unsigned int tile_width, tile_height;
2846                 unsigned int width, height;
2847                 unsigned int pitch_tiles;
2848                 unsigned int x, y;
2849                 u32 offset;
2850
2851                 intel_tile_dims(fb, i, &tile_width, &tile_height);
2852
2853                 x = src_x / hsub;
2854                 y = src_y / vsub;
2855                 width = src_w / hsub;
2856                 height = src_h / vsub;
2857
2858                 /*
2859                  * First pixel of the src viewport from the
2860                  * start of the normal gtt mapping.
2861                  */
2862                 x += intel_fb->normal[i].x;
2863                 y += intel_fb->normal[i].y;
2864
2865                 offset = intel_compute_aligned_offset(dev_priv, &x, &y,
2866                                                       fb, i, fb->pitches[i],
2867                                                       DRM_MODE_ROTATE_0, tile_size);
2868                 offset /= tile_size;
2869
2870                 info->plane[i].offset = offset;
2871                 info->plane[i].stride = DIV_ROUND_UP(fb->pitches[i],
2872                                                      tile_width * cpp);
2873                 info->plane[i].width = DIV_ROUND_UP(x + width, tile_width);
2874                 info->plane[i].height = DIV_ROUND_UP(y + height, tile_height);
2875
2876                 if (drm_rotation_90_or_270(rotation)) {
2877                         struct drm_rect r;
2878
2879                         /* rotate the x/y offsets to match the GTT view */
2880                         r.x1 = x;
2881                         r.y1 = y;
2882                         r.x2 = x + width;
2883                         r.y2 = y + height;
2884                         drm_rect_rotate(&r,
2885                                         info->plane[i].width * tile_width,
2886                                         info->plane[i].height * tile_height,
2887                                         DRM_MODE_ROTATE_270);
2888                         x = r.x1;
2889                         y = r.y1;
2890
2891                         pitch_tiles = info->plane[i].height;
2892                         plane_state->color_plane[i].stride = pitch_tiles * tile_height;
2893
2894                         /* rotate the tile dimensions to match the GTT view */
2895                         swap(tile_width, tile_height);
2896                 } else {
2897                         pitch_tiles = info->plane[i].width;
2898                         plane_state->color_plane[i].stride = pitch_tiles * tile_width * cpp;
2899                 }
2900
2901                 /*
2902                  * We only keep the x/y offsets, so push all of the
2903                  * gtt offset into the x/y offsets.
2904                  */
2905                 intel_adjust_tile_offset(&x, &y,
2906                                          tile_width, tile_height,
2907                                          tile_size, pitch_tiles,
2908                                          gtt_offset * tile_size, 0);
2909
2910                 gtt_offset += info->plane[i].width * info->plane[i].height;
2911
2912                 plane_state->color_plane[i].offset = 0;
2913                 plane_state->color_plane[i].x = x;
2914                 plane_state->color_plane[i].y = y;
2915         }
2916 }
2917
2918 static int
2919 intel_plane_compute_gtt(struct intel_plane_state *plane_state)
2920 {
2921         const struct intel_framebuffer *fb =
2922                 to_intel_framebuffer(plane_state->base.fb);
2923         unsigned int rotation = plane_state->base.rotation;
2924         int i, num_planes;
2925
2926         if (!fb)
2927                 return 0;
2928
2929         num_planes = fb->base.format->num_planes;
2930
2931         if (intel_plane_needs_remap(plane_state)) {
2932                 intel_plane_remap_gtt(plane_state);
2933
2934                 /*
2935                  * Sometimes even remapping can't overcome
2936                  * the stride limitations :( Can happen with
2937                  * big plane sizes and suitably misaligned
2938                  * offsets.
2939                  */
2940                 return intel_plane_check_stride(plane_state);
2941         }
2942
2943         intel_fill_fb_ggtt_view(&plane_state->view, &fb->base, rotation);
2944
2945         for (i = 0; i < num_planes; i++) {
2946                 plane_state->color_plane[i].stride = intel_fb_pitch(&fb->base, i, rotation);
2947                 plane_state->color_plane[i].offset = 0;
2948
2949                 if (drm_rotation_90_or_270(rotation)) {
2950                         plane_state->color_plane[i].x = fb->rotated[i].x;
2951                         plane_state->color_plane[i].y = fb->rotated[i].y;
2952                 } else {
2953                         plane_state->color_plane[i].x = fb->normal[i].x;
2954                         plane_state->color_plane[i].y = fb->normal[i].y;
2955                 }
2956         }
2957
2958         /* Rotate src coordinates to match rotated GTT view */
2959         if (drm_rotation_90_or_270(rotation))
2960                 drm_rect_rotate(&plane_state->base.src,
2961                                 fb->base.width << 16, fb->base.height << 16,
2962                                 DRM_MODE_ROTATE_270);
2963
2964         return intel_plane_check_stride(plane_state);
2965 }
2966
2967 static int i9xx_format_to_fourcc(int format)
2968 {
2969         switch (format) {
2970         case DISPPLANE_8BPP:
2971                 return DRM_FORMAT_C8;
2972         case DISPPLANE_BGRX555:
2973                 return DRM_FORMAT_XRGB1555;
2974         case DISPPLANE_BGRX565:
2975                 return DRM_FORMAT_RGB565;
2976         default:
2977         case DISPPLANE_BGRX888:
2978                 return DRM_FORMAT_XRGB8888;
2979         case DISPPLANE_RGBX888:
2980                 return DRM_FORMAT_XBGR8888;
2981         case DISPPLANE_BGRX101010:
2982                 return DRM_FORMAT_XRGB2101010;
2983         case DISPPLANE_RGBX101010:
2984                 return DRM_FORMAT_XBGR2101010;
2985         }
2986 }
2987
2988 int skl_format_to_fourcc(int format, bool rgb_order, bool alpha)
2989 {
2990         switch (format) {
2991         case PLANE_CTL_FORMAT_RGB_565:
2992                 return DRM_FORMAT_RGB565;
2993         case PLANE_CTL_FORMAT_NV12:
2994                 return DRM_FORMAT_NV12;
2995         case PLANE_CTL_FORMAT_P010:
2996                 return DRM_FORMAT_P010;
2997         case PLANE_CTL_FORMAT_P012:
2998                 return DRM_FORMAT_P012;
2999         case PLANE_CTL_FORMAT_P016:
3000                 return DRM_FORMAT_P016;
3001         case PLANE_CTL_FORMAT_Y210:
3002                 return DRM_FORMAT_Y210;
3003         case PLANE_CTL_FORMAT_Y212:
3004                 return DRM_FORMAT_Y212;
3005         case PLANE_CTL_FORMAT_Y216:
3006                 return DRM_FORMAT_Y216;
3007         case PLANE_CTL_FORMAT_Y410:
3008                 return DRM_FORMAT_XVYU2101010;
3009         case PLANE_CTL_FORMAT_Y412:
3010                 return DRM_FORMAT_XVYU12_16161616;
3011         case PLANE_CTL_FORMAT_Y416:
3012                 return DRM_FORMAT_XVYU16161616;
3013         default:
3014         case PLANE_CTL_FORMAT_XRGB_8888:
3015                 if (rgb_order) {
3016                         if (alpha)
3017                                 return DRM_FORMAT_ABGR8888;
3018                         else
3019                                 return DRM_FORMAT_XBGR8888;
3020                 } else {
3021                         if (alpha)
3022                                 return DRM_FORMAT_ARGB8888;
3023                         else
3024                                 return DRM_FORMAT_XRGB8888;
3025                 }
3026         case PLANE_CTL_FORMAT_XRGB_2101010:
3027                 if (rgb_order)
3028                         return DRM_FORMAT_XBGR2101010;
3029                 else
3030                         return DRM_FORMAT_XRGB2101010;
3031         case PLANE_CTL_FORMAT_XRGB_16161616F:
3032                 if (rgb_order) {
3033                         if (alpha)
3034                                 return DRM_FORMAT_ABGR16161616F;
3035                         else
3036                                 return DRM_FORMAT_XBGR16161616F;
3037                 } else {
3038                         if (alpha)
3039                                 return DRM_FORMAT_ARGB16161616F;
3040                         else
3041                                 return DRM_FORMAT_XRGB16161616F;
3042                 }
3043         }
3044 }
3045
3046 static bool
3047 intel_alloc_initial_plane_obj(struct intel_crtc *crtc,
3048                               struct intel_initial_plane_config *plane_config)
3049 {
3050         struct drm_device *dev = crtc->base.dev;
3051         struct drm_i915_private *dev_priv = to_i915(dev);
3052         struct drm_i915_gem_object *obj = NULL;
3053         struct drm_mode_fb_cmd2 mode_cmd = { 0 };
3054         struct drm_framebuffer *fb = &plane_config->fb->base;
3055         u32 base_aligned = round_down(plane_config->base, PAGE_SIZE);
3056         u32 size_aligned = round_up(plane_config->base + plane_config->size,
3057                                     PAGE_SIZE);
3058
3059         size_aligned -= base_aligned;
3060
3061         if (plane_config->size == 0)
3062                 return false;
3063
3064         /* If the FB is too big, just don't use it since fbdev is not very
3065          * important and we should probably use that space with FBC or other
3066          * features. */
3067         if (size_aligned * 2 > dev_priv->stolen_usable_size)
3068                 return false;
3069
3070         switch (fb->modifier) {
3071         case DRM_FORMAT_MOD_LINEAR:
3072         case I915_FORMAT_MOD_X_TILED:
3073         case I915_FORMAT_MOD_Y_TILED:
3074                 break;
3075         default:
3076                 DRM_DEBUG_DRIVER("Unsupported modifier for initial FB: 0x%llx\n",
3077                                  fb->modifier);
3078                 return false;
3079         }
3080
3081         mutex_lock(&dev->struct_mutex);
3082         obj = i915_gem_object_create_stolen_for_preallocated(dev_priv,
3083                                                              base_aligned,
3084                                                              base_aligned,
3085                                                              size_aligned);
3086         mutex_unlock(&dev->struct_mutex);
3087         if (!obj)
3088                 return false;
3089
3090         switch (plane_config->tiling) {
3091         case I915_TILING_NONE:
3092                 break;
3093         case I915_TILING_X:
3094         case I915_TILING_Y:
3095                 obj->tiling_and_stride = fb->pitches[0] | plane_config->tiling;
3096                 break;
3097         default:
3098                 MISSING_CASE(plane_config->tiling);
3099                 return false;
3100         }
3101
3102         mode_cmd.pixel_format = fb->format->format;
3103         mode_cmd.width = fb->width;
3104         mode_cmd.height = fb->height;
3105         mode_cmd.pitches[0] = fb->pitches[0];
3106         mode_cmd.modifier[0] = fb->modifier;
3107         mode_cmd.flags = DRM_MODE_FB_MODIFIERS;
3108
3109         if (intel_framebuffer_init(to_intel_framebuffer(fb), obj, &mode_cmd)) {
3110                 DRM_DEBUG_KMS("intel fb init failed\n");
3111                 goto out_unref_obj;
3112         }
3113
3114
3115         DRM_DEBUG_KMS("initial plane fb obj %p\n", obj);
3116         return true;
3117
3118 out_unref_obj:
3119         i915_gem_object_put(obj);
3120         return false;
3121 }
3122
3123 static void
3124 intel_set_plane_visible(struct intel_crtc_state *crtc_state,
3125                         struct intel_plane_state *plane_state,
3126                         bool visible)
3127 {
3128         struct intel_plane *plane = to_intel_plane(plane_state->base.plane);
3129
3130         plane_state->base.visible = visible;
3131
3132         if (visible)
3133                 crtc_state->base.plane_mask |= drm_plane_mask(&plane->base);
3134         else
3135                 crtc_state->base.plane_mask &= ~drm_plane_mask(&plane->base);
3136 }
3137
3138 static void fixup_active_planes(struct intel_crtc_state *crtc_state)
3139 {
3140         struct drm_i915_private *dev_priv = to_i915(crtc_state->base.crtc->dev);
3141         struct drm_plane *plane;
3142
3143         /*
3144          * Active_planes aliases if multiple "primary" or cursor planes
3145          * have been used on the same (or wrong) pipe. plane_mask uses
3146          * unique ids, hence we can use that to reconstruct active_planes.
3147          */
3148         crtc_state->active_planes = 0;
3149
3150         drm_for_each_plane_mask(plane, &dev_priv->drm,
3151                                 crtc_state->base.plane_mask)
3152                 crtc_state->active_planes |= BIT(to_intel_plane(plane)->id);
3153 }
3154
3155 static void intel_plane_disable_noatomic(struct intel_crtc *crtc,
3156                                          struct intel_plane *plane)
3157 {
3158         struct intel_crtc_state *crtc_state =
3159                 to_intel_crtc_state(crtc->base.state);
3160         struct intel_plane_state *plane_state =
3161                 to_intel_plane_state(plane->base.state);
3162
3163         DRM_DEBUG_KMS("Disabling [PLANE:%d:%s] on [CRTC:%d:%s]\n",
3164                       plane->base.base.id, plane->base.name,
3165                       crtc->base.base.id, crtc->base.name);
3166
3167         intel_set_plane_visible(crtc_state, plane_state, false);
3168         fixup_active_planes(crtc_state);
3169         crtc_state->data_rate[plane->id] = 0;
3170
3171         if (plane->id == PLANE_PRIMARY)
3172                 intel_pre_disable_primary_noatomic(&crtc->base);
3173
3174         intel_disable_plane(plane, crtc_state);
3175 }
3176
3177 static void
3178 intel_find_initial_plane_obj(struct intel_crtc *intel_crtc,
3179                              struct intel_initial_plane_config *plane_config)
3180 {
3181         struct drm_device *dev = intel_crtc->base.dev;
3182         struct drm_i915_private *dev_priv = to_i915(dev);
3183         struct drm_crtc *c;
3184         struct drm_i915_gem_object *obj;
3185         struct drm_plane *primary = intel_crtc->base.primary;
3186         struct drm_plane_state *plane_state = primary->state;
3187         struct intel_plane *intel_plane = to_intel_plane(primary);
3188         struct intel_plane_state *intel_state =
3189                 to_intel_plane_state(plane_state);
3190         struct drm_framebuffer *fb;
3191
3192         if (!plane_config->fb)
3193                 return;
3194
3195         if (intel_alloc_initial_plane_obj(intel_crtc, plane_config)) {
3196                 fb = &plane_config->fb->base;
3197                 goto valid_fb;
3198         }
3199
3200         kfree(plane_config->fb);
3201
3202         /*
3203          * Failed to alloc the obj, check to see if we should share
3204          * an fb with another CRTC instead
3205          */
3206         for_each_crtc(dev, c) {
3207                 struct intel_plane_state *state;
3208
3209                 if (c == &intel_crtc->base)
3210                         continue;
3211
3212                 if (!to_intel_crtc(c)->active)
3213                         continue;
3214
3215                 state = to_intel_plane_state(c->primary->state);
3216                 if (!state->vma)
3217                         continue;
3218
3219                 if (intel_plane_ggtt_offset(state) == plane_config->base) {
3220                         fb = state->base.fb;
3221                         drm_framebuffer_get(fb);
3222                         goto valid_fb;
3223                 }
3224         }
3225
3226         /*
3227          * We've failed to reconstruct the BIOS FB.  Current display state
3228          * indicates that the primary plane is visible, but has a NULL FB,
3229          * which will lead to problems later if we don't fix it up.  The
3230          * simplest solution is to just disable the primary plane now and
3231          * pretend the BIOS never had it enabled.
3232          */
3233         intel_plane_disable_noatomic(intel_crtc, intel_plane);
3234
3235         return;
3236
3237 valid_fb:
3238         intel_state->base.rotation = plane_config->rotation;
3239         intel_fill_fb_ggtt_view(&intel_state->view, fb,
3240                                 intel_state->base.rotation);
3241         intel_state->color_plane[0].stride =
3242                 intel_fb_pitch(fb, 0, intel_state->base.rotation);
3243
3244         mutex_lock(&dev->struct_mutex);
3245         intel_state->vma =
3246                 intel_pin_and_fence_fb_obj(fb,
3247                                            &intel_state->view,
3248                                            intel_plane_uses_fence(intel_state),
3249                                            &intel_state->flags);
3250         mutex_unlock(&dev->struct_mutex);
3251         if (IS_ERR(intel_state->vma)) {
3252                 DRM_ERROR("failed to pin boot fb on pipe %d: %li\n",
3253                           intel_crtc->pipe, PTR_ERR(intel_state->vma));
3254
3255                 intel_state->vma = NULL;
3256                 drm_framebuffer_put(fb);
3257                 return;
3258         }
3259
3260         obj = intel_fb_obj(fb);
3261         intel_fb_obj_flush(obj, ORIGIN_DIRTYFB);
3262
3263         plane_state->src_x = 0;
3264         plane_state->src_y = 0;
3265         plane_state->src_w = fb->width << 16;
3266         plane_state->src_h = fb->height << 16;
3267
3268         plane_state->crtc_x = 0;
3269         plane_state->crtc_y = 0;
3270         plane_state->crtc_w = fb->width;
3271         plane_state->crtc_h = fb->height;
3272
3273         intel_state->base.src = drm_plane_state_src(plane_state);
3274         intel_state->base.dst = drm_plane_state_dest(plane_state);
3275
3276         if (i915_gem_object_is_tiled(obj))
3277                 dev_priv->preserve_bios_swizzle = true;
3278
3279         plane_state->fb = fb;
3280         plane_state->crtc = &intel_crtc->base;
3281
3282         atomic_or(to_intel_plane(primary)->frontbuffer_bit,
3283                   &obj->frontbuffer_bits);
3284 }
3285
3286 static int skl_max_plane_width(const struct drm_framebuffer *fb,
3287                                int color_plane,
3288                                unsigned int rotation)
3289 {
3290         int cpp = fb->format->cpp[color_plane];
3291
3292         switch (fb->modifier) {
3293         case DRM_FORMAT_MOD_LINEAR:
3294         case I915_FORMAT_MOD_X_TILED:
3295                 return 4096;
3296         case I915_FORMAT_MOD_Y_TILED_CCS:
3297         case I915_FORMAT_MOD_Yf_TILED_CCS:
3298                 /* FIXME AUX plane? */
3299         case I915_FORMAT_MOD_Y_TILED:
3300         case I915_FORMAT_MOD_Yf_TILED:
3301                 if (cpp == 8)
3302                         return 2048;
3303                 else
3304                         return 4096;
3305         default:
3306                 MISSING_CASE(fb->modifier);
3307                 return 2048;
3308         }
3309 }
3310
3311 static int glk_max_plane_width(const struct drm_framebuffer *fb,
3312                                int color_plane,
3313                                unsigned int rotation)
3314 {
3315         int cpp = fb->format->cpp[color_plane];
3316
3317         switch (fb->modifier) {
3318         case DRM_FORMAT_MOD_LINEAR:
3319         case I915_FORMAT_MOD_X_TILED:
3320                 if (cpp == 8)
3321                         return 4096;
3322                 else
3323                         return 5120;
3324         case I915_FORMAT_MOD_Y_TILED_CCS:
3325         case I915_FORMAT_MOD_Yf_TILED_CCS:
3326                 /* FIXME AUX plane? */
3327         case I915_FORMAT_MOD_Y_TILED:
3328         case I915_FORMAT_MOD_Yf_TILED:
3329                 if (cpp == 8)
3330                         return 2048;
3331                 else
3332                         return 5120;
3333         default:
3334                 MISSING_CASE(fb->modifier);
3335                 return 2048;
3336         }
3337 }
3338
3339 static int icl_max_plane_width(const struct drm_framebuffer *fb,
3340                                int color_plane,
3341                                unsigned int rotation)
3342 {
3343         return 5120;
3344 }
3345
3346 static bool skl_check_main_ccs_coordinates(struct intel_plane_state *plane_state,
3347                                            int main_x, int main_y, u32 main_offset)
3348 {
3349         const struct drm_framebuffer *fb = plane_state->base.fb;
3350         int hsub = fb->format->hsub;
3351         int vsub = fb->format->vsub;
3352         int aux_x = plane_state->color_plane[1].x;
3353         int aux_y = plane_state->color_plane[1].y;
3354         u32 aux_offset = plane_state->color_plane[1].offset;
3355         u32 alignment = intel_surf_alignment(fb, 1);
3356
3357         while (aux_offset >= main_offset && aux_y <= main_y) {
3358                 int x, y;
3359
3360                 if (aux_x == main_x && aux_y == main_y)
3361                         break;
3362
3363                 if (aux_offset == 0)
3364                         break;
3365
3366                 x = aux_x / hsub;
3367                 y = aux_y / vsub;
3368                 aux_offset = intel_plane_adjust_aligned_offset(&x, &y, plane_state, 1,
3369                                                                aux_offset, aux_offset - alignment);
3370                 aux_x = x * hsub + aux_x % hsub;
3371                 aux_y = y * vsub + aux_y % vsub;
3372         }
3373
3374         if (aux_x != main_x || aux_y != main_y)
3375                 return false;
3376
3377         plane_state->color_plane[1].offset = aux_offset;
3378         plane_state->color_plane[1].x = aux_x;
3379         plane_state->color_plane[1].y = aux_y;
3380
3381         return true;
3382 }
3383
3384 static int skl_check_main_surface(struct intel_plane_state *plane_state)
3385 {
3386         struct drm_i915_private *dev_priv = to_i915(plane_state->base.plane->dev);
3387         const struct drm_framebuffer *fb = plane_state->base.fb;
3388         unsigned int rotation = plane_state->base.rotation;
3389         int x = plane_state->base.src.x1 >> 16;
3390         int y = plane_state->base.src.y1 >> 16;
3391         int w = drm_rect_width(&plane_state->base.src) >> 16;
3392         int h = drm_rect_height(&plane_state->base.src) >> 16;
3393         int max_width;
3394         int max_height = 4096;
3395         u32 alignment, offset, aux_offset = plane_state->color_plane[1].offset;
3396
3397         if (INTEL_GEN(dev_priv) >= 11)
3398                 max_width = icl_max_plane_width(fb, 0, rotation);
3399         else if (INTEL_GEN(dev_priv) >= 10 || IS_GEMINILAKE(dev_priv))
3400                 max_width = glk_max_plane_width(fb, 0, rotation);
3401         else
3402                 max_width = skl_max_plane_width(fb, 0, rotation);
3403
3404         if (w > max_width || h > max_height) {
3405                 DRM_DEBUG_KMS("requested Y/RGB source size %dx%d too big (limit %dx%d)\n",
3406                               w, h, max_width, max_height);
3407                 return -EINVAL;
3408         }
3409
3410         intel_add_fb_offsets(&x, &y, plane_state, 0);
3411         offset = intel_plane_compute_aligned_offset(&x, &y, plane_state, 0);
3412         alignment = intel_surf_alignment(fb, 0);
3413
3414         /*
3415          * AUX surface offset is specified as the distance from the
3416          * main surface offset, and it must be non-negative. Make
3417          * sure that is what we will get.
3418          */
3419         if (offset > aux_offset)
3420                 offset = intel_plane_adjust_aligned_offset(&x, &y, plane_state, 0,
3421                                                            offset, aux_offset & ~(alignment - 1));
3422
3423         /*
3424          * When using an X-tiled surface, the plane blows up
3425          * if the x offset + width exceed the stride.
3426          *
3427          * TODO: linear and Y-tiled seem fine, Yf untested,
3428          */
3429         if (fb->modifier == I915_FORMAT_MOD_X_TILED) {
3430                 int cpp = fb->format->cpp[0];
3431
3432                 while ((x + w) * cpp > plane_state->color_plane[0].stride) {
3433                         if (offset == 0) {
3434                                 DRM_DEBUG_KMS("Unable to find suitable display surface offset due to X-tiling\n");
3435                                 return -EINVAL;
3436                         }
3437
3438                         offset = intel_plane_adjust_aligned_offset(&x, &y, plane_state, 0,
3439                                                                    offset, offset - alignment);
3440                 }
3441         }
3442
3443         /*
3444          * CCS AUX surface doesn't have its own x/y offsets, we must make sure
3445          * they match with the main surface x/y offsets.
3446          */
3447         if (is_ccs_modifier(fb->modifier)) {
3448                 while (!skl_check_main_ccs_coordinates(plane_state, x, y, offset)) {
3449                         if (offset == 0)
3450                                 break;
3451
3452                         offset = intel_plane_adjust_aligned_offset(&x, &y, plane_state, 0,
3453                                                                    offset, offset - alignment);
3454                 }
3455
3456                 if (x != plane_state->color_plane[1].x || y != plane_state->color_plane[1].y) {
3457                         DRM_DEBUG_KMS("Unable to find suitable display surface offset due to CCS\n");
3458                         return -EINVAL;
3459                 }
3460         }
3461
3462         plane_state->color_plane[0].offset = offset;
3463         plane_state->color_plane[0].x = x;
3464         plane_state->color_plane[0].y = y;
3465
3466         /*
3467          * Put the final coordinates back so that the src
3468          * coordinate checks will see the right values.
3469          */
3470         drm_rect_translate(&plane_state->base.src,
3471                            (x << 16) - plane_state->base.src.x1,
3472                            (y << 16) - plane_state->base.src.y1);
3473
3474         return 0;
3475 }
3476
3477 static int skl_check_nv12_aux_surface(struct intel_plane_state *plane_state)
3478 {
3479         const struct drm_framebuffer *fb = plane_state->base.fb;
3480         unsigned int rotation = plane_state->base.rotation;
3481         int max_width = skl_max_plane_width(fb, 1, rotation);
3482         int max_height = 4096;
3483         int x = plane_state->base.src.x1 >> 17;
3484         int y = plane_state->base.src.y1 >> 17;
3485         int w = drm_rect_width(&plane_state->base.src) >> 17;
3486         int h = drm_rect_height(&plane_state->base.src) >> 17;
3487         u32 offset;
3488
3489         intel_add_fb_offsets(&x, &y, plane_state, 1);
3490         offset = intel_plane_compute_aligned_offset(&x, &y, plane_state, 1);
3491
3492         /* FIXME not quite sure how/if these apply to the chroma plane */
3493         if (w > max_width || h > max_height) {
3494                 DRM_DEBUG_KMS("CbCr source size %dx%d too big (limit %dx%d)\n",
3495                               w, h, max_width, max_height);
3496                 return -EINVAL;
3497         }
3498
3499         plane_state->color_plane[1].offset = offset;
3500         plane_state->color_plane[1].x = x;
3501         plane_state->color_plane[1].y = y;
3502
3503         return 0;
3504 }
3505
3506 static int skl_check_ccs_aux_surface(struct intel_plane_state *plane_state)
3507 {
3508         const struct drm_framebuffer *fb = plane_state->base.fb;
3509         int src_x = plane_state->base.src.x1 >> 16;
3510         int src_y = plane_state->base.src.y1 >> 16;
3511         int hsub = fb->format->hsub;
3512         int vsub = fb->format->vsub;
3513         int x = src_x / hsub;
3514         int y = src_y / vsub;
3515         u32 offset;
3516
3517         intel_add_fb_offsets(&x, &y, plane_state, 1);
3518         offset = intel_plane_compute_aligned_offset(&x, &y, plane_state, 1);
3519
3520         plane_state->color_plane[1].offset = offset;
3521         plane_state->color_plane[1].x = x * hsub + src_x % hsub;
3522         plane_state->color_plane[1].y = y * vsub + src_y % vsub;
3523
3524         return 0;
3525 }
3526
3527 int skl_check_plane_surface(struct intel_plane_state *plane_state)
3528 {
3529         const struct drm_framebuffer *fb = plane_state->base.fb;
3530         int ret;
3531
3532         ret = intel_plane_compute_gtt(plane_state);
3533         if (ret)
3534                 return ret;
3535
3536         if (!plane_state->base.visible)
3537                 return 0;
3538
3539         /*
3540          * Handle the AUX surface first since
3541          * the main surface setup depends on it.
3542          */
3543         if (is_planar_yuv_format(fb->format->format)) {
3544                 ret = skl_check_nv12_aux_surface(plane_state);
3545                 if (ret)
3546                         return ret;
3547         } else if (is_ccs_modifier(fb->modifier)) {
3548                 ret = skl_check_ccs_aux_surface(plane_state);
3549                 if (ret)
3550                         return ret;
3551         } else {
3552                 plane_state->color_plane[1].offset = ~0xfff;
3553                 plane_state->color_plane[1].x = 0;
3554                 plane_state->color_plane[1].y = 0;
3555         }
3556
3557         ret = skl_check_main_surface(plane_state);
3558         if (ret)
3559                 return ret;
3560
3561         return 0;
3562 }
3563
3564 unsigned int
3565 i9xx_plane_max_stride(struct intel_plane *plane,
3566                       u32 pixel_format, u64 modifier,
3567                       unsigned int rotation)
3568 {
3569         struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
3570
3571         if (!HAS_GMCH(dev_priv)) {
3572                 return 32*1024;
3573         } else if (INTEL_GEN(dev_priv) >= 4) {
3574                 if (modifier == I915_FORMAT_MOD_X_TILED)
3575                         return 16*1024;
3576                 else
3577                         return 32*1024;
3578         } else if (INTEL_GEN(dev_priv) >= 3) {
3579                 if (modifier == I915_FORMAT_MOD_X_TILED)
3580                         return 8*1024;
3581                 else
3582                         return 16*1024;
3583         } else {
3584                 if (plane->i9xx_plane == PLANE_C)
3585                         return 4*1024;
3586                 else
3587                         return 8*1024;
3588         }
3589 }
3590
3591 static u32 i9xx_plane_ctl_crtc(const struct intel_crtc_state *crtc_state)
3592 {
3593         struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
3594         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
3595         u32 dspcntr = 0;
3596
3597         if (crtc_state->gamma_enable)
3598                 dspcntr |= DISPPLANE_GAMMA_ENABLE;
3599
3600         if (crtc_state->csc_enable)
3601                 dspcntr |= DISPPLANE_PIPE_CSC_ENABLE;
3602
3603         if (INTEL_GEN(dev_priv) < 5)
3604                 dspcntr |= DISPPLANE_SEL_PIPE(crtc->pipe);
3605
3606         return dspcntr;
3607 }
3608
3609 static u32 i9xx_plane_ctl(const struct intel_crtc_state *crtc_state,
3610                           const struct intel_plane_state *plane_state)
3611 {
3612         struct drm_i915_private *dev_priv =
3613                 to_i915(plane_state->base.plane->dev);
3614         const struct drm_framebuffer *fb = plane_state->base.fb;
3615         unsigned int rotation = plane_state->base.rotation;
3616         u32 dspcntr;
3617
3618         dspcntr = DISPLAY_PLANE_ENABLE;
3619
3620         if (IS_G4X(dev_priv) || IS_GEN(dev_priv, 5) ||
3621             IS_GEN(dev_priv, 6) || IS_IVYBRIDGE(dev_priv))
3622                 dspcntr |= DISPPLANE_TRICKLE_FEED_DISABLE;
3623
3624         switch (fb->format->format) {
3625         case DRM_FORMAT_C8:
3626                 dspcntr |= DISPPLANE_8BPP;
3627                 break;
3628         case DRM_FORMAT_XRGB1555:
3629                 dspcntr |= DISPPLANE_BGRX555;
3630                 break;
3631         case DRM_FORMAT_RGB565:
3632                 dspcntr |= DISPPLANE_BGRX565;
3633                 break;
3634         case DRM_FORMAT_XRGB8888:
3635                 dspcntr |= DISPPLANE_BGRX888;
3636                 break;
3637         case DRM_FORMAT_XBGR8888:
3638                 dspcntr |= DISPPLANE_RGBX888;
3639                 break;
3640         case DRM_FORMAT_XRGB2101010:
3641                 dspcntr |= DISPPLANE_BGRX101010;
3642                 break;
3643         case DRM_FORMAT_XBGR2101010:
3644                 dspcntr |= DISPPLANE_RGBX101010;
3645                 break;
3646         default:
3647                 MISSING_CASE(fb->format->format);
3648                 return 0;
3649         }
3650
3651         if (INTEL_GEN(dev_priv) >= 4 &&
3652             fb->modifier == I915_FORMAT_MOD_X_TILED)
3653                 dspcntr |= DISPPLANE_TILED;
3654
3655         if (rotation & DRM_MODE_ROTATE_180)
3656                 dspcntr |= DISPPLANE_ROTATE_180;
3657
3658         if (rotation & DRM_MODE_REFLECT_X)
3659                 dspcntr |= DISPPLANE_MIRROR;
3660
3661         return dspcntr;
3662 }
3663
3664 int i9xx_check_plane_surface(struct intel_plane_state *plane_state)
3665 {
3666         struct drm_i915_private *dev_priv =
3667                 to_i915(plane_state->base.plane->dev);
3668         int src_x, src_y;
3669         u32 offset;
3670         int ret;
3671
3672         ret = intel_plane_compute_gtt(plane_state);
3673         if (ret)
3674                 return ret;
3675
3676         if (!plane_state->base.visible)
3677                 return 0;
3678
3679         src_x = plane_state->base.src.x1 >> 16;
3680         src_y = plane_state->base.src.y1 >> 16;
3681
3682         intel_add_fb_offsets(&src_x, &src_y, plane_state, 0);
3683
3684         if (INTEL_GEN(dev_priv) >= 4)
3685                 offset = intel_plane_compute_aligned_offset(&src_x, &src_y,
3686                                                             plane_state, 0);
3687         else
3688                 offset = 0;
3689
3690         /*
3691          * Put the final coordinates back so that the src
3692          * coordinate checks will see the right values.
3693          */
3694         drm_rect_translate(&plane_state->base.src,
3695                            (src_x << 16) - plane_state->base.src.x1,
3696                            (src_y << 16) - plane_state->base.src.y1);
3697
3698         /* HSW/BDW do this automagically in hardware */
3699         if (!IS_HASWELL(dev_priv) && !IS_BROADWELL(dev_priv)) {
3700                 unsigned int rotation = plane_state->base.rotation;
3701                 int src_w = drm_rect_width(&plane_state->base.src) >> 16;
3702                 int src_h = drm_rect_height(&plane_state->base.src) >> 16;
3703
3704                 if (rotation & DRM_MODE_ROTATE_180) {
3705                         src_x += src_w - 1;
3706                         src_y += src_h - 1;
3707                 } else if (rotation & DRM_MODE_REFLECT_X) {
3708                         src_x += src_w - 1;
3709                 }
3710         }
3711
3712         plane_state->color_plane[0].offset = offset;
3713         plane_state->color_plane[0].x = src_x;
3714         plane_state->color_plane[0].y = src_y;
3715
3716         return 0;
3717 }
3718
3719 static bool i9xx_plane_has_windowing(struct intel_plane *plane)
3720 {
3721         struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
3722         enum i9xx_plane_id i9xx_plane = plane->i9xx_plane;
3723
3724         if (IS_CHERRYVIEW(dev_priv))
3725                 return i9xx_plane == PLANE_B;
3726         else if (INTEL_GEN(dev_priv) >= 5 || IS_G4X(dev_priv))
3727                 return false;
3728         else if (IS_GEN(dev_priv, 4))
3729                 return i9xx_plane == PLANE_C;
3730         else
3731                 return i9xx_plane == PLANE_B ||
3732                         i9xx_plane == PLANE_C;
3733 }
3734
3735 static int
3736 i9xx_plane_check(struct intel_crtc_state *crtc_state,
3737                  struct intel_plane_state *plane_state)
3738 {
3739         struct intel_plane *plane = to_intel_plane(plane_state->base.plane);
3740         int ret;
3741
3742         ret = chv_plane_check_rotation(plane_state);
3743         if (ret)
3744                 return ret;
3745
3746         ret = drm_atomic_helper_check_plane_state(&plane_state->base,
3747                                                   &crtc_state->base,
3748                                                   DRM_PLANE_HELPER_NO_SCALING,
3749                                                   DRM_PLANE_HELPER_NO_SCALING,
3750                                                   i9xx_plane_has_windowing(plane),
3751                                                   true);
3752         if (ret)
3753                 return ret;
3754
3755         ret = i9xx_check_plane_surface(plane_state);
3756         if (ret)
3757                 return ret;
3758
3759         if (!plane_state->base.visible)
3760                 return 0;
3761
3762         ret = intel_plane_check_src_coordinates(plane_state);
3763         if (ret)
3764                 return ret;
3765
3766         plane_state->ctl = i9xx_plane_ctl(crtc_state, plane_state);
3767
3768         return 0;
3769 }
3770
3771 static void i9xx_update_plane(struct intel_plane *plane,
3772                               const struct intel_crtc_state *crtc_state,
3773                               const struct intel_plane_state *plane_state)
3774 {
3775         struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
3776         enum i9xx_plane_id i9xx_plane = plane->i9xx_plane;
3777         u32 linear_offset;
3778         int x = plane_state->color_plane[0].x;
3779         int y = plane_state->color_plane[0].y;
3780         int crtc_x = plane_state->base.dst.x1;
3781         int crtc_y = plane_state->base.dst.y1;
3782         int crtc_w = drm_rect_width(&plane_state->base.dst);
3783         int crtc_h = drm_rect_height(&plane_state->base.dst);
3784         unsigned long irqflags;
3785         u32 dspaddr_offset;
3786         u32 dspcntr;
3787
3788         dspcntr = plane_state->ctl | i9xx_plane_ctl_crtc(crtc_state);
3789
3790         linear_offset = intel_fb_xy_to_linear(x, y, plane_state, 0);
3791
3792         if (INTEL_GEN(dev_priv) >= 4)
3793                 dspaddr_offset = plane_state->color_plane[0].offset;
3794         else
3795                 dspaddr_offset = linear_offset;
3796
3797         spin_lock_irqsave(&dev_priv->uncore.lock, irqflags);
3798
3799         I915_WRITE_FW(DSPSTRIDE(i9xx_plane), plane_state->color_plane[0].stride);
3800
3801         if (INTEL_GEN(dev_priv) < 4) {
3802                 /*
3803                  * PLANE_A doesn't actually have a full window
3804                  * generator but let's assume we still need to
3805                  * program whatever is there.
3806                  */
3807                 I915_WRITE_FW(DSPPOS(i9xx_plane), (crtc_y << 16) | crtc_x);
3808                 I915_WRITE_FW(DSPSIZE(i9xx_plane),
3809                               ((crtc_h - 1) << 16) | (crtc_w - 1));
3810         } else if (IS_CHERRYVIEW(dev_priv) && i9xx_plane == PLANE_B) {
3811                 I915_WRITE_FW(PRIMPOS(i9xx_plane), (crtc_y << 16) | crtc_x);
3812                 I915_WRITE_FW(PRIMSIZE(i9xx_plane),
3813                               ((crtc_h - 1) << 16) | (crtc_w - 1));
3814                 I915_WRITE_FW(PRIMCNSTALPHA(i9xx_plane), 0);
3815         }
3816
3817         if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv)) {
3818                 I915_WRITE_FW(DSPOFFSET(i9xx_plane), (y << 16) | x);
3819         } else if (INTEL_GEN(dev_priv) >= 4) {
3820                 I915_WRITE_FW(DSPLINOFF(i9xx_plane), linear_offset);
3821                 I915_WRITE_FW(DSPTILEOFF(i9xx_plane), (y << 16) | x);
3822         }
3823
3824         /*
3825          * The control register self-arms if the plane was previously
3826          * disabled. Try to make the plane enable atomic by writing
3827          * the control register just before the surface register.
3828          */
3829         I915_WRITE_FW(DSPCNTR(i9xx_plane), dspcntr);
3830         if (INTEL_GEN(dev_priv) >= 4)
3831                 I915_WRITE_FW(DSPSURF(i9xx_plane),
3832                               intel_plane_ggtt_offset(plane_state) +
3833                               dspaddr_offset);
3834         else
3835                 I915_WRITE_FW(DSPADDR(i9xx_plane),
3836                               intel_plane_ggtt_offset(plane_state) +
3837                               dspaddr_offset);
3838
3839         spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags);
3840 }
3841
3842 static void i9xx_disable_plane(struct intel_plane *plane,
3843                                const struct intel_crtc_state *crtc_state)
3844 {
3845         struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
3846         enum i9xx_plane_id i9xx_plane = plane->i9xx_plane;
3847         unsigned long irqflags;
3848         u32 dspcntr;
3849
3850         /*
3851          * DSPCNTR pipe gamma enable on g4x+ and pipe csc
3852          * enable on ilk+ affect the pipe bottom color as
3853          * well, so we must configure them even if the plane
3854          * is disabled.
3855          *
3856          * On pre-g4x there is no way to gamma correct the
3857          * pipe bottom color but we'll keep on doing this
3858          * anyway so that the crtc state readout works correctly.
3859          */
3860         dspcntr = i9xx_plane_ctl_crtc(crtc_state);
3861
3862         spin_lock_irqsave(&dev_priv->uncore.lock, irqflags);
3863
3864         I915_WRITE_FW(DSPCNTR(i9xx_plane), dspcntr);
3865         if (INTEL_GEN(dev_priv) >= 4)
3866                 I915_WRITE_FW(DSPSURF(i9xx_plane), 0);
3867         else
3868                 I915_WRITE_FW(DSPADDR(i9xx_plane), 0);
3869
3870         spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags);
3871 }
3872
3873 static bool i9xx_plane_get_hw_state(struct intel_plane *plane,
3874                                     enum pipe *pipe)
3875 {
3876         struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
3877         enum intel_display_power_domain power_domain;
3878         enum i9xx_plane_id i9xx_plane = plane->i9xx_plane;
3879         intel_wakeref_t wakeref;
3880         bool ret;
3881         u32 val;
3882
3883         /*
3884          * Not 100% correct for planes that can move between pipes,
3885          * but that's only the case for gen2-4 which don't have any
3886          * display power wells.
3887          */
3888         power_domain = POWER_DOMAIN_PIPE(plane->pipe);
3889         wakeref = intel_display_power_get_if_enabled(dev_priv, power_domain);
3890         if (!wakeref)
3891                 return false;
3892
3893         val = I915_READ(DSPCNTR(i9xx_plane));
3894
3895         ret = val & DISPLAY_PLANE_ENABLE;
3896
3897         if (INTEL_GEN(dev_priv) >= 5)
3898                 *pipe = plane->pipe;
3899         else
3900                 *pipe = (val & DISPPLANE_SEL_PIPE_MASK) >>
3901                         DISPPLANE_SEL_PIPE_SHIFT;
3902
3903         intel_display_power_put(dev_priv, power_domain, wakeref);
3904
3905         return ret;
3906 }
3907
3908 static void skl_detach_scaler(struct intel_crtc *intel_crtc, int id)
3909 {
3910         struct drm_device *dev = intel_crtc->base.dev;
3911         struct drm_i915_private *dev_priv = to_i915(dev);
3912
3913         I915_WRITE(SKL_PS_CTRL(intel_crtc->pipe, id), 0);
3914         I915_WRITE(SKL_PS_WIN_POS(intel_crtc->pipe, id), 0);
3915         I915_WRITE(SKL_PS_WIN_SZ(intel_crtc->pipe, id), 0);
3916 }
3917
3918 /*
3919  * This function detaches (aka. unbinds) unused scalers in hardware
3920  */
3921 static void skl_detach_scalers(const struct intel_crtc_state *crtc_state)
3922 {
3923         struct intel_crtc *intel_crtc = to_intel_crtc(crtc_state->base.crtc);
3924         const struct intel_crtc_scaler_state *scaler_state =
3925                 &crtc_state->scaler_state;
3926         int i;
3927
3928         /* loop through and disable scalers that aren't in use */
3929         for (i = 0; i < intel_crtc->num_scalers; i++) {
3930                 if (!scaler_state->scalers[i].in_use)
3931                         skl_detach_scaler(intel_crtc, i);
3932         }
3933 }
3934
3935 static unsigned int skl_plane_stride_mult(const struct drm_framebuffer *fb,
3936                                           int color_plane, unsigned int rotation)
3937 {
3938         /*
3939          * The stride is either expressed as a multiple of 64 bytes chunks for
3940          * linear buffers or in number of tiles for tiled buffers.
3941          */
3942         if (fb->modifier == DRM_FORMAT_MOD_LINEAR)
3943                 return 64;
3944         else if (drm_rotation_90_or_270(rotation))
3945                 return intel_tile_height(fb, color_plane);
3946         else
3947                 return intel_tile_width_bytes(fb, color_plane);
3948 }
3949
3950 u32 skl_plane_stride(const struct intel_plane_state *plane_state,
3951                      int color_plane)
3952 {
3953         const struct drm_framebuffer *fb = plane_state->base.fb;
3954         unsigned int rotation = plane_state->base.rotation;
3955         u32 stride = plane_state->color_plane[color_plane].stride;
3956
3957         if (color_plane >= fb->format->num_planes)
3958                 return 0;
3959
3960         return stride / skl_plane_stride_mult(fb, color_plane, rotation);
3961 }
3962
3963 static u32 skl_plane_ctl_format(u32 pixel_format)
3964 {
3965         switch (pixel_format) {
3966         case DRM_FORMAT_C8:
3967                 return PLANE_CTL_FORMAT_INDEXED;
3968         case DRM_FORMAT_RGB565:
3969                 return PLANE_CTL_FORMAT_RGB_565;
3970         case DRM_FORMAT_XBGR8888:
3971         case DRM_FORMAT_ABGR8888:
3972                 return PLANE_CTL_FORMAT_XRGB_8888 | PLANE_CTL_ORDER_RGBX;
3973         case DRM_FORMAT_XRGB8888:
3974         case DRM_FORMAT_ARGB8888:
3975                 return PLANE_CTL_FORMAT_XRGB_8888;
3976         case DRM_FORMAT_XBGR2101010:
3977                 return PLANE_CTL_FORMAT_XRGB_2101010 | PLANE_CTL_ORDER_RGBX;
3978         case DRM_FORMAT_XRGB2101010:
3979                 return PLANE_CTL_FORMAT_XRGB_2101010;
3980         case DRM_FORMAT_XBGR16161616F:
3981         case DRM_FORMAT_ABGR16161616F:
3982                 return PLANE_CTL_FORMAT_XRGB_16161616F | PLANE_CTL_ORDER_RGBX;
3983         case DRM_FORMAT_XRGB16161616F:
3984         case DRM_FORMAT_ARGB16161616F:
3985                 return PLANE_CTL_FORMAT_XRGB_16161616F;
3986         case DRM_FORMAT_YUYV:
3987                 return PLANE_CTL_FORMAT_YUV422 | PLANE_CTL_YUV422_YUYV;
3988         case DRM_FORMAT_YVYU:
3989                 return PLANE_CTL_FORMAT_YUV422 | PLANE_CTL_YUV422_YVYU;
3990         case DRM_FORMAT_UYVY:
3991                 return PLANE_CTL_FORMAT_YUV422 | PLANE_CTL_YUV422_UYVY;
3992         case DRM_FORMAT_VYUY:
3993                 return PLANE_CTL_FORMAT_YUV422 | PLANE_CTL_YUV422_VYUY;
3994         case DRM_FORMAT_NV12:
3995                 return PLANE_CTL_FORMAT_NV12;
3996         case DRM_FORMAT_P010:
3997                 return PLANE_CTL_FORMAT_P010;
3998         case DRM_FORMAT_P012:
3999                 return PLANE_CTL_FORMAT_P012;
4000         case DRM_FORMAT_P016:
4001                 return PLANE_CTL_FORMAT_P016;
4002         case DRM_FORMAT_Y210:
4003                 return PLANE_CTL_FORMAT_Y210;
4004         case DRM_FORMAT_Y212:
4005                 return PLANE_CTL_FORMAT_Y212;
4006         case DRM_FORMAT_Y216:
4007                 return PLANE_CTL_FORMAT_Y216;
4008         case DRM_FORMAT_XVYU2101010:
4009                 return PLANE_CTL_FORMAT_Y410;
4010         case DRM_FORMAT_XVYU12_16161616:
4011                 return PLANE_CTL_FORMAT_Y412;
4012         case DRM_FORMAT_XVYU16161616:
4013                 return PLANE_CTL_FORMAT_Y416;
4014         default:
4015                 MISSING_CASE(pixel_format);
4016         }
4017
4018         return 0;
4019 }
4020
4021 static u32 skl_plane_ctl_alpha(const struct intel_plane_state *plane_state)
4022 {
4023         if (!plane_state->base.fb->format->has_alpha)
4024                 return PLANE_CTL_ALPHA_DISABLE;
4025
4026         switch (plane_state->base.pixel_blend_mode) {
4027         case DRM_MODE_BLEND_PIXEL_NONE:
4028                 return PLANE_CTL_ALPHA_DISABLE;
4029         case DRM_MODE_BLEND_PREMULTI:
4030                 return PLANE_CTL_ALPHA_SW_PREMULTIPLY;
4031         case DRM_MODE_BLEND_COVERAGE:
4032                 return PLANE_CTL_ALPHA_HW_PREMULTIPLY;
4033         default:
4034                 MISSING_CASE(plane_state->base.pixel_blend_mode);
4035                 return PLANE_CTL_ALPHA_DISABLE;
4036         }
4037 }
4038
4039 static u32 glk_plane_color_ctl_alpha(const struct intel_plane_state *plane_state)
4040 {
4041         if (!plane_state->base.fb->format->has_alpha)
4042                 return PLANE_COLOR_ALPHA_DISABLE;
4043
4044         switch (plane_state->base.pixel_blend_mode) {
4045         case DRM_MODE_BLEND_PIXEL_NONE:
4046                 return PLANE_COLOR_ALPHA_DISABLE;
4047         case DRM_MODE_BLEND_PREMULTI:
4048                 return PLANE_COLOR_ALPHA_SW_PREMULTIPLY;
4049         case DRM_MODE_BLEND_COVERAGE:
4050                 return PLANE_COLOR_ALPHA_HW_PREMULTIPLY;
4051         default:
4052                 MISSING_CASE(plane_state->base.pixel_blend_mode);
4053                 return PLANE_COLOR_ALPHA_DISABLE;
4054         }
4055 }
4056
4057 static u32 skl_plane_ctl_tiling(u64 fb_modifier)
4058 {
4059         switch (fb_modifier) {
4060         case DRM_FORMAT_MOD_LINEAR:
4061                 break;
4062         case I915_FORMAT_MOD_X_TILED:
4063                 return PLANE_CTL_TILED_X;
4064         case I915_FORMAT_MOD_Y_TILED:
4065                 return PLANE_CTL_TILED_Y;
4066         case I915_FORMAT_MOD_Y_TILED_CCS:
4067                 return PLANE_CTL_TILED_Y | PLANE_CTL_RENDER_DECOMPRESSION_ENABLE;
4068         case I915_FORMAT_MOD_Yf_TILED:
4069                 return PLANE_CTL_TILED_YF;
4070         case I915_FORMAT_MOD_Yf_TILED_CCS:
4071                 return PLANE_CTL_TILED_YF | PLANE_CTL_RENDER_DECOMPRESSION_ENABLE;
4072         default:
4073                 MISSING_CASE(fb_modifier);
4074         }
4075
4076         return 0;
4077 }
4078
4079 static u32 skl_plane_ctl_rotate(unsigned int rotate)
4080 {
4081         switch (rotate) {
4082         case DRM_MODE_ROTATE_0:
4083                 break;
4084         /*
4085          * DRM_MODE_ROTATE_ is counter clockwise to stay compatible with Xrandr
4086          * while i915 HW rotation is clockwise, thats why this swapping.
4087          */
4088         case DRM_MODE_ROTATE_90:
4089                 return PLANE_CTL_ROTATE_270;
4090         case DRM_MODE_ROTATE_180:
4091                 return PLANE_CTL_ROTATE_180;
4092         case DRM_MODE_ROTATE_270:
4093                 return PLANE_CTL_ROTATE_90;
4094         default:
4095                 MISSING_CASE(rotate);
4096         }
4097
4098         return 0;
4099 }
4100
4101 static u32 cnl_plane_ctl_flip(unsigned int reflect)
4102 {
4103         switch (reflect) {
4104         case 0:
4105                 break;
4106         case DRM_MODE_REFLECT_X:
4107                 return PLANE_CTL_FLIP_HORIZONTAL;
4108         case DRM_MODE_REFLECT_Y:
4109         default:
4110                 MISSING_CASE(reflect);
4111         }
4112
4113         return 0;
4114 }
4115
4116 u32 skl_plane_ctl_crtc(const struct intel_crtc_state *crtc_state)
4117 {
4118         struct drm_i915_private *dev_priv = to_i915(crtc_state->base.crtc->dev);
4119         u32 plane_ctl = 0;
4120
4121         if (INTEL_GEN(dev_priv) >= 10 || IS_GEMINILAKE(dev_priv))
4122                 return plane_ctl;
4123
4124         if (crtc_state->gamma_enable)
4125                 plane_ctl |= PLANE_CTL_PIPE_GAMMA_ENABLE;
4126
4127         if (crtc_state->csc_enable)
4128                 plane_ctl |= PLANE_CTL_PIPE_CSC_ENABLE;
4129
4130         return plane_ctl;
4131 }
4132
4133 u32 skl_plane_ctl(const struct intel_crtc_state *crtc_state,
4134                   const struct intel_plane_state *plane_state)
4135 {
4136         struct drm_i915_private *dev_priv =
4137                 to_i915(plane_state->base.plane->dev);
4138         const struct drm_framebuffer *fb = plane_state->base.fb;
4139         unsigned int rotation = plane_state->base.rotation;
4140         const struct drm_intel_sprite_colorkey *key = &plane_state->ckey;
4141         u32 plane_ctl;
4142
4143         plane_ctl = PLANE_CTL_ENABLE;
4144
4145         if (INTEL_GEN(dev_priv) < 10 && !IS_GEMINILAKE(dev_priv)) {
4146                 plane_ctl |= skl_plane_ctl_alpha(plane_state);
4147                 plane_ctl |= PLANE_CTL_PLANE_GAMMA_DISABLE;
4148
4149                 if (plane_state->base.color_encoding == DRM_COLOR_YCBCR_BT709)
4150                         plane_ctl |= PLANE_CTL_YUV_TO_RGB_CSC_FORMAT_BT709;
4151
4152                 if (plane_state->base.color_range == DRM_COLOR_YCBCR_FULL_RANGE)
4153                         plane_ctl |= PLANE_CTL_YUV_RANGE_CORRECTION_DISABLE;
4154         }
4155
4156         plane_ctl |= skl_plane_ctl_format(fb->format->format);
4157         plane_ctl |= skl_plane_ctl_tiling(fb->modifier);
4158         plane_ctl |= skl_plane_ctl_rotate(rotation & DRM_MODE_ROTATE_MASK);
4159
4160         if (INTEL_GEN(dev_priv) >= 10)
4161                 plane_ctl |= cnl_plane_ctl_flip(rotation &
4162                                                 DRM_MODE_REFLECT_MASK);
4163
4164         if (key->flags & I915_SET_COLORKEY_DESTINATION)
4165                 plane_ctl |= PLANE_CTL_KEY_ENABLE_DESTINATION;
4166         else if (key->flags & I915_SET_COLORKEY_SOURCE)
4167                 plane_ctl |= PLANE_CTL_KEY_ENABLE_SOURCE;
4168
4169         return plane_ctl;
4170 }
4171
4172 u32 glk_plane_color_ctl_crtc(const struct intel_crtc_state *crtc_state)
4173 {
4174         struct drm_i915_private *dev_priv = to_i915(crtc_state->base.crtc->dev);
4175         u32 plane_color_ctl = 0;
4176
4177         if (INTEL_GEN(dev_priv) >= 11)
4178                 return plane_color_ctl;
4179
4180         if (crtc_state->gamma_enable)
4181                 plane_color_ctl |= PLANE_COLOR_PIPE_GAMMA_ENABLE;
4182
4183         if (crtc_state->csc_enable)
4184                 plane_color_ctl |= PLANE_COLOR_PIPE_CSC_ENABLE;
4185
4186         return plane_color_ctl;
4187 }
4188
4189 u32 glk_plane_color_ctl(const struct intel_crtc_state *crtc_state,
4190                         const struct intel_plane_state *plane_state)
4191 {
4192         struct drm_i915_private *dev_priv =
4193                 to_i915(plane_state->base.plane->dev);
4194         const struct drm_framebuffer *fb = plane_state->base.fb;
4195         struct intel_plane *plane = to_intel_plane(plane_state->base.plane);
4196         u32 plane_color_ctl = 0;
4197
4198         plane_color_ctl |= PLANE_COLOR_PLANE_GAMMA_DISABLE;
4199         plane_color_ctl |= glk_plane_color_ctl_alpha(plane_state);
4200
4201         if (fb->format->is_yuv && !icl_is_hdr_plane(dev_priv, plane->id)) {
4202                 if (plane_state->base.color_encoding == DRM_COLOR_YCBCR_BT709)
4203                         plane_color_ctl |= PLANE_COLOR_CSC_MODE_YUV709_TO_RGB709;
4204                 else
4205                         plane_color_ctl |= PLANE_COLOR_CSC_MODE_YUV601_TO_RGB709;
4206
4207                 if (plane_state->base.color_range == DRM_COLOR_YCBCR_FULL_RANGE)
4208                         plane_color_ctl |= PLANE_COLOR_YUV_RANGE_CORRECTION_DISABLE;
4209         } else if (fb->format->is_yuv) {
4210                 plane_color_ctl |= PLANE_COLOR_INPUT_CSC_ENABLE;
4211         }
4212
4213         return plane_color_ctl;
4214 }
4215
4216 static int
4217 __intel_display_resume(struct drm_device *dev,
4218                        struct drm_atomic_state *state,
4219                        struct drm_modeset_acquire_ctx *ctx)
4220 {
4221         struct drm_crtc_state *crtc_state;
4222         struct drm_crtc *crtc;
4223         int i, ret;
4224
4225         intel_modeset_setup_hw_state(dev, ctx);
4226         i915_redisable_vga(to_i915(dev));
4227
4228         if (!state)
4229                 return 0;
4230
4231         /*
4232          * We've duplicated the state, pointers to the old state are invalid.
4233          *
4234          * Don't attempt to use the old state until we commit the duplicated state.
4235          */
4236         for_each_new_crtc_in_state(state, crtc, crtc_state, i) {
4237                 /*
4238                  * Force recalculation even if we restore
4239                  * current state. With fast modeset this may not result
4240                  * in a modeset when the state is compatible.
4241                  */
4242                 crtc_state->mode_changed = true;
4243         }
4244
4245         /* ignore any reset values/BIOS leftovers in the WM registers */
4246         if (!HAS_GMCH(to_i915(dev)))
4247                 to_intel_atomic_state(state)->skip_intermediate_wm = true;
4248
4249         ret = drm_atomic_helper_commit_duplicated_state(state, ctx);
4250
4251         WARN_ON(ret == -EDEADLK);
4252         return ret;
4253 }
4254
4255 static bool gpu_reset_clobbers_display(struct drm_i915_private *dev_priv)
4256 {
4257         return (INTEL_INFO(dev_priv)->gpu_reset_clobbers_display &&
4258                 intel_has_gpu_reset(dev_priv));
4259 }
4260
4261 void intel_prepare_reset(struct drm_i915_private *dev_priv)
4262 {
4263         struct drm_device *dev = &dev_priv->drm;
4264         struct drm_modeset_acquire_ctx *ctx = &dev_priv->reset_ctx;
4265         struct drm_atomic_state *state;
4266         int ret;
4267
4268         /* reset doesn't touch the display */
4269         if (!i915_modparams.force_reset_modeset_test &&
4270             !gpu_reset_clobbers_display(dev_priv))
4271                 return;
4272
4273         /* We have a modeset vs reset deadlock, defensively unbreak it. */
4274         set_bit(I915_RESET_MODESET, &dev_priv->gpu_error.flags);
4275         wake_up_all(&dev_priv->gpu_error.wait_queue);
4276
4277         if (atomic_read(&dev_priv->gpu_error.pending_fb_pin)) {
4278                 DRM_DEBUG_KMS("Modeset potentially stuck, unbreaking through wedging\n");
4279                 i915_gem_set_wedged(dev_priv);
4280         }
4281
4282         /*
4283          * Need mode_config.mutex so that we don't
4284          * trample ongoing ->detect() and whatnot.
4285          */
4286         mutex_lock(&dev->mode_config.mutex);
4287         drm_modeset_acquire_init(ctx, 0);
4288         while (1) {
4289                 ret = drm_modeset_lock_all_ctx(dev, ctx);
4290                 if (ret != -EDEADLK)
4291                         break;
4292
4293                 drm_modeset_backoff(ctx);
4294         }
4295         /*
4296          * Disabling the crtcs gracefully seems nicer. Also the
4297          * g33 docs say we should at least disable all the planes.
4298          */
4299         state = drm_atomic_helper_duplicate_state(dev, ctx);
4300         if (IS_ERR(state)) {
4301                 ret = PTR_ERR(state);
4302                 DRM_ERROR("Duplicating state failed with %i\n", ret);
4303                 return;
4304         }
4305
4306         ret = drm_atomic_helper_disable_all(dev, ctx);
4307         if (ret) {
4308                 DRM_ERROR("Suspending crtc's failed with %i\n", ret);
4309                 drm_atomic_state_put(state);
4310                 return;
4311         }
4312
4313         dev_priv->modeset_restore_state = state;
4314         state->acquire_ctx = ctx;
4315 }
4316
4317 void intel_finish_reset(struct drm_i915_private *dev_priv)
4318 {
4319         struct drm_device *dev = &dev_priv->drm;
4320         struct drm_modeset_acquire_ctx *ctx = &dev_priv->reset_ctx;
4321         struct drm_atomic_state *state;
4322         int ret;
4323
4324         /* reset doesn't touch the display */
4325         if (!test_bit(I915_RESET_MODESET, &dev_priv->gpu_error.flags))
4326                 return;
4327
4328         state = fetch_and_zero(&dev_priv->modeset_restore_state);
4329         if (!state)
4330                 goto unlock;
4331
4332         /* reset doesn't touch the display */
4333         if (!gpu_reset_clobbers_display(dev_priv)) {
4334                 /* for testing only restore the display */
4335                 ret = __intel_display_resume(dev, state, ctx);
4336                 if (ret)
4337                         DRM_ERROR("Restoring old state failed with %i\n", ret);
4338         } else {
4339                 /*
4340                  * The display has been reset as well,
4341                  * so need a full re-initialization.
4342                  */
4343                 intel_pps_unlock_regs_wa(dev_priv);
4344                 intel_modeset_init_hw(dev);
4345                 intel_init_clock_gating(dev_priv);
4346
4347                 spin_lock_irq(&dev_priv->irq_lock);
4348                 if (dev_priv->display.hpd_irq_setup)
4349                         dev_priv->display.hpd_irq_setup(dev_priv);
4350                 spin_unlock_irq(&dev_priv->irq_lock);
4351
4352                 ret = __intel_display_resume(dev, state, ctx);
4353                 if (ret)
4354                         DRM_ERROR("Restoring old state failed with %i\n", ret);
4355
4356                 intel_hpd_init(dev_priv);
4357         }
4358
4359         drm_atomic_state_put(state);
4360 unlock:
4361         drm_modeset_drop_locks(ctx);
4362         drm_modeset_acquire_fini(ctx);
4363         mutex_unlock(&dev->mode_config.mutex);
4364
4365         clear_bit(I915_RESET_MODESET, &dev_priv->gpu_error.flags);
4366 }
4367
4368 static void icl_set_pipe_chicken(struct intel_crtc *crtc)
4369 {
4370         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
4371         enum pipe pipe = crtc->pipe;
4372         u32 tmp;
4373
4374         tmp = I915_READ(PIPE_CHICKEN(pipe));
4375
4376         /*
4377          * Display WA #1153: icl
4378          * enable hardware to bypass the alpha math
4379          * and rounding for per-pixel values 00 and 0xff
4380          */
4381         tmp |= PER_PIXEL_ALPHA_BYPASS_EN;
4382         /*
4383          * Display WA # 1605353570: icl
4384          * Set the pixel rounding bit to 1 for allowing
4385          * passthrough of Frame buffer pixels unmodified
4386          * across pipe
4387          */
4388         tmp |= PIXEL_ROUNDING_TRUNC_FB_PASSTHRU;
4389         I915_WRITE(PIPE_CHICKEN(pipe), tmp);
4390 }
4391
4392 static void intel_update_pipe_config(const struct intel_crtc_state *old_crtc_state,
4393                                      const struct intel_crtc_state *new_crtc_state)
4394 {
4395         struct intel_crtc *crtc = to_intel_crtc(new_crtc_state->base.crtc);
4396         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
4397
4398         /* drm_atomic_helper_update_legacy_modeset_state might not be called. */
4399         crtc->base.mode = new_crtc_state->base.mode;
4400
4401         /*
4402          * Update pipe size and adjust fitter if needed: the reason for this is
4403          * that in compute_mode_changes we check the native mode (not the pfit
4404          * mode) to see if we can flip rather than do a full mode set. In the
4405          * fastboot case, we'll flip, but if we don't update the pipesrc and
4406          * pfit state, we'll end up with a big fb scanned out into the wrong
4407          * sized surface.
4408          */
4409
4410         I915_WRITE(PIPESRC(crtc->pipe),
4411                    ((new_crtc_state->pipe_src_w - 1) << 16) |
4412                    (new_crtc_state->pipe_src_h - 1));
4413
4414         /* on skylake this is done by detaching scalers */
4415         if (INTEL_GEN(dev_priv) >= 9) {
4416                 skl_detach_scalers(new_crtc_state);
4417
4418                 if (new_crtc_state->pch_pfit.enabled)
4419                         skylake_pfit_enable(new_crtc_state);
4420         } else if (HAS_PCH_SPLIT(dev_priv)) {
4421                 if (new_crtc_state->pch_pfit.enabled)
4422                         ironlake_pfit_enable(new_crtc_state);
4423                 else if (old_crtc_state->pch_pfit.enabled)
4424                         ironlake_pfit_disable(old_crtc_state);
4425         }
4426
4427         if (INTEL_GEN(dev_priv) >= 11)
4428                 icl_set_pipe_chicken(crtc);
4429 }
4430
4431 static void intel_fdi_normal_train(struct intel_crtc *crtc)
4432 {
4433         struct drm_device *dev = crtc->base.dev;
4434         struct drm_i915_private *dev_priv = to_i915(dev);
4435         int pipe = crtc->pipe;
4436         i915_reg_t reg;
4437         u32 temp;
4438
4439         /* enable normal train */
4440         reg = FDI_TX_CTL(pipe);
4441         temp = I915_READ(reg);
4442         if (IS_IVYBRIDGE(dev_priv)) {
4443                 temp &= ~FDI_LINK_TRAIN_NONE_IVB;
4444                 temp |= FDI_LINK_TRAIN_NONE_IVB | FDI_TX_ENHANCE_FRAME_ENABLE;
4445         } else {
4446                 temp &= ~FDI_LINK_TRAIN_NONE;
4447                 temp |= FDI_LINK_TRAIN_NONE | FDI_TX_ENHANCE_FRAME_ENABLE;
4448         }
4449         I915_WRITE(reg, temp);
4450
4451         reg = FDI_RX_CTL(pipe);
4452         temp = I915_READ(reg);
4453         if (HAS_PCH_CPT(dev_priv)) {
4454                 temp &= ~FDI_LINK_TRAIN_PATTERN_MASK_CPT;
4455                 temp |= FDI_LINK_TRAIN_NORMAL_CPT;
4456         } else {
4457                 temp &= ~FDI_LINK_TRAIN_NONE;
4458                 temp |= FDI_LINK_TRAIN_NONE;
4459         }
4460         I915_WRITE(reg, temp | FDI_RX_ENHANCE_FRAME_ENABLE);
4461
4462         /* wait one idle pattern time */
4463         POSTING_READ(reg);
4464         udelay(1000);
4465
4466         /* IVB wants error correction enabled */
4467         if (IS_IVYBRIDGE(dev_priv))
4468                 I915_WRITE(reg, I915_READ(reg) | FDI_FS_ERRC_ENABLE |
4469                            FDI_FE_ERRC_ENABLE);
4470 }
4471
4472 /* The FDI link training functions for ILK/Ibexpeak. */
4473 static void ironlake_fdi_link_train(struct intel_crtc *crtc,
4474                                     const struct intel_crtc_state *crtc_state)
4475 {
4476         struct drm_device *dev = crtc->base.dev;
4477         struct drm_i915_private *dev_priv = to_i915(dev);
4478         int pipe = crtc->pipe;
4479         i915_reg_t reg;
4480         u32 temp, tries;
4481
4482         /* FDI needs bits from pipe first */
4483         assert_pipe_enabled(dev_priv, pipe);
4484
4485         /* Train 1: umask FDI RX Interrupt symbol_lock and bit_lock bit
4486            for train result */
4487         reg = FDI_RX_IMR(pipe);
4488         temp = I915_READ(reg);
4489         temp &= ~FDI_RX_SYMBOL_LOCK;
4490         temp &= ~FDI_RX_BIT_LOCK;
4491         I915_WRITE(reg, temp);
4492         I915_READ(reg);
4493         udelay(150);
4494
4495         /* enable CPU FDI TX and PCH FDI RX */
4496         reg = FDI_TX_CTL(pipe);
4497         temp = I915_READ(reg);
4498         temp &= ~FDI_DP_PORT_WIDTH_MASK;
4499         temp |= FDI_DP_PORT_WIDTH(crtc_state->fdi_lanes);
4500         temp &= ~FDI_LINK_TRAIN_NONE;
4501         temp |= FDI_LINK_TRAIN_PATTERN_1;
4502         I915_WRITE(reg, temp | FDI_TX_ENABLE);
4503
4504         reg = FDI_RX_CTL(pipe);
4505         temp = I915_READ(reg);
4506         temp &= ~FDI_LINK_TRAIN_NONE;
4507         temp |= FDI_LINK_TRAIN_PATTERN_1;
4508         I915_WRITE(reg, temp | FDI_RX_ENABLE);
4509
4510         POSTING_READ(reg);
4511         udelay(150);
4512
4513         /* Ironlake workaround, enable clock pointer after FDI enable*/
4514         I915_WRITE(FDI_RX_CHICKEN(pipe), FDI_RX_PHASE_SYNC_POINTER_OVR);
4515         I915_WRITE(FDI_RX_CHICKEN(pipe), FDI_RX_PHASE_SYNC_POINTER_OVR |
4516                    FDI_RX_PHASE_SYNC_POINTER_EN);
4517
4518         reg = FDI_RX_IIR(pipe);
4519         for (tries = 0; tries < 5; tries++) {
4520                 temp = I915_READ(reg);
4521                 DRM_DEBUG_KMS("FDI_RX_IIR 0x%x\n", temp);
4522
4523                 if ((temp & FDI_RX_BIT_LOCK)) {
4524                         DRM_DEBUG_KMS("FDI train 1 done.\n");
4525                         I915_WRITE(reg, temp | FDI_RX_BIT_LOCK);
4526                         break;
4527                 }
4528         }
4529         if (tries == 5)
4530                 DRM_ERROR("FDI train 1 fail!\n");
4531
4532         /* Train 2 */
4533         reg = FDI_TX_CTL(pipe);
4534         temp = I915_READ(reg);
4535         temp &= ~FDI_LINK_TRAIN_NONE;
4536         temp |= FDI_LINK_TRAIN_PATTERN_2;
4537         I915_WRITE(reg, temp);
4538
4539         reg = FDI_RX_CTL(pipe);
4540         temp = I915_READ(reg);
4541         temp &= ~FDI_LINK_TRAIN_NONE;
4542         temp |= FDI_LINK_TRAIN_PATTERN_2;
4543         I915_WRITE(reg, temp);
4544
4545         POSTING_READ(reg);
4546         udelay(150);
4547
4548         reg = FDI_RX_IIR(pipe);
4549         for (tries = 0; tries < 5; tries++) {
4550                 temp = I915_READ(reg);
4551                 DRM_DEBUG_KMS("FDI_RX_IIR 0x%x\n", temp);
4552
4553                 if (temp & FDI_RX_SYMBOL_LOCK) {
4554                         I915_WRITE(reg, temp | FDI_RX_SYMBOL_LOCK);
4555                         DRM_DEBUG_KMS("FDI train 2 done.\n");
4556                         break;
4557                 }
4558         }
4559         if (tries == 5)
4560                 DRM_ERROR("FDI train 2 fail!\n");
4561
4562         DRM_DEBUG_KMS("FDI train done\n");
4563
4564 }
4565
4566 static const int snb_b_fdi_train_param[] = {
4567         FDI_LINK_TRAIN_400MV_0DB_SNB_B,
4568         FDI_LINK_TRAIN_400MV_6DB_SNB_B,
4569         FDI_LINK_TRAIN_600MV_3_5DB_SNB_B,
4570         FDI_LINK_TRAIN_800MV_0DB_SNB_B,
4571 };
4572
4573 /* The FDI link training functions for SNB/Cougarpoint. */
4574 static void gen6_fdi_link_train(struct intel_crtc *crtc,
4575                                 const struct intel_crtc_state *crtc_state)
4576 {
4577         struct drm_device *dev = crtc->base.dev;
4578         struct drm_i915_private *dev_priv = to_i915(dev);
4579         int pipe = crtc->pipe;
4580         i915_reg_t reg;
4581         u32 temp, i, retry;
4582
4583         /* Train 1: umask FDI RX Interrupt symbol_lock and bit_lock bit
4584            for train result */
4585         reg = FDI_RX_IMR(pipe);
4586         temp = I915_READ(reg);
4587         temp &= ~FDI_RX_SYMBOL_LOCK;
4588         temp &= ~FDI_RX_BIT_LOCK;
4589         I915_WRITE(reg, temp);
4590
4591         POSTING_READ(reg);
4592         udelay(150);
4593
4594         /* enable CPU FDI TX and PCH FDI RX */
4595         reg = FDI_TX_CTL(pipe);
4596         temp = I915_READ(reg);
4597         temp &= ~FDI_DP_PORT_WIDTH_MASK;
4598         temp |= FDI_DP_PORT_WIDTH(crtc_state->fdi_lanes);
4599         temp &= ~FDI_LINK_TRAIN_NONE;
4600         temp |= FDI_LINK_TRAIN_PATTERN_1;
4601         temp &= ~FDI_LINK_TRAIN_VOL_EMP_MASK;
4602         /* SNB-B */
4603         temp |= FDI_LINK_TRAIN_400MV_0DB_SNB_B;
4604         I915_WRITE(reg, temp | FDI_TX_ENABLE);
4605
4606         I915_WRITE(FDI_RX_MISC(pipe),
4607                    FDI_RX_TP1_TO_TP2_48 | FDI_RX_FDI_DELAY_90);
4608
4609         reg = FDI_RX_CTL(pipe);
4610         temp = I915_READ(reg);
4611         if (HAS_PCH_CPT(dev_priv)) {
4612                 temp &= ~FDI_LINK_TRAIN_PATTERN_MASK_CPT;
4613                 temp |= FDI_LINK_TRAIN_PATTERN_1_CPT;
4614         } else {
4615                 temp &= ~FDI_LINK_TRAIN_NONE;
4616                 temp |= FDI_LINK_TRAIN_PATTERN_1;
4617         }
4618         I915_WRITE(reg, temp | FDI_RX_ENABLE);
4619
4620         POSTING_READ(reg);
4621         udelay(150);
4622
4623         for (i = 0; i < 4; i++) {
4624                 reg = FDI_TX_CTL(pipe);
4625                 temp = I915_READ(reg);
4626                 temp &= ~FDI_LINK_TRAIN_VOL_EMP_MASK;
4627                 temp |= snb_b_fdi_train_param[i];
4628                 I915_WRITE(reg, temp);
4629
4630                 POSTING_READ(reg);
4631                 udelay(500);
4632
4633                 for (retry = 0; retry < 5; retry++) {
4634                         reg = FDI_RX_IIR(pipe);
4635                         temp = I915_READ(reg);
4636                         DRM_DEBUG_KMS("FDI_RX_IIR 0x%x\n", temp);
4637                         if (temp & FDI_RX_BIT_LOCK) {
4638                                 I915_WRITE(reg, temp | FDI_RX_BIT_LOCK);
4639                                 DRM_DEBUG_KMS("FDI train 1 done.\n");
4640                                 break;
4641                         }
4642                         udelay(50);
4643                 }
4644                 if (retry < 5)
4645                         break;
4646         }
4647         if (i == 4)
4648                 DRM_ERROR("FDI train 1 fail!\n");
4649
4650         /* Train 2 */
4651         reg = FDI_TX_CTL(pipe);
4652         temp = I915_READ(reg);
4653         temp &= ~FDI_LINK_TRAIN_NONE;
4654         temp |= FDI_LINK_TRAIN_PATTERN_2;
4655         if (IS_GEN(dev_priv, 6)) {
4656                 temp &= ~FDI_LINK_TRAIN_VOL_EMP_MASK;
4657                 /* SNB-B */
4658                 temp |= FDI_LINK_TRAIN_400MV_0DB_SNB_B;
4659         }
4660         I915_WRITE(reg, temp);
4661
4662         reg = FDI_RX_CTL(pipe);
4663         temp = I915_READ(reg);
4664         if (HAS_PCH_CPT(dev_priv)) {
4665                 temp &= ~FDI_LINK_TRAIN_PATTERN_MASK_CPT;
4666                 temp |= FDI_LINK_TRAIN_PATTERN_2_CPT;
4667         } else {
4668                 temp &= ~FDI_LINK_TRAIN_NONE;
4669                 temp |= FDI_LINK_TRAIN_PATTERN_2;
4670         }
4671         I915_WRITE(reg, temp);
4672
4673         POSTING_READ(reg);
4674         udelay(150);
4675
4676         for (i = 0; i < 4; i++) {
4677                 reg = FDI_TX_CTL(pipe);
4678                 temp = I915_READ(reg);
4679                 temp &= ~FDI_LINK_TRAIN_VOL_EMP_MASK;
4680                 temp |= snb_b_fdi_train_param[i];
4681                 I915_WRITE(reg, temp);
4682
4683                 POSTING_READ(reg);
4684                 udelay(500);
4685
4686                 for (retry = 0; retry < 5; retry++) {
4687                         reg = FDI_RX_IIR(pipe);
4688                         temp = I915_READ(reg);
4689                         DRM_DEBUG_KMS("FDI_RX_IIR 0x%x\n", temp);
4690                         if (temp & FDI_RX_SYMBOL_LOCK) {
4691                                 I915_WRITE(reg, temp | FDI_RX_SYMBOL_LOCK);
4692                                 DRM_DEBUG_KMS("FDI train 2 done.\n");
4693                                 break;
4694                         }
4695                         udelay(50);
4696                 }
4697                 if (retry < 5)
4698                         break;
4699         }
4700         if (i == 4)
4701                 DRM_ERROR("FDI train 2 fail!\n");
4702
4703         DRM_DEBUG_KMS("FDI train done.\n");
4704 }
4705
4706 /* Manual link training for Ivy Bridge A0 parts */
4707 static void ivb_manual_fdi_link_train(struct intel_crtc *crtc,
4708                                       const struct intel_crtc_state *crtc_state)
4709 {
4710         struct drm_device *dev = crtc->base.dev;
4711         struct drm_i915_private *dev_priv = to_i915(dev);
4712         int pipe = crtc->pipe;
4713         i915_reg_t reg;
4714         u32 temp, i, j;
4715
4716         /* Train 1: umask FDI RX Interrupt symbol_lock and bit_lock bit
4717            for train result */
4718         reg = FDI_RX_IMR(pipe);
4719         temp = I915_READ(reg);
4720         temp &= ~FDI_RX_SYMBOL_LOCK;
4721         temp &= ~FDI_RX_BIT_LOCK;
4722         I915_WRITE(reg, temp);
4723
4724         POSTING_READ(reg);
4725         udelay(150);
4726
4727         DRM_DEBUG_KMS("FDI_RX_IIR before link train 0x%x\n",
4728                       I915_READ(FDI_RX_IIR(pipe)));
4729
4730         /* Try each vswing and preemphasis setting twice before moving on */
4731         for (j = 0; j < ARRAY_SIZE(snb_b_fdi_train_param) * 2; j++) {
4732                 /* disable first in case we need to retry */
4733                 reg = FDI_TX_CTL(pipe);
4734                 temp = I915_READ(reg);
4735                 temp &= ~(FDI_LINK_TRAIN_AUTO | FDI_LINK_TRAIN_NONE_IVB);
4736                 temp &= ~FDI_TX_ENABLE;
4737                 I915_WRITE(reg, temp);
4738
4739                 reg = FDI_RX_CTL(pipe);
4740                 temp = I915_READ(reg);
4741                 temp &= ~FDI_LINK_TRAIN_AUTO;
4742                 temp &= ~FDI_LINK_TRAIN_PATTERN_MASK_CPT;
4743                 temp &= ~FDI_RX_ENABLE;
4744                 I915_WRITE(reg, temp);
4745
4746                 /* enable CPU FDI TX and PCH FDI RX */
4747                 reg = FDI_TX_CTL(pipe);
4748                 temp = I915_READ(reg);
4749                 temp &= ~FDI_DP_PORT_WIDTH_MASK;
4750                 temp |= FDI_DP_PORT_WIDTH(crtc_state->fdi_lanes);
4751                 temp |= FDI_LINK_TRAIN_PATTERN_1_IVB;
4752                 temp &= ~FDI_LINK_TRAIN_VOL_EMP_MASK;
4753                 temp |= snb_b_fdi_train_param[j/2];
4754                 temp |= FDI_COMPOSITE_SYNC;
4755                 I915_WRITE(reg, temp | FDI_TX_ENABLE);
4756
4757                 I915_WRITE(FDI_RX_MISC(pipe),
4758                            FDI_RX_TP1_TO_TP2_48 | FDI_RX_FDI_DELAY_90);
4759
4760                 reg = FDI_RX_CTL(pipe);
4761                 temp = I915_READ(reg);
4762                 temp |= FDI_LINK_TRAIN_PATTERN_1_CPT;
4763                 temp |= FDI_COMPOSITE_SYNC;
4764                 I915_WRITE(reg, temp | FDI_RX_ENABLE);
4765
4766                 POSTING_READ(reg);
4767                 udelay(1); /* should be 0.5us */
4768
4769                 for (i = 0; i < 4; i++) {
4770                         reg = FDI_RX_IIR(pipe);
4771                         temp = I915_READ(reg);
4772                         DRM_DEBUG_KMS("FDI_RX_IIR 0x%x\n", temp);
4773
4774                         if (temp & FDI_RX_BIT_LOCK ||
4775                             (I915_READ(reg) & FDI_RX_BIT_LOCK)) {
4776                                 I915_WRITE(reg, temp | FDI_RX_BIT_LOCK);
4777                                 DRM_DEBUG_KMS("FDI train 1 done, level %i.\n",
4778                                               i);
4779                                 break;
4780                         }
4781                         udelay(1); /* should be 0.5us */
4782                 }
4783                 if (i == 4) {
4784                         DRM_DEBUG_KMS("FDI train 1 fail on vswing %d\n", j / 2);
4785                         continue;
4786                 }
4787
4788                 /* Train 2 */
4789                 reg = FDI_TX_CTL(pipe);
4790                 temp = I915_READ(reg);
4791                 temp &= ~FDI_LINK_TRAIN_NONE_IVB;
4792                 temp |= FDI_LINK_TRAIN_PATTERN_2_IVB;
4793                 I915_WRITE(reg, temp);
4794
4795                 reg = FDI_RX_CTL(pipe);
4796                 temp = I915_READ(reg);
4797                 temp &= ~FDI_LINK_TRAIN_PATTERN_MASK_CPT;
4798                 temp |= FDI_LINK_TRAIN_PATTERN_2_CPT;
4799                 I915_WRITE(reg, temp);
4800
4801                 POSTING_READ(reg);
4802                 udelay(2); /* should be 1.5us */
4803
4804                 for (i = 0; i < 4; i++) {
4805                         reg = FDI_RX_IIR(pipe);
4806                         temp = I915_READ(reg);
4807                         DRM_DEBUG_KMS("FDI_RX_IIR 0x%x\n", temp);
4808
4809                         if (temp & FDI_RX_SYMBOL_LOCK ||
4810                             (I915_READ(reg) & FDI_RX_SYMBOL_LOCK)) {
4811                                 I915_WRITE(reg, temp | FDI_RX_SYMBOL_LOCK);
4812                                 DRM_DEBUG_KMS("FDI train 2 done, level %i.\n",
4813                                               i);
4814                                 goto train_done;
4815                         }
4816                         udelay(2); /* should be 1.5us */
4817                 }
4818                 if (i == 4)
4819                         DRM_DEBUG_KMS("FDI train 2 fail on vswing %d\n", j / 2);
4820         }
4821
4822 train_done:
4823         DRM_DEBUG_KMS("FDI train done.\n");
4824 }
4825
4826 static void ironlake_fdi_pll_enable(const struct intel_crtc_state *crtc_state)
4827 {
4828         struct intel_crtc *intel_crtc = to_intel_crtc(crtc_state->base.crtc);
4829         struct drm_i915_private *dev_priv = to_i915(intel_crtc->base.dev);
4830         int pipe = intel_crtc->pipe;
4831         i915_reg_t reg;
4832         u32 temp;
4833
4834         /* enable PCH FDI RX PLL, wait warmup plus DMI latency */
4835         reg = FDI_RX_CTL(pipe);
4836         temp = I915_READ(reg);
4837         temp &= ~(FDI_DP_PORT_WIDTH_MASK | (0x7 << 16));
4838         temp |= FDI_DP_PORT_WIDTH(crtc_state->fdi_lanes);
4839         temp |= (I915_READ(PIPECONF(pipe)) & PIPECONF_BPC_MASK) << 11;
4840         I915_WRITE(reg, temp | FDI_RX_PLL_ENABLE);
4841
4842         POSTING_READ(reg);
4843         udelay(200);
4844
4845         /* Switch from Rawclk to PCDclk */
4846         temp = I915_READ(reg);
4847         I915_WRITE(reg, temp | FDI_PCDCLK);
4848
4849         POSTING_READ(reg);
4850         udelay(200);
4851
4852         /* Enable CPU FDI TX PLL, always on for Ironlake */
4853         reg = FDI_TX_CTL(pipe);
4854         temp = I915_READ(reg);
4855         if ((temp & FDI_TX_PLL_ENABLE) == 0) {
4856                 I915_WRITE(reg, temp | FDI_TX_PLL_ENABLE);
4857
4858                 POSTING_READ(reg);
4859                 udelay(100);
4860         }
4861 }
4862
4863 static void ironlake_fdi_pll_disable(struct intel_crtc *intel_crtc)
4864 {
4865         struct drm_device *dev = intel_crtc->base.dev;
4866         struct drm_i915_private *dev_priv = to_i915(dev);
4867         int pipe = intel_crtc->pipe;
4868         i915_reg_t reg;
4869         u32 temp;
4870
4871         /* Switch from PCDclk to Rawclk */
4872         reg = FDI_RX_CTL(pipe);
4873         temp = I915_READ(reg);
4874         I915_WRITE(reg, temp & ~FDI_PCDCLK);
4875
4876         /* Disable CPU FDI TX PLL */
4877         reg = FDI_TX_CTL(pipe);
4878         temp = I915_READ(reg);
4879         I915_WRITE(reg, temp & ~FDI_TX_PLL_ENABLE);
4880
4881         POSTING_READ(reg);
4882         udelay(100);
4883
4884         reg = FDI_RX_CTL(pipe);
4885         temp = I915_READ(reg);
4886         I915_WRITE(reg, temp & ~FDI_RX_PLL_ENABLE);
4887
4888         /* Wait for the clocks to turn off. */
4889         POSTING_READ(reg);
4890         udelay(100);
4891 }
4892
4893 static void ironlake_fdi_disable(struct drm_crtc *crtc)
4894 {
4895         struct drm_device *dev = crtc->dev;
4896         struct drm_i915_private *dev_priv = to_i915(dev);
4897         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
4898         int pipe = intel_crtc->pipe;
4899         i915_reg_t reg;
4900         u32 temp;
4901
4902         /* disable CPU FDI tx and PCH FDI rx */
4903         reg = FDI_TX_CTL(pipe);
4904         temp = I915_READ(reg);
4905         I915_WRITE(reg, temp & ~FDI_TX_ENABLE);
4906         POSTING_READ(reg);
4907
4908         reg = FDI_RX_CTL(pipe);
4909         temp = I915_READ(reg);
4910         temp &= ~(0x7 << 16);
4911         temp |= (I915_READ(PIPECONF(pipe)) & PIPECONF_BPC_MASK) << 11;
4912         I915_WRITE(reg, temp & ~FDI_RX_ENABLE);
4913
4914         POSTING_READ(reg);
4915         udelay(100);
4916
4917         /* Ironlake workaround, disable clock pointer after downing FDI */
4918         if (HAS_PCH_IBX(dev_priv))
4919                 I915_WRITE(FDI_RX_CHICKEN(pipe), FDI_RX_PHASE_SYNC_POINTER_OVR);
4920
4921         /* still set train pattern 1 */
4922         reg = FDI_TX_CTL(pipe);
4923         temp = I915_READ(reg);
4924         temp &= ~FDI_LINK_TRAIN_NONE;
4925         temp |= FDI_LINK_TRAIN_PATTERN_1;
4926         I915_WRITE(reg, temp);
4927
4928         reg = FDI_RX_CTL(pipe);
4929         temp = I915_READ(reg);
4930         if (HAS_PCH_CPT(dev_priv)) {
4931                 temp &= ~FDI_LINK_TRAIN_PATTERN_MASK_CPT;
4932                 temp |= FDI_LINK_TRAIN_PATTERN_1_CPT;
4933         } else {
4934                 temp &= ~FDI_LINK_TRAIN_NONE;
4935                 temp |= FDI_LINK_TRAIN_PATTERN_1;
4936         }
4937         /* BPC in FDI rx is consistent with that in PIPECONF */
4938         temp &= ~(0x07 << 16);
4939         temp |= (I915_READ(PIPECONF(pipe)) & PIPECONF_BPC_MASK) << 11;
4940         I915_WRITE(reg, temp);
4941
4942         POSTING_READ(reg);
4943         udelay(100);
4944 }
4945
4946 bool intel_has_pending_fb_unpin(struct drm_i915_private *dev_priv)
4947 {
4948         struct drm_crtc *crtc;
4949         bool cleanup_done;
4950
4951         drm_for_each_crtc(crtc, &dev_priv->drm) {
4952                 struct drm_crtc_commit *commit;
4953                 spin_lock(&crtc->commit_lock);
4954                 commit = list_first_entry_or_null(&crtc->commit_list,
4955                                                   struct drm_crtc_commit, commit_entry);
4956                 cleanup_done = commit ?
4957                         try_wait_for_completion(&commit->cleanup_done) : true;
4958                 spin_unlock(&crtc->commit_lock);
4959
4960                 if (cleanup_done)
4961                         continue;
4962
4963                 drm_crtc_wait_one_vblank(crtc);
4964
4965                 return true;
4966         }
4967
4968         return false;
4969 }
4970
4971 void lpt_disable_iclkip(struct drm_i915_private *dev_priv)
4972 {
4973         u32 temp;
4974
4975         I915_WRITE(PIXCLK_GATE, PIXCLK_GATE_GATE);
4976
4977         mutex_lock(&dev_priv->sb_lock);
4978
4979         temp = intel_sbi_read(dev_priv, SBI_SSCCTL6, SBI_ICLK);
4980         temp |= SBI_SSCCTL_DISABLE;
4981         intel_sbi_write(dev_priv, SBI_SSCCTL6, temp, SBI_ICLK);
4982
4983         mutex_unlock(&dev_priv->sb_lock);
4984 }
4985
4986 /* Program iCLKIP clock to the desired frequency */
4987 static void lpt_program_iclkip(const struct intel_crtc_state *crtc_state)
4988 {
4989         struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
4990         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
4991         int clock = crtc_state->base.adjusted_mode.crtc_clock;
4992         u32 divsel, phaseinc, auxdiv, phasedir = 0;
4993         u32 temp;
4994
4995         lpt_disable_iclkip(dev_priv);
4996
4997         /* The iCLK virtual clock root frequency is in MHz,
4998          * but the adjusted_mode->crtc_clock in in KHz. To get the
4999          * divisors, it is necessary to divide one by another, so we
5000          * convert the virtual clock precision to KHz here for higher
5001          * precision.
5002          */
5003         for (auxdiv = 0; auxdiv < 2; auxdiv++) {
5004                 u32 iclk_virtual_root_freq = 172800 * 1000;
5005                 u32 iclk_pi_range = 64;
5006                 u32 desired_divisor;
5007
5008                 desired_divisor = DIV_ROUND_CLOSEST(iclk_virtual_root_freq,
5009                                                     clock << auxdiv);
5010                 divsel = (desired_divisor / iclk_pi_range) - 2;
5011                 phaseinc = desired_divisor % iclk_pi_range;
5012
5013                 /*
5014                  * Near 20MHz is a corner case which is
5015                  * out of range for the 7-bit divisor
5016                  */
5017                 if (divsel <= 0x7f)
5018                         break;
5019         }
5020
5021         /* This should not happen with any sane values */
5022         WARN_ON(SBI_SSCDIVINTPHASE_DIVSEL(divsel) &
5023                 ~SBI_SSCDIVINTPHASE_DIVSEL_MASK);
5024         WARN_ON(SBI_SSCDIVINTPHASE_DIR(phasedir) &
5025                 ~SBI_SSCDIVINTPHASE_INCVAL_MASK);
5026
5027         DRM_DEBUG_KMS("iCLKIP clock: found settings for %dKHz refresh rate: auxdiv=%x, divsel=%x, phasedir=%x, phaseinc=%x\n",
5028                         clock,
5029                         auxdiv,
5030                         divsel,
5031                         phasedir,
5032                         phaseinc);
5033
5034         mutex_lock(&dev_priv->sb_lock);
5035
5036         /* Program SSCDIVINTPHASE6 */
5037         temp = intel_sbi_read(dev_priv, SBI_SSCDIVINTPHASE6, SBI_ICLK);
5038         temp &= ~SBI_SSCDIVINTPHASE_DIVSEL_MASK;
5039         temp |= SBI_SSCDIVINTPHASE_DIVSEL(divsel);
5040         temp &= ~SBI_SSCDIVINTPHASE_INCVAL_MASK;
5041         temp |= SBI_SSCDIVINTPHASE_INCVAL(phaseinc);
5042         temp |= SBI_SSCDIVINTPHASE_DIR(phasedir);
5043         temp |= SBI_SSCDIVINTPHASE_PROPAGATE;
5044         intel_sbi_write(dev_priv, SBI_SSCDIVINTPHASE6, temp, SBI_ICLK);
5045
5046         /* Program SSCAUXDIV */
5047         temp = intel_sbi_read(dev_priv, SBI_SSCAUXDIV6, SBI_ICLK);
5048         temp &= ~SBI_SSCAUXDIV_FINALDIV2SEL(1);
5049         temp |= SBI_SSCAUXDIV_FINALDIV2SEL(auxdiv);
5050         intel_sbi_write(dev_priv, SBI_SSCAUXDIV6, temp, SBI_ICLK);
5051
5052         /* Enable modulator and associated divider */
5053         temp = intel_sbi_read(dev_priv, SBI_SSCCTL6, SBI_ICLK);
5054         temp &= ~SBI_SSCCTL_DISABLE;
5055         intel_sbi_write(dev_priv, SBI_SSCCTL6, temp, SBI_ICLK);
5056
5057         mutex_unlock(&dev_priv->sb_lock);
5058
5059         /* Wait for initialization time */
5060         udelay(24);
5061
5062         I915_WRITE(PIXCLK_GATE, PIXCLK_GATE_UNGATE);
5063 }
5064
5065 int lpt_get_iclkip(struct drm_i915_private *dev_priv)
5066 {
5067         u32 divsel, phaseinc, auxdiv;
5068         u32 iclk_virtual_root_freq = 172800 * 1000;
5069         u32 iclk_pi_range = 64;
5070         u32 desired_divisor;
5071         u32 temp;
5072
5073         if ((I915_READ(PIXCLK_GATE) & PIXCLK_GATE_UNGATE) == 0)
5074                 return 0;
5075
5076         mutex_lock(&dev_priv->sb_lock);
5077
5078         temp = intel_sbi_read(dev_priv, SBI_SSCCTL6, SBI_ICLK);
5079         if (temp & SBI_SSCCTL_DISABLE) {
5080                 mutex_unlock(&dev_priv->sb_lock);
5081                 return 0;
5082         }
5083
5084         temp = intel_sbi_read(dev_priv, SBI_SSCDIVINTPHASE6, SBI_ICLK);
5085         divsel = (temp & SBI_SSCDIVINTPHASE_DIVSEL_MASK) >>
5086                 SBI_SSCDIVINTPHASE_DIVSEL_SHIFT;
5087         phaseinc = (temp & SBI_SSCDIVINTPHASE_INCVAL_MASK) >>
5088                 SBI_SSCDIVINTPHASE_INCVAL_SHIFT;
5089
5090         temp = intel_sbi_read(dev_priv, SBI_SSCAUXDIV6, SBI_ICLK);
5091         auxdiv = (temp & SBI_SSCAUXDIV_FINALDIV2SEL_MASK) >>
5092                 SBI_SSCAUXDIV_FINALDIV2SEL_SHIFT;
5093
5094         mutex_unlock(&dev_priv->sb_lock);
5095
5096         desired_divisor = (divsel + 2) * iclk_pi_range + phaseinc;
5097
5098         return DIV_ROUND_CLOSEST(iclk_virtual_root_freq,
5099                                  desired_divisor << auxdiv);
5100 }
5101
5102 static void ironlake_pch_transcoder_set_timings(const struct intel_crtc_state *crtc_state,
5103                                                 enum pipe pch_transcoder)
5104 {
5105         struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
5106         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
5107         enum transcoder cpu_transcoder = crtc_state->cpu_transcoder;
5108
5109         I915_WRITE(PCH_TRANS_HTOTAL(pch_transcoder),
5110                    I915_READ(HTOTAL(cpu_transcoder)));
5111         I915_WRITE(PCH_TRANS_HBLANK(pch_transcoder),
5112                    I915_READ(HBLANK(cpu_transcoder)));
5113         I915_WRITE(PCH_TRANS_HSYNC(pch_transcoder),
5114                    I915_READ(HSYNC(cpu_transcoder)));
5115
5116         I915_WRITE(PCH_TRANS_VTOTAL(pch_transcoder),
5117                    I915_READ(VTOTAL(cpu_transcoder)));
5118         I915_WRITE(PCH_TRANS_VBLANK(pch_transcoder),
5119                    I915_READ(VBLANK(cpu_transcoder)));
5120         I915_WRITE(PCH_TRANS_VSYNC(pch_transcoder),
5121                    I915_READ(VSYNC(cpu_transcoder)));
5122         I915_WRITE(PCH_TRANS_VSYNCSHIFT(pch_transcoder),
5123                    I915_READ(VSYNCSHIFT(cpu_transcoder)));
5124 }
5125
5126 static void cpt_set_fdi_bc_bifurcation(struct drm_i915_private *dev_priv, bool enable)
5127 {
5128         u32 temp;
5129
5130         temp = I915_READ(SOUTH_CHICKEN1);
5131         if (!!(temp & FDI_BC_BIFURCATION_SELECT) == enable)
5132                 return;
5133
5134         WARN_ON(I915_READ(FDI_RX_CTL(PIPE_B)) & FDI_RX_ENABLE);
5135         WARN_ON(I915_READ(FDI_RX_CTL(PIPE_C)) & FDI_RX_ENABLE);
5136
5137         temp &= ~FDI_BC_BIFURCATION_SELECT;
5138         if (enable)
5139                 temp |= FDI_BC_BIFURCATION_SELECT;
5140
5141         DRM_DEBUG_KMS("%sabling fdi C rx\n", enable ? "en" : "dis");
5142         I915_WRITE(SOUTH_CHICKEN1, temp);
5143         POSTING_READ(SOUTH_CHICKEN1);
5144 }
5145
5146 static void ivybridge_update_fdi_bc_bifurcation(const struct intel_crtc_state *crtc_state)
5147 {
5148         struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
5149         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
5150
5151         switch (crtc->pipe) {
5152         case PIPE_A:
5153                 break;
5154         case PIPE_B:
5155                 if (crtc_state->fdi_lanes > 2)
5156                         cpt_set_fdi_bc_bifurcation(dev_priv, false);
5157                 else
5158                         cpt_set_fdi_bc_bifurcation(dev_priv, true);
5159
5160                 break;
5161         case PIPE_C:
5162                 cpt_set_fdi_bc_bifurcation(dev_priv, true);
5163
5164                 break;
5165         default:
5166                 BUG();
5167         }
5168 }
5169
5170 /*
5171  * Finds the encoder associated with the given CRTC. This can only be
5172  * used when we know that the CRTC isn't feeding multiple encoders!
5173  */
5174 static struct intel_encoder *
5175 intel_get_crtc_new_encoder(const struct intel_atomic_state *state,
5176                            const struct intel_crtc_state *crtc_state)
5177 {
5178         struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
5179         const struct drm_connector_state *connector_state;
5180         const struct drm_connector *connector;
5181         struct intel_encoder *encoder = NULL;
5182         int num_encoders = 0;
5183         int i;
5184
5185         for_each_new_connector_in_state(&state->base, connector, connector_state, i) {
5186                 if (connector_state->crtc != &crtc->base)
5187                         continue;
5188
5189                 encoder = to_intel_encoder(connector_state->best_encoder);
5190                 num_encoders++;
5191         }
5192
5193         WARN(num_encoders != 1, "%d encoders for pipe %c\n",
5194              num_encoders, pipe_name(crtc->pipe));
5195
5196         return encoder;
5197 }
5198
5199 /*
5200  * Enable PCH resources required for PCH ports:
5201  *   - PCH PLLs
5202  *   - FDI training & RX/TX
5203  *   - update transcoder timings
5204  *   - DP transcoding bits
5205  *   - transcoder
5206  */
5207 static void ironlake_pch_enable(const struct intel_atomic_state *state,
5208                                 const struct intel_crtc_state *crtc_state)
5209 {
5210         struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
5211         struct drm_device *dev = crtc->base.dev;
5212         struct drm_i915_private *dev_priv = to_i915(dev);
5213         int pipe = crtc->pipe;
5214         u32 temp;
5215
5216         assert_pch_transcoder_disabled(dev_priv, pipe);
5217
5218         if (IS_IVYBRIDGE(dev_priv))
5219                 ivybridge_update_fdi_bc_bifurcation(crtc_state);
5220
5221         /* Write the TU size bits before fdi link training, so that error
5222          * detection works. */
5223         I915_WRITE(FDI_RX_TUSIZE1(pipe),
5224                    I915_READ(PIPE_DATA_M1(pipe)) & TU_SIZE_MASK);
5225
5226         /* For PCH output, training FDI link */
5227         dev_priv->display.fdi_link_train(crtc, crtc_state);
5228
5229         /* We need to program the right clock selection before writing the pixel
5230          * mutliplier into the DPLL. */
5231         if (HAS_PCH_CPT(dev_priv)) {
5232                 u32 sel;
5233
5234                 temp = I915_READ(PCH_DPLL_SEL);
5235                 temp |= TRANS_DPLL_ENABLE(pipe);
5236                 sel = TRANS_DPLLB_SEL(pipe);
5237                 if (crtc_state->shared_dpll ==
5238                     intel_get_shared_dpll_by_id(dev_priv, DPLL_ID_PCH_PLL_B))
5239                         temp |= sel;
5240                 else
5241                         temp &= ~sel;
5242                 I915_WRITE(PCH_DPLL_SEL, temp);
5243         }
5244
5245         /* XXX: pch pll's can be enabled any time before we enable the PCH
5246          * transcoder, and we actually should do this to not upset any PCH
5247          * transcoder that already use the clock when we share it.
5248          *
5249          * Note that enable_shared_dpll tries to do the right thing, but
5250          * get_shared_dpll unconditionally resets the pll - we need that to have
5251          * the right LVDS enable sequence. */
5252         intel_enable_shared_dpll(crtc_state);
5253
5254         /* set transcoder timing, panel must allow it */
5255         assert_panel_unlocked(dev_priv, pipe);
5256         ironlake_pch_transcoder_set_timings(crtc_state, pipe);
5257
5258         intel_fdi_normal_train(crtc);
5259
5260         /* For PCH DP, enable TRANS_DP_CTL */
5261         if (HAS_PCH_CPT(dev_priv) &&
5262             intel_crtc_has_dp_encoder(crtc_state)) {
5263                 const struct drm_display_mode *adjusted_mode =
5264                         &crtc_state->base.adjusted_mode;
5265                 u32 bpc = (I915_READ(PIPECONF(pipe)) & PIPECONF_BPC_MASK) >> 5;
5266                 i915_reg_t reg = TRANS_DP_CTL(pipe);
5267                 enum port port;
5268
5269                 temp = I915_READ(reg);
5270                 temp &= ~(TRANS_DP_PORT_SEL_MASK |
5271                           TRANS_DP_SYNC_MASK |
5272                           TRANS_DP_BPC_MASK);
5273                 temp |= TRANS_DP_OUTPUT_ENABLE;
5274                 temp |= bpc << 9; /* same format but at 11:9 */
5275
5276                 if (adjusted_mode->flags & DRM_MODE_FLAG_PHSYNC)
5277                         temp |= TRANS_DP_HSYNC_ACTIVE_HIGH;
5278                 if (adjusted_mode->flags & DRM_MODE_FLAG_PVSYNC)
5279                         temp |= TRANS_DP_VSYNC_ACTIVE_HIGH;
5280
5281                 port = intel_get_crtc_new_encoder(state, crtc_state)->port;
5282                 WARN_ON(port < PORT_B || port > PORT_D);
5283                 temp |= TRANS_DP_PORT_SEL(port);
5284
5285                 I915_WRITE(reg, temp);
5286         }
5287
5288         ironlake_enable_pch_transcoder(crtc_state);
5289 }
5290
5291 static void lpt_pch_enable(const struct intel_atomic_state *state,
5292                            const struct intel_crtc_state *crtc_state)
5293 {
5294         struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
5295         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
5296         enum transcoder cpu_transcoder = crtc_state->cpu_transcoder;
5297
5298         assert_pch_transcoder_disabled(dev_priv, PIPE_A);
5299
5300         lpt_program_iclkip(crtc_state);
5301
5302         /* Set transcoder timing. */
5303         ironlake_pch_transcoder_set_timings(crtc_state, PIPE_A);
5304
5305         lpt_enable_pch_transcoder(dev_priv, cpu_transcoder);
5306 }
5307
5308 static void cpt_verify_modeset(struct drm_device *dev, int pipe)
5309 {
5310         struct drm_i915_private *dev_priv = to_i915(dev);
5311         i915_reg_t dslreg = PIPEDSL(pipe);
5312         u32 temp;
5313
5314         temp = I915_READ(dslreg);
5315         udelay(500);
5316         if (wait_for(I915_READ(dslreg) != temp, 5)) {
5317                 if (wait_for(I915_READ(dslreg) != temp, 5))
5318                         DRM_ERROR("mode set failed: pipe %c stuck\n", pipe_name(pipe));
5319         }
5320 }
5321
5322 /*
5323  * The hardware phase 0.0 refers to the center of the pixel.
5324  * We want to start from the top/left edge which is phase
5325  * -0.5. That matches how the hardware calculates the scaling
5326  * factors (from top-left of the first pixel to bottom-right
5327  * of the last pixel, as opposed to the pixel centers).
5328  *
5329  * For 4:2:0 subsampled chroma planes we obviously have to
5330  * adjust that so that the chroma sample position lands in
5331  * the right spot.
5332  *
5333  * Note that for packed YCbCr 4:2:2 formats there is no way to
5334  * control chroma siting. The hardware simply replicates the
5335  * chroma samples for both of the luma samples, and thus we don't
5336  * actually get the expected MPEG2 chroma siting convention :(
5337  * The same behaviour is observed on pre-SKL platforms as well.
5338  *
5339  * Theory behind the formula (note that we ignore sub-pixel
5340  * source coordinates):
5341  * s = source sample position
5342  * d = destination sample position
5343  *
5344  * Downscaling 4:1:
5345  * -0.5
5346  * | 0.0
5347  * | |     1.5 (initial phase)
5348  * | |     |
5349  * v v     v
5350  * | s | s | s | s |
5351  * |       d       |
5352  *
5353  * Upscaling 1:4:
5354  * -0.5
5355  * | -0.375 (initial phase)
5356  * | |     0.0
5357  * | |     |
5358  * v v     v
5359  * |       s       |
5360  * | d | d | d | d |
5361  */
5362 u16 skl_scaler_calc_phase(int sub, int scale, bool chroma_cosited)
5363 {
5364         int phase = -0x8000;
5365         u16 trip = 0;
5366
5367         if (chroma_cosited)
5368                 phase += (sub - 1) * 0x8000 / sub;
5369
5370         phase += scale / (2 * sub);
5371
5372         /*
5373          * Hardware initial phase limited to [-0.5:1.5].
5374          * Since the max hardware scale factor is 3.0, we
5375          * should never actually excdeed 1.0 here.
5376          */
5377         WARN_ON(phase < -0x8000 || phase > 0x18000);
5378
5379         if (phase < 0)
5380                 phase = 0x10000 + phase;
5381         else
5382                 trip = PS_PHASE_TRIP;
5383
5384         return ((phase >> 2) & PS_PHASE_MASK) | trip;
5385 }
5386
5387 #define SKL_MIN_SRC_W 8
5388 #define SKL_MAX_SRC_W 4096
5389 #define SKL_MIN_SRC_H 8
5390 #define SKL_MAX_SRC_H 4096
5391 #define SKL_MIN_DST_W 8
5392 #define SKL_MAX_DST_W 4096
5393 #define SKL_MIN_DST_H 8
5394 #define SKL_MAX_DST_H 4096
5395 #define ICL_MAX_SRC_W 5120
5396 #define ICL_MAX_SRC_H 4096
5397 #define ICL_MAX_DST_W 5120
5398 #define ICL_MAX_DST_H 4096
5399 #define SKL_MIN_YUV_420_SRC_W 16
5400 #define SKL_MIN_YUV_420_SRC_H 16
5401
5402 static int
5403 skl_update_scaler(struct intel_crtc_state *crtc_state, bool force_detach,
5404                   unsigned int scaler_user, int *scaler_id,
5405                   int src_w, int src_h, int dst_w, int dst_h,
5406                   const struct drm_format_info *format, bool need_scaler)
5407 {
5408         struct intel_crtc_scaler_state *scaler_state =
5409                 &crtc_state->scaler_state;
5410         struct intel_crtc *intel_crtc =
5411                 to_intel_crtc(crtc_state->base.crtc);
5412         struct drm_i915_private *dev_priv = to_i915(intel_crtc->base.dev);
5413         const struct drm_display_mode *adjusted_mode =
5414                 &crtc_state->base.adjusted_mode;
5415
5416         /*
5417          * Src coordinates are already rotated by 270 degrees for
5418          * the 90/270 degree plane rotation cases (to match the
5419          * GTT mapping), hence no need to account for rotation here.
5420          */
5421         if (src_w != dst_w || src_h != dst_h)
5422                 need_scaler = true;
5423
5424         /*
5425          * Scaling/fitting not supported in IF-ID mode in GEN9+
5426          * TODO: Interlace fetch mode doesn't support YUV420 planar formats.
5427          * Once NV12 is enabled, handle it here while allocating scaler
5428          * for NV12.
5429          */
5430         if (INTEL_GEN(dev_priv) >= 9 && crtc_state->base.enable &&
5431             need_scaler && adjusted_mode->flags & DRM_MODE_FLAG_INTERLACE) {
5432                 DRM_DEBUG_KMS("Pipe/Plane scaling not supported with IF-ID mode\n");
5433                 return -EINVAL;
5434         }
5435
5436         /*
5437          * if plane is being disabled or scaler is no more required or force detach
5438          *  - free scaler binded to this plane/crtc
5439          *  - in order to do this, update crtc->scaler_usage
5440          *
5441          * Here scaler state in crtc_state is set free so that
5442          * scaler can be assigned to other user. Actual register
5443          * update to free the scaler is done in plane/panel-fit programming.
5444          * For this purpose crtc/plane_state->scaler_id isn't reset here.
5445          */
5446         if (force_detach || !need_scaler) {
5447                 if (*scaler_id >= 0) {
5448                         scaler_state->scaler_users &= ~(1 << scaler_user);
5449                         scaler_state->scalers[*scaler_id].in_use = 0;
5450
5451                         DRM_DEBUG_KMS("scaler_user index %u.%u: "
5452                                 "Staged freeing scaler id %d scaler_users = 0x%x\n",
5453                                 intel_crtc->pipe, scaler_user, *scaler_id,
5454                                 scaler_state->scaler_users);
5455                         *scaler_id = -1;
5456                 }
5457                 return 0;
5458         }
5459
5460         if (format && is_planar_yuv_format(format->format) &&
5461             (src_h < SKL_MIN_YUV_420_SRC_H || src_w < SKL_MIN_YUV_420_SRC_W)) {
5462                 DRM_DEBUG_KMS("Planar YUV: src dimensions not met\n");
5463                 return -EINVAL;
5464         }
5465
5466         /* range checks */
5467         if (src_w < SKL_MIN_SRC_W || src_h < SKL_MIN_SRC_H ||
5468             dst_w < SKL_MIN_DST_W || dst_h < SKL_MIN_DST_H ||
5469             (INTEL_GEN(dev_priv) >= 11 &&
5470              (src_w > ICL_MAX_SRC_W || src_h > ICL_MAX_SRC_H ||
5471               dst_w > ICL_MAX_DST_W || dst_h > ICL_MAX_DST_H)) ||
5472             (INTEL_GEN(dev_priv) < 11 &&
5473              (src_w > SKL_MAX_SRC_W || src_h > SKL_MAX_SRC_H ||
5474               dst_w > SKL_MAX_DST_W || dst_h > SKL_MAX_DST_H))) {
5475                 DRM_DEBUG_KMS("scaler_user index %u.%u: src %ux%u dst %ux%u "
5476                         "size is out of scaler range\n",
5477                         intel_crtc->pipe, scaler_user, src_w, src_h, dst_w, dst_h);
5478                 return -EINVAL;
5479         }
5480
5481         /* mark this plane as a scaler user in crtc_state */
5482         scaler_state->scaler_users |= (1 << scaler_user);
5483         DRM_DEBUG_KMS("scaler_user index %u.%u: "
5484                 "staged scaling request for %ux%u->%ux%u scaler_users = 0x%x\n",
5485                 intel_crtc->pipe, scaler_user, src_w, src_h, dst_w, dst_h,
5486                 scaler_state->scaler_users);
5487
5488         return 0;
5489 }
5490
5491 /**
5492  * skl_update_scaler_crtc - Stages update to scaler state for a given crtc.
5493  *
5494  * @state: crtc's scaler state
5495  *
5496  * Return
5497  *     0 - scaler_usage updated successfully
5498  *    error - requested scaling cannot be supported or other error condition
5499  */
5500 int skl_update_scaler_crtc(struct intel_crtc_state *state)
5501 {
5502         const struct drm_display_mode *adjusted_mode = &state->base.adjusted_mode;
5503         bool need_scaler = false;
5504
5505         if (state->output_format == INTEL_OUTPUT_FORMAT_YCBCR420)
5506                 need_scaler = true;
5507
5508         return skl_update_scaler(state, !state->base.active, SKL_CRTC_INDEX,
5509                                  &state->scaler_state.scaler_id,
5510                                  state->pipe_src_w, state->pipe_src_h,
5511                                  adjusted_mode->crtc_hdisplay,
5512                                  adjusted_mode->crtc_vdisplay, NULL, need_scaler);
5513 }
5514
5515 /**
5516  * skl_update_scaler_plane - Stages update to scaler state for a given plane.
5517  * @crtc_state: crtc's scaler state
5518  * @plane_state: atomic plane state to update
5519  *
5520  * Return
5521  *     0 - scaler_usage updated successfully
5522  *    error - requested scaling cannot be supported or other error condition
5523  */
5524 static int skl_update_scaler_plane(struct intel_crtc_state *crtc_state,
5525                                    struct intel_plane_state *plane_state)
5526 {
5527         struct intel_plane *intel_plane =
5528                 to_intel_plane(plane_state->base.plane);
5529         struct drm_i915_private *dev_priv = to_i915(intel_plane->base.dev);
5530         struct drm_framebuffer *fb = plane_state->base.fb;
5531         int ret;
5532         bool force_detach = !fb || !plane_state->base.visible;
5533         bool need_scaler = false;
5534
5535         /* Pre-gen11 and SDR planes always need a scaler for planar formats. */
5536         if (!icl_is_hdr_plane(dev_priv, intel_plane->id) &&
5537             fb && is_planar_yuv_format(fb->format->format))
5538                 need_scaler = true;
5539
5540         ret = skl_update_scaler(crtc_state, force_detach,
5541                                 drm_plane_index(&intel_plane->base),
5542                                 &plane_state->scaler_id,
5543                                 drm_rect_width(&plane_state->base.src) >> 16,
5544                                 drm_rect_height(&plane_state->base.src) >> 16,
5545                                 drm_rect_width(&plane_state->base.dst),
5546                                 drm_rect_height(&plane_state->base.dst),
5547                                 fb ? fb->format : NULL, need_scaler);
5548
5549         if (ret || plane_state->scaler_id < 0)
5550                 return ret;
5551
5552         /* check colorkey */
5553         if (plane_state->ckey.flags) {
5554                 DRM_DEBUG_KMS("[PLANE:%d:%s] scaling with color key not allowed",
5555                               intel_plane->base.base.id,
5556                               intel_plane->base.name);
5557                 return -EINVAL;
5558         }
5559
5560         /* Check src format */
5561         switch (fb->format->format) {
5562         case DRM_FORMAT_RGB565:
5563         case DRM_FORMAT_XBGR8888:
5564         case DRM_FORMAT_XRGB8888:
5565         case DRM_FORMAT_ABGR8888:
5566         case DRM_FORMAT_ARGB8888:
5567         case DRM_FORMAT_XRGB2101010:
5568         case DRM_FORMAT_XBGR2101010:
5569         case DRM_FORMAT_XBGR16161616F:
5570         case DRM_FORMAT_ABGR16161616F:
5571         case DRM_FORMAT_XRGB16161616F:
5572         case DRM_FORMAT_ARGB16161616F:
5573         case DRM_FORMAT_YUYV:
5574         case DRM_FORMAT_YVYU:
5575         case DRM_FORMAT_UYVY:
5576         case DRM_FORMAT_VYUY:
5577         case DRM_FORMAT_NV12:
5578         case DRM_FORMAT_P010:
5579         case DRM_FORMAT_P012:
5580         case DRM_FORMAT_P016:
5581         case DRM_FORMAT_Y210:
5582         case DRM_FORMAT_Y212:
5583         case DRM_FORMAT_Y216:
5584         case DRM_FORMAT_XVYU2101010:
5585         case DRM_FORMAT_XVYU12_16161616:
5586         case DRM_FORMAT_XVYU16161616:
5587                 break;
5588         default:
5589                 DRM_DEBUG_KMS("[PLANE:%d:%s] FB:%d unsupported scaling format 0x%x\n",
5590                               intel_plane->base.base.id, intel_plane->base.name,
5591                               fb->base.id, fb->format->format);
5592                 return -EINVAL;
5593         }
5594
5595         return 0;
5596 }
5597
5598 static void skylake_scaler_disable(struct intel_crtc *crtc)
5599 {
5600         int i;
5601
5602         for (i = 0; i < crtc->num_scalers; i++)
5603                 skl_detach_scaler(crtc, i);
5604 }
5605
5606 static void skylake_pfit_enable(const struct intel_crtc_state *crtc_state)
5607 {
5608         struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
5609         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
5610         enum pipe pipe = crtc->pipe;
5611         const struct intel_crtc_scaler_state *scaler_state =
5612                 &crtc_state->scaler_state;
5613
5614         if (crtc_state->pch_pfit.enabled) {
5615                 u16 uv_rgb_hphase, uv_rgb_vphase;
5616                 int pfit_w, pfit_h, hscale, vscale;
5617                 int id;
5618
5619                 if (WARN_ON(crtc_state->scaler_state.scaler_id < 0))
5620                         return;
5621
5622                 pfit_w = (crtc_state->pch_pfit.size >> 16) & 0xFFFF;
5623                 pfit_h = crtc_state->pch_pfit.size & 0xFFFF;
5624
5625                 hscale = (crtc_state->pipe_src_w << 16) / pfit_w;
5626                 vscale = (crtc_state->pipe_src_h << 16) / pfit_h;
5627
5628                 uv_rgb_hphase = skl_scaler_calc_phase(1, hscale, false);
5629                 uv_rgb_vphase = skl_scaler_calc_phase(1, vscale, false);
5630
5631                 id = scaler_state->scaler_id;
5632                 I915_WRITE(SKL_PS_CTRL(pipe, id), PS_SCALER_EN |
5633                         PS_FILTER_MEDIUM | scaler_state->scalers[id].mode);
5634                 I915_WRITE_FW(SKL_PS_VPHASE(pipe, id),
5635                               PS_Y_PHASE(0) | PS_UV_RGB_PHASE(uv_rgb_vphase));
5636                 I915_WRITE_FW(SKL_PS_HPHASE(pipe, id),
5637                               PS_Y_PHASE(0) | PS_UV_RGB_PHASE(uv_rgb_hphase));
5638                 I915_WRITE(SKL_PS_WIN_POS(pipe, id), crtc_state->pch_pfit.pos);
5639                 I915_WRITE(SKL_PS_WIN_SZ(pipe, id), crtc_state->pch_pfit.size);
5640         }
5641 }
5642
5643 static void ironlake_pfit_enable(const struct intel_crtc_state *crtc_state)
5644 {
5645         struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
5646         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
5647         int pipe = crtc->pipe;
5648
5649         if (crtc_state->pch_pfit.enabled) {
5650                 /* Force use of hard-coded filter coefficients
5651                  * as some pre-programmed values are broken,
5652                  * e.g. x201.
5653                  */
5654                 if (IS_IVYBRIDGE(dev_priv) || IS_HASWELL(dev_priv))
5655                         I915_WRITE(PF_CTL(pipe), PF_ENABLE | PF_FILTER_MED_3x3 |
5656                                                  PF_PIPE_SEL_IVB(pipe));
5657                 else
5658                         I915_WRITE(PF_CTL(pipe), PF_ENABLE | PF_FILTER_MED_3x3);
5659                 I915_WRITE(PF_WIN_POS(pipe), crtc_state->pch_pfit.pos);
5660                 I915_WRITE(PF_WIN_SZ(pipe), crtc_state->pch_pfit.size);
5661         }
5662 }
5663
5664 void hsw_enable_ips(const struct intel_crtc_state *crtc_state)
5665 {
5666         struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
5667         struct drm_device *dev = crtc->base.dev;
5668         struct drm_i915_private *dev_priv = to_i915(dev);
5669
5670         if (!crtc_state->ips_enabled)
5671                 return;
5672
5673         /*
5674          * We can only enable IPS after we enable a plane and wait for a vblank
5675          * This function is called from post_plane_update, which is run after
5676          * a vblank wait.
5677          */
5678         WARN_ON(!(crtc_state->active_planes & ~BIT(PLANE_CURSOR)));
5679
5680         if (IS_BROADWELL(dev_priv)) {
5681                 WARN_ON(sandybridge_pcode_write(dev_priv, DISPLAY_IPS_CONTROL,
5682                                                 IPS_ENABLE | IPS_PCODE_CONTROL));
5683                 /* Quoting Art Runyan: "its not safe to expect any particular
5684                  * value in IPS_CTL bit 31 after enabling IPS through the
5685                  * mailbox." Moreover, the mailbox may return a bogus state,
5686                  * so we need to just enable it and continue on.
5687                  */
5688         } else {
5689                 I915_WRITE(IPS_CTL, IPS_ENABLE);
5690                 /* The bit only becomes 1 in the next vblank, so this wait here
5691                  * is essentially intel_wait_for_vblank. If we don't have this
5692                  * and don't wait for vblanks until the end of crtc_enable, then
5693                  * the HW state readout code will complain that the expected
5694                  * IPS_CTL value is not the one we read. */
5695                 if (intel_wait_for_register(&dev_priv->uncore,
5696                                             IPS_CTL, IPS_ENABLE, IPS_ENABLE,
5697                                             50))
5698                         DRM_ERROR("Timed out waiting for IPS enable\n");
5699         }
5700 }
5701
5702 void hsw_disable_ips(const struct intel_crtc_state *crtc_state)
5703 {
5704         struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
5705         struct drm_device *dev = crtc->base.dev;
5706         struct drm_i915_private *dev_priv = to_i915(dev);
5707
5708         if (!crtc_state->ips_enabled)
5709                 return;
5710
5711         if (IS_BROADWELL(dev_priv)) {
5712                 WARN_ON(sandybridge_pcode_write(dev_priv, DISPLAY_IPS_CONTROL, 0));
5713                 /*
5714                  * Wait for PCODE to finish disabling IPS. The BSpec specified
5715                  * 42ms timeout value leads to occasional timeouts so use 100ms
5716                  * instead.
5717                  */
5718                 if (intel_wait_for_register(&dev_priv->uncore,
5719                                             IPS_CTL, IPS_ENABLE, 0,
5720                                             100))
5721                         DRM_ERROR("Timed out waiting for IPS disable\n");
5722         } else {
5723                 I915_WRITE(IPS_CTL, 0);
5724                 POSTING_READ(IPS_CTL);
5725         }
5726
5727         /* We need to wait for a vblank before we can disable the plane. */
5728         intel_wait_for_vblank(dev_priv, crtc->pipe);
5729 }
5730
5731 static void intel_crtc_dpms_overlay_disable(struct intel_crtc *intel_crtc)
5732 {
5733         if (intel_crtc->overlay) {
5734                 struct drm_device *dev = intel_crtc->base.dev;
5735
5736                 mutex_lock(&dev->struct_mutex);
5737                 (void) intel_overlay_switch_off(intel_crtc->overlay);
5738                 mutex_unlock(&dev->struct_mutex);
5739         }
5740
5741         /* Let userspace switch the overlay on again. In most cases userspace
5742          * has to recompute where to put it anyway.
5743          */
5744 }
5745
5746 /**
5747  * intel_post_enable_primary - Perform operations after enabling primary plane
5748  * @crtc: the CRTC whose primary plane was just enabled
5749  * @new_crtc_state: the enabling state
5750  *
5751  * Performs potentially sleeping operations that must be done after the primary
5752  * plane is enabled, such as updating FBC and IPS.  Note that this may be
5753  * called due to an explicit primary plane update, or due to an implicit
5754  * re-enable that is caused when a sprite plane is updated to no longer
5755  * completely hide the primary plane.
5756  */
5757 static void
5758 intel_post_enable_primary(struct drm_crtc *crtc,
5759                           const struct intel_crtc_state *new_crtc_state)
5760 {
5761         struct drm_device *dev = crtc->dev;
5762         struct drm_i915_private *dev_priv = to_i915(dev);
5763         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
5764         int pipe = intel_crtc->pipe;
5765
5766         /*
5767          * Gen2 reports pipe underruns whenever all planes are disabled.
5768          * So don't enable underrun reporting before at least some planes
5769          * are enabled.
5770          * FIXME: Need to fix the logic to work when we turn off all planes
5771          * but leave the pipe running.
5772          */
5773         if (IS_GEN(dev_priv, 2))
5774                 intel_set_cpu_fifo_underrun_reporting(dev_priv, pipe, true);
5775
5776         /* Underruns don't always raise interrupts, so check manually. */
5777         intel_check_cpu_fifo_underruns(dev_priv);
5778         intel_check_pch_fifo_underruns(dev_priv);
5779 }
5780
5781 /* FIXME get rid of this and use pre_plane_update */
5782 static void
5783 intel_pre_disable_primary_noatomic(struct drm_crtc *crtc)
5784 {
5785         struct drm_device *dev = crtc->dev;
5786         struct drm_i915_private *dev_priv = to_i915(dev);
5787         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
5788         int pipe = intel_crtc->pipe;
5789
5790         /*
5791          * Gen2 reports pipe underruns whenever all planes are disabled.
5792          * So disable underrun reporting before all the planes get disabled.
5793          */
5794         if (IS_GEN(dev_priv, 2))
5795                 intel_set_cpu_fifo_underrun_reporting(dev_priv, pipe, false);
5796
5797         hsw_disable_ips(to_intel_crtc_state(crtc->state));
5798
5799         /*
5800          * Vblank time updates from the shadow to live plane control register
5801          * are blocked if the memory self-refresh mode is active at that
5802          * moment. So to make sure the plane gets truly disabled, disable
5803          * first the self-refresh mode. The self-refresh enable bit in turn
5804          * will be checked/applied by the HW only at the next frame start
5805          * event which is after the vblank start event, so we need to have a
5806          * wait-for-vblank between disabling the plane and the pipe.
5807          */
5808         if (HAS_GMCH(dev_priv) &&
5809             intel_set_memory_cxsr(dev_priv, false))
5810                 intel_wait_for_vblank(dev_priv, pipe);
5811 }
5812
5813 static bool hsw_pre_update_disable_ips(const struct intel_crtc_state *old_crtc_state,
5814                                        const struct intel_crtc_state *new_crtc_state)
5815 {
5816         struct intel_crtc *crtc = to_intel_crtc(new_crtc_state->base.crtc);
5817         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
5818
5819         if (!old_crtc_state->ips_enabled)
5820                 return false;
5821
5822         if (needs_modeset(new_crtc_state))
5823                 return true;
5824
5825         /*
5826          * Workaround : Do not read or write the pipe palette/gamma data while
5827          * GAMMA_MODE is configured for split gamma and IPS_CTL has IPS enabled.
5828          *
5829          * Disable IPS before we program the LUT.
5830          */
5831         if (IS_HASWELL(dev_priv) &&
5832             (new_crtc_state->base.color_mgmt_changed ||
5833              new_crtc_state->update_pipe) &&
5834             new_crtc_state->gamma_mode == GAMMA_MODE_MODE_SPLIT)
5835                 return true;
5836
5837         return !new_crtc_state->ips_enabled;
5838 }
5839
5840 static bool hsw_post_update_enable_ips(const struct intel_crtc_state *old_crtc_state,
5841                                        const struct intel_crtc_state *new_crtc_state)
5842 {
5843         struct intel_crtc *crtc = to_intel_crtc(new_crtc_state->base.crtc);
5844         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
5845
5846         if (!new_crtc_state->ips_enabled)
5847                 return false;
5848
5849         if (needs_modeset(new_crtc_state))
5850                 return true;
5851
5852         /*
5853          * Workaround : Do not read or write the pipe palette/gamma data while
5854          * GAMMA_MODE is configured for split gamma and IPS_CTL has IPS enabled.
5855          *
5856          * Re-enable IPS after the LUT has been programmed.
5857          */
5858         if (IS_HASWELL(dev_priv) &&
5859             (new_crtc_state->base.color_mgmt_changed ||
5860              new_crtc_state->update_pipe) &&
5861             new_crtc_state->gamma_mode == GAMMA_MODE_MODE_SPLIT)
5862                 return true;
5863
5864         /*
5865          * We can't read out IPS on broadwell, assume the worst and
5866          * forcibly enable IPS on the first fastset.
5867          */
5868         if (new_crtc_state->update_pipe &&
5869             old_crtc_state->base.adjusted_mode.private_flags & I915_MODE_FLAG_INHERITED)
5870                 return true;
5871
5872         return !old_crtc_state->ips_enabled;
5873 }
5874
5875 static bool needs_nv12_wa(struct drm_i915_private *dev_priv,
5876                           const struct intel_crtc_state *crtc_state)
5877 {
5878         if (!crtc_state->nv12_planes)
5879                 return false;
5880
5881         /* WA Display #0827: Gen9:all */
5882         if (IS_GEN(dev_priv, 9) && !IS_GEMINILAKE(dev_priv))
5883                 return true;
5884
5885         return false;
5886 }
5887
5888 static bool needs_scalerclk_wa(struct drm_i915_private *dev_priv,
5889                                const struct intel_crtc_state *crtc_state)
5890 {
5891         /* Wa_2006604312:icl */
5892         if (crtc_state->scaler_state.scaler_users > 0 && IS_ICELAKE(dev_priv))
5893                 return true;
5894
5895         return false;
5896 }
5897
5898 static void intel_post_plane_update(struct intel_crtc_state *old_crtc_state)
5899 {
5900         struct intel_crtc *crtc = to_intel_crtc(old_crtc_state->base.crtc);
5901         struct drm_device *dev = crtc->base.dev;
5902         struct drm_i915_private *dev_priv = to_i915(dev);
5903         struct drm_atomic_state *state = old_crtc_state->base.state;
5904         struct intel_crtc_state *pipe_config =
5905                 intel_atomic_get_new_crtc_state(to_intel_atomic_state(state),
5906                                                 crtc);
5907         struct drm_plane *primary = crtc->base.primary;
5908         struct drm_plane_state *old_primary_state =
5909                 drm_atomic_get_old_plane_state(state, primary);
5910
5911         intel_frontbuffer_flip(to_i915(crtc->base.dev), pipe_config->fb_bits);
5912
5913         if (pipe_config->update_wm_post && pipe_config->base.active)
5914                 intel_update_watermarks(crtc);
5915
5916         if (hsw_post_update_enable_ips(old_crtc_state, pipe_config))
5917                 hsw_enable_ips(pipe_config);
5918
5919         if (old_primary_state) {
5920                 struct drm_plane_state *new_primary_state =
5921                         drm_atomic_get_new_plane_state(state, primary);
5922
5923                 intel_fbc_post_update(crtc);
5924
5925                 if (new_primary_state->visible &&
5926                     (needs_modeset(pipe_config) ||
5927                      !old_primary_state->visible))
5928                         intel_post_enable_primary(&crtc->base, pipe_config);
5929         }
5930
5931         if (needs_nv12_wa(dev_priv, old_crtc_state) &&
5932             !needs_nv12_wa(dev_priv, pipe_config))
5933                 skl_wa_827(dev_priv, crtc->pipe, false);
5934
5935         if (needs_scalerclk_wa(dev_priv, old_crtc_state) &&
5936             !needs_scalerclk_wa(dev_priv, pipe_config))
5937                 icl_wa_scalerclkgating(dev_priv, crtc->pipe, false);
5938 }
5939
5940 static void intel_pre_plane_update(struct intel_crtc_state *old_crtc_state,
5941                                    struct intel_crtc_state *pipe_config)
5942 {
5943         struct intel_crtc *crtc = to_intel_crtc(old_crtc_state->base.crtc);
5944         struct drm_device *dev = crtc->base.dev;
5945         struct drm_i915_private *dev_priv = to_i915(dev);
5946         struct drm_atomic_state *state = old_crtc_state->base.state;
5947         struct drm_plane *primary = crtc->base.primary;
5948         struct drm_plane_state *old_primary_state =
5949                 drm_atomic_get_old_plane_state(state, primary);
5950         bool modeset = needs_modeset(pipe_config);
5951         struct intel_atomic_state *intel_state =
5952                 to_intel_atomic_state(state);
5953
5954         if (hsw_pre_update_disable_ips(old_crtc_state, pipe_config))
5955                 hsw_disable_ips(old_crtc_state);
5956
5957         if (old_primary_state) {
5958                 struct intel_plane_state *new_primary_state =
5959                         intel_atomic_get_new_plane_state(intel_state,
5960                                                          to_intel_plane(primary));
5961
5962                 intel_fbc_pre_update(crtc, pipe_config, new_primary_state);
5963                 /*
5964                  * Gen2 reports pipe underruns whenever all planes are disabled.
5965                  * So disable underrun reporting before all the planes get disabled.
5966                  */
5967                 if (IS_GEN(dev_priv, 2) && old_primary_state->visible &&
5968                     (modeset || !new_primary_state->base.visible))
5969                         intel_set_cpu_fifo_underrun_reporting(dev_priv, crtc->pipe, false);
5970         }
5971
5972         /* Display WA 827 */
5973         if (!needs_nv12_wa(dev_priv, old_crtc_state) &&
5974             needs_nv12_wa(dev_priv, pipe_config))
5975                 skl_wa_827(dev_priv, crtc->pipe, true);
5976
5977         /* Wa_2006604312:icl */
5978         if (!needs_scalerclk_wa(dev_priv, old_crtc_state) &&
5979             needs_scalerclk_wa(dev_priv, pipe_config))
5980                 icl_wa_scalerclkgating(dev_priv, crtc->pipe, true);
5981
5982         /*
5983          * Vblank time updates from the shadow to live plane control register
5984          * are blocked if the memory self-refresh mode is active at that
5985          * moment. So to make sure the plane gets truly disabled, disable
5986          * first the self-refresh mode. The self-refresh enable bit in turn
5987          * will be checked/applied by the HW only at the next frame start
5988          * event which is after the vblank start event, so we need to have a
5989          * wait-for-vblank between disabling the plane and the pipe.
5990          */
5991         if (HAS_GMCH(dev_priv) && old_crtc_state->base.active &&
5992             pipe_config->disable_cxsr && intel_set_memory_cxsr(dev_priv, false))
5993                 intel_wait_for_vblank(dev_priv, crtc->pipe);
5994
5995         /*
5996          * IVB workaround: must disable low power watermarks for at least
5997          * one frame before enabling scaling.  LP watermarks can be re-enabled
5998          * when scaling is disabled.
5999          *
6000          * WaCxSRDisabledForSpriteScaling:ivb
6001          */
6002         if (pipe_config->disable_lp_wm && ilk_disable_lp_wm(dev) &&
6003             old_crtc_state->base.active)
6004                 intel_wait_for_vblank(dev_priv, crtc->pipe);
6005
6006         /*
6007          * If we're doing a modeset, we're done.  No need to do any pre-vblank
6008          * watermark programming here.
6009          */
6010         if (needs_modeset(pipe_config))
6011                 return;
6012
6013         /*
6014          * For platforms that support atomic watermarks, program the
6015          * 'intermediate' watermarks immediately.  On pre-gen9 platforms, these
6016          * will be the intermediate values that are safe for both pre- and
6017          * post- vblank; when vblank happens, the 'active' values will be set
6018          * to the final 'target' values and we'll do this again to get the
6019          * optimal watermarks.  For gen9+ platforms, the values we program here
6020          * will be the final target values which will get automatically latched
6021          * at vblank time; no further programming will be necessary.
6022          *
6023          * If a platform hasn't been transitioned to atomic watermarks yet,
6024          * we'll continue to update watermarks the old way, if flags tell
6025          * us to.
6026          */
6027         if (dev_priv->display.initial_watermarks != NULL)
6028                 dev_priv->display.initial_watermarks(intel_state,
6029                                                      pipe_config);
6030         else if (pipe_config->update_wm_pre)
6031                 intel_update_watermarks(crtc);
6032 }
6033
6034 static void intel_crtc_disable_planes(struct intel_atomic_state *state,
6035                                       struct intel_crtc *crtc)
6036 {
6037         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
6038         const struct intel_crtc_state *new_crtc_state =
6039                 intel_atomic_get_new_crtc_state(state, crtc);
6040         unsigned int update_mask = new_crtc_state->update_planes;
6041         const struct intel_plane_state *old_plane_state;
6042         struct intel_plane *plane;
6043         unsigned fb_bits = 0;
6044         int i;
6045
6046         intel_crtc_dpms_overlay_disable(crtc);
6047
6048         for_each_old_intel_plane_in_state(state, plane, old_plane_state, i) {
6049                 if (crtc->pipe != plane->pipe ||
6050                     !(update_mask & BIT(plane->id)))
6051                         continue;
6052
6053                 intel_disable_plane(plane, new_crtc_state);
6054
6055                 if (old_plane_state->base.visible)
6056                         fb_bits |= plane->frontbuffer_bit;
6057         }
6058
6059         intel_frontbuffer_flip(dev_priv, fb_bits);
6060 }
6061
6062 /*
6063  * intel_connector_primary_encoder - get the primary encoder for a connector
6064  * @connector: connector for which to return the encoder
6065  *
6066  * Returns the primary encoder for a connector. There is a 1:1 mapping from
6067  * all connectors to their encoder, except for DP-MST connectors which have
6068  * both a virtual and a primary encoder. These DP-MST primary encoders can be
6069  * pointed to by as many DP-MST connectors as there are pipes.
6070  */
6071 static struct intel_encoder *
6072 intel_connector_primary_encoder(struct intel_connector *connector)
6073 {
6074         struct intel_encoder *encoder;
6075
6076         if (connector->mst_port)
6077                 return &dp_to_dig_port(connector->mst_port)->base;
6078
6079         encoder = intel_attached_encoder(&connector->base);
6080         WARN_ON(!encoder);
6081
6082         return encoder;
6083 }
6084
6085 static bool
6086 intel_connector_needs_modeset(struct intel_atomic_state *state,
6087                               const struct drm_connector_state *old_conn_state,
6088                               const struct drm_connector_state *new_conn_state)
6089 {
6090         struct intel_crtc *old_crtc = old_conn_state->crtc ?
6091                                       to_intel_crtc(old_conn_state->crtc) : NULL;
6092         struct intel_crtc *new_crtc = new_conn_state->crtc ?
6093                                       to_intel_crtc(new_conn_state->crtc) : NULL;
6094
6095         return new_crtc != old_crtc ||
6096                (new_crtc &&
6097                 needs_modeset(intel_atomic_get_new_crtc_state(state, new_crtc)));
6098 }
6099
6100 static void intel_encoders_update_prepare(struct intel_atomic_state *state)
6101 {
6102         struct drm_connector_state *old_conn_state;
6103         struct drm_connector_state *new_conn_state;
6104         struct drm_connector *conn;
6105         int i;
6106
6107         for_each_oldnew_connector_in_state(&state->base, conn,
6108                                            old_conn_state, new_conn_state, i) {
6109                 struct intel_encoder *encoder;
6110                 struct intel_crtc *crtc;
6111
6112                 if (!intel_connector_needs_modeset(state,
6113                                                    old_conn_state,
6114                                                    new_conn_state))
6115                         continue;
6116
6117                 encoder = intel_connector_primary_encoder(to_intel_connector(conn));
6118                 if (!encoder->update_prepare)
6119                         continue;
6120
6121                 crtc = new_conn_state->crtc ?
6122                         to_intel_crtc(new_conn_state->crtc) : NULL;
6123                 encoder->update_prepare(state, encoder, crtc);
6124         }
6125 }
6126
6127 static void intel_encoders_update_complete(struct intel_atomic_state *state)
6128 {
6129         struct drm_connector_state *old_conn_state;
6130         struct drm_connector_state *new_conn_state;
6131         struct drm_connector *conn;
6132         int i;
6133
6134         for_each_oldnew_connector_in_state(&state->base, conn,
6135                                            old_conn_state, new_conn_state, i) {
6136                 struct intel_encoder *encoder;
6137                 struct intel_crtc *crtc;
6138
6139                 if (!intel_connector_needs_modeset(state,
6140                                                    old_conn_state,
6141                                                    new_conn_state))
6142                         continue;
6143
6144                 encoder = intel_connector_primary_encoder(to_intel_connector(conn));
6145                 if (!encoder->update_complete)
6146                         continue;
6147
6148                 crtc = new_conn_state->crtc ?
6149                         to_intel_crtc(new_conn_state->crtc) : NULL;
6150                 encoder->update_complete(state, encoder, crtc);
6151         }
6152 }
6153
6154 static void intel_encoders_pre_pll_enable(struct intel_crtc *crtc,
6155                                           struct intel_crtc_state *crtc_state,
6156                                           struct intel_atomic_state *state)
6157 {
6158         struct drm_connector_state *conn_state;
6159         struct drm_connector *conn;
6160         int i;
6161
6162         for_each_new_connector_in_state(&state->base, conn, conn_state, i) {
6163                 struct intel_encoder *encoder =
6164                         to_intel_encoder(conn_state->best_encoder);
6165
6166                 if (conn_state->crtc != &crtc->base)
6167                         continue;
6168
6169                 if (encoder->pre_pll_enable)
6170                         encoder->pre_pll_enable(encoder, crtc_state, conn_state);
6171         }
6172 }
6173
6174 static void intel_encoders_pre_enable(struct intel_crtc *crtc,
6175                                       struct intel_crtc_state *crtc_state,
6176                                       struct intel_atomic_state *state)
6177 {
6178         struct drm_connector_state *conn_state;
6179         struct drm_connector *conn;
6180         int i;
6181
6182         for_each_new_connector_in_state(&state->base, conn, conn_state, i) {
6183                 struct intel_encoder *encoder =
6184                         to_intel_encoder(conn_state->best_encoder);
6185
6186                 if (conn_state->crtc != &crtc->base)
6187                         continue;
6188
6189                 if (encoder->pre_enable)
6190                         encoder->pre_enable(encoder, crtc_state, conn_state);
6191         }
6192 }
6193
6194 static void intel_encoders_enable(struct intel_crtc *crtc,
6195                                   struct intel_crtc_state *crtc_state,
6196                                   struct intel_atomic_state *state)
6197 {
6198         struct drm_connector_state *conn_state;
6199         struct drm_connector *conn;
6200         int i;
6201
6202         for_each_new_connector_in_state(&state->base, conn, conn_state, i) {
6203                 struct intel_encoder *encoder =
6204                         to_intel_encoder(conn_state->best_encoder);
6205
6206                 if (conn_state->crtc != &crtc->base)
6207                         continue;
6208
6209                 if (encoder->enable)
6210                         encoder->enable(encoder, crtc_state, conn_state);
6211                 intel_opregion_notify_encoder(encoder, true);
6212         }
6213 }
6214
6215 static void intel_encoders_disable(struct intel_crtc *crtc,
6216                                    struct intel_crtc_state *old_crtc_state,
6217                                    struct intel_atomic_state *state)
6218 {
6219         struct drm_connector_state *old_conn_state;
6220         struct drm_connector *conn;
6221         int i;
6222
6223         for_each_old_connector_in_state(&state->base, conn, old_conn_state, i) {
6224                 struct intel_encoder *encoder =
6225                         to_intel_encoder(old_conn_state->best_encoder);
6226
6227                 if (old_conn_state->crtc != &crtc->base)
6228                         continue;
6229
6230                 intel_opregion_notify_encoder(encoder, false);
6231                 if (encoder->disable)
6232                         encoder->disable(encoder, old_crtc_state, old_conn_state);
6233         }
6234 }
6235
6236 static void intel_encoders_post_disable(struct intel_crtc *crtc,
6237                                         struct intel_crtc_state *old_crtc_state,
6238                                         struct intel_atomic_state *state)
6239 {
6240         struct drm_connector_state *old_conn_state;
6241         struct drm_connector *conn;
6242         int i;
6243
6244         for_each_old_connector_in_state(&state->base, conn, old_conn_state, i) {
6245                 struct intel_encoder *encoder =
6246                         to_intel_encoder(old_conn_state->best_encoder);
6247
6248                 if (old_conn_state->crtc != &crtc->base)
6249                         continue;
6250
6251                 if (encoder->post_disable)
6252                         encoder->post_disable(encoder, old_crtc_state, old_conn_state);
6253         }
6254 }
6255
6256 static void intel_encoders_post_pll_disable(struct intel_crtc *crtc,
6257                                             struct intel_crtc_state *old_crtc_state,
6258                                             struct intel_atomic_state *state)
6259 {
6260         struct drm_connector_state *old_conn_state;
6261         struct drm_connector *conn;
6262         int i;
6263
6264         for_each_old_connector_in_state(&state->base, conn, old_conn_state, i) {
6265                 struct intel_encoder *encoder =
6266                         to_intel_encoder(old_conn_state->best_encoder);
6267
6268                 if (old_conn_state->crtc != &crtc->base)
6269                         continue;
6270
6271                 if (encoder->post_pll_disable)
6272                         encoder->post_pll_disable(encoder, old_crtc_state, old_conn_state);
6273         }
6274 }
6275
6276 static void intel_encoders_update_pipe(struct intel_crtc *crtc,
6277                                        struct intel_crtc_state *crtc_state,
6278                                        struct intel_atomic_state *state)
6279 {
6280         struct drm_connector_state *conn_state;
6281         struct drm_connector *conn;
6282         int i;
6283
6284         for_each_new_connector_in_state(&state->base, conn, conn_state, i) {
6285                 struct intel_encoder *encoder =
6286                         to_intel_encoder(conn_state->best_encoder);
6287
6288                 if (conn_state->crtc != &crtc->base)
6289                         continue;
6290
6291                 if (encoder->update_pipe)
6292                         encoder->update_pipe(encoder, crtc_state, conn_state);
6293         }
6294 }
6295
6296 static void intel_disable_primary_plane(const struct intel_crtc_state *crtc_state)
6297 {
6298         struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
6299         struct intel_plane *plane = to_intel_plane(crtc->base.primary);
6300
6301         plane->disable_plane(plane, crtc_state);
6302 }
6303
6304 static void ironlake_crtc_enable(struct intel_crtc_state *pipe_config,
6305                                  struct intel_atomic_state *state)
6306 {
6307         struct drm_crtc *crtc = pipe_config->base.crtc;
6308         struct drm_device *dev = crtc->dev;
6309         struct drm_i915_private *dev_priv = to_i915(dev);
6310         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
6311         int pipe = intel_crtc->pipe;
6312
6313         if (WARN_ON(intel_crtc->active))
6314                 return;
6315
6316         /*
6317          * Sometimes spurious CPU pipe underruns happen during FDI
6318          * training, at least with VGA+HDMI cloning. Suppress them.
6319          *
6320          * On ILK we get an occasional spurious CPU pipe underruns
6321          * between eDP port A enable and vdd enable. Also PCH port
6322          * enable seems to result in the occasional CPU pipe underrun.
6323          *
6324          * Spurious PCH underruns also occur during PCH enabling.
6325          */
6326         intel_set_cpu_fifo_underrun_reporting(dev_priv, pipe, false);
6327         intel_set_pch_fifo_underrun_reporting(dev_priv, pipe, false);
6328
6329         if (pipe_config->has_pch_encoder)
6330                 intel_prepare_shared_dpll(pipe_config);
6331
6332         if (intel_crtc_has_dp_encoder(pipe_config))
6333                 intel_dp_set_m_n(pipe_config, M1_N1);
6334
6335         intel_set_pipe_timings(pipe_config);
6336         intel_set_pipe_src_size(pipe_config);
6337
6338         if (pipe_config->has_pch_encoder) {
6339                 intel_cpu_transcoder_set_m_n(pipe_config,
6340                                              &pipe_config->fdi_m_n, NULL);
6341         }
6342
6343         ironlake_set_pipeconf(pipe_config);
6344
6345         intel_crtc->active = true;
6346
6347         intel_encoders_pre_enable(intel_crtc, pipe_config, state);
6348
6349         if (pipe_config->has_pch_encoder) {
6350                 /* Note: FDI PLL enabling _must_ be done before we enable the
6351                  * cpu pipes, hence this is separate from all the other fdi/pch
6352                  * enabling. */
6353                 ironlake_fdi_pll_enable(pipe_config);
6354         } else {
6355                 assert_fdi_tx_disabled(dev_priv, pipe);
6356                 assert_fdi_rx_disabled(dev_priv, pipe);
6357         }
6358
6359         ironlake_pfit_enable(pipe_config);
6360
6361         /*
6362          * On ILK+ LUT must be loaded before the pipe is running but with
6363          * clocks enabled
6364          */
6365         intel_color_load_luts(pipe_config);
6366         intel_color_commit(pipe_config);
6367         /* update DSPCNTR to configure gamma for pipe bottom color */
6368         intel_disable_primary_plane(pipe_config);
6369
6370         if (dev_priv->display.initial_watermarks != NULL)
6371                 dev_priv->display.initial_watermarks(state, pipe_config);
6372         intel_enable_pipe(pipe_config);
6373
6374         if (pipe_config->has_pch_encoder)
6375                 ironlake_pch_enable(state, pipe_config);
6376
6377         assert_vblank_disabled(crtc);
6378         intel_crtc_vblank_on(pipe_config);
6379
6380         intel_encoders_enable(intel_crtc, pipe_config, state);
6381
6382         if (HAS_PCH_CPT(dev_priv))
6383                 cpt_verify_modeset(dev, intel_crtc->pipe);
6384
6385         /*
6386          * Must wait for vblank to avoid spurious PCH FIFO underruns.
6387          * And a second vblank wait is needed at least on ILK with
6388          * some interlaced HDMI modes. Let's do the double wait always
6389          * in case there are more corner cases we don't know about.
6390          */
6391         if (pipe_config->has_pch_encoder) {
6392                 intel_wait_for_vblank(dev_priv, pipe);
6393                 intel_wait_for_vblank(dev_priv, pipe);
6394         }
6395         intel_set_cpu_fifo_underrun_reporting(dev_priv, pipe, true);
6396         intel_set_pch_fifo_underrun_reporting(dev_priv, pipe, true);
6397 }
6398
6399 /* IPS only exists on ULT machines and is tied to pipe A. */
6400 static bool hsw_crtc_supports_ips(struct intel_crtc *crtc)
6401 {
6402         return HAS_IPS(to_i915(crtc->base.dev)) && crtc->pipe == PIPE_A;
6403 }
6404
6405 static void glk_pipe_scaler_clock_gating_wa(struct drm_i915_private *dev_priv,
6406                                             enum pipe pipe, bool apply)
6407 {
6408         u32 val = I915_READ(CLKGATE_DIS_PSL(pipe));
6409         u32 mask = DPF_GATING_DIS | DPF_RAM_GATING_DIS | DPFR_GATING_DIS;
6410
6411         if (apply)
6412                 val |= mask;
6413         else
6414                 val &= ~mask;
6415
6416         I915_WRITE(CLKGATE_DIS_PSL(pipe), val);
6417 }
6418
6419 static void icl_pipe_mbus_enable(struct intel_crtc *crtc)
6420 {
6421         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
6422         enum pipe pipe = crtc->pipe;
6423         u32 val;
6424
6425         val = MBUS_DBOX_A_CREDIT(2);
6426         val |= MBUS_DBOX_BW_CREDIT(1);
6427         val |= MBUS_DBOX_B_CREDIT(8);
6428
6429         I915_WRITE(PIPE_MBUS_DBOX_CTL(pipe), val);
6430 }
6431
6432 static void haswell_crtc_enable(struct intel_crtc_state *pipe_config,
6433                                 struct intel_atomic_state *state)
6434 {
6435         struct drm_crtc *crtc = pipe_config->base.crtc;
6436         struct drm_i915_private *dev_priv = to_i915(crtc->dev);
6437         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
6438         int pipe = intel_crtc->pipe, hsw_workaround_pipe;
6439         enum transcoder cpu_transcoder = pipe_config->cpu_transcoder;
6440         bool psl_clkgate_wa;
6441
6442         if (WARN_ON(intel_crtc->active))
6443                 return;
6444
6445         intel_encoders_pre_pll_enable(intel_crtc, pipe_config, state);
6446
6447         if (pipe_config->shared_dpll)
6448                 intel_enable_shared_dpll(pipe_config);
6449
6450         intel_encoders_pre_enable(intel_crtc, pipe_config, state);
6451
6452         if (intel_crtc_has_dp_encoder(pipe_config))
6453                 intel_dp_set_m_n(pipe_config, M1_N1);
6454
6455         if (!transcoder_is_dsi(cpu_transcoder))
6456                 intel_set_pipe_timings(pipe_config);
6457
6458         intel_set_pipe_src_size(pipe_config);
6459
6460         if (cpu_transcoder != TRANSCODER_EDP &&
6461             !transcoder_is_dsi(cpu_transcoder)) {
6462                 I915_WRITE(PIPE_MULT(cpu_transcoder),
6463                            pipe_config->pixel_multiplier - 1);
6464         }
6465
6466         if (pipe_config->has_pch_encoder) {
6467                 intel_cpu_transcoder_set_m_n(pipe_config,
6468                                              &pipe_config->fdi_m_n, NULL);
6469         }
6470
6471         if (!transcoder_is_dsi(cpu_transcoder))
6472                 haswell_set_pipeconf(pipe_config);
6473
6474         if (INTEL_GEN(dev_priv) >= 9 || IS_BROADWELL(dev_priv))
6475                 bdw_set_pipemisc(pipe_config);
6476
6477         intel_crtc->active = true;
6478
6479         /* Display WA #1180: WaDisableScalarClockGating: glk, cnl */
6480         psl_clkgate_wa = (IS_GEMINILAKE(dev_priv) || IS_CANNONLAKE(dev_priv)) &&
6481                          pipe_config->pch_pfit.enabled;
6482         if (psl_clkgate_wa)
6483                 glk_pipe_scaler_clock_gating_wa(dev_priv, pipe, true);
6484
6485         if (INTEL_GEN(dev_priv) >= 9)
6486                 skylake_pfit_enable(pipe_config);
6487         else
6488                 ironlake_pfit_enable(pipe_config);
6489
6490         /*
6491          * On ILK+ LUT must be loaded before the pipe is running but with
6492          * clocks enabled
6493          */
6494         intel_color_load_luts(pipe_config);
6495         intel_color_commit(pipe_config);
6496         /* update DSPCNTR to configure gamma/csc for pipe bottom color */
6497         if (INTEL_GEN(dev_priv) < 9)
6498                 intel_disable_primary_plane(pipe_config);
6499
6500         if (INTEL_GEN(dev_priv) >= 11)
6501                 icl_set_pipe_chicken(intel_crtc);
6502
6503         intel_ddi_set_pipe_settings(pipe_config);
6504         if (!transcoder_is_dsi(cpu_transcoder))
6505                 intel_ddi_enable_transcoder_func(pipe_config);
6506
6507         if (dev_priv->display.initial_watermarks != NULL)
6508                 dev_priv->display.initial_watermarks(state, pipe_config);
6509
6510         if (INTEL_GEN(dev_priv) >= 11)
6511                 icl_pipe_mbus_enable(intel_crtc);
6512
6513         /* XXX: Do the pipe assertions at the right place for BXT DSI. */
6514         if (!transcoder_is_dsi(cpu_transcoder))
6515                 intel_enable_pipe(pipe_config);
6516
6517         if (pipe_config->has_pch_encoder)
6518                 lpt_pch_enable(state, pipe_config);
6519
6520         if (intel_crtc_has_type(pipe_config, INTEL_OUTPUT_DP_MST))
6521                 intel_ddi_set_vc_payload_alloc(pipe_config, true);
6522
6523         assert_vblank_disabled(crtc);
6524         intel_crtc_vblank_on(pipe_config);
6525
6526         intel_encoders_enable(intel_crtc, pipe_config, state);
6527
6528         if (psl_clkgate_wa) {
6529                 intel_wait_for_vblank(dev_priv, pipe);
6530                 glk_pipe_scaler_clock_gating_wa(dev_priv, pipe, false);
6531         }
6532
6533         /* If we change the relative order between pipe/planes enabling, we need
6534          * to change the workaround. */
6535         hsw_workaround_pipe = pipe_config->hsw_workaround_pipe;
6536         if (IS_HASWELL(dev_priv) && hsw_workaround_pipe != INVALID_PIPE) {
6537                 intel_wait_for_vblank(dev_priv, hsw_workaround_pipe);
6538                 intel_wait_for_vblank(dev_priv, hsw_workaround_pipe);
6539         }
6540 }
6541
6542 static void ironlake_pfit_disable(const struct intel_crtc_state *old_crtc_state)
6543 {
6544         struct intel_crtc *crtc = to_intel_crtc(old_crtc_state->base.crtc);
6545         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
6546         enum pipe pipe = crtc->pipe;
6547
6548         /* To avoid upsetting the power well on haswell only disable the pfit if
6549          * it's in use. The hw state code will make sure we get this right. */
6550         if (old_crtc_state->pch_pfit.enabled) {
6551                 I915_WRITE(PF_CTL(pipe), 0);
6552                 I915_WRITE(PF_WIN_POS(pipe), 0);
6553                 I915_WRITE(PF_WIN_SZ(pipe), 0);
6554         }
6555 }
6556
6557 static void ironlake_crtc_disable(struct intel_crtc_state *old_crtc_state,
6558                                   struct intel_atomic_state *state)
6559 {
6560         struct drm_crtc *crtc = old_crtc_state->base.crtc;
6561         struct drm_device *dev = crtc->dev;
6562         struct drm_i915_private *dev_priv = to_i915(dev);
6563         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
6564         int pipe = intel_crtc->pipe;
6565
6566         /*
6567          * Sometimes spurious CPU pipe underruns happen when the
6568          * pipe is already disabled, but FDI RX/TX is still enabled.
6569          * Happens at least with VGA+HDMI cloning. Suppress them.
6570          */
6571         intel_set_cpu_fifo_underrun_reporting(dev_priv, pipe, false);
6572         intel_set_pch_fifo_underrun_reporting(dev_priv, pipe, false);
6573
6574         intel_encoders_disable(intel_crtc, old_crtc_state, state);
6575
6576         drm_crtc_vblank_off(crtc);
6577         assert_vblank_disabled(crtc);
6578
6579         intel_disable_pipe(old_crtc_state);
6580
6581         ironlake_pfit_disable(old_crtc_state);
6582
6583         if (old_crtc_state->has_pch_encoder)
6584                 ironlake_fdi_disable(crtc);
6585
6586         intel_encoders_post_disable(intel_crtc, old_crtc_state, state);
6587
6588         if (old_crtc_state->has_pch_encoder) {
6589                 ironlake_disable_pch_transcoder(dev_priv, pipe);
6590
6591                 if (HAS_PCH_CPT(dev_priv)) {
6592                         i915_reg_t reg;
6593                         u32 temp;
6594
6595                         /* disable TRANS_DP_CTL */
6596                         reg = TRANS_DP_CTL(pipe);
6597                         temp = I915_READ(reg);
6598                         temp &= ~(TRANS_DP_OUTPUT_ENABLE |
6599                                   TRANS_DP_PORT_SEL_MASK);
6600                         temp |= TRANS_DP_PORT_SEL_NONE;
6601                         I915_WRITE(reg, temp);
6602
6603                         /* disable DPLL_SEL */
6604                         temp = I915_READ(PCH_DPLL_SEL);
6605                         temp &= ~(TRANS_DPLL_ENABLE(pipe) | TRANS_DPLLB_SEL(pipe));
6606                         I915_WRITE(PCH_DPLL_SEL, temp);
6607                 }
6608
6609                 ironlake_fdi_pll_disable(intel_crtc);
6610         }
6611
6612         intel_set_cpu_fifo_underrun_reporting(dev_priv, pipe, true);
6613         intel_set_pch_fifo_underrun_reporting(dev_priv, pipe, true);
6614 }
6615
6616 static void haswell_crtc_disable(struct intel_crtc_state *old_crtc_state,
6617                                  struct intel_atomic_state *state)
6618 {
6619         struct drm_crtc *crtc = old_crtc_state->base.crtc;
6620         struct drm_i915_private *dev_priv = to_i915(crtc->dev);
6621         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
6622         enum transcoder cpu_transcoder = old_crtc_state->cpu_transcoder;
6623
6624         intel_encoders_disable(intel_crtc, old_crtc_state, state);
6625
6626         drm_crtc_vblank_off(crtc);
6627         assert_vblank_disabled(crtc);
6628
6629         /* XXX: Do the pipe assertions at the right place for BXT DSI. */
6630         if (!transcoder_is_dsi(cpu_transcoder))
6631                 intel_disable_pipe(old_crtc_state);
6632
6633         if (intel_crtc_has_type(old_crtc_state, INTEL_OUTPUT_DP_MST))
6634                 intel_ddi_set_vc_payload_alloc(old_crtc_state, false);
6635
6636         if (!transcoder_is_dsi(cpu_transcoder))
6637                 intel_ddi_disable_transcoder_func(old_crtc_state);
6638
6639         intel_dsc_disable(old_crtc_state);
6640
6641         if (INTEL_GEN(dev_priv) >= 9)
6642                 skylake_scaler_disable(intel_crtc);
6643         else
6644                 ironlake_pfit_disable(old_crtc_state);
6645
6646         intel_encoders_post_disable(intel_crtc, old_crtc_state, state);
6647
6648         intel_encoders_post_pll_disable(intel_crtc, old_crtc_state, state);
6649 }
6650
6651 static void i9xx_pfit_enable(const struct intel_crtc_state *crtc_state)
6652 {
6653         struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
6654         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
6655
6656         if (!crtc_state->gmch_pfit.control)
6657                 return;
6658
6659         /*
6660          * The panel fitter should only be adjusted whilst the pipe is disabled,
6661          * according to register description and PRM.
6662          */
6663         WARN_ON(I915_READ(PFIT_CONTROL) & PFIT_ENABLE);
6664         assert_pipe_disabled(dev_priv, crtc->pipe);
6665
6666         I915_WRITE(PFIT_PGM_RATIOS, crtc_state->gmch_pfit.pgm_ratios);
6667         I915_WRITE(PFIT_CONTROL, crtc_state->gmch_pfit.control);
6668
6669         /* Border color in case we don't scale up to the full screen. Black by
6670          * default, change to something else for debugging. */
6671         I915_WRITE(BCLRPAT(crtc->pipe), 0);
6672 }
6673
6674 bool intel_port_is_combophy(struct drm_i915_private *dev_priv, enum port port)
6675 {
6676         if (port == PORT_NONE)
6677                 return false;
6678
6679         if (IS_ELKHARTLAKE(dev_priv))
6680                 return port <= PORT_C;
6681
6682         if (INTEL_GEN(dev_priv) >= 11)
6683                 return port <= PORT_B;
6684
6685         return false;
6686 }
6687
6688 bool intel_phy_is_combo(struct drm_i915_private *dev_priv, enum phy phy)
6689 {
6690         if (phy == PHY_NONE)
6691                 return false;
6692
6693         if (IS_ELKHARTLAKE(dev_priv))
6694                 return phy <= PHY_C;
6695
6696         if (INTEL_GEN(dev_priv) >= 11)
6697                 return phy <= PHY_B;
6698
6699         return false;
6700 }
6701
6702 bool intel_port_is_tc(struct drm_i915_private *dev_priv, enum port port)
6703 {
6704         if (INTEL_GEN(dev_priv) >= 11 && !IS_ELKHARTLAKE(dev_priv))
6705                 return port >= PORT_C && port <= PORT_F;
6706
6707         return false;
6708 }
6709
6710 bool intel_phy_is_tc(struct drm_i915_private *dev_priv, enum phy phy)
6711 {
6712         if (INTEL_GEN(dev_priv) >= 11 && !IS_ELKHARTLAKE(dev_priv))
6713                 return phy >= PHY_C && phy <= PHY_F;
6714
6715         return false;
6716 }
6717
6718 enum phy intel_port_to_phy(struct drm_i915_private *i915, enum port port)
6719 {
6720         if (IS_ELKHARTLAKE(i915) && port == PORT_D)
6721                 return PHY_A;
6722
6723         return (enum phy)port;
6724 }
6725
6726 enum tc_port intel_port_to_tc(struct drm_i915_private *dev_priv, enum port port)
6727 {
6728         if (!intel_phy_is_tc(dev_priv, intel_port_to_phy(dev_priv, port)))
6729                 return PORT_TC_NONE;
6730
6731         return port - PORT_C;
6732 }
6733
6734 enum intel_display_power_domain intel_port_to_power_domain(enum port port)
6735 {
6736         switch (port) {
6737         case PORT_A:
6738                 return POWER_DOMAIN_PORT_DDI_A_LANES;
6739         case PORT_B:
6740                 return POWER_DOMAIN_PORT_DDI_B_LANES;
6741         case PORT_C:
6742                 return POWER_DOMAIN_PORT_DDI_C_LANES;
6743         case PORT_D:
6744                 return POWER_DOMAIN_PORT_DDI_D_LANES;
6745         case PORT_E:
6746                 return POWER_DOMAIN_PORT_DDI_E_LANES;
6747         case PORT_F:
6748                 return POWER_DOMAIN_PORT_DDI_F_LANES;
6749         default:
6750                 MISSING_CASE(port);
6751                 return POWER_DOMAIN_PORT_OTHER;
6752         }
6753 }
6754
6755 enum intel_display_power_domain
6756 intel_aux_power_domain(struct intel_digital_port *dig_port)
6757 {
6758         struct drm_i915_private *dev_priv = to_i915(dig_port->base.base.dev);
6759
6760         if (intel_port_is_tc(dev_priv, dig_port->base.port) &&
6761             dig_port->tc_mode == TC_PORT_TBT_ALT) {
6762                 switch (dig_port->aux_ch) {
6763                 case AUX_CH_C:
6764                         return POWER_DOMAIN_AUX_TBT1;
6765                 case AUX_CH_D:
6766                         return POWER_DOMAIN_AUX_TBT2;
6767                 case AUX_CH_E:
6768                         return POWER_DOMAIN_AUX_TBT3;
6769                 case AUX_CH_F:
6770                         return POWER_DOMAIN_AUX_TBT4;
6771                 default:
6772                         MISSING_CASE(dig_port->aux_ch);
6773                         return POWER_DOMAIN_AUX_TBT1;
6774                 }
6775         }
6776
6777         switch (dig_port->aux_ch) {
6778         case AUX_CH_A:
6779                 return POWER_DOMAIN_AUX_A;
6780         case AUX_CH_B:
6781                 return POWER_DOMAIN_AUX_B;
6782         case AUX_CH_C:
6783                 return POWER_DOMAIN_AUX_C;
6784         case AUX_CH_D:
6785                 return POWER_DOMAIN_AUX_D;
6786         case AUX_CH_E:
6787                 return POWER_DOMAIN_AUX_E;
6788         case AUX_CH_F:
6789                 return POWER_DOMAIN_AUX_F;
6790         default:
6791                 MISSING_CASE(dig_port->aux_ch);
6792                 return POWER_DOMAIN_AUX_A;
6793         }
6794 }
6795
6796 static u64 get_crtc_power_domains(struct intel_crtc *crtc,
6797                                   struct intel_crtc_state *crtc_state)
6798 {
6799         struct drm_device *dev = crtc->base.dev;
6800         struct drm_i915_private *dev_priv = to_i915(dev);
6801         struct drm_encoder *encoder;
6802         enum pipe pipe = crtc->pipe;
6803         u64 mask;
6804         enum transcoder transcoder = crtc_state->cpu_transcoder;
6805
6806         if (!crtc_state->base.active)
6807                 return 0;
6808
6809         mask = BIT_ULL(POWER_DOMAIN_PIPE(pipe));
6810         mask |= BIT_ULL(POWER_DOMAIN_TRANSCODER(transcoder));
6811         if (crtc_state->pch_pfit.enabled ||
6812             crtc_state->pch_pfit.force_thru)
6813                 mask |= BIT_ULL(POWER_DOMAIN_PIPE_PANEL_FITTER(pipe));
6814
6815         drm_for_each_encoder_mask(encoder, dev, crtc_state->base.encoder_mask) {
6816                 struct intel_encoder *intel_encoder = to_intel_encoder(encoder);
6817
6818                 mask |= BIT_ULL(intel_encoder->power_domain);
6819         }
6820
6821         if (HAS_DDI(dev_priv) && crtc_state->has_audio)
6822                 mask |= BIT_ULL(POWER_DOMAIN_AUDIO);
6823
6824         if (crtc_state->shared_dpll)
6825                 mask |= BIT_ULL(POWER_DOMAIN_DISPLAY_CORE);
6826
6827         return mask;
6828 }
6829
6830 static u64
6831 modeset_get_crtc_power_domains(struct intel_crtc *crtc,
6832                                struct intel_crtc_state *crtc_state)
6833 {
6834         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
6835         enum intel_display_power_domain domain;
6836         u64 domains, new_domains, old_domains;
6837
6838         old_domains = crtc->enabled_power_domains;
6839         crtc->enabled_power_domains = new_domains =
6840                 get_crtc_power_domains(crtc, crtc_state);
6841
6842         domains = new_domains & ~old_domains;
6843
6844         for_each_power_domain(domain, domains)
6845                 intel_display_power_get(dev_priv, domain);
6846
6847         return old_domains & ~new_domains;
6848 }
6849
6850 static void modeset_put_power_domains(struct drm_i915_private *dev_priv,
6851                                       u64 domains)
6852 {
6853         enum intel_display_power_domain domain;
6854
6855         for_each_power_domain(domain, domains)
6856                 intel_display_power_put_unchecked(dev_priv, domain);
6857 }
6858
6859 static void valleyview_crtc_enable(struct intel_crtc_state *pipe_config,
6860                                    struct intel_atomic_state *state)
6861 {
6862         struct drm_crtc *crtc = pipe_config->base.crtc;
6863         struct drm_device *dev = crtc->dev;
6864         struct drm_i915_private *dev_priv = to_i915(dev);
6865         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
6866         int pipe = intel_crtc->pipe;
6867
6868         if (WARN_ON(intel_crtc->active))
6869                 return;
6870
6871         if (intel_crtc_has_dp_encoder(pipe_config))
6872                 intel_dp_set_m_n(pipe_config, M1_N1);
6873
6874         intel_set_pipe_timings(pipe_config);
6875         intel_set_pipe_src_size(pipe_config);
6876
6877         if (IS_CHERRYVIEW(dev_priv) && pipe == PIPE_B) {
6878                 I915_WRITE(CHV_BLEND(pipe), CHV_BLEND_LEGACY);
6879                 I915_WRITE(CHV_CANVAS(pipe), 0);
6880         }
6881
6882         i9xx_set_pipeconf(pipe_config);
6883
6884         intel_crtc->active = true;
6885
6886         intel_set_cpu_fifo_underrun_reporting(dev_priv, pipe, true);
6887
6888         intel_encoders_pre_pll_enable(intel_crtc, pipe_config, state);
6889
6890         if (IS_CHERRYVIEW(dev_priv)) {
6891                 chv_prepare_pll(intel_crtc, pipe_config);
6892                 chv_enable_pll(intel_crtc, pipe_config);
6893         } else {
6894                 vlv_prepare_pll(intel_crtc, pipe_config);
6895                 vlv_enable_pll(intel_crtc, pipe_config);
6896         }
6897
6898         intel_encoders_pre_enable(intel_crtc, pipe_config, state);
6899
6900         i9xx_pfit_enable(pipe_config);
6901
6902         intel_color_load_luts(pipe_config);
6903         intel_color_commit(pipe_config);
6904         /* update DSPCNTR to configure gamma for pipe bottom color */
6905         intel_disable_primary_plane(pipe_config);
6906
6907         dev_priv->display.initial_watermarks(state, pipe_config);
6908         intel_enable_pipe(pipe_config);
6909
6910         assert_vblank_disabled(crtc);
6911         intel_crtc_vblank_on(pipe_config);
6912
6913         intel_encoders_enable(intel_crtc, pipe_config, state);
6914 }
6915
6916 static void i9xx_set_pll_dividers(const struct intel_crtc_state *crtc_state)
6917 {
6918         struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
6919         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
6920
6921         I915_WRITE(FP0(crtc->pipe), crtc_state->dpll_hw_state.fp0);
6922         I915_WRITE(FP1(crtc->pipe), crtc_state->dpll_hw_state.fp1);
6923 }
6924
6925 static void i9xx_crtc_enable(struct intel_crtc_state *pipe_config,
6926                              struct intel_atomic_state *state)
6927 {
6928         struct drm_crtc *crtc = pipe_config->base.crtc;
6929         struct drm_device *dev = crtc->dev;
6930         struct drm_i915_private *dev_priv = to_i915(dev);
6931         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
6932         enum pipe pipe = intel_crtc->pipe;
6933
6934         if (WARN_ON(intel_crtc->active))
6935                 return;
6936
6937         i9xx_set_pll_dividers(pipe_config);
6938
6939         if (intel_crtc_has_dp_encoder(pipe_config))
6940                 intel_dp_set_m_n(pipe_config, M1_N1);
6941
6942         intel_set_pipe_timings(pipe_config);
6943         intel_set_pipe_src_size(pipe_config);
6944
6945         i9xx_set_pipeconf(pipe_config);
6946
6947         intel_crtc->active = true;
6948
6949         if (!IS_GEN(dev_priv, 2))
6950                 intel_set_cpu_fifo_underrun_reporting(dev_priv, pipe, true);
6951
6952         intel_encoders_pre_enable(intel_crtc, pipe_config, state);
6953
6954         i9xx_enable_pll(intel_crtc, pipe_config);
6955
6956         i9xx_pfit_enable(pipe_config);
6957
6958         intel_color_load_luts(pipe_config);
6959         intel_color_commit(pipe_config);
6960         /* update DSPCNTR to configure gamma for pipe bottom color */
6961         intel_disable_primary_plane(pipe_config);
6962
6963         if (dev_priv->display.initial_watermarks != NULL)
6964                 dev_priv->display.initial_watermarks(state,
6965                                                      pipe_config);
6966         else
6967                 intel_update_watermarks(intel_crtc);
6968         intel_enable_pipe(pipe_config);
6969
6970         assert_vblank_disabled(crtc);
6971         intel_crtc_vblank_on(pipe_config);
6972
6973         intel_encoders_enable(intel_crtc, pipe_config, state);
6974 }
6975
6976 static void i9xx_pfit_disable(const struct intel_crtc_state *old_crtc_state)
6977 {
6978         struct intel_crtc *crtc = to_intel_crtc(old_crtc_state->base.crtc);
6979         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
6980
6981         if (!old_crtc_state->gmch_pfit.control)
6982                 return;
6983
6984         assert_pipe_disabled(dev_priv, crtc->pipe);
6985
6986         DRM_DEBUG_KMS("disabling pfit, current: 0x%08x\n",
6987                       I915_READ(PFIT_CONTROL));
6988         I915_WRITE(PFIT_CONTROL, 0);
6989 }
6990
6991 static void i9xx_crtc_disable(struct intel_crtc_state *old_crtc_state,
6992                               struct intel_atomic_state *state)
6993 {
6994         struct drm_crtc *crtc = old_crtc_state->base.crtc;
6995         struct drm_device *dev = crtc->dev;
6996         struct drm_i915_private *dev_priv = to_i915(dev);
6997         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
6998         int pipe = intel_crtc->pipe;
6999
7000         /*
7001          * On gen2 planes are double buffered but the pipe isn't, so we must
7002          * wait for planes to fully turn off before disabling the pipe.
7003          */
7004         if (IS_GEN(dev_priv, 2))
7005                 intel_wait_for_vblank(dev_priv, pipe);
7006
7007         intel_encoders_disable(intel_crtc, old_crtc_state, state);
7008
7009         drm_crtc_vblank_off(crtc);
7010         assert_vblank_disabled(crtc);
7011
7012         intel_disable_pipe(old_crtc_state);
7013
7014         i9xx_pfit_disable(old_crtc_state);
7015
7016         intel_encoders_post_disable(intel_crtc, old_crtc_state, state);
7017
7018         if (!intel_crtc_has_type(old_crtc_state, INTEL_OUTPUT_DSI)) {
7019                 if (IS_CHERRYVIEW(dev_priv))
7020                         chv_disable_pll(dev_priv, pipe);
7021                 else if (IS_VALLEYVIEW(dev_priv))
7022                         vlv_disable_pll(dev_priv, pipe);
7023                 else
7024                         i9xx_disable_pll(old_crtc_state);
7025         }
7026
7027         intel_encoders_post_pll_disable(intel_crtc, old_crtc_state, state);
7028
7029         if (!IS_GEN(dev_priv, 2))
7030                 intel_set_cpu_fifo_underrun_reporting(dev_priv, pipe, false);
7031
7032         if (!dev_priv->display.initial_watermarks)
7033                 intel_update_watermarks(intel_crtc);
7034
7035         /* clock the pipe down to 640x480@60 to potentially save power */
7036         if (IS_I830(dev_priv))
7037                 i830_enable_pipe(dev_priv, pipe);
7038 }
7039
7040 static void intel_crtc_disable_noatomic(struct drm_crtc *crtc,
7041                                         struct drm_modeset_acquire_ctx *ctx)
7042 {
7043         struct intel_encoder *encoder;
7044         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
7045         struct drm_i915_private *dev_priv = to_i915(crtc->dev);
7046         struct intel_bw_state *bw_state =
7047                 to_intel_bw_state(dev_priv->bw_obj.state);
7048         enum intel_display_power_domain domain;
7049         struct intel_plane *plane;
7050         u64 domains;
7051         struct drm_atomic_state *state;
7052         struct intel_crtc_state *crtc_state;
7053         int ret;
7054
7055         if (!intel_crtc->active)
7056                 return;
7057
7058         for_each_intel_plane_on_crtc(&dev_priv->drm, intel_crtc, plane) {
7059                 const struct intel_plane_state *plane_state =
7060                         to_intel_plane_state(plane->base.state);
7061
7062                 if (plane_state->base.visible)
7063                         intel_plane_disable_noatomic(intel_crtc, plane);
7064         }
7065
7066         state = drm_atomic_state_alloc(crtc->dev);
7067         if (!state) {
7068                 DRM_DEBUG_KMS("failed to disable [CRTC:%d:%s], out of memory",
7069                               crtc->base.id, crtc->name);
7070                 return;
7071         }
7072
7073         state->acquire_ctx = ctx;
7074
7075         /* Everything's already locked, -EDEADLK can't happen. */
7076         crtc_state = intel_atomic_get_crtc_state(state, intel_crtc);
7077         ret = drm_atomic_add_affected_connectors(state, crtc);
7078
7079         WARN_ON(IS_ERR(crtc_state) || ret);
7080
7081         dev_priv->display.crtc_disable(crtc_state, to_intel_atomic_state(state));
7082
7083         drm_atomic_state_put(state);
7084
7085         DRM_DEBUG_KMS("[CRTC:%d:%s] hw state adjusted, was enabled, now disabled\n",
7086                       crtc->base.id, crtc->name);
7087
7088         WARN_ON(drm_atomic_set_mode_for_crtc(crtc->state, NULL) < 0);
7089         crtc->state->active = false;
7090         intel_crtc->active = false;
7091         crtc->enabled = false;
7092         crtc->state->connector_mask = 0;
7093         crtc->state->encoder_mask = 0;
7094
7095         for_each_encoder_on_crtc(crtc->dev, crtc, encoder)
7096                 encoder->base.crtc = NULL;
7097
7098         intel_fbc_disable(intel_crtc);
7099         intel_update_watermarks(intel_crtc);
7100         intel_disable_shared_dpll(to_intel_crtc_state(crtc->state));
7101
7102         domains = intel_crtc->enabled_power_domains;
7103         for_each_power_domain(domain, domains)
7104                 intel_display_power_put_unchecked(dev_priv, domain);
7105         intel_crtc->enabled_power_domains = 0;
7106
7107         dev_priv->active_crtcs &= ~(1 << intel_crtc->pipe);
7108         dev_priv->min_cdclk[intel_crtc->pipe] = 0;
7109         dev_priv->min_voltage_level[intel_crtc->pipe] = 0;
7110
7111         bw_state->data_rate[intel_crtc->pipe] = 0;
7112         bw_state->num_active_planes[intel_crtc->pipe] = 0;
7113 }
7114
7115 /*
7116  * turn all crtc's off, but do not adjust state
7117  * This has to be paired with a call to intel_modeset_setup_hw_state.
7118  */
7119 int intel_display_suspend(struct drm_device *dev)
7120 {
7121         struct drm_i915_private *dev_priv = to_i915(dev);
7122         struct drm_atomic_state *state;
7123         int ret;
7124
7125         state = drm_atomic_helper_suspend(dev);
7126         ret = PTR_ERR_OR_ZERO(state);
7127         if (ret)
7128                 DRM_ERROR("Suspending crtc's failed with %i\n", ret);
7129         else
7130                 dev_priv->modeset_restore_state = state;
7131         return ret;
7132 }
7133
7134 void intel_encoder_destroy(struct drm_encoder *encoder)
7135 {
7136         struct intel_encoder *intel_encoder = to_intel_encoder(encoder);
7137
7138         drm_encoder_cleanup(encoder);
7139         kfree(intel_encoder);
7140 }
7141
7142 /* Cross check the actual hw state with our own modeset state tracking (and it's
7143  * internal consistency). */
7144 static void intel_connector_verify_state(struct intel_crtc_state *crtc_state,
7145                                          struct drm_connector_state *conn_state)
7146 {
7147         struct intel_connector *connector = to_intel_connector(conn_state->connector);
7148
7149         DRM_DEBUG_KMS("[CONNECTOR:%d:%s]\n",
7150                       connector->base.base.id,
7151                       connector->base.name);
7152
7153         if (connector->get_hw_state(connector)) {
7154                 struct intel_encoder *encoder = connector->encoder;
7155
7156                 I915_STATE_WARN(!crtc_state,
7157                          "connector enabled without attached crtc\n");
7158
7159                 if (!crtc_state)
7160                         return;
7161
7162                 I915_STATE_WARN(!crtc_state->base.active,
7163                       "connector is active, but attached crtc isn't\n");
7164
7165                 if (!encoder || encoder->type == INTEL_OUTPUT_DP_MST)
7166                         return;
7167
7168                 I915_STATE_WARN(conn_state->best_encoder != &encoder->base,
7169                         "atomic encoder doesn't match attached encoder\n");
7170
7171                 I915_STATE_WARN(conn_state->crtc != encoder->base.crtc,
7172                         "attached encoder crtc differs from connector crtc\n");
7173         } else {
7174                 I915_STATE_WARN(crtc_state && crtc_state->base.active,
7175                         "attached crtc is active, but connector isn't\n");
7176                 I915_STATE_WARN(!crtc_state && conn_state->best_encoder,
7177                         "best encoder set without crtc!\n");
7178         }
7179 }
7180
7181 static int pipe_required_fdi_lanes(struct intel_crtc_state *crtc_state)
7182 {
7183         if (crtc_state->base.enable && crtc_state->has_pch_encoder)
7184                 return crtc_state->fdi_lanes;
7185
7186         return 0;
7187 }
7188
7189 static int ironlake_check_fdi_lanes(struct drm_device *dev, enum pipe pipe,
7190                                      struct intel_crtc_state *pipe_config)
7191 {
7192         struct drm_i915_private *dev_priv = to_i915(dev);
7193         struct drm_atomic_state *state = pipe_config->base.state;
7194         struct intel_crtc *other_crtc;
7195         struct intel_crtc_state *other_crtc_state;
7196
7197         DRM_DEBUG_KMS("checking fdi config on pipe %c, lanes %i\n",
7198                       pipe_name(pipe), pipe_config->fdi_lanes);
7199         if (pipe_config->fdi_lanes > 4) {
7200                 DRM_DEBUG_KMS("invalid fdi lane config on pipe %c: %i lanes\n",
7201                               pipe_name(pipe), pipe_config->fdi_lanes);
7202                 return -EINVAL;
7203         }
7204
7205         if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv)) {
7206                 if (pipe_config->fdi_lanes > 2) {
7207                         DRM_DEBUG_KMS("only 2 lanes on haswell, required: %i lanes\n",
7208                                       pipe_config->fdi_lanes);
7209                         return -EINVAL;
7210                 } else {
7211                         return 0;
7212                 }
7213         }
7214
7215         if (INTEL_INFO(dev_priv)->num_pipes == 2)
7216                 return 0;
7217
7218         /* Ivybridge 3 pipe is really complicated */
7219         switch (pipe) {
7220         case PIPE_A:
7221                 return 0;
7222         case PIPE_B:
7223                 if (pipe_config->fdi_lanes <= 2)
7224                         return 0;
7225
7226                 other_crtc = intel_get_crtc_for_pipe(dev_priv, PIPE_C);
7227                 other_crtc_state =
7228                         intel_atomic_get_crtc_state(state, other_crtc);
7229                 if (IS_ERR(other_crtc_state))
7230                         return PTR_ERR(other_crtc_state);
7231
7232                 if (pipe_required_fdi_lanes(other_crtc_state) > 0) {
7233                         DRM_DEBUG_KMS("invalid shared fdi lane config on pipe %c: %i lanes\n",
7234                                       pipe_name(pipe), pipe_config->fdi_lanes);
7235                         return -EINVAL;
7236                 }
7237                 return 0;
7238         case PIPE_C:
7239                 if (pipe_config->fdi_lanes > 2) {
7240                         DRM_DEBUG_KMS("only 2 lanes on pipe %c: required %i lanes\n",
7241                                       pipe_name(pipe), pipe_config->fdi_lanes);
7242                         return -EINVAL;
7243                 }
7244
7245                 other_crtc = intel_get_crtc_for_pipe(dev_priv, PIPE_B);
7246                 other_crtc_state =
7247                         intel_atomic_get_crtc_state(state, other_crtc);
7248                 if (IS_ERR(other_crtc_state))
7249                         return PTR_ERR(other_crtc_state);
7250
7251                 if (pipe_required_fdi_lanes(other_crtc_state) > 2) {
7252                         DRM_DEBUG_KMS("fdi link B uses too many lanes to enable link C\n");
7253                         return -EINVAL;
7254                 }
7255                 return 0;
7256         default:
7257                 BUG();
7258         }
7259 }
7260
7261 #define RETRY 1
7262 static int ironlake_fdi_compute_config(struct intel_crtc *intel_crtc,
7263                                        struct intel_crtc_state *pipe_config)
7264 {
7265         struct drm_device *dev = intel_crtc->base.dev;
7266         const struct drm_display_mode *adjusted_mode = &pipe_config->base.adjusted_mode;
7267         int lane, link_bw, fdi_dotclock, ret;
7268         bool needs_recompute = false;
7269
7270 retry:
7271         /* FDI is a binary signal running at ~2.7GHz, encoding
7272          * each output octet as 10 bits. The actual frequency
7273          * is stored as a divider into a 100MHz clock, and the
7274          * mode pixel clock is stored in units of 1KHz.
7275          * Hence the bw of each lane in terms of the mode signal
7276          * is:
7277          */
7278         link_bw = intel_fdi_link_freq(to_i915(dev), pipe_config);
7279
7280         fdi_dotclock = adjusted_mode->crtc_clock;
7281
7282         lane = ironlake_get_lanes_required(fdi_dotclock, link_bw,
7283                                            pipe_config->pipe_bpp);
7284
7285         pipe_config->fdi_lanes = lane;
7286
7287         intel_link_compute_m_n(pipe_config->pipe_bpp, lane, fdi_dotclock,
7288                                link_bw, &pipe_config->fdi_m_n, false);
7289
7290         ret = ironlake_check_fdi_lanes(dev, intel_crtc->pipe, pipe_config);
7291         if (ret == -EDEADLK)
7292                 return ret;
7293
7294         if (ret == -EINVAL && pipe_config->pipe_bpp > 6*3) {
7295                 pipe_config->pipe_bpp -= 2*3;
7296                 DRM_DEBUG_KMS("fdi link bw constraint, reducing pipe bpp to %i\n",
7297                               pipe_config->pipe_bpp);
7298                 needs_recompute = true;
7299                 pipe_config->bw_constrained = true;
7300
7301                 goto retry;
7302         }
7303
7304         if (needs_recompute)
7305                 return RETRY;
7306
7307         return ret;
7308 }
7309
7310 bool hsw_crtc_state_ips_capable(const struct intel_crtc_state *crtc_state)
7311 {
7312         struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
7313         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
7314
7315         /* IPS only exists on ULT machines and is tied to pipe A. */
7316         if (!hsw_crtc_supports_ips(crtc))
7317                 return false;
7318
7319         if (!i915_modparams.enable_ips)
7320                 return false;
7321
7322         if (crtc_state->pipe_bpp > 24)
7323                 return false;
7324
7325         /*
7326          * We compare against max which means we must take
7327          * the increased cdclk requirement into account when
7328          * calculating the new cdclk.
7329          *
7330          * Should measure whether using a lower cdclk w/o IPS
7331          */
7332         if (IS_BROADWELL(dev_priv) &&
7333             crtc_state->pixel_rate > dev_priv->max_cdclk_freq * 95 / 100)
7334                 return false;
7335
7336         return true;
7337 }
7338
7339 static bool hsw_compute_ips_config(struct intel_crtc_state *crtc_state)
7340 {
7341         struct drm_i915_private *dev_priv =
7342                 to_i915(crtc_state->base.crtc->dev);
7343         struct intel_atomic_state *intel_state =
7344                 to_intel_atomic_state(crtc_state->base.state);
7345
7346         if (!hsw_crtc_state_ips_capable(crtc_state))
7347                 return false;
7348
7349         /*
7350          * When IPS gets enabled, the pipe CRC changes. Since IPS gets
7351          * enabled and disabled dynamically based on package C states,
7352          * user space can't make reliable use of the CRCs, so let's just
7353          * completely disable it.
7354          */
7355         if (crtc_state->crc_enabled)
7356                 return false;
7357
7358         /* IPS should be fine as long as at least one plane is enabled. */
7359         if (!(crtc_state->active_planes & ~BIT(PLANE_CURSOR)))
7360                 return false;
7361
7362         /* pixel rate mustn't exceed 95% of cdclk with IPS on BDW */
7363         if (IS_BROADWELL(dev_priv) &&
7364             crtc_state->pixel_rate > intel_state->cdclk.logical.cdclk * 95 / 100)
7365                 return false;
7366
7367         return true;
7368 }
7369
7370 static bool intel_crtc_supports_double_wide(const struct intel_crtc *crtc)
7371 {
7372         const struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
7373
7374         /* GDG double wide on either pipe, otherwise pipe A only */
7375         return INTEL_GEN(dev_priv) < 4 &&
7376                 (crtc->pipe == PIPE_A || IS_I915G(dev_priv));
7377 }
7378
7379 static u32 ilk_pipe_pixel_rate(const struct intel_crtc_state *pipe_config)
7380 {
7381         u32 pixel_rate;
7382
7383         pixel_rate = pipe_config->base.adjusted_mode.crtc_clock;
7384
7385         /*
7386          * We only use IF-ID interlacing. If we ever use
7387          * PF-ID we'll need to adjust the pixel_rate here.
7388          */
7389
7390         if (pipe_config->pch_pfit.enabled) {
7391                 u64 pipe_w, pipe_h, pfit_w, pfit_h;
7392                 u32 pfit_size = pipe_config->pch_pfit.size;
7393
7394                 pipe_w = pipe_config->pipe_src_w;
7395                 pipe_h = pipe_config->pipe_src_h;
7396
7397                 pfit_w = (pfit_size >> 16) & 0xFFFF;
7398                 pfit_h = pfit_size & 0xFFFF;
7399                 if (pipe_w < pfit_w)
7400                         pipe_w = pfit_w;
7401                 if (pipe_h < pfit_h)
7402                         pipe_h = pfit_h;
7403
7404                 if (WARN_ON(!pfit_w || !pfit_h))
7405                         return pixel_rate;
7406
7407                 pixel_rate = div_u64(mul_u32_u32(pixel_rate, pipe_w * pipe_h),
7408                                      pfit_w * pfit_h);
7409         }
7410
7411         return pixel_rate;
7412 }
7413
7414 static void intel_crtc_compute_pixel_rate(struct intel_crtc_state *crtc_state)
7415 {
7416         struct drm_i915_private *dev_priv = to_i915(crtc_state->base.crtc->dev);
7417
7418         if (HAS_GMCH(dev_priv))
7419                 /* FIXME calculate proper pipe pixel rate for GMCH pfit */
7420                 crtc_state->pixel_rate =
7421                         crtc_state->base.adjusted_mode.crtc_clock;
7422         else
7423                 crtc_state->pixel_rate =
7424                         ilk_pipe_pixel_rate(crtc_state);
7425 }
7426
7427 static int intel_crtc_compute_config(struct intel_crtc *crtc,
7428                                      struct intel_crtc_state *pipe_config)
7429 {
7430         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
7431         const struct drm_display_mode *adjusted_mode = &pipe_config->base.adjusted_mode;
7432         int clock_limit = dev_priv->max_dotclk_freq;
7433
7434         if (INTEL_GEN(dev_priv) < 4) {
7435                 clock_limit = dev_priv->max_cdclk_freq * 9 / 10;
7436
7437                 /*
7438                  * Enable double wide mode when the dot clock
7439                  * is > 90% of the (display) core speed.
7440                  */
7441                 if (intel_crtc_supports_double_wide(crtc) &&
7442                     adjusted_mode->crtc_clock > clock_limit) {
7443                         clock_limit = dev_priv->max_dotclk_freq;
7444                         pipe_config->double_wide = true;
7445                 }
7446         }
7447
7448         if (adjusted_mode->crtc_clock > clock_limit) {
7449                 DRM_DEBUG_KMS("requested pixel clock (%d kHz) too high (max: %d kHz, double wide: %s)\n",
7450                               adjusted_mode->crtc_clock, clock_limit,
7451                               yesno(pipe_config->double_wide));
7452                 return -EINVAL;
7453         }
7454
7455         if ((pipe_config->output_format == INTEL_OUTPUT_FORMAT_YCBCR420 ||
7456              pipe_config->output_format == INTEL_OUTPUT_FORMAT_YCBCR444) &&
7457              pipe_config->base.ctm) {
7458                 /*
7459                  * There is only one pipe CSC unit per pipe, and we need that
7460                  * for output conversion from RGB->YCBCR. So if CTM is already
7461                  * applied we can't support YCBCR420 output.
7462                  */
7463                 DRM_DEBUG_KMS("YCBCR420 and CTM together are not possible\n");
7464                 return -EINVAL;
7465         }
7466
7467         /*
7468          * Pipe horizontal size must be even in:
7469          * - DVO ganged mode
7470          * - LVDS dual channel mode
7471          * - Double wide pipe
7472          */
7473         if (pipe_config->pipe_src_w & 1) {
7474                 if (pipe_config->double_wide) {
7475                         DRM_DEBUG_KMS("Odd pipe source width not supported with double wide pipe\n");
7476                         return -EINVAL;
7477                 }
7478
7479                 if (intel_crtc_has_type(pipe_config, INTEL_OUTPUT_LVDS) &&
7480                     intel_is_dual_link_lvds(dev_priv)) {
7481                         DRM_DEBUG_KMS("Odd pipe source width not supported with dual link LVDS\n");
7482                         return -EINVAL;
7483                 }
7484         }
7485
7486         /* Cantiga+ cannot handle modes with a hsync front porch of 0.
7487          * WaPruneModeWithIncorrectHsyncOffset:ctg,elk,ilk,snb,ivb,vlv,hsw.
7488          */
7489         if ((INTEL_GEN(dev_priv) > 4 || IS_G4X(dev_priv)) &&
7490                 adjusted_mode->crtc_hsync_start == adjusted_mode->crtc_hdisplay)
7491                 return -EINVAL;
7492
7493         intel_crtc_compute_pixel_rate(pipe_config);
7494
7495         if (pipe_config->has_pch_encoder)
7496                 return ironlake_fdi_compute_config(crtc, pipe_config);
7497
7498         return 0;
7499 }
7500
7501 static void
7502 intel_reduce_m_n_ratio(u32 *num, u32 *den)
7503 {
7504         while (*num > DATA_LINK_M_N_MASK ||
7505                *den > DATA_LINK_M_N_MASK) {
7506                 *num >>= 1;
7507                 *den >>= 1;
7508         }
7509 }
7510
7511 static void compute_m_n(unsigned int m, unsigned int n,
7512                         u32 *ret_m, u32 *ret_n,
7513                         bool constant_n)
7514 {
7515         /*
7516          * Several DP dongles in particular seem to be fussy about
7517          * too large link M/N values. Give N value as 0x8000 that
7518          * should be acceptable by specific devices. 0x8000 is the
7519          * specified fixed N value for asynchronous clock mode,
7520          * which the devices expect also in synchronous clock mode.
7521          */
7522         if (constant_n)
7523                 *ret_n = 0x8000;
7524         else
7525                 *ret_n = min_t(unsigned int, roundup_pow_of_two(n), DATA_LINK_N_MAX);
7526
7527         *ret_m = div_u64(mul_u32_u32(m, *ret_n), n);
7528         intel_reduce_m_n_ratio(ret_m, ret_n);
7529 }
7530
7531 void
7532 intel_link_compute_m_n(u16 bits_per_pixel, int nlanes,
7533                        int pixel_clock, int link_clock,
7534                        struct intel_link_m_n *m_n,
7535                        bool constant_n)
7536 {
7537         m_n->tu = 64;
7538
7539         compute_m_n(bits_per_pixel * pixel_clock,
7540                     link_clock * nlanes * 8,
7541                     &m_n->gmch_m, &m_n->gmch_n,
7542                     constant_n);
7543
7544         compute_m_n(pixel_clock, link_clock,
7545                     &m_n->link_m, &m_n->link_n,
7546                     constant_n);
7547 }
7548
7549 static inline bool intel_panel_use_ssc(struct drm_i915_private *dev_priv)
7550 {
7551         if (i915_modparams.panel_use_ssc >= 0)
7552                 return i915_modparams.panel_use_ssc != 0;
7553         return dev_priv->vbt.lvds_use_ssc
7554                 && !(dev_priv->quirks & QUIRK_LVDS_SSC_DISABLE);
7555 }
7556
7557 static u32 pnv_dpll_compute_fp(struct dpll *dpll)
7558 {
7559         return (1 << dpll->n) << 16 | dpll->m2;
7560 }
7561
7562 static u32 i9xx_dpll_compute_fp(struct dpll *dpll)
7563 {
7564         return dpll->n << 16 | dpll->m1 << 8 | dpll->m2;
7565 }
7566
7567 static void i9xx_update_pll_dividers(struct intel_crtc *crtc,
7568                                      struct intel_crtc_state *crtc_state,
7569                                      struct dpll *reduced_clock)
7570 {
7571         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
7572         u32 fp, fp2 = 0;
7573
7574         if (IS_PINEVIEW(dev_priv)) {
7575                 fp = pnv_dpll_compute_fp(&crtc_state->dpll);
7576                 if (reduced_clock)
7577                         fp2 = pnv_dpll_compute_fp(reduced_clock);
7578         } else {
7579                 fp = i9xx_dpll_compute_fp(&crtc_state->dpll);
7580                 if (reduced_clock)
7581                         fp2 = i9xx_dpll_compute_fp(reduced_clock);
7582         }
7583
7584         crtc_state->dpll_hw_state.fp0 = fp;
7585
7586         if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_LVDS) &&
7587             reduced_clock) {
7588                 crtc_state->dpll_hw_state.fp1 = fp2;
7589         } else {
7590                 crtc_state->dpll_hw_state.fp1 = fp;
7591         }
7592 }
7593
7594 static void vlv_pllb_recal_opamp(struct drm_i915_private *dev_priv, enum pipe
7595                 pipe)
7596 {
7597         u32 reg_val;
7598
7599         /*
7600          * PLLB opamp always calibrates to max value of 0x3f, force enable it
7601          * and set it to a reasonable value instead.
7602          */
7603         reg_val = vlv_dpio_read(dev_priv, pipe, VLV_PLL_DW9(1));
7604         reg_val &= 0xffffff00;
7605         reg_val |= 0x00000030;
7606         vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW9(1), reg_val);
7607
7608         reg_val = vlv_dpio_read(dev_priv, pipe, VLV_REF_DW13);
7609         reg_val &= 0x00ffffff;
7610         reg_val |= 0x8c000000;
7611         vlv_dpio_write(dev_priv, pipe, VLV_REF_DW13, reg_val);
7612
7613         reg_val = vlv_dpio_read(dev_priv, pipe, VLV_PLL_DW9(1));
7614         reg_val &= 0xffffff00;
7615         vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW9(1), reg_val);
7616
7617         reg_val = vlv_dpio_read(dev_priv, pipe, VLV_REF_DW13);
7618         reg_val &= 0x00ffffff;
7619         reg_val |= 0xb0000000;
7620         vlv_dpio_write(dev_priv, pipe, VLV_REF_DW13, reg_val);
7621 }
7622
7623 static void intel_pch_transcoder_set_m_n(const struct intel_crtc_state *crtc_state,
7624                                          const struct intel_link_m_n *m_n)
7625 {
7626         struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
7627         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
7628         enum pipe pipe = crtc->pipe;
7629
7630         I915_WRITE(PCH_TRANS_DATA_M1(pipe), TU_SIZE(m_n->tu) | m_n->gmch_m);
7631         I915_WRITE(PCH_TRANS_DATA_N1(pipe), m_n->gmch_n);
7632         I915_WRITE(PCH_TRANS_LINK_M1(pipe), m_n->link_m);
7633         I915_WRITE(PCH_TRANS_LINK_N1(pipe), m_n->link_n);
7634 }
7635
7636 static bool transcoder_has_m2_n2(struct drm_i915_private *dev_priv,
7637                                  enum transcoder transcoder)
7638 {
7639         if (IS_HASWELL(dev_priv))
7640                 return transcoder == TRANSCODER_EDP;
7641
7642         /*
7643          * Strictly speaking some registers are available before
7644          * gen7, but we only support DRRS on gen7+
7645          */
7646         return IS_GEN(dev_priv, 7) || IS_CHERRYVIEW(dev_priv);
7647 }
7648
7649 static void intel_cpu_transcoder_set_m_n(const struct intel_crtc_state *crtc_state,
7650                                          const struct intel_link_m_n *m_n,
7651                                          const struct intel_link_m_n *m2_n2)
7652 {
7653         struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
7654         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
7655         enum pipe pipe = crtc->pipe;
7656         enum transcoder transcoder = crtc_state->cpu_transcoder;
7657
7658         if (INTEL_GEN(dev_priv) >= 5) {
7659                 I915_WRITE(PIPE_DATA_M1(transcoder), TU_SIZE(m_n->tu) | m_n->gmch_m);
7660                 I915_WRITE(PIPE_DATA_N1(transcoder), m_n->gmch_n);
7661                 I915_WRITE(PIPE_LINK_M1(transcoder), m_n->link_m);
7662                 I915_WRITE(PIPE_LINK_N1(transcoder), m_n->link_n);
7663                 /*
7664                  *  M2_N2 registers are set only if DRRS is supported
7665                  * (to make sure the registers are not unnecessarily accessed).
7666                  */
7667                 if (m2_n2 && crtc_state->has_drrs &&
7668                     transcoder_has_m2_n2(dev_priv, transcoder)) {
7669                         I915_WRITE(PIPE_DATA_M2(transcoder),
7670                                         TU_SIZE(m2_n2->tu) | m2_n2->gmch_m);
7671                         I915_WRITE(PIPE_DATA_N2(transcoder), m2_n2->gmch_n);
7672                         I915_WRITE(PIPE_LINK_M2(transcoder), m2_n2->link_m);
7673                         I915_WRITE(PIPE_LINK_N2(transcoder), m2_n2->link_n);
7674                 }
7675         } else {
7676                 I915_WRITE(PIPE_DATA_M_G4X(pipe), TU_SIZE(m_n->tu) | m_n->gmch_m);
7677                 I915_WRITE(PIPE_DATA_N_G4X(pipe), m_n->gmch_n);
7678                 I915_WRITE(PIPE_LINK_M_G4X(pipe), m_n->link_m);
7679                 I915_WRITE(PIPE_LINK_N_G4X(pipe), m_n->link_n);
7680         }
7681 }
7682
7683 void intel_dp_set_m_n(const struct intel_crtc_state *crtc_state, enum link_m_n_set m_n)
7684 {
7685         const struct intel_link_m_n *dp_m_n, *dp_m2_n2 = NULL;
7686
7687         if (m_n == M1_N1) {
7688                 dp_m_n = &crtc_state->dp_m_n;
7689                 dp_m2_n2 = &crtc_state->dp_m2_n2;
7690         } else if (m_n == M2_N2) {
7691
7692                 /*
7693                  * M2_N2 registers are not supported. Hence m2_n2 divider value
7694                  * needs to be programmed into M1_N1.
7695                  */
7696                 dp_m_n = &crtc_state->dp_m2_n2;
7697         } else {
7698                 DRM_ERROR("Unsupported divider value\n");
7699                 return;
7700         }
7701
7702         if (crtc_state->has_pch_encoder)
7703                 intel_pch_transcoder_set_m_n(crtc_state, &crtc_state->dp_m_n);
7704         else
7705                 intel_cpu_transcoder_set_m_n(crtc_state, dp_m_n, dp_m2_n2);
7706 }
7707
7708 static void vlv_compute_dpll(struct intel_crtc *crtc,
7709                              struct intel_crtc_state *pipe_config)
7710 {
7711         pipe_config->dpll_hw_state.dpll = DPLL_INTEGRATED_REF_CLK_VLV |
7712                 DPLL_REF_CLK_ENABLE_VLV | DPLL_VGA_MODE_DIS;
7713         if (crtc->pipe != PIPE_A)
7714                 pipe_config->dpll_hw_state.dpll |= DPLL_INTEGRATED_CRI_CLK_VLV;
7715
7716         /* DPLL not used with DSI, but still need the rest set up */
7717         if (!intel_crtc_has_type(pipe_config, INTEL_OUTPUT_DSI))
7718                 pipe_config->dpll_hw_state.dpll |= DPLL_VCO_ENABLE |
7719                         DPLL_EXT_BUFFER_ENABLE_VLV;
7720
7721         pipe_config->dpll_hw_state.dpll_md =
7722                 (pipe_config->pixel_multiplier - 1) << DPLL_MD_UDI_MULTIPLIER_SHIFT;
7723 }
7724
7725 static void chv_compute_dpll(struct intel_crtc *crtc,
7726                              struct intel_crtc_state *pipe_config)
7727 {
7728         pipe_config->dpll_hw_state.dpll = DPLL_SSC_REF_CLK_CHV |
7729                 DPLL_REF_CLK_ENABLE_VLV | DPLL_VGA_MODE_DIS;
7730         if (crtc->pipe != PIPE_A)
7731                 pipe_config->dpll_hw_state.dpll |= DPLL_INTEGRATED_CRI_CLK_VLV;
7732
7733         /* DPLL not used with DSI, but still need the rest set up */
7734         if (!intel_crtc_has_type(pipe_config, INTEL_OUTPUT_DSI))
7735                 pipe_config->dpll_hw_state.dpll |= DPLL_VCO_ENABLE;
7736
7737         pipe_config->dpll_hw_state.dpll_md =
7738                 (pipe_config->pixel_multiplier - 1) << DPLL_MD_UDI_MULTIPLIER_SHIFT;
7739 }
7740
7741 static void vlv_prepare_pll(struct intel_crtc *crtc,
7742                             const struct intel_crtc_state *pipe_config)
7743 {
7744         struct drm_device *dev = crtc->base.dev;
7745         struct drm_i915_private *dev_priv = to_i915(dev);
7746         enum pipe pipe = crtc->pipe;
7747         u32 mdiv;
7748         u32 bestn, bestm1, bestm2, bestp1, bestp2;
7749         u32 coreclk, reg_val;
7750
7751         /* Enable Refclk */
7752         I915_WRITE(DPLL(pipe),
7753                    pipe_config->dpll_hw_state.dpll &
7754                    ~(DPLL_VCO_ENABLE | DPLL_EXT_BUFFER_ENABLE_VLV));
7755
7756         /* No need to actually set up the DPLL with DSI */
7757         if ((pipe_config->dpll_hw_state.dpll & DPLL_VCO_ENABLE) == 0)
7758                 return;
7759
7760         vlv_dpio_get(dev_priv);
7761
7762         bestn = pipe_config->dpll.n;
7763         bestm1 = pipe_config->dpll.m1;
7764         bestm2 = pipe_config->dpll.m2;
7765         bestp1 = pipe_config->dpll.p1;
7766         bestp2 = pipe_config->dpll.p2;
7767
7768         /* See eDP HDMI DPIO driver vbios notes doc */
7769
7770         /* PLL B needs special handling */
7771         if (pipe == PIPE_B)
7772                 vlv_pllb_recal_opamp(dev_priv, pipe);
7773
7774         /* Set up Tx target for periodic Rcomp update */
7775         vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW9_BCAST, 0x0100000f);
7776
7777         /* Disable target IRef on PLL */
7778         reg_val = vlv_dpio_read(dev_priv, pipe, VLV_PLL_DW8(pipe));
7779         reg_val &= 0x00ffffff;
7780         vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW8(pipe), reg_val);
7781
7782         /* Disable fast lock */
7783         vlv_dpio_write(dev_priv, pipe, VLV_CMN_DW0, 0x610);
7784
7785         /* Set idtafcrecal before PLL is enabled */
7786         mdiv = ((bestm1 << DPIO_M1DIV_SHIFT) | (bestm2 & DPIO_M2DIV_MASK));
7787         mdiv |= ((bestp1 << DPIO_P1_SHIFT) | (bestp2 << DPIO_P2_SHIFT));
7788         mdiv |= ((bestn << DPIO_N_SHIFT));
7789         mdiv |= (1 << DPIO_K_SHIFT);
7790
7791         /*
7792          * Post divider depends on pixel clock rate, DAC vs digital (and LVDS,
7793          * but we don't support that).
7794          * Note: don't use the DAC post divider as it seems unstable.
7795          */
7796         mdiv |= (DPIO_POST_DIV_HDMIDP << DPIO_POST_DIV_SHIFT);
7797         vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW3(pipe), mdiv);
7798
7799         mdiv |= DPIO_ENABLE_CALIBRATION;
7800         vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW3(pipe), mdiv);
7801
7802         /* Set HBR and RBR LPF coefficients */
7803         if (pipe_config->port_clock == 162000 ||
7804             intel_crtc_has_type(pipe_config, INTEL_OUTPUT_ANALOG) ||
7805             intel_crtc_has_type(pipe_config, INTEL_OUTPUT_HDMI))
7806                 vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW10(pipe),
7807                                  0x009f0003);
7808         else
7809                 vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW10(pipe),
7810                                  0x00d0000f);
7811
7812         if (intel_crtc_has_dp_encoder(pipe_config)) {
7813                 /* Use SSC source */
7814                 if (pipe == PIPE_A)
7815                         vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW5(pipe),
7816                                          0x0df40000);
7817                 else
7818                         vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW5(pipe),
7819                                          0x0df70000);
7820         } else { /* HDMI or VGA */
7821                 /* Use bend source */
7822                 if (pipe == PIPE_A)
7823                         vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW5(pipe),
7824                                          0x0df70000);
7825                 else
7826                         vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW5(pipe),
7827                                          0x0df40000);
7828         }
7829
7830         coreclk = vlv_dpio_read(dev_priv, pipe, VLV_PLL_DW7(pipe));
7831         coreclk = (coreclk & 0x0000ff00) | 0x01c00000;
7832         if (intel_crtc_has_dp_encoder(pipe_config))
7833                 coreclk |= 0x01000000;
7834         vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW7(pipe), coreclk);
7835
7836         vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW11(pipe), 0x87871000);
7837
7838         vlv_dpio_put(dev_priv);
7839 }
7840
7841 static void chv_prepare_pll(struct intel_crtc *crtc,
7842                             const struct intel_crtc_state *pipe_config)
7843 {
7844         struct drm_device *dev = crtc->base.dev;
7845         struct drm_i915_private *dev_priv = to_i915(dev);
7846         enum pipe pipe = crtc->pipe;
7847         enum dpio_channel port = vlv_pipe_to_channel(pipe);
7848         u32 loopfilter, tribuf_calcntr;
7849         u32 bestn, bestm1, bestm2, bestp1, bestp2, bestm2_frac;
7850         u32 dpio_val;
7851         int vco;
7852
7853         /* Enable Refclk and SSC */
7854         I915_WRITE(DPLL(pipe),
7855                    pipe_config->dpll_hw_state.dpll & ~DPLL_VCO_ENABLE);
7856
7857         /* No need to actually set up the DPLL with DSI */
7858         if ((pipe_config->dpll_hw_state.dpll & DPLL_VCO_ENABLE) == 0)
7859                 return;
7860
7861         bestn = pipe_config->dpll.n;
7862         bestm2_frac = pipe_config->dpll.m2 & 0x3fffff;
7863         bestm1 = pipe_config->dpll.m1;
7864         bestm2 = pipe_config->dpll.m2 >> 22;
7865         bestp1 = pipe_config->dpll.p1;
7866         bestp2 = pipe_config->dpll.p2;
7867         vco = pipe_config->dpll.vco;
7868         dpio_val = 0;
7869         loopfilter = 0;
7870
7871         vlv_dpio_get(dev_priv);
7872
7873         /* p1 and p2 divider */
7874         vlv_dpio_write(dev_priv, pipe, CHV_CMN_DW13(port),
7875                         5 << DPIO_CHV_S1_DIV_SHIFT |
7876                         bestp1 << DPIO_CHV_P1_DIV_SHIFT |
7877                         bestp2 << DPIO_CHV_P2_DIV_SHIFT |
7878                         1 << DPIO_CHV_K_DIV_SHIFT);
7879
7880         /* Feedback post-divider - m2 */
7881         vlv_dpio_write(dev_priv, pipe, CHV_PLL_DW0(port), bestm2);
7882
7883         /* Feedback refclk divider - n and m1 */
7884         vlv_dpio_write(dev_priv, pipe, CHV_PLL_DW1(port),
7885                         DPIO_CHV_M1_DIV_BY_2 |
7886                         1 << DPIO_CHV_N_DIV_SHIFT);
7887
7888         /* M2 fraction division */
7889         vlv_dpio_write(dev_priv, pipe, CHV_PLL_DW2(port), bestm2_frac);
7890
7891         /* M2 fraction division enable */
7892         dpio_val = vlv_dpio_read(dev_priv, pipe, CHV_PLL_DW3(port));
7893         dpio_val &= ~(DPIO_CHV_FEEDFWD_GAIN_MASK | DPIO_CHV_FRAC_DIV_EN);
7894         dpio_val |= (2 << DPIO_CHV_FEEDFWD_GAIN_SHIFT);
7895         if (bestm2_frac)
7896                 dpio_val |= DPIO_CHV_FRAC_DIV_EN;
7897         vlv_dpio_write(dev_priv, pipe, CHV_PLL_DW3(port), dpio_val);
7898
7899         /* Program digital lock detect threshold */
7900         dpio_val = vlv_dpio_read(dev_priv, pipe, CHV_PLL_DW9(port));
7901         dpio_val &= ~(DPIO_CHV_INT_LOCK_THRESHOLD_MASK |
7902                                         DPIO_CHV_INT_LOCK_THRESHOLD_SEL_COARSE);
7903         dpio_val |= (0x5 << DPIO_CHV_INT_LOCK_THRESHOLD_SHIFT);
7904         if (!bestm2_frac)
7905                 dpio_val |= DPIO_CHV_INT_LOCK_THRESHOLD_SEL_COARSE;
7906         vlv_dpio_write(dev_priv, pipe, CHV_PLL_DW9(port), dpio_val);
7907
7908         /* Loop filter */
7909         if (vco == 5400000) {
7910                 loopfilter |= (0x3 << DPIO_CHV_PROP_COEFF_SHIFT);
7911                 loopfilter |= (0x8 << DPIO_CHV_INT_COEFF_SHIFT);
7912                 loopfilter |= (0x1 << DPIO_CHV_GAIN_CTRL_SHIFT);
7913                 tribuf_calcntr = 0x9;
7914         } else if (vco <= 6200000) {
7915                 loopfilter |= (0x5 << DPIO_CHV_PROP_COEFF_SHIFT);
7916                 loopfilter |= (0xB << DPIO_CHV_INT_COEFF_SHIFT);
7917                 loopfilter |= (0x3 << DPIO_CHV_GAIN_CTRL_SHIFT);
7918                 tribuf_calcntr = 0x9;
7919         } else if (vco <= 6480000) {
7920                 loopfilter |= (0x4 << DPIO_CHV_PROP_COEFF_SHIFT);
7921                 loopfilter |= (0x9 << DPIO_CHV_INT_COEFF_SHIFT);
7922                 loopfilter |= (0x3 << DPIO_CHV_GAIN_CTRL_SHIFT);
7923                 tribuf_calcntr = 0x8;
7924         } else {
7925                 /* Not supported. Apply the same limits as in the max case */
7926                 loopfilter |= (0x4 << DPIO_CHV_PROP_COEFF_SHIFT);
7927                 loopfilter |= (0x9 << DPIO_CHV_INT_COEFF_SHIFT);
7928                 loopfilter |= (0x3 << DPIO_CHV_GAIN_CTRL_SHIFT);
7929                 tribuf_calcntr = 0;
7930         }
7931         vlv_dpio_write(dev_priv, pipe, CHV_PLL_DW6(port), loopfilter);
7932
7933         dpio_val = vlv_dpio_read(dev_priv, pipe, CHV_PLL_DW8(port));
7934         dpio_val &= ~DPIO_CHV_TDC_TARGET_CNT_MASK;
7935         dpio_val |= (tribuf_calcntr << DPIO_CHV_TDC_TARGET_CNT_SHIFT);
7936         vlv_dpio_write(dev_priv, pipe, CHV_PLL_DW8(port), dpio_val);
7937
7938         /* AFC Recal */
7939         vlv_dpio_write(dev_priv, pipe, CHV_CMN_DW14(port),
7940                         vlv_dpio_read(dev_priv, pipe, CHV_CMN_DW14(port)) |
7941                         DPIO_AFC_RECAL);
7942
7943         vlv_dpio_put(dev_priv);
7944 }
7945
7946 /**
7947  * vlv_force_pll_on - forcibly enable just the PLL
7948  * @dev_priv: i915 private structure
7949  * @pipe: pipe PLL to enable
7950  * @dpll: PLL configuration
7951  *
7952  * Enable the PLL for @pipe using the supplied @dpll config. To be used
7953  * in cases where we need the PLL enabled even when @pipe is not going to
7954  * be enabled.
7955  */
7956 int vlv_force_pll_on(struct drm_i915_private *dev_priv, enum pipe pipe,
7957                      const struct dpll *dpll)
7958 {
7959         struct intel_crtc *crtc = intel_get_crtc_for_pipe(dev_priv, pipe);
7960         struct intel_crtc_state *pipe_config;
7961
7962         pipe_config = kzalloc(sizeof(*pipe_config), GFP_KERNEL);
7963         if (!pipe_config)
7964                 return -ENOMEM;
7965
7966         pipe_config->base.crtc = &crtc->base;
7967         pipe_config->pixel_multiplier = 1;
7968         pipe_config->dpll = *dpll;
7969
7970         if (IS_CHERRYVIEW(dev_priv)) {
7971                 chv_compute_dpll(crtc, pipe_config);
7972                 chv_prepare_pll(crtc, pipe_config);
7973                 chv_enable_pll(crtc, pipe_config);
7974         } else {
7975                 vlv_compute_dpll(crtc, pipe_config);
7976                 vlv_prepare_pll(crtc, pipe_config);
7977                 vlv_enable_pll(crtc, pipe_config);
7978         }
7979
7980         kfree(pipe_config);
7981
7982         return 0;
7983 }
7984
7985 /**
7986  * vlv_force_pll_off - forcibly disable just the PLL
7987  * @dev_priv: i915 private structure
7988  * @pipe: pipe PLL to disable
7989  *
7990  * Disable the PLL for @pipe. To be used in cases where we need
7991  * the PLL enabled even when @pipe is not going to be enabled.
7992  */
7993 void vlv_force_pll_off(struct drm_i915_private *dev_priv, enum pipe pipe)
7994 {
7995         if (IS_CHERRYVIEW(dev_priv))
7996                 chv_disable_pll(dev_priv, pipe);
7997         else
7998                 vlv_disable_pll(dev_priv, pipe);
7999 }
8000
8001 static void i9xx_compute_dpll(struct intel_crtc *crtc,
8002                               struct intel_crtc_state *crtc_state,
8003                               struct dpll *reduced_clock)
8004 {
8005         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
8006         u32 dpll;
8007         struct dpll *clock = &crtc_state->dpll;
8008
8009         i9xx_update_pll_dividers(crtc, crtc_state, reduced_clock);
8010
8011         dpll = DPLL_VGA_MODE_DIS;
8012
8013         if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_LVDS))
8014                 dpll |= DPLLB_MODE_LVDS;
8015         else
8016                 dpll |= DPLLB_MODE_DAC_SERIAL;
8017
8018         if (IS_I945G(dev_priv) || IS_I945GM(dev_priv) ||
8019             IS_G33(dev_priv) || IS_PINEVIEW(dev_priv)) {
8020                 dpll |= (crtc_state->pixel_multiplier - 1)
8021                         << SDVO_MULTIPLIER_SHIFT_HIRES;
8022         }
8023
8024         if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_SDVO) ||
8025             intel_crtc_has_type(crtc_state, INTEL_OUTPUT_HDMI))
8026                 dpll |= DPLL_SDVO_HIGH_SPEED;
8027
8028         if (intel_crtc_has_dp_encoder(crtc_state))
8029                 dpll |= DPLL_SDVO_HIGH_SPEED;
8030
8031         /* compute bitmask from p1 value */
8032         if (IS_PINEVIEW(dev_priv))
8033                 dpll |= (1 << (clock->p1 - 1)) << DPLL_FPA01_P1_POST_DIV_SHIFT_PINEVIEW;
8034         else {
8035                 dpll |= (1 << (clock->p1 - 1)) << DPLL_FPA01_P1_POST_DIV_SHIFT;
8036                 if (IS_G4X(dev_priv) && reduced_clock)
8037                         dpll |= (1 << (reduced_clock->p1 - 1)) << DPLL_FPA1_P1_POST_DIV_SHIFT;
8038         }
8039         switch (clock->p2) {
8040         case 5:
8041                 dpll |= DPLL_DAC_SERIAL_P2_CLOCK_DIV_5;
8042                 break;
8043         case 7:
8044                 dpll |= DPLLB_LVDS_P2_CLOCK_DIV_7;
8045                 break;
8046         case 10:
8047                 dpll |= DPLL_DAC_SERIAL_P2_CLOCK_DIV_10;
8048                 break;
8049         case 14:
8050                 dpll |= DPLLB_LVDS_P2_CLOCK_DIV_14;
8051                 break;
8052         }
8053         if (INTEL_GEN(dev_priv) >= 4)
8054                 dpll |= (6 << PLL_LOAD_PULSE_PHASE_SHIFT);
8055
8056         if (crtc_state->sdvo_tv_clock)
8057                 dpll |= PLL_REF_INPUT_TVCLKINBC;
8058         else if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_LVDS) &&
8059                  intel_panel_use_ssc(dev_priv))
8060                 dpll |= PLLB_REF_INPUT_SPREADSPECTRUMIN;
8061         else
8062                 dpll |= PLL_REF_INPUT_DREFCLK;
8063
8064         dpll |= DPLL_VCO_ENABLE;
8065         crtc_state->dpll_hw_state.dpll = dpll;
8066
8067         if (INTEL_GEN(dev_priv) >= 4) {
8068                 u32 dpll_md = (crtc_state->pixel_multiplier - 1)
8069                         << DPLL_MD_UDI_MULTIPLIER_SHIFT;
8070                 crtc_state->dpll_hw_state.dpll_md = dpll_md;
8071         }
8072 }
8073
8074 static void i8xx_compute_dpll(struct intel_crtc *crtc,
8075                               struct intel_crtc_state *crtc_state,
8076                               struct dpll *reduced_clock)
8077 {
8078         struct drm_device *dev = crtc->base.dev;
8079         struct drm_i915_private *dev_priv = to_i915(dev);
8080         u32 dpll;
8081         struct dpll *clock = &crtc_state->dpll;
8082
8083         i9xx_update_pll_dividers(crtc, crtc_state, reduced_clock);
8084
8085         dpll = DPLL_VGA_MODE_DIS;
8086
8087         if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_LVDS)) {
8088                 dpll |= (1 << (clock->p1 - 1)) << DPLL_FPA01_P1_POST_DIV_SHIFT;
8089         } else {
8090                 if (clock->p1 == 2)
8091                         dpll |= PLL_P1_DIVIDE_BY_TWO;
8092                 else
8093                         dpll |= (clock->p1 - 2) << DPLL_FPA01_P1_POST_DIV_SHIFT;
8094                 if (clock->p2 == 4)
8095                         dpll |= PLL_P2_DIVIDE_BY_4;
8096         }
8097
8098         /*
8099          * Bspec:
8100          * "[Almador Errata}: For the correct operation of the muxed DVO pins
8101          *  (GDEVSELB/I2Cdata, GIRDBY/I2CClk) and (GFRAMEB/DVI_Data,
8102          *  GTRDYB/DVI_Clk): Bit 31 (DPLL VCO Enable) and Bit 30 (2X Clock
8103          *  Enable) must be set to “1” in both the DPLL A Control Register
8104          *  (06014h-06017h) and DPLL B Control Register (06018h-0601Bh)."
8105          *
8106          * For simplicity We simply keep both bits always enabled in
8107          * both DPLLS. The spec says we should disable the DVO 2X clock
8108          * when not needed, but this seems to work fine in practice.
8109          */
8110         if (IS_I830(dev_priv) ||
8111             intel_crtc_has_type(crtc_state, INTEL_OUTPUT_DVO))
8112                 dpll |= DPLL_DVO_2X_MODE;
8113
8114         if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_LVDS) &&
8115             intel_panel_use_ssc(dev_priv))
8116                 dpll |= PLLB_REF_INPUT_SPREADSPECTRUMIN;
8117         else
8118                 dpll |= PLL_REF_INPUT_DREFCLK;
8119
8120         dpll |= DPLL_VCO_ENABLE;
8121         crtc_state->dpll_hw_state.dpll = dpll;
8122 }
8123
8124 static void intel_set_pipe_timings(const struct intel_crtc_state *crtc_state)
8125 {
8126         struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
8127         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
8128         enum pipe pipe = crtc->pipe;
8129         enum transcoder cpu_transcoder = crtc_state->cpu_transcoder;
8130         const struct drm_display_mode *adjusted_mode = &crtc_state->base.adjusted_mode;
8131         u32 crtc_vtotal, crtc_vblank_end;
8132         int vsyncshift = 0;
8133
8134         /* We need to be careful not to changed the adjusted mode, for otherwise
8135          * the hw state checker will get angry at the mismatch. */
8136         crtc_vtotal = adjusted_mode->crtc_vtotal;
8137         crtc_vblank_end = adjusted_mode->crtc_vblank_end;
8138
8139         if (adjusted_mode->flags & DRM_MODE_FLAG_INTERLACE) {
8140                 /* the chip adds 2 halflines automatically */
8141                 crtc_vtotal -= 1;
8142                 crtc_vblank_end -= 1;
8143
8144                 if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_SDVO))
8145                         vsyncshift = (adjusted_mode->crtc_htotal - 1) / 2;
8146                 else
8147                         vsyncshift = adjusted_mode->crtc_hsync_start -
8148                                 adjusted_mode->crtc_htotal / 2;
8149                 if (vsyncshift < 0)
8150                         vsyncshift += adjusted_mode->crtc_htotal;
8151         }
8152
8153         if (INTEL_GEN(dev_priv) > 3)
8154                 I915_WRITE(VSYNCSHIFT(cpu_transcoder), vsyncshift);
8155
8156         I915_WRITE(HTOTAL(cpu_transcoder),
8157                    (adjusted_mode->crtc_hdisplay - 1) |
8158                    ((adjusted_mode->crtc_htotal - 1) << 16));
8159         I915_WRITE(HBLANK(cpu_transcoder),
8160                    (adjusted_mode->crtc_hblank_start - 1) |
8161                    ((adjusted_mode->crtc_hblank_end - 1) << 16));
8162         I915_WRITE(HSYNC(cpu_transcoder),
8163                    (adjusted_mode->crtc_hsync_start - 1) |
8164                    ((adjusted_mode->crtc_hsync_end - 1) << 16));
8165
8166         I915_WRITE(VTOTAL(cpu_transcoder),
8167                    (adjusted_mode->crtc_vdisplay - 1) |
8168                    ((crtc_vtotal - 1) << 16));
8169         I915_WRITE(VBLANK(cpu_transcoder),
8170                    (adjusted_mode->crtc_vblank_start - 1) |
8171                    ((crtc_vblank_end - 1) << 16));
8172         I915_WRITE(VSYNC(cpu_transcoder),
8173                    (adjusted_mode->crtc_vsync_start - 1) |
8174                    ((adjusted_mode->crtc_vsync_end - 1) << 16));
8175
8176         /* Workaround: when the EDP input selection is B, the VTOTAL_B must be
8177          * programmed with the VTOTAL_EDP value. Same for VTOTAL_C. This is
8178          * documented on the DDI_FUNC_CTL register description, EDP Input Select
8179          * bits. */
8180         if (IS_HASWELL(dev_priv) && cpu_transcoder == TRANSCODER_EDP &&
8181             (pipe == PIPE_B || pipe == PIPE_C))
8182                 I915_WRITE(VTOTAL(pipe), I915_READ(VTOTAL(cpu_transcoder)));
8183
8184 }
8185
8186 static void intel_set_pipe_src_size(const struct intel_crtc_state *crtc_state)
8187 {
8188         struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
8189         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
8190         enum pipe pipe = crtc->pipe;
8191
8192         /* pipesrc controls the size that is scaled from, which should
8193          * always be the user's requested size.
8194          */
8195         I915_WRITE(PIPESRC(pipe),
8196                    ((crtc_state->pipe_src_w - 1) << 16) |
8197                    (crtc_state->pipe_src_h - 1));
8198 }
8199
8200 static void intel_get_pipe_timings(struct intel_crtc *crtc,
8201                                    struct intel_crtc_state *pipe_config)
8202 {
8203         struct drm_device *dev = crtc->base.dev;
8204         struct drm_i915_private *dev_priv = to_i915(dev);
8205         enum transcoder cpu_transcoder = pipe_config->cpu_transcoder;
8206         u32 tmp;
8207
8208         tmp = I915_READ(HTOTAL(cpu_transcoder));
8209         pipe_config->base.adjusted_mode.crtc_hdisplay = (tmp & 0xffff) + 1;
8210         pipe_config->base.adjusted_mode.crtc_htotal = ((tmp >> 16) & 0xffff) + 1;
8211
8212         if (!transcoder_is_dsi(cpu_transcoder)) {
8213                 tmp = I915_READ(HBLANK(cpu_transcoder));
8214                 pipe_config->base.adjusted_mode.crtc_hblank_start =
8215                                                         (tmp & 0xffff) + 1;
8216                 pipe_config->base.adjusted_mode.crtc_hblank_end =
8217                                                 ((tmp >> 16) & 0xffff) + 1;
8218         }
8219         tmp = I915_READ(HSYNC(cpu_transcoder));
8220         pipe_config->base.adjusted_mode.crtc_hsync_start = (tmp & 0xffff) + 1;
8221         pipe_config->base.adjusted_mode.crtc_hsync_end = ((tmp >> 16) & 0xffff) + 1;
8222
8223         tmp = I915_READ(VTOTAL(cpu_transcoder));
8224         pipe_config->base.adjusted_mode.crtc_vdisplay = (tmp & 0xffff) + 1;
8225         pipe_config->base.adjusted_mode.crtc_vtotal = ((tmp >> 16) & 0xffff) + 1;
8226
8227         if (!transcoder_is_dsi(cpu_transcoder)) {
8228                 tmp = I915_READ(VBLANK(cpu_transcoder));
8229                 pipe_config->base.adjusted_mode.crtc_vblank_start =
8230                                                         (tmp & 0xffff) + 1;
8231                 pipe_config->base.adjusted_mode.crtc_vblank_end =
8232                                                 ((tmp >> 16) & 0xffff) + 1;
8233         }
8234         tmp = I915_READ(VSYNC(cpu_transcoder));
8235         pipe_config->base.adjusted_mode.crtc_vsync_start = (tmp & 0xffff) + 1;
8236         pipe_config->base.adjusted_mode.crtc_vsync_end = ((tmp >> 16) & 0xffff) + 1;
8237
8238         if (I915_READ(PIPECONF(cpu_transcoder)) & PIPECONF_INTERLACE_MASK) {
8239                 pipe_config->base.adjusted_mode.flags |= DRM_MODE_FLAG_INTERLACE;
8240                 pipe_config->base.adjusted_mode.crtc_vtotal += 1;
8241                 pipe_config->base.adjusted_mode.crtc_vblank_end += 1;
8242         }
8243 }
8244
8245 static void intel_get_pipe_src_size(struct intel_crtc *crtc,
8246                                     struct intel_crtc_state *pipe_config)
8247 {
8248         struct drm_device *dev = crtc->base.dev;
8249         struct drm_i915_private *dev_priv = to_i915(dev);
8250         u32 tmp;
8251
8252         tmp = I915_READ(PIPESRC(crtc->pipe));
8253         pipe_config->pipe_src_h = (tmp & 0xffff) + 1;
8254         pipe_config->pipe_src_w = ((tmp >> 16) & 0xffff) + 1;
8255
8256         pipe_config->base.mode.vdisplay = pipe_config->pipe_src_h;
8257         pipe_config->base.mode.hdisplay = pipe_config->pipe_src_w;
8258 }
8259
8260 void intel_mode_from_pipe_config(struct drm_display_mode *mode,
8261                                  struct intel_crtc_state *pipe_config)
8262 {
8263         mode->hdisplay = pipe_config->base.adjusted_mode.crtc_hdisplay;
8264         mode->htotal = pipe_config->base.adjusted_mode.crtc_htotal;
8265         mode->hsync_start = pipe_config->base.adjusted_mode.crtc_hsync_start;
8266         mode->hsync_end = pipe_config->base.adjusted_mode.crtc_hsync_end;
8267
8268         mode->vdisplay = pipe_config->base.adjusted_mode.crtc_vdisplay;
8269         mode->vtotal = pipe_config->base.adjusted_mode.crtc_vtotal;
8270         mode->vsync_start = pipe_config->base.adjusted_mode.crtc_vsync_start;
8271         mode->vsync_end = pipe_config->base.adjusted_mode.crtc_vsync_end;
8272
8273         mode->flags = pipe_config->base.adjusted_mode.flags;
8274         mode->type = DRM_MODE_TYPE_DRIVER;
8275
8276         mode->clock = pipe_config->base.adjusted_mode.crtc_clock;
8277
8278         mode->hsync = drm_mode_hsync(mode);
8279         mode->vrefresh = drm_mode_vrefresh(mode);
8280         drm_mode_set_name(mode);
8281 }
8282
8283 static void i9xx_set_pipeconf(const struct intel_crtc_state *crtc_state)
8284 {
8285         struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
8286         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
8287         u32 pipeconf;
8288
8289         pipeconf = 0;
8290
8291         /* we keep both pipes enabled on 830 */
8292         if (IS_I830(dev_priv))
8293                 pipeconf |= I915_READ(PIPECONF(crtc->pipe)) & PIPECONF_ENABLE;
8294
8295         if (crtc_state->double_wide)
8296                 pipeconf |= PIPECONF_DOUBLE_WIDE;
8297
8298         /* only g4x and later have fancy bpc/dither controls */
8299         if (IS_G4X(dev_priv) || IS_VALLEYVIEW(dev_priv) ||
8300             IS_CHERRYVIEW(dev_priv)) {
8301                 /* Bspec claims that we can't use dithering for 30bpp pipes. */
8302                 if (crtc_state->dither && crtc_state->pipe_bpp != 30)
8303                         pipeconf |= PIPECONF_DITHER_EN |
8304                                     PIPECONF_DITHER_TYPE_SP;
8305
8306                 switch (crtc_state->pipe_bpp) {
8307                 case 18:
8308                         pipeconf |= PIPECONF_6BPC;
8309                         break;
8310                 case 24:
8311                         pipeconf |= PIPECONF_8BPC;
8312                         break;
8313                 case 30:
8314                         pipeconf |= PIPECONF_10BPC;
8315                         break;
8316                 default:
8317                         /* Case prevented by intel_choose_pipe_bpp_dither. */
8318                         BUG();
8319                 }
8320         }
8321
8322         if (crtc_state->base.adjusted_mode.flags & DRM_MODE_FLAG_INTERLACE) {
8323                 if (INTEL_GEN(dev_priv) < 4 ||
8324                     intel_crtc_has_type(crtc_state, INTEL_OUTPUT_SDVO))
8325                         pipeconf |= PIPECONF_INTERLACE_W_FIELD_INDICATION;
8326                 else
8327                         pipeconf |= PIPECONF_INTERLACE_W_SYNC_SHIFT;
8328         } else {
8329                 pipeconf |= PIPECONF_PROGRESSIVE;
8330         }
8331
8332         if ((IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) &&
8333              crtc_state->limited_color_range)
8334                 pipeconf |= PIPECONF_COLOR_RANGE_SELECT;
8335
8336         pipeconf |= PIPECONF_GAMMA_MODE(crtc_state->gamma_mode);
8337
8338         I915_WRITE(PIPECONF(crtc->pipe), pipeconf);
8339         POSTING_READ(PIPECONF(crtc->pipe));
8340 }
8341
8342 static int i8xx_crtc_compute_clock(struct intel_crtc *crtc,
8343                                    struct intel_crtc_state *crtc_state)
8344 {
8345         struct drm_device *dev = crtc->base.dev;
8346         struct drm_i915_private *dev_priv = to_i915(dev);
8347         const struct intel_limit *limit;
8348         int refclk = 48000;
8349
8350         memset(&crtc_state->dpll_hw_state, 0,
8351                sizeof(crtc_state->dpll_hw_state));
8352
8353         if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_LVDS)) {
8354                 if (intel_panel_use_ssc(dev_priv)) {
8355                         refclk = dev_priv->vbt.lvds_ssc_freq;
8356                         DRM_DEBUG_KMS("using SSC reference clock of %d kHz\n", refclk);
8357                 }
8358
8359                 limit = &intel_limits_i8xx_lvds;
8360         } else if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_DVO)) {
8361                 limit = &intel_limits_i8xx_dvo;
8362         } else {
8363                 limit = &intel_limits_i8xx_dac;
8364         }
8365
8366         if (!crtc_state->clock_set &&
8367             !i9xx_find_best_dpll(limit, crtc_state, crtc_state->port_clock,
8368                                  refclk, NULL, &crtc_state->dpll)) {
8369                 DRM_ERROR("Couldn't find PLL settings for mode!\n");
8370                 return -EINVAL;
8371         }
8372
8373         i8xx_compute_dpll(crtc, crtc_state, NULL);
8374
8375         return 0;
8376 }
8377
8378 static int g4x_crtc_compute_clock(struct intel_crtc *crtc,
8379                                   struct intel_crtc_state *crtc_state)
8380 {
8381         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
8382         const struct intel_limit *limit;
8383         int refclk = 96000;
8384
8385         memset(&crtc_state->dpll_hw_state, 0,
8386                sizeof(crtc_state->dpll_hw_state));
8387
8388         if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_LVDS)) {
8389                 if (intel_panel_use_ssc(dev_priv)) {
8390                         refclk = dev_priv->vbt.lvds_ssc_freq;
8391                         DRM_DEBUG_KMS("using SSC reference clock of %d kHz\n", refclk);
8392                 }
8393
8394                 if (intel_is_dual_link_lvds(dev_priv))
8395                         limit = &intel_limits_g4x_dual_channel_lvds;
8396                 else
8397                         limit = &intel_limits_g4x_single_channel_lvds;
8398         } else if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_HDMI) ||
8399                    intel_crtc_has_type(crtc_state, INTEL_OUTPUT_ANALOG)) {
8400                 limit = &intel_limits_g4x_hdmi;
8401         } else if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_SDVO)) {
8402                 limit = &intel_limits_g4x_sdvo;
8403         } else {
8404                 /* The option is for other outputs */
8405                 limit = &intel_limits_i9xx_sdvo;
8406         }
8407
8408         if (!crtc_state->clock_set &&
8409             !g4x_find_best_dpll(limit, crtc_state, crtc_state->port_clock,
8410                                 refclk, NULL, &crtc_state->dpll)) {
8411                 DRM_ERROR("Couldn't find PLL settings for mode!\n");
8412                 return -EINVAL;
8413         }
8414
8415         i9xx_compute_dpll(crtc, crtc_state, NULL);
8416
8417         return 0;
8418 }
8419
8420 static int pnv_crtc_compute_clock(struct intel_crtc *crtc,
8421                                   struct intel_crtc_state *crtc_state)
8422 {
8423         struct drm_device *dev = crtc->base.dev;
8424         struct drm_i915_private *dev_priv = to_i915(dev);
8425         const struct intel_limit *limit;
8426         int refclk = 96000;
8427
8428         memset(&crtc_state->dpll_hw_state, 0,
8429                sizeof(crtc_state->dpll_hw_state));
8430
8431         if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_LVDS)) {
8432                 if (intel_panel_use_ssc(dev_priv)) {
8433                         refclk = dev_priv->vbt.lvds_ssc_freq;
8434                         DRM_DEBUG_KMS("using SSC reference clock of %d kHz\n", refclk);
8435                 }
8436
8437                 limit = &intel_limits_pineview_lvds;
8438         } else {
8439                 limit = &intel_limits_pineview_sdvo;
8440         }
8441
8442         if (!crtc_state->clock_set &&
8443             !pnv_find_best_dpll(limit, crtc_state, crtc_state->port_clock,
8444                                 refclk, NULL, &crtc_state->dpll)) {
8445                 DRM_ERROR("Couldn't find PLL settings for mode!\n");
8446                 return -EINVAL;
8447         }
8448
8449         i9xx_compute_dpll(crtc, crtc_state, NULL);
8450
8451         return 0;
8452 }
8453
8454 static int i9xx_crtc_compute_clock(struct intel_crtc *crtc,
8455                                    struct intel_crtc_state *crtc_state)
8456 {
8457         struct drm_device *dev = crtc->base.dev;
8458         struct drm_i915_private *dev_priv = to_i915(dev);
8459         const struct intel_limit *limit;
8460         int refclk = 96000;
8461
8462         memset(&crtc_state->dpll_hw_state, 0,
8463                sizeof(crtc_state->dpll_hw_state));
8464
8465         if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_LVDS)) {
8466                 if (intel_panel_use_ssc(dev_priv)) {
8467                         refclk = dev_priv->vbt.lvds_ssc_freq;
8468                         DRM_DEBUG_KMS("using SSC reference clock of %d kHz\n", refclk);
8469                 }
8470
8471                 limit = &intel_limits_i9xx_lvds;
8472         } else {
8473                 limit = &intel_limits_i9xx_sdvo;
8474         }
8475
8476         if (!crtc_state->clock_set &&
8477             !i9xx_find_best_dpll(limit, crtc_state, crtc_state->port_clock,
8478                                  refclk, NULL, &crtc_state->dpll)) {
8479                 DRM_ERROR("Couldn't find PLL settings for mode!\n");
8480                 return -EINVAL;
8481         }
8482
8483         i9xx_compute_dpll(crtc, crtc_state, NULL);
8484
8485         return 0;
8486 }
8487
8488 static int chv_crtc_compute_clock(struct intel_crtc *crtc,
8489                                   struct intel_crtc_state *crtc_state)
8490 {
8491         int refclk = 100000;
8492         const struct intel_limit *limit = &intel_limits_chv;
8493
8494         memset(&crtc_state->dpll_hw_state, 0,
8495                sizeof(crtc_state->dpll_hw_state));
8496
8497         if (!crtc_state->clock_set &&
8498             !chv_find_best_dpll(limit, crtc_state, crtc_state->port_clock,
8499                                 refclk, NULL, &crtc_state->dpll)) {
8500                 DRM_ERROR("Couldn't find PLL settings for mode!\n");
8501                 return -EINVAL;
8502         }
8503
8504         chv_compute_dpll(crtc, crtc_state);
8505
8506         return 0;
8507 }
8508
8509 static int vlv_crtc_compute_clock(struct intel_crtc *crtc,
8510                                   struct intel_crtc_state *crtc_state)
8511 {
8512         int refclk = 100000;
8513         const struct intel_limit *limit = &intel_limits_vlv;
8514
8515         memset(&crtc_state->dpll_hw_state, 0,
8516                sizeof(crtc_state->dpll_hw_state));
8517
8518         if (!crtc_state->clock_set &&
8519             !vlv_find_best_dpll(limit, crtc_state, crtc_state->port_clock,
8520                                 refclk, NULL, &crtc_state->dpll)) {
8521                 DRM_ERROR("Couldn't find PLL settings for mode!\n");
8522                 return -EINVAL;
8523         }
8524
8525         vlv_compute_dpll(crtc, crtc_state);
8526
8527         return 0;
8528 }
8529
8530 static bool i9xx_has_pfit(struct drm_i915_private *dev_priv)
8531 {
8532         if (IS_I830(dev_priv))
8533                 return false;
8534
8535         return INTEL_GEN(dev_priv) >= 4 ||
8536                 IS_PINEVIEW(dev_priv) || IS_MOBILE(dev_priv);
8537 }
8538
8539 static void i9xx_get_pfit_config(struct intel_crtc *crtc,
8540                                  struct intel_crtc_state *pipe_config)
8541 {
8542         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
8543         u32 tmp;
8544
8545         if (!i9xx_has_pfit(dev_priv))
8546                 return;
8547
8548         tmp = I915_READ(PFIT_CONTROL);
8549         if (!(tmp & PFIT_ENABLE))
8550                 return;
8551
8552         /* Check whether the pfit is attached to our pipe. */
8553         if (INTEL_GEN(dev_priv) < 4) {
8554                 if (crtc->pipe != PIPE_B)
8555                         return;
8556         } else {
8557                 if ((tmp & PFIT_PIPE_MASK) != (crtc->pipe << PFIT_PIPE_SHIFT))
8558                         return;
8559         }
8560
8561         pipe_config->gmch_pfit.control = tmp;
8562         pipe_config->gmch_pfit.pgm_ratios = I915_READ(PFIT_PGM_RATIOS);
8563 }
8564
8565 static void vlv_crtc_clock_get(struct intel_crtc *crtc,
8566                                struct intel_crtc_state *pipe_config)
8567 {
8568         struct drm_device *dev = crtc->base.dev;
8569         struct drm_i915_private *dev_priv = to_i915(dev);
8570         int pipe = pipe_config->cpu_transcoder;
8571         struct dpll clock;
8572         u32 mdiv;
8573         int refclk = 100000;
8574
8575         /* In case of DSI, DPLL will not be used */
8576         if ((pipe_config->dpll_hw_state.dpll & DPLL_VCO_ENABLE) == 0)
8577                 return;
8578
8579         vlv_dpio_get(dev_priv);
8580         mdiv = vlv_dpio_read(dev_priv, pipe, VLV_PLL_DW3(pipe));
8581         vlv_dpio_put(dev_priv);
8582
8583         clock.m1 = (mdiv >> DPIO_M1DIV_SHIFT) & 7;
8584         clock.m2 = mdiv & DPIO_M2DIV_MASK;
8585         clock.n = (mdiv >> DPIO_N_SHIFT) & 0xf;
8586         clock.p1 = (mdiv >> DPIO_P1_SHIFT) & 7;
8587         clock.p2 = (mdiv >> DPIO_P2_SHIFT) & 0x1f;
8588
8589         pipe_config->port_clock = vlv_calc_dpll_params(refclk, &clock);
8590 }
8591
8592 static void
8593 i9xx_get_initial_plane_config(struct intel_crtc *crtc,
8594                               struct intel_initial_plane_config *plane_config)
8595 {
8596         struct drm_device *dev = crtc->base.dev;
8597         struct drm_i915_private *dev_priv = to_i915(dev);
8598         struct intel_plane *plane = to_intel_plane(crtc->base.primary);
8599         enum i9xx_plane_id i9xx_plane = plane->i9xx_plane;
8600         enum pipe pipe;
8601         u32 val, base, offset;
8602         int fourcc, pixel_format;
8603         unsigned int aligned_height;
8604         struct drm_framebuffer *fb;
8605         struct intel_framebuffer *intel_fb;
8606
8607         if (!plane->get_hw_state(plane, &pipe))
8608                 return;
8609
8610         WARN_ON(pipe != crtc->pipe);
8611
8612         intel_fb = kzalloc(sizeof(*intel_fb), GFP_KERNEL);
8613         if (!intel_fb) {
8614                 DRM_DEBUG_KMS("failed to alloc fb\n");
8615                 return;
8616         }
8617
8618         fb = &intel_fb->base;
8619
8620         fb->dev = dev;
8621
8622         val = I915_READ(DSPCNTR(i9xx_plane));
8623
8624         if (INTEL_GEN(dev_priv) >= 4) {
8625                 if (val & DISPPLANE_TILED) {
8626                         plane_config->tiling = I915_TILING_X;
8627                         fb->modifier = I915_FORMAT_MOD_X_TILED;
8628                 }
8629
8630                 if (val & DISPPLANE_ROTATE_180)
8631                         plane_config->rotation = DRM_MODE_ROTATE_180;
8632         }
8633
8634         if (IS_CHERRYVIEW(dev_priv) && pipe == PIPE_B &&
8635             val & DISPPLANE_MIRROR)
8636                 plane_config->rotation |= DRM_MODE_REFLECT_X;
8637
8638         pixel_format = val & DISPPLANE_PIXFORMAT_MASK;
8639         fourcc = i9xx_format_to_fourcc(pixel_format);
8640         fb->format = drm_format_info(fourcc);
8641
8642         if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv)) {
8643                 offset = I915_READ(DSPOFFSET(i9xx_plane));
8644                 base = I915_READ(DSPSURF(i9xx_plane)) & 0xfffff000;
8645         } else if (INTEL_GEN(dev_priv) >= 4) {
8646                 if (plane_config->tiling)
8647                         offset = I915_READ(DSPTILEOFF(i9xx_plane));
8648                 else
8649                         offset = I915_READ(DSPLINOFF(i9xx_plane));
8650                 base = I915_READ(DSPSURF(i9xx_plane)) & 0xfffff000;
8651         } else {
8652                 base = I915_READ(DSPADDR(i9xx_plane));
8653         }
8654         plane_config->base = base;
8655
8656         val = I915_READ(PIPESRC(pipe));
8657         fb->width = ((val >> 16) & 0xfff) + 1;
8658         fb->height = ((val >> 0) & 0xfff) + 1;
8659
8660         val = I915_READ(DSPSTRIDE(i9xx_plane));
8661         fb->pitches[0] = val & 0xffffffc0;
8662
8663         aligned_height = intel_fb_align_height(fb, 0, fb->height);
8664
8665         plane_config->size = fb->pitches[0] * aligned_height;
8666
8667         DRM_DEBUG_KMS("%s/%s with fb: size=%dx%d@%d, offset=%x, pitch %d, size 0x%x\n",
8668                       crtc->base.name, plane->base.name, fb->width, fb->height,
8669                       fb->format->cpp[0] * 8, base, fb->pitches[0],
8670                       plane_config->size);
8671
8672         plane_config->fb = intel_fb;
8673 }
8674
8675 static void chv_crtc_clock_get(struct intel_crtc *crtc,
8676                                struct intel_crtc_state *pipe_config)
8677 {
8678         struct drm_device *dev = crtc->base.dev;
8679         struct drm_i915_private *dev_priv = to_i915(dev);
8680         int pipe = pipe_config->cpu_transcoder;
8681         enum dpio_channel port = vlv_pipe_to_channel(pipe);
8682         struct dpll clock;
8683         u32 cmn_dw13, pll_dw0, pll_dw1, pll_dw2, pll_dw3;
8684         int refclk = 100000;
8685
8686         /* In case of DSI, DPLL will not be used */
8687         if ((pipe_config->dpll_hw_state.dpll & DPLL_VCO_ENABLE) == 0)
8688                 return;
8689
8690         vlv_dpio_get(dev_priv);
8691         cmn_dw13 = vlv_dpio_read(dev_priv, pipe, CHV_CMN_DW13(port));
8692         pll_dw0 = vlv_dpio_read(dev_priv, pipe, CHV_PLL_DW0(port));
8693         pll_dw1 = vlv_dpio_read(dev_priv, pipe, CHV_PLL_DW1(port));
8694         pll_dw2 = vlv_dpio_read(dev_priv, pipe, CHV_PLL_DW2(port));
8695         pll_dw3 = vlv_dpio_read(dev_priv, pipe, CHV_PLL_DW3(port));
8696         vlv_dpio_put(dev_priv);
8697
8698         clock.m1 = (pll_dw1 & 0x7) == DPIO_CHV_M1_DIV_BY_2 ? 2 : 0;
8699         clock.m2 = (pll_dw0 & 0xff) << 22;
8700         if (pll_dw3 & DPIO_CHV_FRAC_DIV_EN)
8701                 clock.m2 |= pll_dw2 & 0x3fffff;
8702         clock.n = (pll_dw1 >> DPIO_CHV_N_DIV_SHIFT) & 0xf;
8703         clock.p1 = (cmn_dw13 >> DPIO_CHV_P1_DIV_SHIFT) & 0x7;
8704         clock.p2 = (cmn_dw13 >> DPIO_CHV_P2_DIV_SHIFT) & 0x1f;
8705
8706         pipe_config->port_clock = chv_calc_dpll_params(refclk, &clock);
8707 }
8708
8709 static void intel_get_crtc_ycbcr_config(struct intel_crtc *crtc,
8710                                         struct intel_crtc_state *pipe_config)
8711 {
8712         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
8713         enum intel_output_format output = INTEL_OUTPUT_FORMAT_RGB;
8714
8715         pipe_config->lspcon_downsampling = false;
8716
8717         if (IS_BROADWELL(dev_priv) || INTEL_GEN(dev_priv) >= 9) {
8718                 u32 tmp = I915_READ(PIPEMISC(crtc->pipe));
8719
8720                 if (tmp & PIPEMISC_OUTPUT_COLORSPACE_YUV) {
8721                         bool ycbcr420_enabled = tmp & PIPEMISC_YUV420_ENABLE;
8722                         bool blend = tmp & PIPEMISC_YUV420_MODE_FULL_BLEND;
8723
8724                         if (ycbcr420_enabled) {
8725                                 /* We support 4:2:0 in full blend mode only */
8726                                 if (!blend)
8727                                         output = INTEL_OUTPUT_FORMAT_INVALID;
8728                                 else if (!(IS_GEMINILAKE(dev_priv) ||
8729                                            INTEL_GEN(dev_priv) >= 10))
8730                                         output = INTEL_OUTPUT_FORMAT_INVALID;
8731                                 else
8732                                         output = INTEL_OUTPUT_FORMAT_YCBCR420;
8733                         } else {
8734                                 /*
8735                                  * Currently there is no interface defined to
8736                                  * check user preference between RGB/YCBCR444
8737                                  * or YCBCR420. So the only possible case for
8738                                  * YCBCR444 usage is driving YCBCR420 output
8739                                  * with LSPCON, when pipe is configured for
8740                                  * YCBCR444 output and LSPCON takes care of
8741                                  * downsampling it.
8742                                  */
8743                                 pipe_config->lspcon_downsampling = true;
8744                                 output = INTEL_OUTPUT_FORMAT_YCBCR444;
8745                         }
8746                 }
8747         }
8748
8749         pipe_config->output_format = output;
8750 }
8751
8752 static void i9xx_get_pipe_color_config(struct intel_crtc_state *crtc_state)
8753 {
8754         struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
8755         struct intel_plane *plane = to_intel_plane(crtc->base.primary);
8756         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
8757         enum i9xx_plane_id i9xx_plane = plane->i9xx_plane;
8758         u32 tmp;
8759
8760         tmp = I915_READ(DSPCNTR(i9xx_plane));
8761
8762         if (tmp & DISPPLANE_GAMMA_ENABLE)
8763                 crtc_state->gamma_enable = true;
8764
8765         if (!HAS_GMCH(dev_priv) &&
8766             tmp & DISPPLANE_PIPE_CSC_ENABLE)
8767                 crtc_state->csc_enable = true;
8768 }
8769
8770 static bool i9xx_get_pipe_config(struct intel_crtc *crtc,
8771                                  struct intel_crtc_state *pipe_config)
8772 {
8773         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
8774         enum intel_display_power_domain power_domain;
8775         intel_wakeref_t wakeref;
8776         u32 tmp;
8777         bool ret;
8778
8779         power_domain = POWER_DOMAIN_PIPE(crtc->pipe);
8780         wakeref = intel_display_power_get_if_enabled(dev_priv, power_domain);
8781         if (!wakeref)
8782                 return false;
8783
8784         pipe_config->output_format = INTEL_OUTPUT_FORMAT_RGB;
8785         pipe_config->cpu_transcoder = (enum transcoder) crtc->pipe;
8786         pipe_config->shared_dpll = NULL;
8787
8788         ret = false;
8789
8790         tmp = I915_READ(PIPECONF(crtc->pipe));
8791         if (!(tmp & PIPECONF_ENABLE))
8792                 goto out;
8793
8794         if (IS_G4X(dev_priv) || IS_VALLEYVIEW(dev_priv) ||
8795             IS_CHERRYVIEW(dev_priv)) {
8796                 switch (tmp & PIPECONF_BPC_MASK) {
8797                 case PIPECONF_6BPC:
8798                         pipe_config->pipe_bpp = 18;
8799                         break;
8800                 case PIPECONF_8BPC:
8801                         pipe_config->pipe_bpp = 24;
8802                         break;
8803                 case PIPECONF_10BPC:
8804                         pipe_config->pipe_bpp = 30;
8805                         break;
8806                 default:
8807                         break;
8808                 }
8809         }
8810
8811         if ((IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) &&
8812             (tmp & PIPECONF_COLOR_RANGE_SELECT))
8813                 pipe_config->limited_color_range = true;
8814
8815         pipe_config->gamma_mode = (tmp & PIPECONF_GAMMA_MODE_MASK_I9XX) >>
8816                 PIPECONF_GAMMA_MODE_SHIFT;
8817
8818         if (IS_CHERRYVIEW(dev_priv))
8819                 pipe_config->cgm_mode = I915_READ(CGM_PIPE_MODE(crtc->pipe));
8820
8821         i9xx_get_pipe_color_config(pipe_config);
8822         intel_color_get_config(pipe_config);
8823
8824         if (INTEL_GEN(dev_priv) < 4)
8825                 pipe_config->double_wide = tmp & PIPECONF_DOUBLE_WIDE;
8826
8827         intel_get_pipe_timings(crtc, pipe_config);
8828         intel_get_pipe_src_size(crtc, pipe_config);
8829
8830         i9xx_get_pfit_config(crtc, pipe_config);
8831
8832         if (INTEL_GEN(dev_priv) >= 4) {
8833                 /* No way to read it out on pipes B and C */
8834                 if (IS_CHERRYVIEW(dev_priv) && crtc->pipe != PIPE_A)
8835                         tmp = dev_priv->chv_dpll_md[crtc->pipe];
8836                 else
8837                         tmp = I915_READ(DPLL_MD(crtc->pipe));
8838                 pipe_config->pixel_multiplier =
8839                         ((tmp & DPLL_MD_UDI_MULTIPLIER_MASK)
8840                          >> DPLL_MD_UDI_MULTIPLIER_SHIFT) + 1;
8841                 pipe_config->dpll_hw_state.dpll_md = tmp;
8842         } else if (IS_I945G(dev_priv) || IS_I945GM(dev_priv) ||
8843                    IS_G33(dev_priv) || IS_PINEVIEW(dev_priv)) {
8844                 tmp = I915_READ(DPLL(crtc->pipe));
8845                 pipe_config->pixel_multiplier =
8846                         ((tmp & SDVO_MULTIPLIER_MASK)
8847                          >> SDVO_MULTIPLIER_SHIFT_HIRES) + 1;
8848         } else {
8849                 /* Note that on i915G/GM the pixel multiplier is in the sdvo
8850                  * port and will be fixed up in the encoder->get_config
8851                  * function. */
8852                 pipe_config->pixel_multiplier = 1;
8853         }
8854         pipe_config->dpll_hw_state.dpll = I915_READ(DPLL(crtc->pipe));
8855         if (!IS_VALLEYVIEW(dev_priv) && !IS_CHERRYVIEW(dev_priv)) {
8856                 pipe_config->dpll_hw_state.fp0 = I915_READ(FP0(crtc->pipe));
8857                 pipe_config->dpll_hw_state.fp1 = I915_READ(FP1(crtc->pipe));
8858         } else {
8859                 /* Mask out read-only status bits. */
8860                 pipe_config->dpll_hw_state.dpll &= ~(DPLL_LOCK_VLV |
8861                                                      DPLL_PORTC_READY_MASK |
8862                                                      DPLL_PORTB_READY_MASK);
8863         }
8864
8865         if (IS_CHERRYVIEW(dev_priv))
8866                 chv_crtc_clock_get(crtc, pipe_config);
8867         else if (IS_VALLEYVIEW(dev_priv))
8868                 vlv_crtc_clock_get(crtc, pipe_config);
8869         else
8870                 i9xx_crtc_clock_get(crtc, pipe_config);
8871
8872         /*
8873          * Normally the dotclock is filled in by the encoder .get_config()
8874          * but in case the pipe is enabled w/o any ports we need a sane
8875          * default.
8876          */
8877         pipe_config->base.adjusted_mode.crtc_clock =
8878                 pipe_config->port_clock / pipe_config->pixel_multiplier;
8879
8880         ret = true;
8881
8882 out:
8883         intel_display_power_put(dev_priv, power_domain, wakeref);
8884
8885         return ret;
8886 }
8887
8888 static void ironlake_init_pch_refclk(struct drm_i915_private *dev_priv)
8889 {
8890         struct intel_encoder *encoder;
8891         int i;
8892         u32 val, final;
8893         bool has_lvds = false;
8894         bool has_cpu_edp = false;
8895         bool has_panel = false;
8896         bool has_ck505 = false;
8897         bool can_ssc = false;
8898         bool using_ssc_source = false;
8899
8900         /* We need to take the global config into account */
8901         for_each_intel_encoder(&dev_priv->drm, encoder) {
8902                 switch (encoder->type) {
8903                 case INTEL_OUTPUT_LVDS:
8904                         has_panel = true;
8905                         has_lvds = true;
8906                         break;
8907                 case INTEL_OUTPUT_EDP:
8908                         has_panel = true;
8909                         if (encoder->port == PORT_A)
8910                                 has_cpu_edp = true;
8911                         break;
8912                 default:
8913                         break;
8914                 }
8915         }
8916
8917         if (HAS_PCH_IBX(dev_priv)) {
8918                 has_ck505 = dev_priv->vbt.display_clock_mode;
8919                 can_ssc = has_ck505;
8920         } else {
8921                 has_ck505 = false;
8922                 can_ssc = true;
8923         }
8924
8925         /* Check if any DPLLs are using the SSC source */
8926         for (i = 0; i < dev_priv->num_shared_dpll; i++) {
8927                 u32 temp = I915_READ(PCH_DPLL(i));
8928
8929                 if (!(temp & DPLL_VCO_ENABLE))
8930                         continue;
8931
8932                 if ((temp & PLL_REF_INPUT_MASK) ==
8933                     PLLB_REF_INPUT_SPREADSPECTRUMIN) {
8934                         using_ssc_source = true;
8935                         break;
8936                 }
8937         }
8938
8939         DRM_DEBUG_KMS("has_panel %d has_lvds %d has_ck505 %d using_ssc_source %d\n",
8940                       has_panel, has_lvds, has_ck505, using_ssc_source);
8941
8942         /* Ironlake: try to setup display ref clock before DPLL
8943          * enabling. This is only under driver's control after
8944          * PCH B stepping, previous chipset stepping should be
8945          * ignoring this setting.
8946          */
8947         val = I915_READ(PCH_DREF_CONTROL);
8948
8949         /* As we must carefully and slowly disable/enable each source in turn,
8950          * compute the final state we want first and check if we need to
8951          * make any changes at all.
8952          */
8953         final = val;
8954         final &= ~DREF_NONSPREAD_SOURCE_MASK;
8955         if (has_ck505)
8956                 final |= DREF_NONSPREAD_CK505_ENABLE;
8957         else
8958                 final |= DREF_NONSPREAD_SOURCE_ENABLE;
8959
8960         final &= ~DREF_SSC_SOURCE_MASK;
8961         final &= ~DREF_CPU_SOURCE_OUTPUT_MASK;
8962         final &= ~DREF_SSC1_ENABLE;
8963
8964         if (has_panel) {
8965                 final |= DREF_SSC_SOURCE_ENABLE;
8966
8967                 if (intel_panel_use_ssc(dev_priv) && can_ssc)
8968                         final |= DREF_SSC1_ENABLE;
8969
8970                 if (has_cpu_edp) {
8971                         if (intel_panel_use_ssc(dev_priv) && can_ssc)
8972                                 final |= DREF_CPU_SOURCE_OUTPUT_DOWNSPREAD;
8973                         else
8974                                 final |= DREF_CPU_SOURCE_OUTPUT_NONSPREAD;
8975                 } else
8976                         final |= DREF_CPU_SOURCE_OUTPUT_DISABLE;
8977         } else if (using_ssc_source) {
8978                 final |= DREF_SSC_SOURCE_ENABLE;
8979                 final |= DREF_SSC1_ENABLE;
8980         }
8981
8982         if (final == val)
8983                 return;
8984
8985         /* Always enable nonspread source */
8986         val &= ~DREF_NONSPREAD_SOURCE_MASK;
8987
8988         if (has_ck505)
8989                 val |= DREF_NONSPREAD_CK505_ENABLE;
8990         else
8991                 val |= DREF_NONSPREAD_SOURCE_ENABLE;
8992
8993         if (has_panel) {
8994                 val &= ~DREF_SSC_SOURCE_MASK;
8995                 val |= DREF_SSC_SOURCE_ENABLE;
8996
8997                 /* SSC must be turned on before enabling the CPU output  */
8998                 if (intel_panel_use_ssc(dev_priv) && can_ssc) {
8999                         DRM_DEBUG_KMS("Using SSC on panel\n");
9000                         val |= DREF_SSC1_ENABLE;
9001                 } else
9002                         val &= ~DREF_SSC1_ENABLE;
9003
9004                 /* Get SSC going before enabling the outputs */
9005                 I915_WRITE(PCH_DREF_CONTROL, val);
9006                 POSTING_READ(PCH_DREF_CONTROL);
9007                 udelay(200);
9008
9009                 val &= ~DREF_CPU_SOURCE_OUTPUT_MASK;
9010
9011                 /* Enable CPU source on CPU attached eDP */
9012                 if (has_cpu_edp) {
9013                         if (intel_panel_use_ssc(dev_priv) && can_ssc) {
9014                                 DRM_DEBUG_KMS("Using SSC on eDP\n");
9015                                 val |= DREF_CPU_SOURCE_OUTPUT_DOWNSPREAD;
9016                         } else
9017                                 val |= DREF_CPU_SOURCE_OUTPUT_NONSPREAD;
9018                 } else
9019                         val |= DREF_CPU_SOURCE_OUTPUT_DISABLE;
9020
9021                 I915_WRITE(PCH_DREF_CONTROL, val);
9022                 POSTING_READ(PCH_DREF_CONTROL);
9023                 udelay(200);
9024         } else {
9025                 DRM_DEBUG_KMS("Disabling CPU source output\n");
9026
9027                 val &= ~DREF_CPU_SOURCE_OUTPUT_MASK;
9028
9029                 /* Turn off CPU output */
9030                 val |= DREF_CPU_SOURCE_OUTPUT_DISABLE;
9031
9032                 I915_WRITE(PCH_DREF_CONTROL, val);
9033                 POSTING_READ(PCH_DREF_CONTROL);
9034                 udelay(200);
9035
9036                 if (!using_ssc_source) {
9037                         DRM_DEBUG_KMS("Disabling SSC source\n");
9038
9039                         /* Turn off the SSC source */
9040                         val &= ~DREF_SSC_SOURCE_MASK;
9041                         val |= DREF_SSC_SOURCE_DISABLE;
9042
9043                         /* Turn off SSC1 */
9044                         val &= ~DREF_SSC1_ENABLE;
9045
9046                         I915_WRITE(PCH_DREF_CONTROL, val);
9047                         POSTING_READ(PCH_DREF_CONTROL);
9048                         udelay(200);
9049                 }
9050         }
9051
9052         BUG_ON(val != final);
9053 }
9054
9055 static void lpt_reset_fdi_mphy(struct drm_i915_private *dev_priv)
9056 {
9057         u32 tmp;
9058
9059         tmp = I915_READ(SOUTH_CHICKEN2);
9060         tmp |= FDI_MPHY_IOSFSB_RESET_CTL;
9061         I915_WRITE(SOUTH_CHICKEN2, tmp);
9062
9063         if (wait_for_us(I915_READ(SOUTH_CHICKEN2) &
9064                         FDI_MPHY_IOSFSB_RESET_STATUS, 100))
9065                 DRM_ERROR("FDI mPHY reset assert timeout\n");
9066
9067         tmp = I915_READ(SOUTH_CHICKEN2);
9068         tmp &= ~FDI_MPHY_IOSFSB_RESET_CTL;
9069         I915_WRITE(SOUTH_CHICKEN2, tmp);
9070
9071         if (wait_for_us((I915_READ(SOUTH_CHICKEN2) &
9072                          FDI_MPHY_IOSFSB_RESET_STATUS) == 0, 100))
9073                 DRM_ERROR("FDI mPHY reset de-assert timeout\n");
9074 }
9075
9076 /* WaMPhyProgramming:hsw */
9077 static void lpt_program_fdi_mphy(struct drm_i915_private *dev_priv)
9078 {
9079         u32 tmp;
9080
9081         tmp = intel_sbi_read(dev_priv, 0x8008, SBI_MPHY);
9082         tmp &= ~(0xFF << 24);
9083         tmp |= (0x12 << 24);
9084         intel_sbi_write(dev_priv, 0x8008, tmp, SBI_MPHY);
9085
9086         tmp = intel_sbi_read(dev_priv, 0x2008, SBI_MPHY);
9087         tmp |= (1 << 11);
9088         intel_sbi_write(dev_priv, 0x2008, tmp, SBI_MPHY);
9089
9090         tmp = intel_sbi_read(dev_priv, 0x2108, SBI_MPHY);
9091         tmp |= (1 << 11);
9092         intel_sbi_write(dev_priv, 0x2108, tmp, SBI_MPHY);
9093
9094         tmp = intel_sbi_read(dev_priv, 0x206C, SBI_MPHY);
9095         tmp |= (1 << 24) | (1 << 21) | (1 << 18);
9096         intel_sbi_write(dev_priv, 0x206C, tmp, SBI_MPHY);
9097
9098         tmp = intel_sbi_read(dev_priv, 0x216C, SBI_MPHY);
9099         tmp |= (1 << 24) | (1 << 21) | (1 << 18);
9100         intel_sbi_write(dev_priv, 0x216C, tmp, SBI_MPHY);
9101
9102         tmp = intel_sbi_read(dev_priv, 0x2080, SBI_MPHY);
9103         tmp &= ~(7 << 13);
9104         tmp |= (5 << 13);
9105         intel_sbi_write(dev_priv, 0x2080, tmp, SBI_MPHY);
9106
9107         tmp = intel_sbi_read(dev_priv, 0x2180, SBI_MPHY);
9108         tmp &= ~(7 << 13);
9109         tmp |= (5 << 13);
9110         intel_sbi_write(dev_priv, 0x2180, tmp, SBI_MPHY);
9111
9112         tmp = intel_sbi_read(dev_priv, 0x208C, SBI_MPHY);
9113         tmp &= ~0xFF;
9114         tmp |= 0x1C;
9115         intel_sbi_write(dev_priv, 0x208C, tmp, SBI_MPHY);
9116
9117         tmp = intel_sbi_read(dev_priv, 0x218C, SBI_MPHY);
9118         tmp &= ~0xFF;
9119         tmp |= 0x1C;
9120         intel_sbi_write(dev_priv, 0x218C, tmp, SBI_MPHY);
9121
9122         tmp = intel_sbi_read(dev_priv, 0x2098, SBI_MPHY);
9123         tmp &= ~(0xFF << 16);
9124         tmp |= (0x1C << 16);
9125         intel_sbi_write(dev_priv, 0x2098, tmp, SBI_MPHY);
9126
9127         tmp = intel_sbi_read(dev_priv, 0x2198, SBI_MPHY);
9128         tmp &= ~(0xFF << 16);
9129         tmp |= (0x1C << 16);
9130         intel_sbi_write(dev_priv, 0x2198, tmp, SBI_MPHY);
9131
9132         tmp = intel_sbi_read(dev_priv, 0x20C4, SBI_MPHY);
9133         tmp |= (1 << 27);
9134         intel_sbi_write(dev_priv, 0x20C4, tmp, SBI_MPHY);
9135
9136         tmp = intel_sbi_read(dev_priv, 0x21C4, SBI_MPHY);
9137         tmp |= (1 << 27);
9138         intel_sbi_write(dev_priv, 0x21C4, tmp, SBI_MPHY);
9139
9140         tmp = intel_sbi_read(dev_priv, 0x20EC, SBI_MPHY);
9141         tmp &= ~(0xF << 28);
9142         tmp |= (4 << 28);
9143         intel_sbi_write(dev_priv, 0x20EC, tmp, SBI_MPHY);
9144
9145         tmp = intel_sbi_read(dev_priv, 0x21EC, SBI_MPHY);
9146         tmp &= ~(0xF << 28);
9147         tmp |= (4 << 28);
9148         intel_sbi_write(dev_priv, 0x21EC, tmp, SBI_MPHY);
9149 }
9150
9151 /* Implements 3 different sequences from BSpec chapter "Display iCLK
9152  * Programming" based on the parameters passed:
9153  * - Sequence to enable CLKOUT_DP
9154  * - Sequence to enable CLKOUT_DP without spread
9155  * - Sequence to enable CLKOUT_DP for FDI usage and configure PCH FDI I/O
9156  */
9157 static void lpt_enable_clkout_dp(struct drm_i915_private *dev_priv,
9158                                  bool with_spread, bool with_fdi)
9159 {
9160         u32 reg, tmp;
9161
9162         if (WARN(with_fdi && !with_spread, "FDI requires downspread\n"))
9163                 with_spread = true;
9164         if (WARN(HAS_PCH_LPT_LP(dev_priv) &&
9165             with_fdi, "LP PCH doesn't have FDI\n"))
9166                 with_fdi = false;
9167
9168         mutex_lock(&dev_priv->sb_lock);
9169
9170         tmp = intel_sbi_read(dev_priv, SBI_SSCCTL, SBI_ICLK);
9171         tmp &= ~SBI_SSCCTL_DISABLE;
9172         tmp |= SBI_SSCCTL_PATHALT;
9173         intel_sbi_write(dev_priv, SBI_SSCCTL, tmp, SBI_ICLK);
9174
9175         udelay(24);
9176
9177         if (with_spread) {
9178                 tmp = intel_sbi_read(dev_priv, SBI_SSCCTL, SBI_ICLK);
9179                 tmp &= ~SBI_SSCCTL_PATHALT;
9180                 intel_sbi_write(dev_priv, SBI_SSCCTL, tmp, SBI_ICLK);
9181
9182                 if (with_fdi) {
9183                         lpt_reset_fdi_mphy(dev_priv);
9184                         lpt_program_fdi_mphy(dev_priv);
9185                 }
9186         }
9187
9188         reg = HAS_PCH_LPT_LP(dev_priv) ? SBI_GEN0 : SBI_DBUFF0;
9189         tmp = intel_sbi_read(dev_priv, reg, SBI_ICLK);
9190         tmp |= SBI_GEN0_CFG_BUFFENABLE_DISABLE;
9191         intel_sbi_write(dev_priv, reg, tmp, SBI_ICLK);
9192
9193         mutex_unlock(&dev_priv->sb_lock);
9194 }
9195
9196 /* Sequence to disable CLKOUT_DP */
9197 void lpt_disable_clkout_dp(struct drm_i915_private *dev_priv)
9198 {
9199         u32 reg, tmp;
9200
9201         mutex_lock(&dev_priv->sb_lock);
9202
9203         reg = HAS_PCH_LPT_LP(dev_priv) ? SBI_GEN0 : SBI_DBUFF0;
9204         tmp = intel_sbi_read(dev_priv, reg, SBI_ICLK);
9205         tmp &= ~SBI_GEN0_CFG_BUFFENABLE_DISABLE;
9206         intel_sbi_write(dev_priv, reg, tmp, SBI_ICLK);
9207
9208         tmp = intel_sbi_read(dev_priv, SBI_SSCCTL, SBI_ICLK);
9209         if (!(tmp & SBI_SSCCTL_DISABLE)) {
9210                 if (!(tmp & SBI_SSCCTL_PATHALT)) {
9211                         tmp |= SBI_SSCCTL_PATHALT;
9212                         intel_sbi_write(dev_priv, SBI_SSCCTL, tmp, SBI_ICLK);
9213                         udelay(32);
9214                 }
9215                 tmp |= SBI_SSCCTL_DISABLE;
9216                 intel_sbi_write(dev_priv, SBI_SSCCTL, tmp, SBI_ICLK);
9217         }
9218
9219         mutex_unlock(&dev_priv->sb_lock);
9220 }
9221
9222 #define BEND_IDX(steps) ((50 + (steps)) / 5)
9223
9224 static const u16 sscdivintphase[] = {
9225         [BEND_IDX( 50)] = 0x3B23,
9226         [BEND_IDX( 45)] = 0x3B23,
9227         [BEND_IDX( 40)] = 0x3C23,
9228         [BEND_IDX( 35)] = 0x3C23,
9229         [BEND_IDX( 30)] = 0x3D23,
9230         [BEND_IDX( 25)] = 0x3D23,
9231         [BEND_IDX( 20)] = 0x3E23,
9232         [BEND_IDX( 15)] = 0x3E23,
9233         [BEND_IDX( 10)] = 0x3F23,
9234         [BEND_IDX(  5)] = 0x3F23,
9235         [BEND_IDX(  0)] = 0x0025,
9236         [BEND_IDX( -5)] = 0x0025,
9237         [BEND_IDX(-10)] = 0x0125,
9238         [BEND_IDX(-15)] = 0x0125,
9239         [BEND_IDX(-20)] = 0x0225,
9240         [BEND_IDX(-25)] = 0x0225,
9241         [BEND_IDX(-30)] = 0x0325,
9242         [BEND_IDX(-35)] = 0x0325,
9243         [BEND_IDX(-40)] = 0x0425,
9244         [BEND_IDX(-45)] = 0x0425,
9245         [BEND_IDX(-50)] = 0x0525,
9246 };
9247
9248 /*
9249  * Bend CLKOUT_DP
9250  * steps -50 to 50 inclusive, in steps of 5
9251  * < 0 slow down the clock, > 0 speed up the clock, 0 == no bend (135MHz)
9252  * change in clock period = -(steps / 10) * 5.787 ps
9253  */
9254 static void lpt_bend_clkout_dp(struct drm_i915_private *dev_priv, int steps)
9255 {
9256         u32 tmp;
9257         int idx = BEND_IDX(steps);
9258
9259         if (WARN_ON(steps % 5 != 0))
9260                 return;
9261
9262         if (WARN_ON(idx >= ARRAY_SIZE(sscdivintphase)))
9263                 return;
9264
9265         mutex_lock(&dev_priv->sb_lock);
9266
9267         if (steps % 10 != 0)
9268                 tmp = 0xAAAAAAAB;
9269         else
9270                 tmp = 0x00000000;
9271         intel_sbi_write(dev_priv, SBI_SSCDITHPHASE, tmp, SBI_ICLK);
9272
9273         tmp = intel_sbi_read(dev_priv, SBI_SSCDIVINTPHASE, SBI_ICLK);
9274         tmp &= 0xffff0000;
9275         tmp |= sscdivintphase[idx];
9276         intel_sbi_write(dev_priv, SBI_SSCDIVINTPHASE, tmp, SBI_ICLK);
9277
9278         mutex_unlock(&dev_priv->sb_lock);
9279 }
9280
9281 #undef BEND_IDX
9282
9283 static bool spll_uses_pch_ssc(struct drm_i915_private *dev_priv)
9284 {
9285         u32 fuse_strap = I915_READ(FUSE_STRAP);
9286         u32 ctl = I915_READ(SPLL_CTL);
9287
9288         if ((ctl & SPLL_PLL_ENABLE) == 0)
9289                 return false;
9290
9291         if ((ctl & SPLL_REF_MASK) == SPLL_REF_MUXED_SSC &&
9292             (fuse_strap & HSW_CPU_SSC_ENABLE) == 0)
9293                 return true;
9294
9295         if (IS_BROADWELL(dev_priv) &&
9296             (ctl & SPLL_REF_MASK) == SPLL_REF_PCH_SSC_BDW)
9297                 return true;
9298
9299         return false;
9300 }
9301
9302 static bool wrpll_uses_pch_ssc(struct drm_i915_private *dev_priv,
9303                                enum intel_dpll_id id)
9304 {
9305         u32 fuse_strap = I915_READ(FUSE_STRAP);
9306         u32 ctl = I915_READ(WRPLL_CTL(id));
9307
9308         if ((ctl & WRPLL_PLL_ENABLE) == 0)
9309                 return false;
9310
9311         if ((ctl & WRPLL_REF_MASK) == WRPLL_REF_PCH_SSC)
9312                 return true;
9313
9314         if ((IS_BROADWELL(dev_priv) || IS_HSW_ULT(dev_priv)) &&
9315             (ctl & WRPLL_REF_MASK) == WRPLL_REF_MUXED_SSC_BDW &&
9316             (fuse_strap & HSW_CPU_SSC_ENABLE) == 0)
9317                 return true;
9318
9319         return false;
9320 }
9321
9322 static void lpt_init_pch_refclk(struct drm_i915_private *dev_priv)
9323 {
9324         struct intel_encoder *encoder;
9325         bool pch_ssc_in_use = false;
9326         bool has_fdi = false;
9327
9328         for_each_intel_encoder(&dev_priv->drm, encoder) {
9329                 switch (encoder->type) {
9330                 case INTEL_OUTPUT_ANALOG:
9331                         has_fdi = true;
9332                         break;
9333                 default:
9334                         break;
9335                 }
9336         }
9337
9338         /*
9339          * The BIOS may have decided to use the PCH SSC
9340          * reference so we must not disable it until the
9341          * relevant PLLs have stopped relying on it. We'll
9342          * just leave the PCH SSC reference enabled in case
9343          * any active PLL is using it. It will get disabled
9344          * after runtime suspend if we don't have FDI.
9345          *
9346          * TODO: Move the whole reference clock handling
9347          * to the modeset sequence proper so that we can
9348          * actually enable/disable/reconfigure these things
9349          * safely. To do that we need to introduce a real
9350          * clock hierarchy. That would also allow us to do
9351          * clock bending finally.
9352          */
9353         if (spll_uses_pch_ssc(dev_priv)) {
9354                 DRM_DEBUG_KMS("SPLL using PCH SSC\n");
9355                 pch_ssc_in_use = true;
9356         }
9357
9358         if (wrpll_uses_pch_ssc(dev_priv, DPLL_ID_WRPLL1)) {
9359                 DRM_DEBUG_KMS("WRPLL1 using PCH SSC\n");
9360                 pch_ssc_in_use = true;
9361         }
9362
9363         if (wrpll_uses_pch_ssc(dev_priv, DPLL_ID_WRPLL2)) {
9364                 DRM_DEBUG_KMS("WRPLL2 using PCH SSC\n");
9365                 pch_ssc_in_use = true;
9366         }
9367
9368         if (pch_ssc_in_use)
9369                 return;
9370
9371         if (has_fdi) {
9372                 lpt_bend_clkout_dp(dev_priv, 0);
9373                 lpt_enable_clkout_dp(dev_priv, true, true);
9374         } else {
9375                 lpt_disable_clkout_dp(dev_priv);
9376         }
9377 }
9378
9379 /*
9380  * Initialize reference clocks when the driver loads
9381  */
9382 void intel_init_pch_refclk(struct drm_i915_private *dev_priv)
9383 {
9384         if (HAS_PCH_IBX(dev_priv) || HAS_PCH_CPT(dev_priv))
9385                 ironlake_init_pch_refclk(dev_priv);
9386         else if (HAS_PCH_LPT(dev_priv))
9387                 lpt_init_pch_refclk(dev_priv);
9388 }
9389
9390 static void ironlake_set_pipeconf(const struct intel_crtc_state *crtc_state)
9391 {
9392         struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
9393         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
9394         enum pipe pipe = crtc->pipe;
9395         u32 val;
9396
9397         val = 0;
9398
9399         switch (crtc_state->pipe_bpp) {
9400         case 18:
9401                 val |= PIPECONF_6BPC;
9402                 break;
9403         case 24:
9404                 val |= PIPECONF_8BPC;
9405                 break;
9406         case 30:
9407                 val |= PIPECONF_10BPC;
9408                 break;
9409         case 36:
9410                 val |= PIPECONF_12BPC;
9411                 break;
9412         default:
9413                 /* Case prevented by intel_choose_pipe_bpp_dither. */
9414                 BUG();
9415         }
9416
9417         if (crtc_state->dither)
9418                 val |= (PIPECONF_DITHER_EN | PIPECONF_DITHER_TYPE_SP);
9419
9420         if (crtc_state->base.adjusted_mode.flags & DRM_MODE_FLAG_INTERLACE)
9421                 val |= PIPECONF_INTERLACED_ILK;
9422         else
9423                 val |= PIPECONF_PROGRESSIVE;
9424
9425         if (crtc_state->limited_color_range)
9426                 val |= PIPECONF_COLOR_RANGE_SELECT;
9427
9428         val |= PIPECONF_GAMMA_MODE(crtc_state->gamma_mode);
9429
9430         I915_WRITE(PIPECONF(pipe), val);
9431         POSTING_READ(PIPECONF(pipe));
9432 }
9433
9434 static void haswell_set_pipeconf(const struct intel_crtc_state *crtc_state)
9435 {
9436         struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
9437         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
9438         enum transcoder cpu_transcoder = crtc_state->cpu_transcoder;
9439         u32 val = 0;
9440
9441         if (IS_HASWELL(dev_priv) && crtc_state->dither)
9442                 val |= (PIPECONF_DITHER_EN | PIPECONF_DITHER_TYPE_SP);
9443
9444         if (crtc_state->base.adjusted_mode.flags & DRM_MODE_FLAG_INTERLACE)
9445                 val |= PIPECONF_INTERLACED_ILK;
9446         else
9447                 val |= PIPECONF_PROGRESSIVE;
9448
9449         I915_WRITE(PIPECONF(cpu_transcoder), val);
9450         POSTING_READ(PIPECONF(cpu_transcoder));
9451 }
9452
9453 static void bdw_set_pipemisc(const struct intel_crtc_state *crtc_state)
9454 {
9455         struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
9456         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
9457         u32 val = 0;
9458
9459         switch (crtc_state->pipe_bpp) {
9460         case 18:
9461                 val |= PIPEMISC_DITHER_6_BPC;
9462                 break;
9463         case 24:
9464                 val |= PIPEMISC_DITHER_8_BPC;
9465                 break;
9466         case 30:
9467                 val |= PIPEMISC_DITHER_10_BPC;
9468                 break;
9469         case 36:
9470                 val |= PIPEMISC_DITHER_12_BPC;
9471                 break;
9472         default:
9473                 MISSING_CASE(crtc_state->pipe_bpp);
9474                 break;
9475         }
9476
9477         if (crtc_state->dither)
9478                 val |= PIPEMISC_DITHER_ENABLE | PIPEMISC_DITHER_TYPE_SP;
9479
9480         if (crtc_state->output_format == INTEL_OUTPUT_FORMAT_YCBCR420 ||
9481             crtc_state->output_format == INTEL_OUTPUT_FORMAT_YCBCR444)
9482                 val |= PIPEMISC_OUTPUT_COLORSPACE_YUV;
9483
9484         if (crtc_state->output_format == INTEL_OUTPUT_FORMAT_YCBCR420)
9485                 val |= PIPEMISC_YUV420_ENABLE |
9486                         PIPEMISC_YUV420_MODE_FULL_BLEND;
9487
9488         if (INTEL_GEN(dev_priv) >= 11 &&
9489             (crtc_state->active_planes & ~(icl_hdr_plane_mask() |
9490                                            BIT(PLANE_CURSOR))) == 0)
9491                 val |= PIPEMISC_HDR_MODE_PRECISION;
9492
9493         I915_WRITE(PIPEMISC(crtc->pipe), val);
9494 }
9495
9496 int bdw_get_pipemisc_bpp(struct intel_crtc *crtc)
9497 {
9498         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
9499         u32 tmp;
9500
9501         tmp = I915_READ(PIPEMISC(crtc->pipe));
9502
9503         switch (tmp & PIPEMISC_DITHER_BPC_MASK) {
9504         case PIPEMISC_DITHER_6_BPC:
9505                 return 18;
9506         case PIPEMISC_DITHER_8_BPC:
9507                 return 24;
9508         case PIPEMISC_DITHER_10_BPC:
9509                 return 30;
9510         case PIPEMISC_DITHER_12_BPC:
9511                 return 36;
9512         default:
9513                 MISSING_CASE(tmp);
9514                 return 0;
9515         }
9516 }
9517
9518 int ironlake_get_lanes_required(int target_clock, int link_bw, int bpp)
9519 {
9520         /*
9521          * Account for spread spectrum to avoid
9522          * oversubscribing the link. Max center spread
9523          * is 2.5%; use 5% for safety's sake.
9524          */
9525         u32 bps = target_clock * bpp * 21 / 20;
9526         return DIV_ROUND_UP(bps, link_bw * 8);
9527 }
9528
9529 static bool ironlake_needs_fb_cb_tune(struct dpll *dpll, int factor)
9530 {
9531         return i9xx_dpll_compute_m(dpll) < factor * dpll->n;
9532 }
9533
9534 static void ironlake_compute_dpll(struct intel_crtc *crtc,
9535                                   struct intel_crtc_state *crtc_state,
9536                                   struct dpll *reduced_clock)
9537 {
9538         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
9539         u32 dpll, fp, fp2;
9540         int factor;
9541
9542         /* Enable autotuning of the PLL clock (if permissible) */
9543         factor = 21;
9544         if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_LVDS)) {
9545                 if ((intel_panel_use_ssc(dev_priv) &&
9546                      dev_priv->vbt.lvds_ssc_freq == 100000) ||
9547                     (HAS_PCH_IBX(dev_priv) &&
9548                      intel_is_dual_link_lvds(dev_priv)))
9549                         factor = 25;
9550         } else if (crtc_state->sdvo_tv_clock) {
9551                 factor = 20;
9552         }
9553
9554         fp = i9xx_dpll_compute_fp(&crtc_state->dpll);
9555
9556         if (ironlake_needs_fb_cb_tune(&crtc_state->dpll, factor))
9557                 fp |= FP_CB_TUNE;
9558
9559         if (reduced_clock) {
9560                 fp2 = i9xx_dpll_compute_fp(reduced_clock);
9561
9562                 if (reduced_clock->m < factor * reduced_clock->n)
9563                         fp2 |= FP_CB_TUNE;
9564         } else {
9565                 fp2 = fp;
9566         }
9567
9568         dpll = 0;
9569
9570         if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_LVDS))
9571                 dpll |= DPLLB_MODE_LVDS;
9572         else
9573                 dpll |= DPLLB_MODE_DAC_SERIAL;
9574
9575         dpll |= (crtc_state->pixel_multiplier - 1)
9576                 << PLL_REF_SDVO_HDMI_MULTIPLIER_SHIFT;
9577
9578         if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_SDVO) ||
9579             intel_crtc_has_type(crtc_state, INTEL_OUTPUT_HDMI))
9580                 dpll |= DPLL_SDVO_HIGH_SPEED;
9581
9582         if (intel_crtc_has_dp_encoder(crtc_state))
9583                 dpll |= DPLL_SDVO_HIGH_SPEED;
9584
9585         /*
9586          * The high speed IO clock is only really required for
9587          * SDVO/HDMI/DP, but we also enable it for CRT to make it
9588          * possible to share the DPLL between CRT and HDMI. Enabling
9589          * the clock needlessly does no real harm, except use up a
9590          * bit of power potentially.
9591          *
9592          * We'll limit this to IVB with 3 pipes, since it has only two
9593          * DPLLs and so DPLL sharing is the only way to get three pipes
9594          * driving PCH ports at the same time. On SNB we could do this,
9595          * and potentially avoid enabling the second DPLL, but it's not
9596          * clear if it''s a win or loss power wise. No point in doing
9597          * this on ILK at all since it has a fixed DPLL<->pipe mapping.
9598          */
9599         if (INTEL_INFO(dev_priv)->num_pipes == 3 &&
9600             intel_crtc_has_type(crtc_state, INTEL_OUTPUT_ANALOG))
9601                 dpll |= DPLL_SDVO_HIGH_SPEED;
9602
9603         /* compute bitmask from p1 value */
9604         dpll |= (1 << (crtc_state->dpll.p1 - 1)) << DPLL_FPA01_P1_POST_DIV_SHIFT;
9605         /* also FPA1 */
9606         dpll |= (1 << (crtc_state->dpll.p1 - 1)) << DPLL_FPA1_P1_POST_DIV_SHIFT;
9607
9608         switch (crtc_state->dpll.p2) {
9609         case 5:
9610                 dpll |= DPLL_DAC_SERIAL_P2_CLOCK_DIV_5;
9611                 break;
9612         case 7:
9613                 dpll |= DPLLB_LVDS_P2_CLOCK_DIV_7;
9614                 break;
9615         case 10:
9616                 dpll |= DPLL_DAC_SERIAL_P2_CLOCK_DIV_10;
9617                 break;
9618         case 14:
9619                 dpll |= DPLLB_LVDS_P2_CLOCK_DIV_14;
9620                 break;
9621         }
9622
9623         if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_LVDS) &&
9624             intel_panel_use_ssc(dev_priv))
9625                 dpll |= PLLB_REF_INPUT_SPREADSPECTRUMIN;
9626         else
9627                 dpll |= PLL_REF_INPUT_DREFCLK;
9628
9629         dpll |= DPLL_VCO_ENABLE;
9630
9631         crtc_state->dpll_hw_state.dpll = dpll;
9632         crtc_state->dpll_hw_state.fp0 = fp;
9633         crtc_state->dpll_hw_state.fp1 = fp2;
9634 }
9635
9636 static int ironlake_crtc_compute_clock(struct intel_crtc *crtc,
9637                                        struct intel_crtc_state *crtc_state)
9638 {
9639         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
9640         struct intel_atomic_state *state =
9641                 to_intel_atomic_state(crtc_state->base.state);
9642         const struct intel_limit *limit;
9643         int refclk = 120000;
9644
9645         memset(&crtc_state->dpll_hw_state, 0,
9646                sizeof(crtc_state->dpll_hw_state));
9647
9648         /* CPU eDP is the only output that doesn't need a PCH PLL of its own. */
9649         if (!crtc_state->has_pch_encoder)
9650                 return 0;
9651
9652         if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_LVDS)) {
9653                 if (intel_panel_use_ssc(dev_priv)) {
9654                         DRM_DEBUG_KMS("using SSC reference clock of %d kHz\n",
9655                                       dev_priv->vbt.lvds_ssc_freq);
9656                         refclk = dev_priv->vbt.lvds_ssc_freq;
9657                 }
9658
9659                 if (intel_is_dual_link_lvds(dev_priv)) {
9660                         if (refclk == 100000)
9661                                 limit = &intel_limits_ironlake_dual_lvds_100m;
9662                         else
9663                                 limit = &intel_limits_ironlake_dual_lvds;
9664                 } else {
9665                         if (refclk == 100000)
9666                                 limit = &intel_limits_ironlake_single_lvds_100m;
9667                         else
9668                                 limit = &intel_limits_ironlake_single_lvds;
9669                 }
9670         } else {
9671                 limit = &intel_limits_ironlake_dac;
9672         }
9673
9674         if (!crtc_state->clock_set &&
9675             !g4x_find_best_dpll(limit, crtc_state, crtc_state->port_clock,
9676                                 refclk, NULL, &crtc_state->dpll)) {
9677                 DRM_ERROR("Couldn't find PLL settings for mode!\n");
9678                 return -EINVAL;
9679         }
9680
9681         ironlake_compute_dpll(crtc, crtc_state, NULL);
9682
9683         if (!intel_reserve_shared_dplls(state, crtc, NULL)) {
9684                 DRM_DEBUG_KMS("failed to find PLL for pipe %c\n",
9685                               pipe_name(crtc->pipe));
9686                 return -EINVAL;
9687         }
9688
9689         return 0;
9690 }
9691
9692 static void intel_pch_transcoder_get_m_n(struct intel_crtc *crtc,
9693                                          struct intel_link_m_n *m_n)
9694 {
9695         struct drm_device *dev = crtc->base.dev;
9696         struct drm_i915_private *dev_priv = to_i915(dev);
9697         enum pipe pipe = crtc->pipe;
9698
9699         m_n->link_m = I915_READ(PCH_TRANS_LINK_M1(pipe));
9700         m_n->link_n = I915_READ(PCH_TRANS_LINK_N1(pipe));
9701         m_n->gmch_m = I915_READ(PCH_TRANS_DATA_M1(pipe))
9702                 & ~TU_SIZE_MASK;
9703         m_n->gmch_n = I915_READ(PCH_TRANS_DATA_N1(pipe));
9704         m_n->tu = ((I915_READ(PCH_TRANS_DATA_M1(pipe))
9705                     & TU_SIZE_MASK) >> TU_SIZE_SHIFT) + 1;
9706 }
9707
9708 static void intel_cpu_transcoder_get_m_n(struct intel_crtc *crtc,
9709                                          enum transcoder transcoder,
9710                                          struct intel_link_m_n *m_n,
9711                                          struct intel_link_m_n *m2_n2)
9712 {
9713         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
9714         enum pipe pipe = crtc->pipe;
9715
9716         if (INTEL_GEN(dev_priv) >= 5) {
9717                 m_n->link_m = I915_READ(PIPE_LINK_M1(transcoder));
9718                 m_n->link_n = I915_READ(PIPE_LINK_N1(transcoder));
9719                 m_n->gmch_m = I915_READ(PIPE_DATA_M1(transcoder))
9720                         & ~TU_SIZE_MASK;
9721                 m_n->gmch_n = I915_READ(PIPE_DATA_N1(transcoder));
9722                 m_n->tu = ((I915_READ(PIPE_DATA_M1(transcoder))
9723                             & TU_SIZE_MASK) >> TU_SIZE_SHIFT) + 1;
9724
9725                 if (m2_n2 && transcoder_has_m2_n2(dev_priv, transcoder)) {
9726                         m2_n2->link_m = I915_READ(PIPE_LINK_M2(transcoder));
9727                         m2_n2->link_n = I915_READ(PIPE_LINK_N2(transcoder));
9728                         m2_n2->gmch_m = I915_READ(PIPE_DATA_M2(transcoder))
9729                                         & ~TU_SIZE_MASK;
9730                         m2_n2->gmch_n = I915_READ(PIPE_DATA_N2(transcoder));
9731                         m2_n2->tu = ((I915_READ(PIPE_DATA_M2(transcoder))
9732                                         & TU_SIZE_MASK) >> TU_SIZE_SHIFT) + 1;
9733                 }
9734         } else {
9735                 m_n->link_m = I915_READ(PIPE_LINK_M_G4X(pipe));
9736                 m_n->link_n = I915_READ(PIPE_LINK_N_G4X(pipe));
9737                 m_n->gmch_m = I915_READ(PIPE_DATA_M_G4X(pipe))
9738                         & ~TU_SIZE_MASK;
9739                 m_n->gmch_n = I915_READ(PIPE_DATA_N_G4X(pipe));
9740                 m_n->tu = ((I915_READ(PIPE_DATA_M_G4X(pipe))
9741                             & TU_SIZE_MASK) >> TU_SIZE_SHIFT) + 1;
9742         }
9743 }
9744
9745 void intel_dp_get_m_n(struct intel_crtc *crtc,
9746                       struct intel_crtc_state *pipe_config)
9747 {
9748         if (pipe_config->has_pch_encoder)
9749                 intel_pch_transcoder_get_m_n(crtc, &pipe_config->dp_m_n);
9750         else
9751                 intel_cpu_transcoder_get_m_n(crtc, pipe_config->cpu_transcoder,
9752                                              &pipe_config->dp_m_n,
9753                                              &pipe_config->dp_m2_n2);
9754 }
9755
9756 static void ironlake_get_fdi_m_n_config(struct intel_crtc *crtc,
9757                                         struct intel_crtc_state *pipe_config)
9758 {
9759         intel_cpu_transcoder_get_m_n(crtc, pipe_config->cpu_transcoder,
9760                                      &pipe_config->fdi_m_n, NULL);
9761 }
9762
9763 static void skylake_get_pfit_config(struct intel_crtc *crtc,
9764                                     struct intel_crtc_state *pipe_config)
9765 {
9766         struct drm_device *dev = crtc->base.dev;
9767         struct drm_i915_private *dev_priv = to_i915(dev);
9768         struct intel_crtc_scaler_state *scaler_state = &pipe_config->scaler_state;
9769         u32 ps_ctrl = 0;
9770         int id = -1;
9771         int i;
9772
9773         /* find scaler attached to this pipe */
9774         for (i = 0; i < crtc->num_scalers; i++) {
9775                 ps_ctrl = I915_READ(SKL_PS_CTRL(crtc->pipe, i));
9776                 if (ps_ctrl & PS_SCALER_EN && !(ps_ctrl & PS_PLANE_SEL_MASK)) {
9777                         id = i;
9778                         pipe_config->pch_pfit.enabled = true;
9779                         pipe_config->pch_pfit.pos = I915_READ(SKL_PS_WIN_POS(crtc->pipe, i));
9780                         pipe_config->pch_pfit.size = I915_READ(SKL_PS_WIN_SZ(crtc->pipe, i));
9781                         scaler_state->scalers[i].in_use = true;
9782                         break;
9783                 }
9784         }
9785
9786         scaler_state->scaler_id = id;
9787         if (id >= 0) {
9788                 scaler_state->scaler_users |= (1 << SKL_CRTC_INDEX);
9789         } else {
9790                 scaler_state->scaler_users &= ~(1 << SKL_CRTC_INDEX);
9791         }
9792 }
9793
9794 static void
9795 skylake_get_initial_plane_config(struct intel_crtc *crtc,
9796                                  struct intel_initial_plane_config *plane_config)
9797 {
9798         struct drm_device *dev = crtc->base.dev;
9799         struct drm_i915_private *dev_priv = to_i915(dev);
9800         struct intel_plane *plane = to_intel_plane(crtc->base.primary);
9801         enum plane_id plane_id = plane->id;
9802         enum pipe pipe;
9803         u32 val, base, offset, stride_mult, tiling, alpha;
9804         int fourcc, pixel_format;
9805         unsigned int aligned_height;
9806         struct drm_framebuffer *fb;
9807         struct intel_framebuffer *intel_fb;
9808
9809         if (!plane->get_hw_state(plane, &pipe))
9810                 return;
9811
9812         WARN_ON(pipe != crtc->pipe);
9813
9814         intel_fb = kzalloc(sizeof(*intel_fb), GFP_KERNEL);
9815         if (!intel_fb) {
9816                 DRM_DEBUG_KMS("failed to alloc fb\n");
9817                 return;
9818         }
9819
9820         fb = &intel_fb->base;
9821
9822         fb->dev = dev;
9823
9824         val = I915_READ(PLANE_CTL(pipe, plane_id));
9825
9826         if (INTEL_GEN(dev_priv) >= 11)
9827                 pixel_format = val & ICL_PLANE_CTL_FORMAT_MASK;
9828         else
9829                 pixel_format = val & PLANE_CTL_FORMAT_MASK;
9830
9831         if (INTEL_GEN(dev_priv) >= 10 || IS_GEMINILAKE(dev_priv)) {
9832                 alpha = I915_READ(PLANE_COLOR_CTL(pipe, plane_id));
9833                 alpha &= PLANE_COLOR_ALPHA_MASK;
9834         } else {
9835                 alpha = val & PLANE_CTL_ALPHA_MASK;
9836         }
9837
9838         fourcc = skl_format_to_fourcc(pixel_format,
9839                                       val & PLANE_CTL_ORDER_RGBX, alpha);
9840         fb->format = drm_format_info(fourcc);
9841
9842         tiling = val & PLANE_CTL_TILED_MASK;
9843         switch (tiling) {
9844         case PLANE_CTL_TILED_LINEAR:
9845                 fb->modifier = DRM_FORMAT_MOD_LINEAR;
9846                 break;
9847         case PLANE_CTL_TILED_X:
9848                 plane_config->tiling = I915_TILING_X;
9849                 fb->modifier = I915_FORMAT_MOD_X_TILED;
9850                 break;
9851         case PLANE_CTL_TILED_Y:
9852                 plane_config->tiling = I915_TILING_Y;
9853                 if (val & PLANE_CTL_RENDER_DECOMPRESSION_ENABLE)
9854                         fb->modifier = I915_FORMAT_MOD_Y_TILED_CCS;
9855                 else
9856                         fb->modifier = I915_FORMAT_MOD_Y_TILED;
9857                 break;
9858         case PLANE_CTL_TILED_YF:
9859                 if (val & PLANE_CTL_RENDER_DECOMPRESSION_ENABLE)
9860                         fb->modifier = I915_FORMAT_MOD_Yf_TILED_CCS;
9861                 else
9862                         fb->modifier = I915_FORMAT_MOD_Yf_TILED;
9863                 break;
9864         default:
9865                 MISSING_CASE(tiling);
9866                 goto error;
9867         }
9868
9869         /*
9870          * DRM_MODE_ROTATE_ is counter clockwise to stay compatible with Xrandr
9871          * while i915 HW rotation is clockwise, thats why this swapping.
9872          */
9873         switch (val & PLANE_CTL_ROTATE_MASK) {
9874         case PLANE_CTL_ROTATE_0:
9875                 plane_config->rotation = DRM_MODE_ROTATE_0;
9876                 break;
9877         case PLANE_CTL_ROTATE_90:
9878                 plane_config->rotation = DRM_MODE_ROTATE_270;
9879                 break;
9880         case PLANE_CTL_ROTATE_180:
9881                 plane_config->rotation = DRM_MODE_ROTATE_180;
9882                 break;
9883         case PLANE_CTL_ROTATE_270:
9884                 plane_config->rotation = DRM_MODE_ROTATE_90;
9885                 break;
9886         }
9887
9888         if (INTEL_GEN(dev_priv) >= 10 &&
9889             val & PLANE_CTL_FLIP_HORIZONTAL)
9890                 plane_config->rotation |= DRM_MODE_REFLECT_X;
9891
9892         base = I915_READ(PLANE_SURF(pipe, plane_id)) & 0xfffff000;
9893         plane_config->base = base;
9894
9895         offset = I915_READ(PLANE_OFFSET(pipe, plane_id));
9896
9897         val = I915_READ(PLANE_SIZE(pipe, plane_id));
9898         fb->height = ((val >> 16) & 0xfff) + 1;
9899         fb->width = ((val >> 0) & 0x1fff) + 1;
9900
9901         val = I915_READ(PLANE_STRIDE(pipe, plane_id));
9902         stride_mult = skl_plane_stride_mult(fb, 0, DRM_MODE_ROTATE_0);
9903         fb->pitches[0] = (val & 0x3ff) * stride_mult;
9904
9905         aligned_height = intel_fb_align_height(fb, 0, fb->height);
9906
9907         plane_config->size = fb->pitches[0] * aligned_height;
9908
9909         DRM_DEBUG_KMS("%s/%s with fb: size=%dx%d@%d, offset=%x, pitch %d, size 0x%x\n",
9910                       crtc->base.name, plane->base.name, fb->width, fb->height,
9911                       fb->format->cpp[0] * 8, base, fb->pitches[0],
9912                       plane_config->size);
9913
9914         plane_config->fb = intel_fb;
9915         return;
9916
9917 error:
9918         kfree(intel_fb);
9919 }
9920
9921 static void ironlake_get_pfit_config(struct intel_crtc *crtc,
9922                                      struct intel_crtc_state *pipe_config)
9923 {
9924         struct drm_device *dev = crtc->base.dev;
9925         struct drm_i915_private *dev_priv = to_i915(dev);
9926         u32 tmp;
9927
9928         tmp = I915_READ(PF_CTL(crtc->pipe));
9929
9930         if (tmp & PF_ENABLE) {
9931                 pipe_config->pch_pfit.enabled = true;
9932                 pipe_config->pch_pfit.pos = I915_READ(PF_WIN_POS(crtc->pipe));
9933                 pipe_config->pch_pfit.size = I915_READ(PF_WIN_SZ(crtc->pipe));
9934
9935                 /* We currently do not free assignements of panel fitters on
9936                  * ivb/hsw (since we don't use the higher upscaling modes which
9937                  * differentiates them) so just WARN about this case for now. */
9938                 if (IS_GEN(dev_priv, 7)) {
9939                         WARN_ON((tmp & PF_PIPE_SEL_MASK_IVB) !=
9940                                 PF_PIPE_SEL_IVB(crtc->pipe));
9941                 }
9942         }
9943 }
9944
9945 static bool ironlake_get_pipe_config(struct intel_crtc *crtc,
9946                                      struct intel_crtc_state *pipe_config)
9947 {
9948         struct drm_device *dev = crtc->base.dev;
9949         struct drm_i915_private *dev_priv = to_i915(dev);
9950         enum intel_display_power_domain power_domain;
9951         intel_wakeref_t wakeref;
9952         u32 tmp;
9953         bool ret;
9954
9955         power_domain = POWER_DOMAIN_PIPE(crtc->pipe);
9956         wakeref = intel_display_power_get_if_enabled(dev_priv, power_domain);
9957         if (!wakeref)
9958                 return false;
9959
9960         pipe_config->output_format = INTEL_OUTPUT_FORMAT_RGB;
9961         pipe_config->cpu_transcoder = (enum transcoder) crtc->pipe;
9962         pipe_config->shared_dpll = NULL;
9963
9964         ret = false;
9965         tmp = I915_READ(PIPECONF(crtc->pipe));
9966         if (!(tmp & PIPECONF_ENABLE))
9967                 goto out;
9968
9969         switch (tmp & PIPECONF_BPC_MASK) {
9970         case PIPECONF_6BPC:
9971                 pipe_config->pipe_bpp = 18;
9972                 break;
9973         case PIPECONF_8BPC:
9974                 pipe_config->pipe_bpp = 24;
9975                 break;
9976         case PIPECONF_10BPC:
9977                 pipe_config->pipe_bpp = 30;
9978                 break;
9979         case PIPECONF_12BPC:
9980                 pipe_config->pipe_bpp = 36;
9981                 break;
9982         default:
9983                 break;
9984         }
9985
9986         if (tmp & PIPECONF_COLOR_RANGE_SELECT)
9987                 pipe_config->limited_color_range = true;
9988
9989         pipe_config->gamma_mode = (tmp & PIPECONF_GAMMA_MODE_MASK_ILK) >>
9990                 PIPECONF_GAMMA_MODE_SHIFT;
9991
9992         pipe_config->csc_mode = I915_READ(PIPE_CSC_MODE(crtc->pipe));
9993
9994         i9xx_get_pipe_color_config(pipe_config);
9995         intel_color_get_config(pipe_config);
9996
9997         if (I915_READ(PCH_TRANSCONF(crtc->pipe)) & TRANS_ENABLE) {
9998                 struct intel_shared_dpll *pll;
9999                 enum intel_dpll_id pll_id;
10000
10001                 pipe_config->has_pch_encoder = true;
10002
10003                 tmp = I915_READ(FDI_RX_CTL(crtc->pipe));
10004                 pipe_config->fdi_lanes = ((FDI_DP_PORT_WIDTH_MASK & tmp) >>
10005                                           FDI_DP_PORT_WIDTH_SHIFT) + 1;
10006
10007                 ironlake_get_fdi_m_n_config(crtc, pipe_config);
10008
10009                 if (HAS_PCH_IBX(dev_priv)) {
10010                         /*
10011                          * The pipe->pch transcoder and pch transcoder->pll
10012                          * mapping is fixed.
10013                          */
10014                         pll_id = (enum intel_dpll_id) crtc->pipe;
10015                 } else {
10016                         tmp = I915_READ(PCH_DPLL_SEL);
10017                         if (tmp & TRANS_DPLLB_SEL(crtc->pipe))
10018                                 pll_id = DPLL_ID_PCH_PLL_B;
10019                         else
10020                                 pll_id= DPLL_ID_PCH_PLL_A;
10021                 }
10022
10023                 pipe_config->shared_dpll =
10024                         intel_get_shared_dpll_by_id(dev_priv, pll_id);
10025                 pll = pipe_config->shared_dpll;
10026
10027                 WARN_ON(!pll->info->funcs->get_hw_state(dev_priv, pll,
10028                                                 &pipe_config->dpll_hw_state));
10029
10030                 tmp = pipe_config->dpll_hw_state.dpll;
10031                 pipe_config->pixel_multiplier =
10032                         ((tmp & PLL_REF_SDVO_HDMI_MULTIPLIER_MASK)
10033                          >> PLL_REF_SDVO_HDMI_MULTIPLIER_SHIFT) + 1;
10034
10035                 ironlake_pch_clock_get(crtc, pipe_config);
10036         } else {
10037                 pipe_config->pixel_multiplier = 1;
10038         }
10039
10040         intel_get_pipe_timings(crtc, pipe_config);
10041         intel_get_pipe_src_size(crtc, pipe_config);
10042
10043         ironlake_get_pfit_config(crtc, pipe_config);
10044
10045         ret = true;
10046
10047 out:
10048         intel_display_power_put(dev_priv, power_domain, wakeref);
10049
10050         return ret;
10051 }
10052 static int haswell_crtc_compute_clock(struct intel_crtc *crtc,
10053                                       struct intel_crtc_state *crtc_state)
10054 {
10055         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
10056         struct intel_atomic_state *state =
10057                 to_intel_atomic_state(crtc_state->base.state);
10058
10059         if (!intel_crtc_has_type(crtc_state, INTEL_OUTPUT_DSI) ||
10060             INTEL_GEN(dev_priv) >= 11) {
10061                 struct intel_encoder *encoder =
10062                         intel_get_crtc_new_encoder(state, crtc_state);
10063
10064                 if (!intel_reserve_shared_dplls(state, crtc, encoder)) {
10065                         DRM_DEBUG_KMS("failed to find PLL for pipe %c\n",
10066                                       pipe_name(crtc->pipe));
10067                         return -EINVAL;
10068                 }
10069         }
10070
10071         return 0;
10072 }
10073
10074 static void cannonlake_get_ddi_pll(struct drm_i915_private *dev_priv,
10075                                    enum port port,
10076                                    struct intel_crtc_state *pipe_config)
10077 {
10078         enum intel_dpll_id id;
10079         u32 temp;
10080
10081         temp = I915_READ(DPCLKA_CFGCR0) & DPCLKA_CFGCR0_DDI_CLK_SEL_MASK(port);
10082         id = temp >> DPCLKA_CFGCR0_DDI_CLK_SEL_SHIFT(port);
10083
10084         if (WARN_ON(id < SKL_DPLL0 || id > SKL_DPLL2))
10085                 return;
10086
10087         pipe_config->shared_dpll = intel_get_shared_dpll_by_id(dev_priv, id);
10088 }
10089
10090 static void icelake_get_ddi_pll(struct drm_i915_private *dev_priv,
10091                                 enum port port,
10092                                 struct intel_crtc_state *pipe_config)
10093 {
10094         enum icl_port_dpll_id port_dpll_id;
10095         enum intel_dpll_id id;
10096         u32 temp;
10097
10098         if (intel_port_is_combophy(dev_priv, port)) {
10099                 temp = I915_READ(DPCLKA_CFGCR0_ICL) &
10100                        DPCLKA_CFGCR0_DDI_CLK_SEL_MASK(port);
10101                 id = temp >> DPCLKA_CFGCR0_DDI_CLK_SEL_SHIFT(port);
10102                 port_dpll_id = ICL_PORT_DPLL_DEFAULT;
10103         } else if (intel_port_is_tc(dev_priv, port)) {
10104                 u32 clk_sel = I915_READ(DDI_CLK_SEL(port)) & DDI_CLK_SEL_MASK;
10105
10106                 if (clk_sel == DDI_CLK_SEL_MG) {
10107                         id = icl_tc_port_to_pll_id(intel_port_to_tc(dev_priv,
10108                                                                     port));
10109                         port_dpll_id = ICL_PORT_DPLL_MG_PHY;
10110                 } else {
10111                         WARN_ON(clk_sel < DDI_CLK_SEL_TBT_162);
10112                         id = DPLL_ID_ICL_TBTPLL;
10113                         port_dpll_id = ICL_PORT_DPLL_DEFAULT;
10114                 }
10115         } else {
10116                 WARN(1, "Invalid port %x\n", port);
10117                 return;
10118         }
10119
10120         pipe_config->icl_port_dplls[port_dpll_id].pll =
10121                 intel_get_shared_dpll_by_id(dev_priv, id);
10122
10123         icl_set_active_port_dpll(pipe_config, port_dpll_id);
10124 }
10125
10126 static void bxt_get_ddi_pll(struct drm_i915_private *dev_priv,
10127                                 enum port port,
10128                                 struct intel_crtc_state *pipe_config)
10129 {
10130         enum intel_dpll_id id;
10131
10132         switch (port) {
10133         case PORT_A:
10134                 id = DPLL_ID_SKL_DPLL0;
10135                 break;
10136         case PORT_B:
10137                 id = DPLL_ID_SKL_DPLL1;
10138                 break;
10139         case PORT_C:
10140                 id = DPLL_ID_SKL_DPLL2;
10141                 break;
10142         default:
10143                 DRM_ERROR("Incorrect port type\n");
10144                 return;
10145         }
10146
10147         pipe_config->shared_dpll = intel_get_shared_dpll_by_id(dev_priv, id);
10148 }
10149
10150 static void skylake_get_ddi_pll(struct drm_i915_private *dev_priv,
10151                                 enum port port,
10152                                 struct intel_crtc_state *pipe_config)
10153 {
10154         enum intel_dpll_id id;
10155         u32 temp;
10156
10157         temp = I915_READ(DPLL_CTRL2) & DPLL_CTRL2_DDI_CLK_SEL_MASK(port);
10158         id = temp >> (port * 3 + 1);
10159
10160         if (WARN_ON(id < SKL_DPLL0 || id > SKL_DPLL3))
10161                 return;
10162
10163         pipe_config->shared_dpll = intel_get_shared_dpll_by_id(dev_priv, id);
10164 }
10165
10166 static void haswell_get_ddi_pll(struct drm_i915_private *dev_priv,
10167                                 enum port port,
10168                                 struct intel_crtc_state *pipe_config)
10169 {
10170         enum intel_dpll_id id;
10171         u32 ddi_pll_sel = I915_READ(PORT_CLK_SEL(port));
10172
10173         switch (ddi_pll_sel) {
10174         case PORT_CLK_SEL_WRPLL1:
10175                 id = DPLL_ID_WRPLL1;
10176                 break;
10177         case PORT_CLK_SEL_WRPLL2:
10178                 id = DPLL_ID_WRPLL2;
10179                 break;
10180         case PORT_CLK_SEL_SPLL:
10181                 id = DPLL_ID_SPLL;
10182                 break;
10183         case PORT_CLK_SEL_LCPLL_810:
10184                 id = DPLL_ID_LCPLL_810;
10185                 break;
10186         case PORT_CLK_SEL_LCPLL_1350:
10187                 id = DPLL_ID_LCPLL_1350;
10188                 break;
10189         case PORT_CLK_SEL_LCPLL_2700:
10190                 id = DPLL_ID_LCPLL_2700;
10191                 break;
10192         default:
10193                 MISSING_CASE(ddi_pll_sel);
10194                 /* fall through */
10195         case PORT_CLK_SEL_NONE:
10196                 return;
10197         }
10198
10199         pipe_config->shared_dpll = intel_get_shared_dpll_by_id(dev_priv, id);
10200 }
10201
10202 static bool hsw_get_transcoder_state(struct intel_crtc *crtc,
10203                                      struct intel_crtc_state *pipe_config,
10204                                      u64 *power_domain_mask,
10205                                      intel_wakeref_t *wakerefs)
10206 {
10207         struct drm_device *dev = crtc->base.dev;
10208         struct drm_i915_private *dev_priv = to_i915(dev);
10209         enum intel_display_power_domain power_domain;
10210         unsigned long panel_transcoder_mask = 0;
10211         unsigned long enabled_panel_transcoders = 0;
10212         enum transcoder panel_transcoder;
10213         intel_wakeref_t wf;
10214         u32 tmp;
10215
10216         if (INTEL_GEN(dev_priv) >= 11)
10217                 panel_transcoder_mask |=
10218                         BIT(TRANSCODER_DSI_0) | BIT(TRANSCODER_DSI_1);
10219
10220         if (HAS_TRANSCODER_EDP(dev_priv))
10221                 panel_transcoder_mask |= BIT(TRANSCODER_EDP);
10222
10223         /*
10224          * The pipe->transcoder mapping is fixed with the exception of the eDP
10225          * and DSI transcoders handled below.
10226          */
10227         pipe_config->cpu_transcoder = (enum transcoder) crtc->pipe;
10228
10229         /*
10230          * XXX: Do intel_display_power_get_if_enabled before reading this (for
10231          * consistency and less surprising code; it's in always on power).
10232          */
10233         for_each_set_bit(panel_transcoder,
10234                          &panel_transcoder_mask,
10235                          ARRAY_SIZE(INTEL_INFO(dev_priv)->trans_offsets)) {
10236                 bool force_thru = false;
10237                 enum pipe trans_pipe;
10238
10239                 tmp = I915_READ(TRANS_DDI_FUNC_CTL(panel_transcoder));
10240                 if (!(tmp & TRANS_DDI_FUNC_ENABLE))
10241                         continue;
10242
10243                 /*
10244                  * Log all enabled ones, only use the first one.
10245                  *
10246                  * FIXME: This won't work for two separate DSI displays.
10247                  */
10248                 enabled_panel_transcoders |= BIT(panel_transcoder);
10249                 if (enabled_panel_transcoders != BIT(panel_transcoder))
10250                         continue;
10251
10252                 switch (tmp & TRANS_DDI_EDP_INPUT_MASK) {
10253                 default:
10254                         WARN(1, "unknown pipe linked to transcoder %s\n",
10255                              transcoder_name(panel_transcoder));
10256                         /* fall through */
10257                 case TRANS_DDI_EDP_INPUT_A_ONOFF:
10258                         force_thru = true;
10259                         /* fall through */
10260                 case TRANS_DDI_EDP_INPUT_A_ON:
10261                         trans_pipe = PIPE_A;
10262                         break;
10263                 case TRANS_DDI_EDP_INPUT_B_ONOFF:
10264                         trans_pipe = PIPE_B;
10265                         break;
10266                 case TRANS_DDI_EDP_INPUT_C_ONOFF:
10267                         trans_pipe = PIPE_C;
10268                         break;
10269                 }
10270
10271                 if (trans_pipe == crtc->pipe) {
10272                         pipe_config->cpu_transcoder = panel_transcoder;
10273                         pipe_config->pch_pfit.force_thru = force_thru;
10274                 }
10275         }
10276
10277         /*
10278          * Valid combos: none, eDP, DSI0, DSI1, DSI0+DSI1
10279          */
10280         WARN_ON((enabled_panel_transcoders & BIT(TRANSCODER_EDP)) &&
10281                 enabled_panel_transcoders != BIT(TRANSCODER_EDP));
10282
10283         power_domain = POWER_DOMAIN_TRANSCODER(pipe_config->cpu_transcoder);
10284         WARN_ON(*power_domain_mask & BIT_ULL(power_domain));
10285
10286         wf = intel_display_power_get_if_enabled(dev_priv, power_domain);
10287         if (!wf)
10288                 return false;
10289
10290         wakerefs[power_domain] = wf;
10291         *power_domain_mask |= BIT_ULL(power_domain);
10292
10293         tmp = I915_READ(PIPECONF(pipe_config->cpu_transcoder));
10294
10295         return tmp & PIPECONF_ENABLE;
10296 }
10297
10298 static bool bxt_get_dsi_transcoder_state(struct intel_crtc *crtc,
10299                                          struct intel_crtc_state *pipe_config,
10300                                          u64 *power_domain_mask,
10301                                          intel_wakeref_t *wakerefs)
10302 {
10303         struct drm_device *dev = crtc->base.dev;
10304         struct drm_i915_private *dev_priv = to_i915(dev);
10305         enum intel_display_power_domain power_domain;
10306         enum transcoder cpu_transcoder;
10307         intel_wakeref_t wf;
10308         enum port port;
10309         u32 tmp;
10310
10311         for_each_port_masked(port, BIT(PORT_A) | BIT(PORT_C)) {
10312                 if (port == PORT_A)
10313                         cpu_transcoder = TRANSCODER_DSI_A;
10314                 else
10315                         cpu_transcoder = TRANSCODER_DSI_C;
10316
10317                 power_domain = POWER_DOMAIN_TRANSCODER(cpu_transcoder);
10318                 WARN_ON(*power_domain_mask & BIT_ULL(power_domain));
10319
10320                 wf = intel_display_power_get_if_enabled(dev_priv, power_domain);
10321                 if (!wf)
10322                         continue;
10323
10324                 wakerefs[power_domain] = wf;
10325                 *power_domain_mask |= BIT_ULL(power_domain);
10326
10327                 /*
10328                  * The PLL needs to be enabled with a valid divider
10329                  * configuration, otherwise accessing DSI registers will hang
10330                  * the machine. See BSpec North Display Engine
10331                  * registers/MIPI[BXT]. We can break out here early, since we
10332                  * need the same DSI PLL to be enabled for both DSI ports.
10333                  */
10334                 if (!bxt_dsi_pll_is_enabled(dev_priv))
10335                         break;
10336
10337                 /* XXX: this works for video mode only */
10338                 tmp = I915_READ(BXT_MIPI_PORT_CTRL(port));
10339                 if (!(tmp & DPI_ENABLE))
10340                         continue;
10341
10342                 tmp = I915_READ(MIPI_CTRL(port));
10343                 if ((tmp & BXT_PIPE_SELECT_MASK) != BXT_PIPE_SELECT(crtc->pipe))
10344                         continue;
10345
10346                 pipe_config->cpu_transcoder = cpu_transcoder;
10347                 break;
10348         }
10349
10350         return transcoder_is_dsi(pipe_config->cpu_transcoder);
10351 }
10352
10353 static void haswell_get_ddi_port_state(struct intel_crtc *crtc,
10354                                        struct intel_crtc_state *pipe_config)
10355 {
10356         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
10357         struct intel_shared_dpll *pll;
10358         enum port port;
10359         u32 tmp;
10360
10361         tmp = I915_READ(TRANS_DDI_FUNC_CTL(pipe_config->cpu_transcoder));
10362
10363         port = (tmp & TRANS_DDI_PORT_MASK) >> TRANS_DDI_PORT_SHIFT;
10364
10365         if (INTEL_GEN(dev_priv) >= 11)
10366                 icelake_get_ddi_pll(dev_priv, port, pipe_config);
10367         else if (IS_CANNONLAKE(dev_priv))
10368                 cannonlake_get_ddi_pll(dev_priv, port, pipe_config);
10369         else if (IS_GEN9_BC(dev_priv))
10370                 skylake_get_ddi_pll(dev_priv, port, pipe_config);
10371         else if (IS_GEN9_LP(dev_priv))
10372                 bxt_get_ddi_pll(dev_priv, port, pipe_config);
10373         else
10374                 haswell_get_ddi_pll(dev_priv, port, pipe_config);
10375
10376         pll = pipe_config->shared_dpll;
10377         if (pll) {
10378                 WARN_ON(!pll->info->funcs->get_hw_state(dev_priv, pll,
10379                                                 &pipe_config->dpll_hw_state));
10380         }
10381
10382         /*
10383          * Haswell has only FDI/PCH transcoder A. It is which is connected to
10384          * DDI E. So just check whether this pipe is wired to DDI E and whether
10385          * the PCH transcoder is on.
10386          */
10387         if (INTEL_GEN(dev_priv) < 9 &&
10388             (port == PORT_E) && I915_READ(LPT_TRANSCONF) & TRANS_ENABLE) {
10389                 pipe_config->has_pch_encoder = true;
10390
10391                 tmp = I915_READ(FDI_RX_CTL(PIPE_A));
10392                 pipe_config->fdi_lanes = ((FDI_DP_PORT_WIDTH_MASK & tmp) >>
10393                                           FDI_DP_PORT_WIDTH_SHIFT) + 1;
10394
10395                 ironlake_get_fdi_m_n_config(crtc, pipe_config);
10396         }
10397 }
10398
10399 static bool haswell_get_pipe_config(struct intel_crtc *crtc,
10400                                     struct intel_crtc_state *pipe_config)
10401 {
10402         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
10403         intel_wakeref_t wakerefs[POWER_DOMAIN_NUM], wf;
10404         enum intel_display_power_domain power_domain;
10405         u64 power_domain_mask;
10406         bool active;
10407
10408         intel_crtc_init_scalers(crtc, pipe_config);
10409
10410         power_domain = POWER_DOMAIN_PIPE(crtc->pipe);
10411         wf = intel_display_power_get_if_enabled(dev_priv, power_domain);
10412         if (!wf)
10413                 return false;
10414
10415         wakerefs[power_domain] = wf;
10416         power_domain_mask = BIT_ULL(power_domain);
10417
10418         pipe_config->shared_dpll = NULL;
10419
10420         active = hsw_get_transcoder_state(crtc, pipe_config,
10421                                           &power_domain_mask, wakerefs);
10422
10423         if (IS_GEN9_LP(dev_priv) &&
10424             bxt_get_dsi_transcoder_state(crtc, pipe_config,
10425                                          &power_domain_mask, wakerefs)) {
10426                 WARN_ON(active);
10427                 active = true;
10428         }
10429
10430         if (!active)
10431                 goto out;
10432
10433         if (!transcoder_is_dsi(pipe_config->cpu_transcoder) ||
10434             INTEL_GEN(dev_priv) >= 11) {
10435                 haswell_get_ddi_port_state(crtc, pipe_config);
10436                 intel_get_pipe_timings(crtc, pipe_config);
10437         }
10438
10439         intel_get_pipe_src_size(crtc, pipe_config);
10440         intel_get_crtc_ycbcr_config(crtc, pipe_config);
10441
10442         pipe_config->gamma_mode = I915_READ(GAMMA_MODE(crtc->pipe));
10443
10444         pipe_config->csc_mode = I915_READ(PIPE_CSC_MODE(crtc->pipe));
10445
10446         if (INTEL_GEN(dev_priv) >= 9) {
10447                 u32 tmp = I915_READ(SKL_BOTTOM_COLOR(crtc->pipe));
10448
10449                 if (tmp & SKL_BOTTOM_COLOR_GAMMA_ENABLE)
10450                         pipe_config->gamma_enable = true;
10451
10452                 if (tmp & SKL_BOTTOM_COLOR_CSC_ENABLE)
10453                         pipe_config->csc_enable = true;
10454         } else {
10455                 i9xx_get_pipe_color_config(pipe_config);
10456         }
10457
10458         intel_color_get_config(pipe_config);
10459
10460         power_domain = POWER_DOMAIN_PIPE_PANEL_FITTER(crtc->pipe);
10461         WARN_ON(power_domain_mask & BIT_ULL(power_domain));
10462
10463         wf = intel_display_power_get_if_enabled(dev_priv, power_domain);
10464         if (wf) {
10465                 wakerefs[power_domain] = wf;
10466                 power_domain_mask |= BIT_ULL(power_domain);
10467
10468                 if (INTEL_GEN(dev_priv) >= 9)
10469                         skylake_get_pfit_config(crtc, pipe_config);
10470                 else
10471                         ironlake_get_pfit_config(crtc, pipe_config);
10472         }
10473
10474         if (hsw_crtc_supports_ips(crtc)) {
10475                 if (IS_HASWELL(dev_priv))
10476                         pipe_config->ips_enabled = I915_READ(IPS_CTL) & IPS_ENABLE;
10477                 else {
10478                         /*
10479                          * We cannot readout IPS state on broadwell, set to
10480                          * true so we can set it to a defined state on first
10481                          * commit.
10482                          */
10483                         pipe_config->ips_enabled = true;
10484                 }
10485         }
10486
10487         if (pipe_config->cpu_transcoder != TRANSCODER_EDP &&
10488             !transcoder_is_dsi(pipe_config->cpu_transcoder)) {
10489                 pipe_config->pixel_multiplier =
10490                         I915_READ(PIPE_MULT(pipe_config->cpu_transcoder)) + 1;
10491         } else {
10492                 pipe_config->pixel_multiplier = 1;
10493         }
10494
10495 out:
10496         for_each_power_domain(power_domain, power_domain_mask)
10497                 intel_display_power_put(dev_priv,
10498                                         power_domain, wakerefs[power_domain]);
10499
10500         return active;
10501 }
10502
10503 static u32 intel_cursor_base(const struct intel_plane_state *plane_state)
10504 {
10505         struct drm_i915_private *dev_priv =
10506                 to_i915(plane_state->base.plane->dev);
10507         const struct drm_framebuffer *fb = plane_state->base.fb;
10508         const struct drm_i915_gem_object *obj = intel_fb_obj(fb);
10509         u32 base;
10510
10511         if (INTEL_INFO(dev_priv)->display.cursor_needs_physical)
10512                 base = obj->phys_handle->busaddr;
10513         else
10514                 base = intel_plane_ggtt_offset(plane_state);
10515
10516         base += plane_state->color_plane[0].offset;
10517
10518         /* ILK+ do this automagically */
10519         if (HAS_GMCH(dev_priv) &&
10520             plane_state->base.rotation & DRM_MODE_ROTATE_180)
10521                 base += (plane_state->base.crtc_h *
10522                          plane_state->base.crtc_w - 1) * fb->format->cpp[0];
10523
10524         return base;
10525 }
10526
10527 static u32 intel_cursor_position(const struct intel_plane_state *plane_state)
10528 {
10529         int x = plane_state->base.crtc_x;
10530         int y = plane_state->base.crtc_y;
10531         u32 pos = 0;
10532
10533         if (x < 0) {
10534                 pos |= CURSOR_POS_SIGN << CURSOR_X_SHIFT;
10535                 x = -x;
10536         }
10537         pos |= x << CURSOR_X_SHIFT;
10538
10539         if (y < 0) {
10540                 pos |= CURSOR_POS_SIGN << CURSOR_Y_SHIFT;
10541                 y = -y;
10542         }
10543         pos |= y << CURSOR_Y_SHIFT;
10544
10545         return pos;
10546 }
10547
10548 static bool intel_cursor_size_ok(const struct intel_plane_state *plane_state)
10549 {
10550         const struct drm_mode_config *config =
10551                 &plane_state->base.plane->dev->mode_config;
10552         int width = plane_state->base.crtc_w;
10553         int height = plane_state->base.crtc_h;
10554
10555         return width > 0 && width <= config->cursor_width &&
10556                 height > 0 && height <= config->cursor_height;
10557 }
10558
10559 static int intel_cursor_check_surface(struct intel_plane_state *plane_state)
10560 {
10561         int src_x, src_y;
10562         u32 offset;
10563         int ret;
10564
10565         ret = intel_plane_compute_gtt(plane_state);
10566         if (ret)
10567                 return ret;
10568
10569         if (!plane_state->base.visible)
10570                 return 0;
10571
10572         src_x = plane_state->base.src_x >> 16;
10573         src_y = plane_state->base.src_y >> 16;
10574
10575         intel_add_fb_offsets(&src_x, &src_y, plane_state, 0);
10576         offset = intel_plane_compute_aligned_offset(&src_x, &src_y,
10577                                                     plane_state, 0);
10578
10579         if (src_x != 0 || src_y != 0) {
10580                 DRM_DEBUG_KMS("Arbitrary cursor panning not supported\n");
10581                 return -EINVAL;
10582         }
10583
10584         plane_state->color_plane[0].offset = offset;
10585
10586         return 0;
10587 }
10588
10589 static int intel_check_cursor(struct intel_crtc_state *crtc_state,
10590                               struct intel_plane_state *plane_state)
10591 {
10592         const struct drm_framebuffer *fb = plane_state->base.fb;
10593         int ret;
10594
10595         if (fb && fb->modifier != DRM_FORMAT_MOD_LINEAR) {
10596                 DRM_DEBUG_KMS("cursor cannot be tiled\n");
10597                 return -EINVAL;
10598         }
10599
10600         ret = drm_atomic_helper_check_plane_state(&plane_state->base,
10601                                                   &crtc_state->base,
10602                                                   DRM_PLANE_HELPER_NO_SCALING,
10603                                                   DRM_PLANE_HELPER_NO_SCALING,
10604                                                   true, true);
10605         if (ret)
10606                 return ret;
10607
10608         ret = intel_cursor_check_surface(plane_state);
10609         if (ret)
10610                 return ret;
10611
10612         if (!plane_state->base.visible)
10613                 return 0;
10614
10615         ret = intel_plane_check_src_coordinates(plane_state);
10616         if (ret)
10617                 return ret;
10618
10619         return 0;
10620 }
10621
10622 static unsigned int
10623 i845_cursor_max_stride(struct intel_plane *plane,
10624                        u32 pixel_format, u64 modifier,
10625                        unsigned int rotation)
10626 {
10627         return 2048;
10628 }
10629
10630 static u32 i845_cursor_ctl_crtc(const struct intel_crtc_state *crtc_state)
10631 {
10632         u32 cntl = 0;
10633
10634         if (crtc_state->gamma_enable)
10635                 cntl |= CURSOR_GAMMA_ENABLE;
10636
10637         return cntl;
10638 }
10639
10640 static u32 i845_cursor_ctl(const struct intel_crtc_state *crtc_state,
10641                            const struct intel_plane_state *plane_state)
10642 {
10643         return CURSOR_ENABLE |
10644                 CURSOR_FORMAT_ARGB |
10645                 CURSOR_STRIDE(plane_state->color_plane[0].stride);
10646 }
10647
10648 static bool i845_cursor_size_ok(const struct intel_plane_state *plane_state)
10649 {
10650         int width = plane_state->base.crtc_w;
10651
10652         /*
10653          * 845g/865g are only limited by the width of their cursors,
10654          * the height is arbitrary up to the precision of the register.
10655          */
10656         return intel_cursor_size_ok(plane_state) && IS_ALIGNED(width, 64);
10657 }
10658
10659 static int i845_check_cursor(struct intel_crtc_state *crtc_state,
10660                              struct intel_plane_state *plane_state)
10661 {
10662         const struct drm_framebuffer *fb = plane_state->base.fb;
10663         int ret;
10664
10665         ret = intel_check_cursor(crtc_state, plane_state);
10666         if (ret)
10667                 return ret;
10668
10669         /* if we want to turn off the cursor ignore width and height */
10670         if (!fb)
10671                 return 0;
10672
10673         /* Check for which cursor types we support */
10674         if (!i845_cursor_size_ok(plane_state)) {
10675                 DRM_DEBUG("Cursor dimension %dx%d not supported\n",
10676                           plane_state->base.crtc_w,
10677                           plane_state->base.crtc_h);
10678                 return -EINVAL;
10679         }
10680
10681         WARN_ON(plane_state->base.visible &&
10682                 plane_state->color_plane[0].stride != fb->pitches[0]);
10683
10684         switch (fb->pitches[0]) {
10685         case 256:
10686         case 512:
10687         case 1024:
10688         case 2048:
10689                 break;
10690         default:
10691                 DRM_DEBUG_KMS("Invalid cursor stride (%u)\n",
10692                               fb->pitches[0]);
10693                 return -EINVAL;
10694         }
10695
10696         plane_state->ctl = i845_cursor_ctl(crtc_state, plane_state);
10697
10698         return 0;
10699 }
10700
10701 static void i845_update_cursor(struct intel_plane *plane,
10702                                const struct intel_crtc_state *crtc_state,
10703                                const struct intel_plane_state *plane_state)
10704 {
10705         struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
10706         u32 cntl = 0, base = 0, pos = 0, size = 0;
10707         unsigned long irqflags;
10708
10709         if (plane_state && plane_state->base.visible) {
10710                 unsigned int width = plane_state->base.crtc_w;
10711                 unsigned int height = plane_state->base.crtc_h;
10712
10713                 cntl = plane_state->ctl |
10714                         i845_cursor_ctl_crtc(crtc_state);
10715
10716                 size = (height << 12) | width;
10717
10718                 base = intel_cursor_base(plane_state);
10719                 pos = intel_cursor_position(plane_state);
10720         }
10721
10722         spin_lock_irqsave(&dev_priv->uncore.lock, irqflags);
10723
10724         /* On these chipsets we can only modify the base/size/stride
10725          * whilst the cursor is disabled.
10726          */
10727         if (plane->cursor.base != base ||
10728             plane->cursor.size != size ||
10729             plane->cursor.cntl != cntl) {
10730                 I915_WRITE_FW(CURCNTR(PIPE_A), 0);
10731                 I915_WRITE_FW(CURBASE(PIPE_A), base);
10732                 I915_WRITE_FW(CURSIZE, size);
10733                 I915_WRITE_FW(CURPOS(PIPE_A), pos);
10734                 I915_WRITE_FW(CURCNTR(PIPE_A), cntl);
10735
10736                 plane->cursor.base = base;
10737                 plane->cursor.size = size;
10738                 plane->cursor.cntl = cntl;
10739         } else {
10740                 I915_WRITE_FW(CURPOS(PIPE_A), pos);
10741         }
10742
10743         spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags);
10744 }
10745
10746 static void i845_disable_cursor(struct intel_plane *plane,
10747                                 const struct intel_crtc_state *crtc_state)
10748 {
10749         i845_update_cursor(plane, crtc_state, NULL);
10750 }
10751
10752 static bool i845_cursor_get_hw_state(struct intel_plane *plane,
10753                                      enum pipe *pipe)
10754 {
10755         struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
10756         enum intel_display_power_domain power_domain;
10757         intel_wakeref_t wakeref;
10758         bool ret;
10759
10760         power_domain = POWER_DOMAIN_PIPE(PIPE_A);
10761         wakeref = intel_display_power_get_if_enabled(dev_priv, power_domain);
10762         if (!wakeref)
10763                 return false;
10764
10765         ret = I915_READ(CURCNTR(PIPE_A)) & CURSOR_ENABLE;
10766
10767         *pipe = PIPE_A;
10768
10769         intel_display_power_put(dev_priv, power_domain, wakeref);
10770
10771         return ret;
10772 }
10773
10774 static unsigned int
10775 i9xx_cursor_max_stride(struct intel_plane *plane,
10776                        u32 pixel_format, u64 modifier,
10777                        unsigned int rotation)
10778 {
10779         return plane->base.dev->mode_config.cursor_width * 4;
10780 }
10781
10782 static u32 i9xx_cursor_ctl_crtc(const struct intel_crtc_state *crtc_state)
10783 {
10784         struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
10785         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
10786         u32 cntl = 0;
10787
10788         if (INTEL_GEN(dev_priv) >= 11)
10789                 return cntl;
10790
10791         if (crtc_state->gamma_enable)
10792                 cntl = MCURSOR_GAMMA_ENABLE;
10793
10794         if (crtc_state->csc_enable)
10795                 cntl |= MCURSOR_PIPE_CSC_ENABLE;
10796
10797         if (INTEL_GEN(dev_priv) < 5 && !IS_G4X(dev_priv))
10798                 cntl |= MCURSOR_PIPE_SELECT(crtc->pipe);
10799
10800         return cntl;
10801 }
10802
10803 static u32 i9xx_cursor_ctl(const struct intel_crtc_state *crtc_state,
10804                            const struct intel_plane_state *plane_state)
10805 {
10806         struct drm_i915_private *dev_priv =
10807                 to_i915(plane_state->base.plane->dev);
10808         u32 cntl = 0;
10809
10810         if (IS_GEN(dev_priv, 6) || IS_IVYBRIDGE(dev_priv))
10811                 cntl |= MCURSOR_TRICKLE_FEED_DISABLE;
10812
10813         switch (plane_state->base.crtc_w) {
10814         case 64:
10815                 cntl |= MCURSOR_MODE_64_ARGB_AX;
10816                 break;
10817         case 128:
10818                 cntl |= MCURSOR_MODE_128_ARGB_AX;
10819                 break;
10820         case 256:
10821                 cntl |= MCURSOR_MODE_256_ARGB_AX;
10822                 break;
10823         default:
10824                 MISSING_CASE(plane_state->base.crtc_w);
10825                 return 0;
10826         }
10827
10828         if (plane_state->base.rotation & DRM_MODE_ROTATE_180)
10829                 cntl |= MCURSOR_ROTATE_180;
10830
10831         return cntl;
10832 }
10833
10834 static bool i9xx_cursor_size_ok(const struct intel_plane_state *plane_state)
10835 {
10836         struct drm_i915_private *dev_priv =
10837                 to_i915(plane_state->base.plane->dev);
10838         int width = plane_state->base.crtc_w;
10839         int height = plane_state->base.crtc_h;
10840
10841         if (!intel_cursor_size_ok(plane_state))
10842                 return false;
10843
10844         /* Cursor width is limited to a few power-of-two sizes */
10845         switch (width) {
10846         case 256:
10847         case 128:
10848         case 64:
10849                 break;
10850         default:
10851                 return false;
10852         }
10853
10854         /*
10855          * IVB+ have CUR_FBC_CTL which allows an arbitrary cursor
10856          * height from 8 lines up to the cursor width, when the
10857          * cursor is not rotated. Everything else requires square
10858          * cursors.
10859          */
10860         if (HAS_CUR_FBC(dev_priv) &&
10861             plane_state->base.rotation & DRM_MODE_ROTATE_0) {
10862                 if (height < 8 || height > width)
10863                         return false;
10864         } else {
10865                 if (height != width)
10866                         return false;
10867         }
10868
10869         return true;
10870 }
10871
10872 static int i9xx_check_cursor(struct intel_crtc_state *crtc_state,
10873                              struct intel_plane_state *plane_state)
10874 {
10875         struct intel_plane *plane = to_intel_plane(plane_state->base.plane);
10876         struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
10877         const struct drm_framebuffer *fb = plane_state->base.fb;
10878         enum pipe pipe = plane->pipe;
10879         int ret;
10880
10881         ret = intel_check_cursor(crtc_state, plane_state);
10882         if (ret)
10883                 return ret;
10884
10885         /* if we want to turn off the cursor ignore width and height */
10886         if (!fb)
10887                 return 0;
10888
10889         /* Check for which cursor types we support */
10890         if (!i9xx_cursor_size_ok(plane_state)) {
10891                 DRM_DEBUG("Cursor dimension %dx%d not supported\n",
10892                           plane_state->base.crtc_w,
10893                           plane_state->base.crtc_h);
10894                 return -EINVAL;
10895         }
10896
10897         WARN_ON(plane_state->base.visible &&
10898                 plane_state->color_plane[0].stride != fb->pitches[0]);
10899
10900         if (fb->pitches[0] != plane_state->base.crtc_w * fb->format->cpp[0]) {
10901                 DRM_DEBUG_KMS("Invalid cursor stride (%u) (cursor width %d)\n",
10902                               fb->pitches[0], plane_state->base.crtc_w);
10903                 return -EINVAL;
10904         }
10905
10906         /*
10907          * There's something wrong with the cursor on CHV pipe C.
10908          * If it straddles the left edge of the screen then
10909          * moving it away from the edge or disabling it often
10910          * results in a pipe underrun, and often that can lead to
10911          * dead pipe (constant underrun reported, and it scans
10912          * out just a solid color). To recover from that, the
10913          * display power well must be turned off and on again.
10914          * Refuse the put the cursor into that compromised position.
10915          */
10916         if (IS_CHERRYVIEW(dev_priv) && pipe == PIPE_C &&
10917             plane_state->base.visible && plane_state->base.crtc_x < 0) {
10918                 DRM_DEBUG_KMS("CHV cursor C not allowed to straddle the left screen edge\n");
10919                 return -EINVAL;
10920         }
10921
10922         plane_state->ctl = i9xx_cursor_ctl(crtc_state, plane_state);
10923
10924         return 0;
10925 }
10926
10927 static void i9xx_update_cursor(struct intel_plane *plane,
10928                                const struct intel_crtc_state *crtc_state,
10929                                const struct intel_plane_state *plane_state)
10930 {
10931         struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
10932         enum pipe pipe = plane->pipe;
10933         u32 cntl = 0, base = 0, pos = 0, fbc_ctl = 0;
10934         unsigned long irqflags;
10935
10936         if (plane_state && plane_state->base.visible) {
10937                 cntl = plane_state->ctl |
10938                         i9xx_cursor_ctl_crtc(crtc_state);
10939
10940                 if (plane_state->base.crtc_h != plane_state->base.crtc_w)
10941                         fbc_ctl = CUR_FBC_CTL_EN | (plane_state->base.crtc_h - 1);
10942
10943                 base = intel_cursor_base(plane_state);
10944                 pos = intel_cursor_position(plane_state);
10945         }
10946
10947         spin_lock_irqsave(&dev_priv->uncore.lock, irqflags);
10948
10949         /*
10950          * On some platforms writing CURCNTR first will also
10951          * cause CURPOS to be armed by the CURBASE write.
10952          * Without the CURCNTR write the CURPOS write would
10953          * arm itself. Thus we always update CURCNTR before
10954          * CURPOS.
10955          *
10956          * On other platforms CURPOS always requires the
10957          * CURBASE write to arm the update. Additonally
10958          * a write to any of the cursor register will cancel
10959          * an already armed cursor update. Thus leaving out
10960          * the CURBASE write after CURPOS could lead to a
10961          * cursor that doesn't appear to move, or even change
10962          * shape. Thus we always write CURBASE.
10963          *
10964          * The other registers are armed by by the CURBASE write
10965          * except when the plane is getting enabled at which time
10966          * the CURCNTR write arms the update.
10967          */
10968
10969         if (INTEL_GEN(dev_priv) >= 9)
10970                 skl_write_cursor_wm(plane, crtc_state);
10971
10972         if (plane->cursor.base != base ||
10973             plane->cursor.size != fbc_ctl ||
10974             plane->cursor.cntl != cntl) {
10975                 if (HAS_CUR_FBC(dev_priv))
10976                         I915_WRITE_FW(CUR_FBC_CTL(pipe), fbc_ctl);
10977                 I915_WRITE_FW(CURCNTR(pipe), cntl);
10978                 I915_WRITE_FW(CURPOS(pipe), pos);
10979                 I915_WRITE_FW(CURBASE(pipe), base);
10980
10981                 plane->cursor.base = base;
10982                 plane->cursor.size = fbc_ctl;
10983                 plane->cursor.cntl = cntl;
10984         } else {
10985                 I915_WRITE_FW(CURPOS(pipe), pos);
10986                 I915_WRITE_FW(CURBASE(pipe), base);
10987         }
10988
10989         spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags);
10990 }
10991
10992 static void i9xx_disable_cursor(struct intel_plane *plane,
10993                                 const struct intel_crtc_state *crtc_state)
10994 {
10995         i9xx_update_cursor(plane, crtc_state, NULL);
10996 }
10997
10998 static bool i9xx_cursor_get_hw_state(struct intel_plane *plane,
10999                                      enum pipe *pipe)
11000 {
11001         struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
11002         enum intel_display_power_domain power_domain;
11003         intel_wakeref_t wakeref;
11004         bool ret;
11005         u32 val;
11006
11007         /*
11008          * Not 100% correct for planes that can move between pipes,
11009          * but that's only the case for gen2-3 which don't have any
11010          * display power wells.
11011          */
11012         power_domain = POWER_DOMAIN_PIPE(plane->pipe);
11013         wakeref = intel_display_power_get_if_enabled(dev_priv, power_domain);
11014         if (!wakeref)
11015                 return false;
11016
11017         val = I915_READ(CURCNTR(plane->pipe));
11018
11019         ret = val & MCURSOR_MODE;
11020
11021         if (INTEL_GEN(dev_priv) >= 5 || IS_G4X(dev_priv))
11022                 *pipe = plane->pipe;
11023         else
11024                 *pipe = (val & MCURSOR_PIPE_SELECT_MASK) >>
11025                         MCURSOR_PIPE_SELECT_SHIFT;
11026
11027         intel_display_power_put(dev_priv, power_domain, wakeref);
11028
11029         return ret;
11030 }
11031
11032 /* VESA 640x480x72Hz mode to set on the pipe */
11033 static const struct drm_display_mode load_detect_mode = {
11034         DRM_MODE("640x480", DRM_MODE_TYPE_DEFAULT, 31500, 640, 664,
11035                  704, 832, 0, 480, 489, 491, 520, 0, DRM_MODE_FLAG_NHSYNC | DRM_MODE_FLAG_NVSYNC),
11036 };
11037
11038 struct drm_framebuffer *
11039 intel_framebuffer_create(struct drm_i915_gem_object *obj,
11040                          struct drm_mode_fb_cmd2 *mode_cmd)
11041 {
11042         struct intel_framebuffer *intel_fb;
11043         int ret;
11044
11045         intel_fb = kzalloc(sizeof(*intel_fb), GFP_KERNEL);
11046         if (!intel_fb)
11047                 return ERR_PTR(-ENOMEM);
11048
11049         ret = intel_framebuffer_init(intel_fb, obj, mode_cmd);
11050         if (ret)
11051                 goto err;
11052
11053         return &intel_fb->base;
11054
11055 err:
11056         kfree(intel_fb);
11057         return ERR_PTR(ret);
11058 }
11059
11060 static int intel_modeset_disable_planes(struct drm_atomic_state *state,
11061                                         struct drm_crtc *crtc)
11062 {
11063         struct drm_plane *plane;
11064         struct drm_plane_state *plane_state;
11065         int ret, i;
11066
11067         ret = drm_atomic_add_affected_planes(state, crtc);
11068         if (ret)
11069                 return ret;
11070
11071         for_each_new_plane_in_state(state, plane, plane_state, i) {
11072                 if (plane_state->crtc != crtc)
11073                         continue;
11074
11075                 ret = drm_atomic_set_crtc_for_plane(plane_state, NULL);
11076                 if (ret)
11077                         return ret;
11078
11079                 drm_atomic_set_fb_for_plane(plane_state, NULL);
11080         }
11081
11082         return 0;
11083 }
11084
11085 int intel_get_load_detect_pipe(struct drm_connector *connector,
11086                                const struct drm_display_mode *mode,
11087                                struct intel_load_detect_pipe *old,
11088                                struct drm_modeset_acquire_ctx *ctx)
11089 {
11090         struct intel_crtc *intel_crtc;
11091         struct intel_encoder *intel_encoder =
11092                 intel_attached_encoder(connector);
11093         struct drm_crtc *possible_crtc;
11094         struct drm_encoder *encoder = &intel_encoder->base;
11095         struct drm_crtc *crtc = NULL;
11096         struct drm_device *dev = encoder->dev;
11097         struct drm_i915_private *dev_priv = to_i915(dev);
11098         struct drm_mode_config *config = &dev->mode_config;
11099         struct drm_atomic_state *state = NULL, *restore_state = NULL;
11100         struct drm_connector_state *connector_state;
11101         struct intel_crtc_state *crtc_state;
11102         int ret, i = -1;
11103
11104         DRM_DEBUG_KMS("[CONNECTOR:%d:%s], [ENCODER:%d:%s]\n",
11105                       connector->base.id, connector->name,
11106                       encoder->base.id, encoder->name);
11107
11108         old->restore_state = NULL;
11109
11110         WARN_ON(!drm_modeset_is_locked(&config->connection_mutex));
11111
11112         /*
11113          * Algorithm gets a little messy:
11114          *
11115          *   - if the connector already has an assigned crtc, use it (but make
11116          *     sure it's on first)
11117          *
11118          *   - try to find the first unused crtc that can drive this connector,
11119          *     and use that if we find one
11120          */
11121
11122         /* See if we already have a CRTC for this connector */
11123         if (connector->state->crtc) {
11124                 crtc = connector->state->crtc;
11125
11126                 ret = drm_modeset_lock(&crtc->mutex, ctx);
11127                 if (ret)
11128                         goto fail;
11129
11130                 /* Make sure the crtc and connector are running */
11131                 goto found;
11132         }
11133
11134         /* Find an unused one (if possible) */
11135         for_each_crtc(dev, possible_crtc) {
11136                 i++;
11137                 if (!(encoder->possible_crtcs & (1 << i)))
11138                         continue;
11139
11140                 ret = drm_modeset_lock(&possible_crtc->mutex, ctx);
11141                 if (ret)
11142                         goto fail;
11143
11144                 if (possible_crtc->state->enable) {
11145                         drm_modeset_unlock(&possible_crtc->mutex);
11146                         continue;
11147                 }
11148
11149                 crtc = possible_crtc;
11150                 break;
11151         }
11152
11153         /*
11154          * If we didn't find an unused CRTC, don't use any.
11155          */
11156         if (!crtc) {
11157                 DRM_DEBUG_KMS("no pipe available for load-detect\n");
11158                 ret = -ENODEV;
11159                 goto fail;
11160         }
11161
11162 found:
11163         intel_crtc = to_intel_crtc(crtc);
11164
11165         state = drm_atomic_state_alloc(dev);
11166         restore_state = drm_atomic_state_alloc(dev);
11167         if (!state || !restore_state) {
11168                 ret = -ENOMEM;
11169                 goto fail;
11170         }
11171
11172         state->acquire_ctx = ctx;
11173         restore_state->acquire_ctx = ctx;
11174
11175         connector_state = drm_atomic_get_connector_state(state, connector);
11176         if (IS_ERR(connector_state)) {
11177                 ret = PTR_ERR(connector_state);
11178                 goto fail;
11179         }
11180
11181         ret = drm_atomic_set_crtc_for_connector(connector_state, crtc);
11182         if (ret)
11183                 goto fail;
11184
11185         crtc_state = intel_atomic_get_crtc_state(state, intel_crtc);
11186         if (IS_ERR(crtc_state)) {
11187                 ret = PTR_ERR(crtc_state);
11188                 goto fail;
11189         }
11190
11191         crtc_state->base.active = crtc_state->base.enable = true;
11192
11193         if (!mode)
11194                 mode = &load_detect_mode;
11195
11196         ret = drm_atomic_set_mode_for_crtc(&crtc_state->base, mode);
11197         if (ret)
11198                 goto fail;
11199
11200         ret = intel_modeset_disable_planes(state, crtc);
11201         if (ret)
11202                 goto fail;
11203
11204         ret = PTR_ERR_OR_ZERO(drm_atomic_get_connector_state(restore_state, connector));
11205         if (!ret)
11206                 ret = PTR_ERR_OR_ZERO(drm_atomic_get_crtc_state(restore_state, crtc));
11207         if (!ret)
11208                 ret = drm_atomic_add_affected_planes(restore_state, crtc);
11209         if (ret) {
11210                 DRM_DEBUG_KMS("Failed to create a copy of old state to restore: %i\n", ret);
11211                 goto fail;
11212         }
11213
11214         ret = drm_atomic_commit(state);
11215         if (ret) {
11216                 DRM_DEBUG_KMS("failed to set mode on load-detect pipe\n");
11217                 goto fail;
11218         }
11219
11220         old->restore_state = restore_state;
11221         drm_atomic_state_put(state);
11222
11223         /* let the connector get through one full cycle before testing */
11224         intel_wait_for_vblank(dev_priv, intel_crtc->pipe);
11225         return true;
11226
11227 fail:
11228         if (state) {
11229                 drm_atomic_state_put(state);
11230                 state = NULL;
11231         }
11232         if (restore_state) {
11233                 drm_atomic_state_put(restore_state);
11234                 restore_state = NULL;
11235         }
11236
11237         if (ret == -EDEADLK)
11238                 return ret;
11239
11240         return false;
11241 }
11242
11243 void intel_release_load_detect_pipe(struct drm_connector *connector,
11244                                     struct intel_load_detect_pipe *old,
11245                                     struct drm_modeset_acquire_ctx *ctx)
11246 {
11247         struct intel_encoder *intel_encoder =
11248                 intel_attached_encoder(connector);
11249         struct drm_encoder *encoder = &intel_encoder->base;
11250         struct drm_atomic_state *state = old->restore_state;
11251         int ret;
11252
11253         DRM_DEBUG_KMS("[CONNECTOR:%d:%s], [ENCODER:%d:%s]\n",
11254                       connector->base.id, connector->name,
11255                       encoder->base.id, encoder->name);
11256
11257         if (!state)
11258                 return;
11259
11260         ret = drm_atomic_helper_commit_duplicated_state(state, ctx);
11261         if (ret)
11262                 DRM_DEBUG_KMS("Couldn't release load detect pipe: %i\n", ret);
11263         drm_atomic_state_put(state);
11264 }
11265
11266 static int i9xx_pll_refclk(struct drm_device *dev,
11267                            const struct intel_crtc_state *pipe_config)
11268 {
11269         struct drm_i915_private *dev_priv = to_i915(dev);
11270         u32 dpll = pipe_config->dpll_hw_state.dpll;
11271
11272         if ((dpll & PLL_REF_INPUT_MASK) == PLLB_REF_INPUT_SPREADSPECTRUMIN)
11273                 return dev_priv->vbt.lvds_ssc_freq;
11274         else if (HAS_PCH_SPLIT(dev_priv))
11275                 return 120000;
11276         else if (!IS_GEN(dev_priv, 2))
11277                 return 96000;
11278         else
11279                 return 48000;
11280 }
11281
11282 /* Returns the clock of the currently programmed mode of the given pipe. */
11283 static void i9xx_crtc_clock_get(struct intel_crtc *crtc,
11284                                 struct intel_crtc_state *pipe_config)
11285 {
11286         struct drm_device *dev = crtc->base.dev;
11287         struct drm_i915_private *dev_priv = to_i915(dev);
11288         int pipe = pipe_config->cpu_transcoder;
11289         u32 dpll = pipe_config->dpll_hw_state.dpll;
11290         u32 fp;
11291         struct dpll clock;
11292         int port_clock;
11293         int refclk = i9xx_pll_refclk(dev, pipe_config);
11294
11295         if ((dpll & DISPLAY_RATE_SELECT_FPA1) == 0)
11296                 fp = pipe_config->dpll_hw_state.fp0;
11297         else
11298                 fp = pipe_config->dpll_hw_state.fp1;
11299
11300         clock.m1 = (fp & FP_M1_DIV_MASK) >> FP_M1_DIV_SHIFT;
11301         if (IS_PINEVIEW(dev_priv)) {
11302                 clock.n = ffs((fp & FP_N_PINEVIEW_DIV_MASK) >> FP_N_DIV_SHIFT) - 1;
11303                 clock.m2 = (fp & FP_M2_PINEVIEW_DIV_MASK) >> FP_M2_DIV_SHIFT;
11304         } else {
11305                 clock.n = (fp & FP_N_DIV_MASK) >> FP_N_DIV_SHIFT;
11306                 clock.m2 = (fp & FP_M2_DIV_MASK) >> FP_M2_DIV_SHIFT;
11307         }
11308
11309         if (!IS_GEN(dev_priv, 2)) {
11310                 if (IS_PINEVIEW(dev_priv))
11311                         clock.p1 = ffs((dpll & DPLL_FPA01_P1_POST_DIV_MASK_PINEVIEW) >>
11312                                 DPLL_FPA01_P1_POST_DIV_SHIFT_PINEVIEW);
11313                 else
11314                         clock.p1 = ffs((dpll & DPLL_FPA01_P1_POST_DIV_MASK) >>
11315                                DPLL_FPA01_P1_POST_DIV_SHIFT);
11316
11317                 switch (dpll & DPLL_MODE_MASK) {
11318                 case DPLLB_MODE_DAC_SERIAL:
11319                         clock.p2 = dpll & DPLL_DAC_SERIAL_P2_CLOCK_DIV_5 ?
11320                                 5 : 10;
11321                         break;
11322                 case DPLLB_MODE_LVDS:
11323                         clock.p2 = dpll & DPLLB_LVDS_P2_CLOCK_DIV_7 ?
11324                                 7 : 14;
11325                         break;
11326                 default:
11327                         DRM_DEBUG_KMS("Unknown DPLL mode %08x in programmed "
11328                                   "mode\n", (int)(dpll & DPLL_MODE_MASK));
11329                         return;
11330                 }
11331
11332                 if (IS_PINEVIEW(dev_priv))
11333                         port_clock = pnv_calc_dpll_params(refclk, &clock);
11334                 else
11335                         port_clock = i9xx_calc_dpll_params(refclk, &clock);
11336         } else {
11337                 u32 lvds = IS_I830(dev_priv) ? 0 : I915_READ(LVDS);
11338                 bool is_lvds = (pipe == 1) && (lvds & LVDS_PORT_EN);
11339
11340                 if (is_lvds) {
11341                         clock.p1 = ffs((dpll & DPLL_FPA01_P1_POST_DIV_MASK_I830_LVDS) >>
11342                                        DPLL_FPA01_P1_POST_DIV_SHIFT);
11343
11344                         if (lvds & LVDS_CLKB_POWER_UP)
11345                                 clock.p2 = 7;
11346                         else
11347                                 clock.p2 = 14;
11348                 } else {
11349                         if (dpll & PLL_P1_DIVIDE_BY_TWO)
11350                                 clock.p1 = 2;
11351                         else {
11352                                 clock.p1 = ((dpll & DPLL_FPA01_P1_POST_DIV_MASK_I830) >>
11353                                             DPLL_FPA01_P1_POST_DIV_SHIFT) + 2;
11354                         }
11355                         if (dpll & PLL_P2_DIVIDE_BY_4)
11356                                 clock.p2 = 4;
11357                         else
11358                                 clock.p2 = 2;
11359                 }
11360
11361                 port_clock = i9xx_calc_dpll_params(refclk, &clock);
11362         }
11363
11364         /*
11365          * This value includes pixel_multiplier. We will use
11366          * port_clock to compute adjusted_mode.crtc_clock in the
11367          * encoder's get_config() function.
11368          */
11369         pipe_config->port_clock = port_clock;
11370 }
11371
11372 int intel_dotclock_calculate(int link_freq,
11373                              const struct intel_link_m_n *m_n)
11374 {
11375         /*
11376          * The calculation for the data clock is:
11377          * pixel_clock = ((m/n)*(link_clock * nr_lanes))/bpp
11378          * But we want to avoid losing precison if possible, so:
11379          * pixel_clock = ((m * link_clock * nr_lanes)/(n*bpp))
11380          *
11381          * and the link clock is simpler:
11382          * link_clock = (m * link_clock) / n
11383          */
11384
11385         if (!m_n->link_n)
11386                 return 0;
11387
11388         return div_u64(mul_u32_u32(m_n->link_m, link_freq), m_n->link_n);
11389 }
11390
11391 static void ironlake_pch_clock_get(struct intel_crtc *crtc,
11392                                    struct intel_crtc_state *pipe_config)
11393 {
11394         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
11395
11396         /* read out port_clock from the DPLL */
11397         i9xx_crtc_clock_get(crtc, pipe_config);
11398
11399         /*
11400          * In case there is an active pipe without active ports,
11401          * we may need some idea for the dotclock anyway.
11402          * Calculate one based on the FDI configuration.
11403          */
11404         pipe_config->base.adjusted_mode.crtc_clock =
11405                 intel_dotclock_calculate(intel_fdi_link_freq(dev_priv, pipe_config),
11406                                          &pipe_config->fdi_m_n);
11407 }
11408
11409 /* Returns the currently programmed mode of the given encoder. */
11410 struct drm_display_mode *
11411 intel_encoder_current_mode(struct intel_encoder *encoder)
11412 {
11413         struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
11414         struct intel_crtc_state *crtc_state;
11415         struct drm_display_mode *mode;
11416         struct intel_crtc *crtc;
11417         enum pipe pipe;
11418
11419         if (!encoder->get_hw_state(encoder, &pipe))
11420                 return NULL;
11421
11422         crtc = intel_get_crtc_for_pipe(dev_priv, pipe);
11423
11424         mode = kzalloc(sizeof(*mode), GFP_KERNEL);
11425         if (!mode)
11426                 return NULL;
11427
11428         crtc_state = kzalloc(sizeof(*crtc_state), GFP_KERNEL);
11429         if (!crtc_state) {
11430                 kfree(mode);
11431                 return NULL;
11432         }
11433
11434         crtc_state->base.crtc = &crtc->base;
11435
11436         if (!dev_priv->display.get_pipe_config(crtc, crtc_state)) {
11437                 kfree(crtc_state);
11438                 kfree(mode);
11439                 return NULL;
11440         }
11441
11442         encoder->get_config(encoder, crtc_state);
11443
11444         intel_mode_from_pipe_config(mode, crtc_state);
11445
11446         kfree(crtc_state);
11447
11448         return mode;
11449 }
11450
11451 static void intel_crtc_destroy(struct drm_crtc *crtc)
11452 {
11453         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
11454
11455         drm_crtc_cleanup(crtc);
11456         kfree(intel_crtc);
11457 }
11458
11459 /**
11460  * intel_wm_need_update - Check whether watermarks need updating
11461  * @cur: current plane state
11462  * @new: new plane state
11463  *
11464  * Check current plane state versus the new one to determine whether
11465  * watermarks need to be recalculated.
11466  *
11467  * Returns true or false.
11468  */
11469 static bool intel_wm_need_update(const struct intel_plane_state *cur,
11470                                  struct intel_plane_state *new)
11471 {
11472         /* Update watermarks on tiling or size changes. */
11473         if (new->base.visible != cur->base.visible)
11474                 return true;
11475
11476         if (!cur->base.fb || !new->base.fb)
11477                 return false;
11478
11479         if (cur->base.fb->modifier != new->base.fb->modifier ||
11480             cur->base.rotation != new->base.rotation ||
11481             drm_rect_width(&new->base.src) != drm_rect_width(&cur->base.src) ||
11482             drm_rect_height(&new->base.src) != drm_rect_height(&cur->base.src) ||
11483             drm_rect_width(&new->base.dst) != drm_rect_width(&cur->base.dst) ||
11484             drm_rect_height(&new->base.dst) != drm_rect_height(&cur->base.dst))
11485                 return true;
11486
11487         return false;
11488 }
11489
11490 static bool needs_scaling(const struct intel_plane_state *state)
11491 {
11492         int src_w = drm_rect_width(&state->base.src) >> 16;
11493         int src_h = drm_rect_height(&state->base.src) >> 16;
11494         int dst_w = drm_rect_width(&state->base.dst);
11495         int dst_h = drm_rect_height(&state->base.dst);
11496
11497         return (src_w != dst_w || src_h != dst_h);
11498 }
11499
11500 int intel_plane_atomic_calc_changes(const struct intel_crtc_state *old_crtc_state,
11501                                     struct intel_crtc_state *crtc_state,
11502                                     const struct intel_plane_state *old_plane_state,
11503                                     struct intel_plane_state *plane_state)
11504 {
11505         struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
11506         struct intel_plane *plane = to_intel_plane(plane_state->base.plane);
11507         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
11508         bool mode_changed = needs_modeset(crtc_state);
11509         bool was_crtc_enabled = old_crtc_state->base.active;
11510         bool is_crtc_enabled = crtc_state->base.active;
11511         bool turn_off, turn_on, visible, was_visible;
11512         struct drm_framebuffer *fb = plane_state->base.fb;
11513         int ret;
11514
11515         if (INTEL_GEN(dev_priv) >= 9 && plane->id != PLANE_CURSOR) {
11516                 ret = skl_update_scaler_plane(crtc_state, plane_state);
11517                 if (ret)
11518                         return ret;
11519         }
11520
11521         was_visible = old_plane_state->base.visible;
11522         visible = plane_state->base.visible;
11523
11524         if (!was_crtc_enabled && WARN_ON(was_visible))
11525                 was_visible = false;
11526
11527         /*
11528          * Visibility is calculated as if the crtc was on, but
11529          * after scaler setup everything depends on it being off
11530          * when the crtc isn't active.
11531          *
11532          * FIXME this is wrong for watermarks. Watermarks should also
11533          * be computed as if the pipe would be active. Perhaps move
11534          * per-plane wm computation to the .check_plane() hook, and
11535          * only combine the results from all planes in the current place?
11536          */
11537         if (!is_crtc_enabled) {
11538                 plane_state->base.visible = visible = false;
11539                 crtc_state->active_planes &= ~BIT(plane->id);
11540                 crtc_state->data_rate[plane->id] = 0;
11541         }
11542
11543         if (!was_visible && !visible)
11544                 return 0;
11545
11546         if (fb != old_plane_state->base.fb)
11547                 crtc_state->fb_changed = true;
11548
11549         turn_off = was_visible && (!visible || mode_changed);
11550         turn_on = visible && (!was_visible || mode_changed);
11551
11552         DRM_DEBUG_ATOMIC("[CRTC:%d:%s] has [PLANE:%d:%s] with fb %i\n",
11553                          crtc->base.base.id, crtc->base.name,
11554                          plane->base.base.id, plane->base.name,
11555                          fb ? fb->base.id : -1);
11556
11557         DRM_DEBUG_ATOMIC("[PLANE:%d:%s] visible %i -> %i, off %i, on %i, ms %i\n",
11558                          plane->base.base.id, plane->base.name,
11559                          was_visible, visible,
11560                          turn_off, turn_on, mode_changed);
11561
11562         if (turn_on) {
11563                 if (INTEL_GEN(dev_priv) < 5 && !IS_G4X(dev_priv))
11564                         crtc_state->update_wm_pre = true;
11565
11566                 /* must disable cxsr around plane enable/disable */
11567                 if (plane->id != PLANE_CURSOR)
11568                         crtc_state->disable_cxsr = true;
11569         } else if (turn_off) {
11570                 if (INTEL_GEN(dev_priv) < 5 && !IS_G4X(dev_priv))
11571                         crtc_state->update_wm_post = true;
11572
11573                 /* must disable cxsr around plane enable/disable */
11574                 if (plane->id != PLANE_CURSOR)
11575                         crtc_state->disable_cxsr = true;
11576         } else if (intel_wm_need_update(old_plane_state, plane_state)) {
11577                 if (INTEL_GEN(dev_priv) < 5 && !IS_G4X(dev_priv)) {
11578                         /* FIXME bollocks */
11579                         crtc_state->update_wm_pre = true;
11580                         crtc_state->update_wm_post = true;
11581                 }
11582         }
11583
11584         if (visible || was_visible)
11585                 crtc_state->fb_bits |= plane->frontbuffer_bit;
11586
11587         /*
11588          * ILK/SNB DVSACNTR/Sprite Enable
11589          * IVB SPR_CTL/Sprite Enable
11590          * "When in Self Refresh Big FIFO mode, a write to enable the
11591          *  plane will be internally buffered and delayed while Big FIFO
11592          *  mode is exiting."
11593          *
11594          * Which means that enabling the sprite can take an extra frame
11595          * when we start in big FIFO mode (LP1+). Thus we need to drop
11596          * down to LP0 and wait for vblank in order to make sure the
11597          * sprite gets enabled on the next vblank after the register write.
11598          * Doing otherwise would risk enabling the sprite one frame after
11599          * we've already signalled flip completion. We can resume LP1+
11600          * once the sprite has been enabled.
11601          *
11602          *
11603          * WaCxSRDisabledForSpriteScaling:ivb
11604          * IVB SPR_SCALE/Scaling Enable
11605          * "Low Power watermarks must be disabled for at least one
11606          *  frame before enabling sprite scaling, and kept disabled
11607          *  until sprite scaling is disabled."
11608          *
11609          * ILK/SNB DVSASCALE/Scaling Enable
11610          * "When in Self Refresh Big FIFO mode, scaling enable will be
11611          *  masked off while Big FIFO mode is exiting."
11612          *
11613          * Despite the w/a only being listed for IVB we assume that
11614          * the ILK/SNB note has similar ramifications, hence we apply
11615          * the w/a on all three platforms.
11616          *
11617          * With experimental results seems this is needed also for primary
11618          * plane, not only sprite plane.
11619          */
11620         if (plane->id != PLANE_CURSOR &&
11621             (IS_GEN_RANGE(dev_priv, 5, 6) ||
11622              IS_IVYBRIDGE(dev_priv)) &&
11623             (turn_on || (!needs_scaling(old_plane_state) &&
11624                          needs_scaling(plane_state))))
11625                 crtc_state->disable_lp_wm = true;
11626
11627         return 0;
11628 }
11629
11630 static bool encoders_cloneable(const struct intel_encoder *a,
11631                                const struct intel_encoder *b)
11632 {
11633         /* masks could be asymmetric, so check both ways */
11634         return a == b || (a->cloneable & (1 << b->type) &&
11635                           b->cloneable & (1 << a->type));
11636 }
11637
11638 static bool check_single_encoder_cloning(struct drm_atomic_state *state,
11639                                          struct intel_crtc *crtc,
11640                                          struct intel_encoder *encoder)
11641 {
11642         struct intel_encoder *source_encoder;
11643         struct drm_connector *connector;
11644         struct drm_connector_state *connector_state;
11645         int i;
11646
11647         for_each_new_connector_in_state(state, connector, connector_state, i) {
11648                 if (connector_state->crtc != &crtc->base)
11649                         continue;
11650
11651                 source_encoder =
11652                         to_intel_encoder(connector_state->best_encoder);
11653                 if (!encoders_cloneable(encoder, source_encoder))
11654                         return false;
11655         }
11656
11657         return true;
11658 }
11659
11660 static int icl_add_linked_planes(struct intel_atomic_state *state)
11661 {
11662         struct intel_plane *plane, *linked;
11663         struct intel_plane_state *plane_state, *linked_plane_state;
11664         int i;
11665
11666         for_each_new_intel_plane_in_state(state, plane, plane_state, i) {
11667                 linked = plane_state->linked_plane;
11668
11669                 if (!linked)
11670                         continue;
11671
11672                 linked_plane_state = intel_atomic_get_plane_state(state, linked);
11673                 if (IS_ERR(linked_plane_state))
11674                         return PTR_ERR(linked_plane_state);
11675
11676                 WARN_ON(linked_plane_state->linked_plane != plane);
11677                 WARN_ON(linked_plane_state->slave == plane_state->slave);
11678         }
11679
11680         return 0;
11681 }
11682
11683 static int icl_check_nv12_planes(struct intel_crtc_state *crtc_state)
11684 {
11685         struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
11686         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
11687         struct intel_atomic_state *state = to_intel_atomic_state(crtc_state->base.state);
11688         struct intel_plane *plane, *linked;
11689         struct intel_plane_state *plane_state;
11690         int i;
11691
11692         if (INTEL_GEN(dev_priv) < 11)
11693                 return 0;
11694
11695         /*
11696          * Destroy all old plane links and make the slave plane invisible
11697          * in the crtc_state->active_planes mask.
11698          */
11699         for_each_new_intel_plane_in_state(state, plane, plane_state, i) {
11700                 if (plane->pipe != crtc->pipe || !plane_state->linked_plane)
11701                         continue;
11702
11703                 plane_state->linked_plane = NULL;
11704                 if (plane_state->slave && !plane_state->base.visible) {
11705                         crtc_state->active_planes &= ~BIT(plane->id);
11706                         crtc_state->update_planes |= BIT(plane->id);
11707                 }
11708
11709                 plane_state->slave = false;
11710         }
11711
11712         if (!crtc_state->nv12_planes)
11713                 return 0;
11714
11715         for_each_new_intel_plane_in_state(state, plane, plane_state, i) {
11716                 struct intel_plane_state *linked_state = NULL;
11717
11718                 if (plane->pipe != crtc->pipe ||
11719                     !(crtc_state->nv12_planes & BIT(plane->id)))
11720                         continue;
11721
11722                 for_each_intel_plane_on_crtc(&dev_priv->drm, crtc, linked) {
11723                         if (!icl_is_nv12_y_plane(linked->id))
11724                                 continue;
11725
11726                         if (crtc_state->active_planes & BIT(linked->id))
11727                                 continue;
11728
11729                         linked_state = intel_atomic_get_plane_state(state, linked);
11730                         if (IS_ERR(linked_state))
11731                                 return PTR_ERR(linked_state);
11732
11733                         break;
11734                 }
11735
11736                 if (!linked_state) {
11737                         DRM_DEBUG_KMS("Need %d free Y planes for planar YUV\n",
11738                                       hweight8(crtc_state->nv12_planes));
11739
11740                         return -EINVAL;
11741                 }
11742
11743                 plane_state->linked_plane = linked;
11744
11745                 linked_state->slave = true;
11746                 linked_state->linked_plane = plane;
11747                 crtc_state->active_planes |= BIT(linked->id);
11748                 crtc_state->update_planes |= BIT(linked->id);
11749                 DRM_DEBUG_KMS("Using %s as Y plane for %s\n", linked->base.name, plane->base.name);
11750         }
11751
11752         return 0;
11753 }
11754
11755 static bool c8_planes_changed(const struct intel_crtc_state *new_crtc_state)
11756 {
11757         struct intel_crtc *crtc = to_intel_crtc(new_crtc_state->base.crtc);
11758         struct intel_atomic_state *state =
11759                 to_intel_atomic_state(new_crtc_state->base.state);
11760         const struct intel_crtc_state *old_crtc_state =
11761                 intel_atomic_get_old_crtc_state(state, crtc);
11762
11763         return !old_crtc_state->c8_planes != !new_crtc_state->c8_planes;
11764 }
11765
11766 static int intel_crtc_atomic_check(struct drm_crtc *crtc,
11767                                    struct drm_crtc_state *crtc_state)
11768 {
11769         struct drm_i915_private *dev_priv = to_i915(crtc->dev);
11770         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
11771         struct intel_crtc_state *pipe_config =
11772                 to_intel_crtc_state(crtc_state);
11773         int ret;
11774         bool mode_changed = needs_modeset(pipe_config);
11775
11776         if (INTEL_GEN(dev_priv) < 5 && !IS_G4X(dev_priv) &&
11777             mode_changed && !crtc_state->active)
11778                 pipe_config->update_wm_post = true;
11779
11780         if (mode_changed && crtc_state->enable &&
11781             dev_priv->display.crtc_compute_clock &&
11782             !WARN_ON(pipe_config->shared_dpll)) {
11783                 ret = dev_priv->display.crtc_compute_clock(intel_crtc,
11784                                                            pipe_config);
11785                 if (ret)
11786                         return ret;
11787         }
11788
11789         /*
11790          * May need to update pipe gamma enable bits
11791          * when C8 planes are getting enabled/disabled.
11792          */
11793         if (c8_planes_changed(pipe_config))
11794                 crtc_state->color_mgmt_changed = true;
11795
11796         if (mode_changed || pipe_config->update_pipe ||
11797             crtc_state->color_mgmt_changed) {
11798                 ret = intel_color_check(pipe_config);
11799                 if (ret)
11800                         return ret;
11801         }
11802
11803         ret = 0;
11804         if (dev_priv->display.compute_pipe_wm) {
11805                 ret = dev_priv->display.compute_pipe_wm(pipe_config);
11806                 if (ret) {
11807                         DRM_DEBUG_KMS("Target pipe watermarks are invalid\n");
11808                         return ret;
11809                 }
11810         }
11811
11812         if (dev_priv->display.compute_intermediate_wm) {
11813                 if (WARN_ON(!dev_priv->display.compute_pipe_wm))
11814                         return 0;
11815
11816                 /*
11817                  * Calculate 'intermediate' watermarks that satisfy both the
11818                  * old state and the new state.  We can program these
11819                  * immediately.
11820                  */
11821                 ret = dev_priv->display.compute_intermediate_wm(pipe_config);
11822                 if (ret) {
11823                         DRM_DEBUG_KMS("No valid intermediate pipe watermarks are possible\n");
11824                         return ret;
11825                 }
11826         }
11827
11828         if (INTEL_GEN(dev_priv) >= 9) {
11829                 if (mode_changed || pipe_config->update_pipe)
11830                         ret = skl_update_scaler_crtc(pipe_config);
11831
11832                 if (!ret)
11833                         ret = icl_check_nv12_planes(pipe_config);
11834                 if (!ret)
11835                         ret = skl_check_pipe_max_pixel_rate(intel_crtc,
11836                                                             pipe_config);
11837                 if (!ret)
11838                         ret = intel_atomic_setup_scalers(dev_priv, intel_crtc,
11839                                                          pipe_config);
11840         }
11841
11842         if (HAS_IPS(dev_priv))
11843                 pipe_config->ips_enabled = hsw_compute_ips_config(pipe_config);
11844
11845         return ret;
11846 }
11847
11848 static const struct drm_crtc_helper_funcs intel_helper_funcs = {
11849         .atomic_check = intel_crtc_atomic_check,
11850 };
11851
11852 static void intel_modeset_update_connector_atomic_state(struct drm_device *dev)
11853 {
11854         struct intel_connector *connector;
11855         struct drm_connector_list_iter conn_iter;
11856
11857         drm_connector_list_iter_begin(dev, &conn_iter);
11858         for_each_intel_connector_iter(connector, &conn_iter) {
11859                 if (connector->base.state->crtc)
11860                         drm_connector_put(&connector->base);
11861
11862                 if (connector->base.encoder) {
11863                         connector->base.state->best_encoder =
11864                                 connector->base.encoder;
11865                         connector->base.state->crtc =
11866                                 connector->base.encoder->crtc;
11867
11868                         drm_connector_get(&connector->base);
11869                 } else {
11870                         connector->base.state->best_encoder = NULL;
11871                         connector->base.state->crtc = NULL;
11872                 }
11873         }
11874         drm_connector_list_iter_end(&conn_iter);
11875 }
11876
11877 static int
11878 compute_sink_pipe_bpp(const struct drm_connector_state *conn_state,
11879                       struct intel_crtc_state *pipe_config)
11880 {
11881         struct drm_connector *connector = conn_state->connector;
11882         const struct drm_display_info *info = &connector->display_info;
11883         int bpp;
11884
11885         switch (conn_state->max_bpc) {
11886         case 6 ... 7:
11887                 bpp = 6 * 3;
11888                 break;
11889         case 8 ... 9:
11890                 bpp = 8 * 3;
11891                 break;
11892         case 10 ... 11:
11893                 bpp = 10 * 3;
11894                 break;
11895         case 12:
11896                 bpp = 12 * 3;
11897                 break;
11898         default:
11899                 return -EINVAL;
11900         }
11901
11902         if (bpp < pipe_config->pipe_bpp) {
11903                 DRM_DEBUG_KMS("[CONNECTOR:%d:%s] Limiting display bpp to %d instead of "
11904                               "EDID bpp %d, requested bpp %d, max platform bpp %d\n",
11905                               connector->base.id, connector->name,
11906                               bpp, 3 * info->bpc, 3 * conn_state->max_requested_bpc,
11907                               pipe_config->pipe_bpp);
11908
11909                 pipe_config->pipe_bpp = bpp;
11910         }
11911
11912         return 0;
11913 }
11914
11915 static int
11916 compute_baseline_pipe_bpp(struct intel_crtc *crtc,
11917                           struct intel_crtc_state *pipe_config)
11918 {
11919         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
11920         struct drm_atomic_state *state = pipe_config->base.state;
11921         struct drm_connector *connector;
11922         struct drm_connector_state *connector_state;
11923         int bpp, i;
11924
11925         if ((IS_G4X(dev_priv) || IS_VALLEYVIEW(dev_priv) ||
11926             IS_CHERRYVIEW(dev_priv)))
11927                 bpp = 10*3;
11928         else if (INTEL_GEN(dev_priv) >= 5)
11929                 bpp = 12*3;
11930         else
11931                 bpp = 8*3;
11932
11933         pipe_config->pipe_bpp = bpp;
11934
11935         /* Clamp display bpp to connector max bpp */
11936         for_each_new_connector_in_state(state, connector, connector_state, i) {
11937                 int ret;
11938
11939                 if (connector_state->crtc != &crtc->base)
11940                         continue;
11941
11942                 ret = compute_sink_pipe_bpp(connector_state, pipe_config);
11943                 if (ret)
11944                         return ret;
11945         }
11946
11947         return 0;
11948 }
11949
11950 static void intel_dump_crtc_timings(const struct drm_display_mode *mode)
11951 {
11952         DRM_DEBUG_KMS("crtc timings: %d %d %d %d %d %d %d %d %d, "
11953                       "type: 0x%x flags: 0x%x\n",
11954                       mode->crtc_clock,
11955                       mode->crtc_hdisplay, mode->crtc_hsync_start,
11956                       mode->crtc_hsync_end, mode->crtc_htotal,
11957                       mode->crtc_vdisplay, mode->crtc_vsync_start,
11958                       mode->crtc_vsync_end, mode->crtc_vtotal,
11959                       mode->type, mode->flags);
11960 }
11961
11962 static inline void
11963 intel_dump_m_n_config(const struct intel_crtc_state *pipe_config,
11964                       const char *id, unsigned int lane_count,
11965                       const struct intel_link_m_n *m_n)
11966 {
11967         DRM_DEBUG_KMS("%s: lanes: %i; gmch_m: %u, gmch_n: %u, link_m: %u, link_n: %u, tu: %u\n",
11968                       id, lane_count,
11969                       m_n->gmch_m, m_n->gmch_n,
11970                       m_n->link_m, m_n->link_n, m_n->tu);
11971 }
11972
11973 static void
11974 intel_dump_infoframe(struct drm_i915_private *dev_priv,
11975                      const union hdmi_infoframe *frame)
11976 {
11977         if ((drm_debug & DRM_UT_KMS) == 0)
11978                 return;
11979
11980         hdmi_infoframe_log(KERN_DEBUG, dev_priv->drm.dev, frame);
11981 }
11982
11983 #define OUTPUT_TYPE(x) [INTEL_OUTPUT_ ## x] = #x
11984
11985 static const char * const output_type_str[] = {
11986         OUTPUT_TYPE(UNUSED),
11987         OUTPUT_TYPE(ANALOG),
11988         OUTPUT_TYPE(DVO),
11989         OUTPUT_TYPE(SDVO),
11990         OUTPUT_TYPE(LVDS),
11991         OUTPUT_TYPE(TVOUT),
11992         OUTPUT_TYPE(HDMI),
11993         OUTPUT_TYPE(DP),
11994         OUTPUT_TYPE(EDP),
11995         OUTPUT_TYPE(DSI),
11996         OUTPUT_TYPE(DDI),
11997         OUTPUT_TYPE(DP_MST),
11998 };
11999
12000 #undef OUTPUT_TYPE
12001
12002 static void snprintf_output_types(char *buf, size_t len,
12003                                   unsigned int output_types)
12004 {
12005         char *str = buf;
12006         int i;
12007
12008         str[0] = '\0';
12009
12010         for (i = 0; i < ARRAY_SIZE(output_type_str); i++) {
12011                 int r;
12012
12013                 if ((output_types & BIT(i)) == 0)
12014                         continue;
12015
12016                 r = snprintf(str, len, "%s%s",
12017                              str != buf ? "," : "", output_type_str[i]);
12018                 if (r >= len)
12019                         break;
12020                 str += r;
12021                 len -= r;
12022
12023                 output_types &= ~BIT(i);
12024         }
12025
12026         WARN_ON_ONCE(output_types != 0);
12027 }
12028
12029 static const char * const output_format_str[] = {
12030         [INTEL_OUTPUT_FORMAT_INVALID] = "Invalid",
12031         [INTEL_OUTPUT_FORMAT_RGB] = "RGB",
12032         [INTEL_OUTPUT_FORMAT_YCBCR420] = "YCBCR4:2:0",
12033         [INTEL_OUTPUT_FORMAT_YCBCR444] = "YCBCR4:4:4",
12034 };
12035
12036 static const char *output_formats(enum intel_output_format format)
12037 {
12038         if (format >= ARRAY_SIZE(output_format_str))
12039                 format = INTEL_OUTPUT_FORMAT_INVALID;
12040         return output_format_str[format];
12041 }
12042
12043 static void intel_dump_plane_state(const struct intel_plane_state *plane_state)
12044 {
12045         struct intel_plane *plane = to_intel_plane(plane_state->base.plane);
12046         const struct drm_framebuffer *fb = plane_state->base.fb;
12047         struct drm_format_name_buf format_name;
12048
12049         if (!fb) {
12050                 DRM_DEBUG_KMS("[PLANE:%d:%s] fb: [NOFB], visible: %s\n",
12051                               plane->base.base.id, plane->base.name,
12052                               yesno(plane_state->base.visible));
12053                 return;
12054         }
12055
12056         DRM_DEBUG_KMS("[PLANE:%d:%s] fb: [FB:%d] %ux%u format = %s, visible: %s\n",
12057                       plane->base.base.id, plane->base.name,
12058                       fb->base.id, fb->width, fb->height,
12059                       drm_get_format_name(fb->format->format, &format_name),
12060                       yesno(plane_state->base.visible));
12061         DRM_DEBUG_KMS("\trotation: 0x%x, scaler: %d\n",
12062                       plane_state->base.rotation, plane_state->scaler_id);
12063         if (plane_state->base.visible)
12064                 DRM_DEBUG_KMS("\tsrc: " DRM_RECT_FP_FMT " dst: " DRM_RECT_FMT "\n",
12065                               DRM_RECT_FP_ARG(&plane_state->base.src),
12066                               DRM_RECT_ARG(&plane_state->base.dst));
12067 }
12068
12069 static void intel_dump_pipe_config(const struct intel_crtc_state *pipe_config,
12070                                    struct intel_atomic_state *state,
12071                                    const char *context)
12072 {
12073         struct intel_crtc *crtc = to_intel_crtc(pipe_config->base.crtc);
12074         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
12075         const struct intel_plane_state *plane_state;
12076         struct intel_plane *plane;
12077         char buf[64];
12078         int i;
12079
12080         DRM_DEBUG_KMS("[CRTC:%d:%s] enable: %s %s\n",
12081                       crtc->base.base.id, crtc->base.name,
12082                       yesno(pipe_config->base.enable), context);
12083
12084         if (!pipe_config->base.enable)
12085                 goto dump_planes;
12086
12087         snprintf_output_types(buf, sizeof(buf), pipe_config->output_types);
12088         DRM_DEBUG_KMS("active: %s, output_types: %s (0x%x), output format: %s\n",
12089                       yesno(pipe_config->base.active),
12090                       buf, pipe_config->output_types,
12091                       output_formats(pipe_config->output_format));
12092
12093         DRM_DEBUG_KMS("cpu_transcoder: %s, pipe bpp: %i, dithering: %i\n",
12094                       transcoder_name(pipe_config->cpu_transcoder),
12095                       pipe_config->pipe_bpp, pipe_config->dither);
12096
12097         if (pipe_config->has_pch_encoder)
12098                 intel_dump_m_n_config(pipe_config, "fdi",
12099                                       pipe_config->fdi_lanes,
12100                                       &pipe_config->fdi_m_n);
12101
12102         if (intel_crtc_has_dp_encoder(pipe_config)) {
12103                 intel_dump_m_n_config(pipe_config, "dp m_n",
12104                                 pipe_config->lane_count, &pipe_config->dp_m_n);
12105                 if (pipe_config->has_drrs)
12106                         intel_dump_m_n_config(pipe_config, "dp m2_n2",
12107                                               pipe_config->lane_count,
12108                                               &pipe_config->dp_m2_n2);
12109         }
12110
12111         DRM_DEBUG_KMS("audio: %i, infoframes: %i, infoframes enabled: 0x%x\n",
12112                       pipe_config->has_audio, pipe_config->has_infoframe,
12113                       pipe_config->infoframes.enable);
12114
12115         if (pipe_config->infoframes.enable &
12116             intel_hdmi_infoframe_enable(HDMI_PACKET_TYPE_GENERAL_CONTROL))
12117                 DRM_DEBUG_KMS("GCP: 0x%x\n", pipe_config->infoframes.gcp);
12118         if (pipe_config->infoframes.enable &
12119             intel_hdmi_infoframe_enable(HDMI_INFOFRAME_TYPE_AVI))
12120                 intel_dump_infoframe(dev_priv, &pipe_config->infoframes.avi);
12121         if (pipe_config->infoframes.enable &
12122             intel_hdmi_infoframe_enable(HDMI_INFOFRAME_TYPE_SPD))
12123                 intel_dump_infoframe(dev_priv, &pipe_config->infoframes.spd);
12124         if (pipe_config->infoframes.enable &
12125             intel_hdmi_infoframe_enable(HDMI_INFOFRAME_TYPE_VENDOR))
12126                 intel_dump_infoframe(dev_priv, &pipe_config->infoframes.hdmi);
12127
12128         DRM_DEBUG_KMS("requested mode:\n");
12129         drm_mode_debug_printmodeline(&pipe_config->base.mode);
12130         DRM_DEBUG_KMS("adjusted mode:\n");
12131         drm_mode_debug_printmodeline(&pipe_config->base.adjusted_mode);
12132         intel_dump_crtc_timings(&pipe_config->base.adjusted_mode);
12133         DRM_DEBUG_KMS("port clock: %d, pipe src size: %dx%d, pixel rate %d\n",
12134                       pipe_config->port_clock,
12135                       pipe_config->pipe_src_w, pipe_config->pipe_src_h,
12136                       pipe_config->pixel_rate);
12137
12138         if (INTEL_GEN(dev_priv) >= 9)
12139                 DRM_DEBUG_KMS("num_scalers: %d, scaler_users: 0x%x, scaler_id: %d\n",
12140                               crtc->num_scalers,
12141                               pipe_config->scaler_state.scaler_users,
12142                               pipe_config->scaler_state.scaler_id);
12143
12144         if (HAS_GMCH(dev_priv))
12145                 DRM_DEBUG_KMS("gmch pfit: control: 0x%08x, ratios: 0x%08x, lvds border: 0x%08x\n",
12146                               pipe_config->gmch_pfit.control,
12147                               pipe_config->gmch_pfit.pgm_ratios,
12148                               pipe_config->gmch_pfit.lvds_border_bits);
12149         else
12150                 DRM_DEBUG_KMS("pch pfit: pos: 0x%08x, size: 0x%08x, %s, force thru: %s\n",
12151                               pipe_config->pch_pfit.pos,
12152                               pipe_config->pch_pfit.size,
12153                               enableddisabled(pipe_config->pch_pfit.enabled),
12154                               yesno(pipe_config->pch_pfit.force_thru));
12155
12156         DRM_DEBUG_KMS("ips: %i, double wide: %i\n",
12157                       pipe_config->ips_enabled, pipe_config->double_wide);
12158
12159         intel_dpll_dump_hw_state(dev_priv, &pipe_config->dpll_hw_state);
12160
12161 dump_planes:
12162         if (!state)
12163                 return;
12164
12165         for_each_new_intel_plane_in_state(state, plane, plane_state, i) {
12166                 if (plane->pipe == crtc->pipe)
12167                         intel_dump_plane_state(plane_state);
12168         }
12169 }
12170
12171 static bool check_digital_port_conflicts(struct intel_atomic_state *state)
12172 {
12173         struct drm_device *dev = state->base.dev;
12174         struct drm_connector *connector;
12175         struct drm_connector_list_iter conn_iter;
12176         unsigned int used_ports = 0;
12177         unsigned int used_mst_ports = 0;
12178         bool ret = true;
12179
12180         /*
12181          * Walk the connector list instead of the encoder
12182          * list to detect the problem on ddi platforms
12183          * where there's just one encoder per digital port.
12184          */
12185         drm_connector_list_iter_begin(dev, &conn_iter);
12186         drm_for_each_connector_iter(connector, &conn_iter) {
12187                 struct drm_connector_state *connector_state;
12188                 struct intel_encoder *encoder;
12189
12190                 connector_state =
12191                         drm_atomic_get_new_connector_state(&state->base,
12192                                                            connector);
12193                 if (!connector_state)
12194                         connector_state = connector->state;
12195
12196                 if (!connector_state->best_encoder)
12197                         continue;
12198
12199                 encoder = to_intel_encoder(connector_state->best_encoder);
12200
12201                 WARN_ON(!connector_state->crtc);
12202
12203                 switch (encoder->type) {
12204                         unsigned int port_mask;
12205                 case INTEL_OUTPUT_DDI:
12206                         if (WARN_ON(!HAS_DDI(to_i915(dev))))
12207                                 break;
12208                         /* else: fall through */
12209                 case INTEL_OUTPUT_DP:
12210                 case INTEL_OUTPUT_HDMI:
12211                 case INTEL_OUTPUT_EDP:
12212                         port_mask = 1 << encoder->port;
12213
12214                         /* the same port mustn't appear more than once */
12215                         if (used_ports & port_mask)
12216                                 ret = false;
12217
12218                         used_ports |= port_mask;
12219                         break;
12220                 case INTEL_OUTPUT_DP_MST:
12221                         used_mst_ports |=
12222                                 1 << encoder->port;
12223                         break;
12224                 default:
12225                         break;
12226                 }
12227         }
12228         drm_connector_list_iter_end(&conn_iter);
12229
12230         /* can't mix MST and SST/HDMI on the same port */
12231         if (used_ports & used_mst_ports)
12232                 return false;
12233
12234         return ret;
12235 }
12236
12237 static int
12238 clear_intel_crtc_state(struct intel_crtc_state *crtc_state)
12239 {
12240         struct drm_i915_private *dev_priv =
12241                 to_i915(crtc_state->base.crtc->dev);
12242         struct intel_crtc_state *saved_state;
12243
12244         saved_state = kzalloc(sizeof(*saved_state), GFP_KERNEL);
12245         if (!saved_state)
12246                 return -ENOMEM;
12247
12248         /* FIXME: before the switch to atomic started, a new pipe_config was
12249          * kzalloc'd. Code that depends on any field being zero should be
12250          * fixed, so that the crtc_state can be safely duplicated. For now,
12251          * only fields that are know to not cause problems are preserved. */
12252
12253         saved_state->scaler_state = crtc_state->scaler_state;
12254         saved_state->shared_dpll = crtc_state->shared_dpll;
12255         saved_state->dpll_hw_state = crtc_state->dpll_hw_state;
12256         memcpy(saved_state->icl_port_dplls, crtc_state->icl_port_dplls,
12257                sizeof(saved_state->icl_port_dplls));
12258         saved_state->crc_enabled = crtc_state->crc_enabled;
12259         if (IS_G4X(dev_priv) ||
12260             IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
12261                 saved_state->wm = crtc_state->wm;
12262
12263         /* Keep base drm_crtc_state intact, only clear our extended struct */
12264         BUILD_BUG_ON(offsetof(struct intel_crtc_state, base));
12265         memcpy(&crtc_state->base + 1, &saved_state->base + 1,
12266                sizeof(*crtc_state) - sizeof(crtc_state->base));
12267
12268         kfree(saved_state);
12269         return 0;
12270 }
12271
12272 static int
12273 intel_modeset_pipe_config(struct intel_crtc_state *pipe_config)
12274 {
12275         struct drm_crtc *crtc = pipe_config->base.crtc;
12276         struct drm_atomic_state *state = pipe_config->base.state;
12277         struct intel_encoder *encoder;
12278         struct drm_connector *connector;
12279         struct drm_connector_state *connector_state;
12280         int base_bpp, ret;
12281         int i;
12282         bool retry = true;
12283
12284         ret = clear_intel_crtc_state(pipe_config);
12285         if (ret)
12286                 return ret;
12287
12288         pipe_config->cpu_transcoder =
12289                 (enum transcoder) to_intel_crtc(crtc)->pipe;
12290
12291         /*
12292          * Sanitize sync polarity flags based on requested ones. If neither
12293          * positive or negative polarity is requested, treat this as meaning
12294          * negative polarity.
12295          */
12296         if (!(pipe_config->base.adjusted_mode.flags &
12297               (DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_NHSYNC)))
12298                 pipe_config->base.adjusted_mode.flags |= DRM_MODE_FLAG_NHSYNC;
12299
12300         if (!(pipe_config->base.adjusted_mode.flags &
12301               (DRM_MODE_FLAG_PVSYNC | DRM_MODE_FLAG_NVSYNC)))
12302                 pipe_config->base.adjusted_mode.flags |= DRM_MODE_FLAG_NVSYNC;
12303
12304         ret = compute_baseline_pipe_bpp(to_intel_crtc(crtc),
12305                                         pipe_config);
12306         if (ret)
12307                 return ret;
12308
12309         base_bpp = pipe_config->pipe_bpp;
12310
12311         /*
12312          * Determine the real pipe dimensions. Note that stereo modes can
12313          * increase the actual pipe size due to the frame doubling and
12314          * insertion of additional space for blanks between the frame. This
12315          * is stored in the crtc timings. We use the requested mode to do this
12316          * computation to clearly distinguish it from the adjusted mode, which
12317          * can be changed by the connectors in the below retry loop.
12318          */
12319         drm_mode_get_hv_timing(&pipe_config->base.mode,
12320                                &pipe_config->pipe_src_w,
12321                                &pipe_config->pipe_src_h);
12322
12323         for_each_new_connector_in_state(state, connector, connector_state, i) {
12324                 if (connector_state->crtc != crtc)
12325                         continue;
12326
12327                 encoder = to_intel_encoder(connector_state->best_encoder);
12328
12329                 if (!check_single_encoder_cloning(state, to_intel_crtc(crtc), encoder)) {
12330                         DRM_DEBUG_KMS("rejecting invalid cloning configuration\n");
12331                         return -EINVAL;
12332                 }
12333
12334                 /*
12335                  * Determine output_types before calling the .compute_config()
12336                  * hooks so that the hooks can use this information safely.
12337                  */
12338                 if (encoder->compute_output_type)
12339                         pipe_config->output_types |=
12340                                 BIT(encoder->compute_output_type(encoder, pipe_config,
12341                                                                  connector_state));
12342                 else
12343                         pipe_config->output_types |= BIT(encoder->type);
12344         }
12345
12346 encoder_retry:
12347         /* Ensure the port clock defaults are reset when retrying. */
12348         pipe_config->port_clock = 0;
12349         pipe_config->pixel_multiplier = 1;
12350
12351         /* Fill in default crtc timings, allow encoders to overwrite them. */
12352         drm_mode_set_crtcinfo(&pipe_config->base.adjusted_mode,
12353                               CRTC_STEREO_DOUBLE);
12354
12355         /* Pass our mode to the connectors and the CRTC to give them a chance to
12356          * adjust it according to limitations or connector properties, and also
12357          * a chance to reject the mode entirely.
12358          */
12359         for_each_new_connector_in_state(state, connector, connector_state, i) {
12360                 if (connector_state->crtc != crtc)
12361                         continue;
12362
12363                 encoder = to_intel_encoder(connector_state->best_encoder);
12364                 ret = encoder->compute_config(encoder, pipe_config,
12365                                               connector_state);
12366                 if (ret < 0) {
12367                         if (ret != -EDEADLK)
12368                                 DRM_DEBUG_KMS("Encoder config failure: %d\n",
12369                                               ret);
12370                         return ret;
12371                 }
12372         }
12373
12374         /* Set default port clock if not overwritten by the encoder. Needs to be
12375          * done afterwards in case the encoder adjusts the mode. */
12376         if (!pipe_config->port_clock)
12377                 pipe_config->port_clock = pipe_config->base.adjusted_mode.crtc_clock
12378                         * pipe_config->pixel_multiplier;
12379
12380         ret = intel_crtc_compute_config(to_intel_crtc(crtc), pipe_config);
12381         if (ret == -EDEADLK)
12382                 return ret;
12383         if (ret < 0) {
12384                 DRM_DEBUG_KMS("CRTC fixup failed\n");
12385                 return ret;
12386         }
12387
12388         if (ret == RETRY) {
12389                 if (WARN(!retry, "loop in pipe configuration computation\n"))
12390                         return -EINVAL;
12391
12392                 DRM_DEBUG_KMS("CRTC bw constrained, retrying\n");
12393                 retry = false;
12394                 goto encoder_retry;
12395         }
12396
12397         /* Dithering seems to not pass-through bits correctly when it should, so
12398          * only enable it on 6bpc panels and when its not a compliance
12399          * test requesting 6bpc video pattern.
12400          */
12401         pipe_config->dither = (pipe_config->pipe_bpp == 6*3) &&
12402                 !pipe_config->dither_force_disable;
12403         DRM_DEBUG_KMS("hw max bpp: %i, pipe bpp: %i, dithering: %i\n",
12404                       base_bpp, pipe_config->pipe_bpp, pipe_config->dither);
12405
12406         return 0;
12407 }
12408
12409 bool intel_fuzzy_clock_check(int clock1, int clock2)
12410 {
12411         int diff;
12412
12413         if (clock1 == clock2)
12414                 return true;
12415
12416         if (!clock1 || !clock2)
12417                 return false;
12418
12419         diff = abs(clock1 - clock2);
12420
12421         if (((((diff + clock1 + clock2) * 100)) / (clock1 + clock2)) < 105)
12422                 return true;
12423
12424         return false;
12425 }
12426
12427 static bool
12428 intel_compare_m_n(unsigned int m, unsigned int n,
12429                   unsigned int m2, unsigned int n2,
12430                   bool exact)
12431 {
12432         if (m == m2 && n == n2)
12433                 return true;
12434
12435         if (exact || !m || !n || !m2 || !n2)
12436                 return false;
12437
12438         BUILD_BUG_ON(DATA_LINK_M_N_MASK > INT_MAX);
12439
12440         if (n > n2) {
12441                 while (n > n2) {
12442                         m2 <<= 1;
12443                         n2 <<= 1;
12444                 }
12445         } else if (n < n2) {
12446                 while (n < n2) {
12447                         m <<= 1;
12448                         n <<= 1;
12449                 }
12450         }
12451
12452         if (n != n2)
12453                 return false;
12454
12455         return intel_fuzzy_clock_check(m, m2);
12456 }
12457
12458 static bool
12459 intel_compare_link_m_n(const struct intel_link_m_n *m_n,
12460                        const struct intel_link_m_n *m2_n2,
12461                        bool exact)
12462 {
12463         return m_n->tu == m2_n2->tu &&
12464                 intel_compare_m_n(m_n->gmch_m, m_n->gmch_n,
12465                                   m2_n2->gmch_m, m2_n2->gmch_n, exact) &&
12466                 intel_compare_m_n(m_n->link_m, m_n->link_n,
12467                                   m2_n2->link_m, m2_n2->link_n, exact);
12468 }
12469
12470 static bool
12471 intel_compare_infoframe(const union hdmi_infoframe *a,
12472                         const union hdmi_infoframe *b)
12473 {
12474         return memcmp(a, b, sizeof(*a)) == 0;
12475 }
12476
12477 static void
12478 pipe_config_infoframe_mismatch(struct drm_i915_private *dev_priv,
12479                                bool fastset, const char *name,
12480                                const union hdmi_infoframe *a,
12481                                const union hdmi_infoframe *b)
12482 {
12483         if (fastset) {
12484                 if ((drm_debug & DRM_UT_KMS) == 0)
12485                         return;
12486
12487                 drm_dbg(DRM_UT_KMS, "fastset mismatch in %s infoframe", name);
12488                 drm_dbg(DRM_UT_KMS, "expected:");
12489                 hdmi_infoframe_log(KERN_DEBUG, dev_priv->drm.dev, a);
12490                 drm_dbg(DRM_UT_KMS, "found");
12491                 hdmi_infoframe_log(KERN_DEBUG, dev_priv->drm.dev, b);
12492         } else {
12493                 drm_err("mismatch in %s infoframe", name);
12494                 drm_err("expected:");
12495                 hdmi_infoframe_log(KERN_ERR, dev_priv->drm.dev, a);
12496                 drm_err("found");
12497                 hdmi_infoframe_log(KERN_ERR, dev_priv->drm.dev, b);
12498         }
12499 }
12500
12501 static void __printf(3, 4)
12502 pipe_config_mismatch(bool fastset, const char *name, const char *format, ...)
12503 {
12504         struct va_format vaf;
12505         va_list args;
12506
12507         va_start(args, format);
12508         vaf.fmt = format;
12509         vaf.va = &args;
12510
12511         if (fastset)
12512                 drm_dbg(DRM_UT_KMS, "fastset mismatch in %s %pV", name, &vaf);
12513         else
12514                 drm_err("mismatch in %s %pV", name, &vaf);
12515
12516         va_end(args);
12517 }
12518
12519 static bool fastboot_enabled(struct drm_i915_private *dev_priv)
12520 {
12521         if (i915_modparams.fastboot != -1)
12522                 return i915_modparams.fastboot;
12523
12524         /* Enable fastboot by default on Skylake and newer */
12525         if (INTEL_GEN(dev_priv) >= 9)
12526                 return true;
12527
12528         /* Enable fastboot by default on VLV and CHV */
12529         if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
12530                 return true;
12531
12532         /* Disabled by default on all others */
12533         return false;
12534 }
12535
12536 static bool
12537 intel_pipe_config_compare(const struct intel_crtc_state *current_config,
12538                           const struct intel_crtc_state *pipe_config,
12539                           bool fastset)
12540 {
12541         struct drm_i915_private *dev_priv = to_i915(current_config->base.crtc->dev);
12542         bool ret = true;
12543         bool fixup_inherited = fastset &&
12544                 (current_config->base.mode.private_flags & I915_MODE_FLAG_INHERITED) &&
12545                 !(pipe_config->base.mode.private_flags & I915_MODE_FLAG_INHERITED);
12546
12547         if (fixup_inherited && !fastboot_enabled(dev_priv)) {
12548                 DRM_DEBUG_KMS("initial modeset and fastboot not set\n");
12549                 ret = false;
12550         }
12551
12552 #define PIPE_CONF_CHECK_X(name) do { \
12553         if (current_config->name != pipe_config->name) { \
12554                 pipe_config_mismatch(fastset, __stringify(name), \
12555                                      "(expected 0x%08x, found 0x%08x)\n", \
12556                                      current_config->name, \
12557                                      pipe_config->name); \
12558                 ret = false; \
12559         } \
12560 } while (0)
12561
12562 #define PIPE_CONF_CHECK_I(name) do { \
12563         if (current_config->name != pipe_config->name) { \
12564                 pipe_config_mismatch(fastset, __stringify(name), \
12565                                      "(expected %i, found %i)\n", \
12566                                      current_config->name, \
12567                                      pipe_config->name); \
12568                 ret = false; \
12569         } \
12570 } while (0)
12571
12572 #define PIPE_CONF_CHECK_BOOL(name) do { \
12573         if (current_config->name != pipe_config->name) { \
12574                 pipe_config_mismatch(fastset, __stringify(name), \
12575                                      "(expected %s, found %s)\n", \
12576                                      yesno(current_config->name), \
12577                                      yesno(pipe_config->name)); \
12578                 ret = false; \
12579         } \
12580 } while (0)
12581
12582 /*
12583  * Checks state where we only read out the enabling, but not the entire
12584  * state itself (like full infoframes or ELD for audio). These states
12585  * require a full modeset on bootup to fix up.
12586  */
12587 #define PIPE_CONF_CHECK_BOOL_INCOMPLETE(name) do { \
12588         if (!fixup_inherited || (!current_config->name && !pipe_config->name)) { \
12589                 PIPE_CONF_CHECK_BOOL(name); \
12590         } else { \
12591                 pipe_config_mismatch(fastset, __stringify(name), \
12592                                      "unable to verify whether state matches exactly, forcing modeset (expected %s, found %s)\n", \
12593                                      yesno(current_config->name), \
12594                                      yesno(pipe_config->name)); \
12595                 ret = false; \
12596         } \
12597 } while (0)
12598
12599 #define PIPE_CONF_CHECK_P(name) do { \
12600         if (current_config->name != pipe_config->name) { \
12601                 pipe_config_mismatch(fastset, __stringify(name), \
12602                                      "(expected %p, found %p)\n", \
12603                                      current_config->name, \
12604                                      pipe_config->name); \
12605                 ret = false; \
12606         } \
12607 } while (0)
12608
12609 #define PIPE_CONF_CHECK_M_N(name) do { \
12610         if (!intel_compare_link_m_n(&current_config->name, \
12611                                     &pipe_config->name,\
12612                                     !fastset)) { \
12613                 pipe_config_mismatch(fastset, __stringify(name), \
12614                                      "(expected tu %i gmch %i/%i link %i/%i, " \
12615                                      "found tu %i, gmch %i/%i link %i/%i)\n", \
12616                                      current_config->name.tu, \
12617                                      current_config->name.gmch_m, \
12618                                      current_config->name.gmch_n, \
12619                                      current_config->name.link_m, \
12620                                      current_config->name.link_n, \
12621                                      pipe_config->name.tu, \
12622                                      pipe_config->name.gmch_m, \
12623                                      pipe_config->name.gmch_n, \
12624                                      pipe_config->name.link_m, \
12625                                      pipe_config->name.link_n); \
12626                 ret = false; \
12627         } \
12628 } while (0)
12629
12630 /* This is required for BDW+ where there is only one set of registers for
12631  * switching between high and low RR.
12632  * This macro can be used whenever a comparison has to be made between one
12633  * hw state and multiple sw state variables.
12634  */
12635 #define PIPE_CONF_CHECK_M_N_ALT(name, alt_name) do { \
12636         if (!intel_compare_link_m_n(&current_config->name, \
12637                                     &pipe_config->name, !fastset) && \
12638             !intel_compare_link_m_n(&current_config->alt_name, \
12639                                     &pipe_config->name, !fastset)) { \
12640                 pipe_config_mismatch(fastset, __stringify(name), \
12641                                      "(expected tu %i gmch %i/%i link %i/%i, " \
12642                                      "or tu %i gmch %i/%i link %i/%i, " \
12643                                      "found tu %i, gmch %i/%i link %i/%i)\n", \
12644                                      current_config->name.tu, \
12645                                      current_config->name.gmch_m, \
12646                                      current_config->name.gmch_n, \
12647                                      current_config->name.link_m, \
12648                                      current_config->name.link_n, \
12649                                      current_config->alt_name.tu, \
12650                                      current_config->alt_name.gmch_m, \
12651                                      current_config->alt_name.gmch_n, \
12652                                      current_config->alt_name.link_m, \
12653                                      current_config->alt_name.link_n, \
12654                                      pipe_config->name.tu, \
12655                                      pipe_config->name.gmch_m, \
12656                                      pipe_config->name.gmch_n, \
12657                                      pipe_config->name.link_m, \
12658                                      pipe_config->name.link_n); \
12659                 ret = false; \
12660         } \
12661 } while (0)
12662
12663 #define PIPE_CONF_CHECK_FLAGS(name, mask) do { \
12664         if ((current_config->name ^ pipe_config->name) & (mask)) { \
12665                 pipe_config_mismatch(fastset, __stringify(name), \
12666                                      "(%x) (expected %i, found %i)\n", \
12667                                      (mask), \
12668                                      current_config->name & (mask), \
12669                                      pipe_config->name & (mask)); \
12670                 ret = false; \
12671         } \
12672 } while (0)
12673
12674 #define PIPE_CONF_CHECK_CLOCK_FUZZY(name) do { \
12675         if (!intel_fuzzy_clock_check(current_config->name, pipe_config->name)) { \
12676                 pipe_config_mismatch(fastset, __stringify(name), \
12677                                      "(expected %i, found %i)\n", \
12678                                      current_config->name, \
12679                                      pipe_config->name); \
12680                 ret = false; \
12681         } \
12682 } while (0)
12683
12684 #define PIPE_CONF_CHECK_INFOFRAME(name) do { \
12685         if (!intel_compare_infoframe(&current_config->infoframes.name, \
12686                                      &pipe_config->infoframes.name)) { \
12687                 pipe_config_infoframe_mismatch(dev_priv, fastset, __stringify(name), \
12688                                                &current_config->infoframes.name, \
12689                                                &pipe_config->infoframes.name); \
12690                 ret = false; \
12691         } \
12692 } while (0)
12693
12694 #define PIPE_CONF_QUIRK(quirk) \
12695         ((current_config->quirks | pipe_config->quirks) & (quirk))
12696
12697         PIPE_CONF_CHECK_I(cpu_transcoder);
12698
12699         PIPE_CONF_CHECK_BOOL(has_pch_encoder);
12700         PIPE_CONF_CHECK_I(fdi_lanes);
12701         PIPE_CONF_CHECK_M_N(fdi_m_n);
12702
12703         PIPE_CONF_CHECK_I(lane_count);
12704         PIPE_CONF_CHECK_X(lane_lat_optim_mask);
12705
12706         if (INTEL_GEN(dev_priv) < 8) {
12707                 PIPE_CONF_CHECK_M_N(dp_m_n);
12708
12709                 if (current_config->has_drrs)
12710                         PIPE_CONF_CHECK_M_N(dp_m2_n2);
12711         } else
12712                 PIPE_CONF_CHECK_M_N_ALT(dp_m_n, dp_m2_n2);
12713
12714         PIPE_CONF_CHECK_X(output_types);
12715
12716         PIPE_CONF_CHECK_I(base.adjusted_mode.crtc_hdisplay);
12717         PIPE_CONF_CHECK_I(base.adjusted_mode.crtc_htotal);
12718         PIPE_CONF_CHECK_I(base.adjusted_mode.crtc_hblank_start);
12719         PIPE_CONF_CHECK_I(base.adjusted_mode.crtc_hblank_end);
12720         PIPE_CONF_CHECK_I(base.adjusted_mode.crtc_hsync_start);
12721         PIPE_CONF_CHECK_I(base.adjusted_mode.crtc_hsync_end);
12722
12723         PIPE_CONF_CHECK_I(base.adjusted_mode.crtc_vdisplay);
12724         PIPE_CONF_CHECK_I(base.adjusted_mode.crtc_vtotal);
12725         PIPE_CONF_CHECK_I(base.adjusted_mode.crtc_vblank_start);
12726         PIPE_CONF_CHECK_I(base.adjusted_mode.crtc_vblank_end);
12727         PIPE_CONF_CHECK_I(base.adjusted_mode.crtc_vsync_start);
12728         PIPE_CONF_CHECK_I(base.adjusted_mode.crtc_vsync_end);
12729
12730         PIPE_CONF_CHECK_I(pixel_multiplier);
12731         PIPE_CONF_CHECK_I(output_format);
12732         PIPE_CONF_CHECK_BOOL(has_hdmi_sink);
12733         if ((INTEL_GEN(dev_priv) < 8 && !IS_HASWELL(dev_priv)) ||
12734             IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
12735                 PIPE_CONF_CHECK_BOOL(limited_color_range);
12736
12737         PIPE_CONF_CHECK_BOOL(hdmi_scrambling);
12738         PIPE_CONF_CHECK_BOOL(hdmi_high_tmds_clock_ratio);
12739         PIPE_CONF_CHECK_BOOL(has_infoframe);
12740
12741         PIPE_CONF_CHECK_BOOL_INCOMPLETE(has_audio);
12742
12743         PIPE_CONF_CHECK_FLAGS(base.adjusted_mode.flags,
12744                               DRM_MODE_FLAG_INTERLACE);
12745
12746         if (!PIPE_CONF_QUIRK(PIPE_CONFIG_QUIRK_MODE_SYNC_FLAGS)) {
12747                 PIPE_CONF_CHECK_FLAGS(base.adjusted_mode.flags,
12748                                       DRM_MODE_FLAG_PHSYNC);
12749                 PIPE_CONF_CHECK_FLAGS(base.adjusted_mode.flags,
12750                                       DRM_MODE_FLAG_NHSYNC);
12751                 PIPE_CONF_CHECK_FLAGS(base.adjusted_mode.flags,
12752                                       DRM_MODE_FLAG_PVSYNC);
12753                 PIPE_CONF_CHECK_FLAGS(base.adjusted_mode.flags,
12754                                       DRM_MODE_FLAG_NVSYNC);
12755         }
12756
12757         PIPE_CONF_CHECK_X(gmch_pfit.control);
12758         /* pfit ratios are autocomputed by the hw on gen4+ */
12759         if (INTEL_GEN(dev_priv) < 4)
12760                 PIPE_CONF_CHECK_X(gmch_pfit.pgm_ratios);
12761         PIPE_CONF_CHECK_X(gmch_pfit.lvds_border_bits);
12762
12763         /*
12764          * Changing the EDP transcoder input mux
12765          * (A_ONOFF vs. A_ON) requires a full modeset.
12766          */
12767         PIPE_CONF_CHECK_BOOL(pch_pfit.force_thru);
12768
12769         if (!fastset) {
12770                 PIPE_CONF_CHECK_I(pipe_src_w);
12771                 PIPE_CONF_CHECK_I(pipe_src_h);
12772
12773                 PIPE_CONF_CHECK_BOOL(pch_pfit.enabled);
12774                 if (current_config->pch_pfit.enabled) {
12775                         PIPE_CONF_CHECK_X(pch_pfit.pos);
12776                         PIPE_CONF_CHECK_X(pch_pfit.size);
12777                 }
12778
12779                 PIPE_CONF_CHECK_I(scaler_state.scaler_id);
12780                 PIPE_CONF_CHECK_CLOCK_FUZZY(pixel_rate);
12781
12782                 PIPE_CONF_CHECK_X(gamma_mode);
12783                 if (IS_CHERRYVIEW(dev_priv))
12784                         PIPE_CONF_CHECK_X(cgm_mode);
12785                 else
12786                         PIPE_CONF_CHECK_X(csc_mode);
12787                 PIPE_CONF_CHECK_BOOL(gamma_enable);
12788                 PIPE_CONF_CHECK_BOOL(csc_enable);
12789         }
12790
12791         PIPE_CONF_CHECK_BOOL(double_wide);
12792
12793         PIPE_CONF_CHECK_P(shared_dpll);
12794         PIPE_CONF_CHECK_X(dpll_hw_state.dpll);
12795         PIPE_CONF_CHECK_X(dpll_hw_state.dpll_md);
12796         PIPE_CONF_CHECK_X(dpll_hw_state.fp0);
12797         PIPE_CONF_CHECK_X(dpll_hw_state.fp1);
12798         PIPE_CONF_CHECK_X(dpll_hw_state.wrpll);
12799         PIPE_CONF_CHECK_X(dpll_hw_state.spll);
12800         PIPE_CONF_CHECK_X(dpll_hw_state.ctrl1);
12801         PIPE_CONF_CHECK_X(dpll_hw_state.cfgcr1);
12802         PIPE_CONF_CHECK_X(dpll_hw_state.cfgcr2);
12803         PIPE_CONF_CHECK_X(dpll_hw_state.cfgcr0);
12804         PIPE_CONF_CHECK_X(dpll_hw_state.ebb0);
12805         PIPE_CONF_CHECK_X(dpll_hw_state.ebb4);
12806         PIPE_CONF_CHECK_X(dpll_hw_state.pll0);
12807         PIPE_CONF_CHECK_X(dpll_hw_state.pll1);
12808         PIPE_CONF_CHECK_X(dpll_hw_state.pll2);
12809         PIPE_CONF_CHECK_X(dpll_hw_state.pll3);
12810         PIPE_CONF_CHECK_X(dpll_hw_state.pll6);
12811         PIPE_CONF_CHECK_X(dpll_hw_state.pll8);
12812         PIPE_CONF_CHECK_X(dpll_hw_state.pll9);
12813         PIPE_CONF_CHECK_X(dpll_hw_state.pll10);
12814         PIPE_CONF_CHECK_X(dpll_hw_state.pcsdw12);
12815         PIPE_CONF_CHECK_X(dpll_hw_state.mg_refclkin_ctl);
12816         PIPE_CONF_CHECK_X(dpll_hw_state.mg_clktop2_coreclkctl1);
12817         PIPE_CONF_CHECK_X(dpll_hw_state.mg_clktop2_hsclkctl);
12818         PIPE_CONF_CHECK_X(dpll_hw_state.mg_pll_div0);
12819         PIPE_CONF_CHECK_X(dpll_hw_state.mg_pll_div1);
12820         PIPE_CONF_CHECK_X(dpll_hw_state.mg_pll_lf);
12821         PIPE_CONF_CHECK_X(dpll_hw_state.mg_pll_frac_lock);
12822         PIPE_CONF_CHECK_X(dpll_hw_state.mg_pll_ssc);
12823         PIPE_CONF_CHECK_X(dpll_hw_state.mg_pll_bias);
12824         PIPE_CONF_CHECK_X(dpll_hw_state.mg_pll_tdc_coldst_bias);
12825
12826         PIPE_CONF_CHECK_X(dsi_pll.ctrl);
12827         PIPE_CONF_CHECK_X(dsi_pll.div);
12828
12829         if (IS_G4X(dev_priv) || INTEL_GEN(dev_priv) >= 5)
12830                 PIPE_CONF_CHECK_I(pipe_bpp);
12831
12832         PIPE_CONF_CHECK_CLOCK_FUZZY(base.adjusted_mode.crtc_clock);
12833         PIPE_CONF_CHECK_CLOCK_FUZZY(port_clock);
12834
12835         PIPE_CONF_CHECK_I(min_voltage_level);
12836
12837         PIPE_CONF_CHECK_X(infoframes.enable);
12838         PIPE_CONF_CHECK_X(infoframes.gcp);
12839         PIPE_CONF_CHECK_INFOFRAME(avi);
12840         PIPE_CONF_CHECK_INFOFRAME(spd);
12841         PIPE_CONF_CHECK_INFOFRAME(hdmi);
12842         PIPE_CONF_CHECK_INFOFRAME(drm);
12843
12844 #undef PIPE_CONF_CHECK_X
12845 #undef PIPE_CONF_CHECK_I
12846 #undef PIPE_CONF_CHECK_BOOL
12847 #undef PIPE_CONF_CHECK_BOOL_INCOMPLETE
12848 #undef PIPE_CONF_CHECK_P
12849 #undef PIPE_CONF_CHECK_FLAGS
12850 #undef PIPE_CONF_CHECK_CLOCK_FUZZY
12851 #undef PIPE_CONF_QUIRK
12852
12853         return ret;
12854 }
12855
12856 static void intel_pipe_config_sanity_check(struct drm_i915_private *dev_priv,
12857                                            const struct intel_crtc_state *pipe_config)
12858 {
12859         if (pipe_config->has_pch_encoder) {
12860                 int fdi_dotclock = intel_dotclock_calculate(intel_fdi_link_freq(dev_priv, pipe_config),
12861                                                             &pipe_config->fdi_m_n);
12862                 int dotclock = pipe_config->base.adjusted_mode.crtc_clock;
12863
12864                 /*
12865                  * FDI already provided one idea for the dotclock.
12866                  * Yell if the encoder disagrees.
12867                  */
12868                 WARN(!intel_fuzzy_clock_check(fdi_dotclock, dotclock),
12869                      "FDI dotclock and encoder dotclock mismatch, fdi: %i, encoder: %i\n",
12870                      fdi_dotclock, dotclock);
12871         }
12872 }
12873
12874 static void verify_wm_state(struct intel_crtc *crtc,
12875                             struct intel_crtc_state *new_crtc_state)
12876 {
12877         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
12878         struct skl_hw_state {
12879                 struct skl_ddb_entry ddb_y[I915_MAX_PLANES];
12880                 struct skl_ddb_entry ddb_uv[I915_MAX_PLANES];
12881                 struct skl_ddb_allocation ddb;
12882                 struct skl_pipe_wm wm;
12883         } *hw;
12884         struct skl_ddb_allocation *sw_ddb;
12885         struct skl_pipe_wm *sw_wm;
12886         struct skl_ddb_entry *hw_ddb_entry, *sw_ddb_entry;
12887         const enum pipe pipe = crtc->pipe;
12888         int plane, level, max_level = ilk_wm_max_level(dev_priv);
12889
12890         if (INTEL_GEN(dev_priv) < 9 || !new_crtc_state->base.active)
12891                 return;
12892
12893         hw = kzalloc(sizeof(*hw), GFP_KERNEL);
12894         if (!hw)
12895                 return;
12896
12897         skl_pipe_wm_get_hw_state(crtc, &hw->wm);
12898         sw_wm = &new_crtc_state->wm.skl.optimal;
12899
12900         skl_pipe_ddb_get_hw_state(crtc, hw->ddb_y, hw->ddb_uv);
12901
12902         skl_ddb_get_hw_state(dev_priv, &hw->ddb);
12903         sw_ddb = &dev_priv->wm.skl_hw.ddb;
12904
12905         if (INTEL_GEN(dev_priv) >= 11 &&
12906             hw->ddb.enabled_slices != sw_ddb->enabled_slices)
12907                 DRM_ERROR("mismatch in DBUF Slices (expected %u, got %u)\n",
12908                           sw_ddb->enabled_slices,
12909                           hw->ddb.enabled_slices);
12910
12911         /* planes */
12912         for_each_universal_plane(dev_priv, pipe, plane) {
12913                 struct skl_plane_wm *hw_plane_wm, *sw_plane_wm;
12914
12915                 hw_plane_wm = &hw->wm.planes[plane];
12916                 sw_plane_wm = &sw_wm->planes[plane];
12917
12918                 /* Watermarks */
12919                 for (level = 0; level <= max_level; level++) {
12920                         if (skl_wm_level_equals(&hw_plane_wm->wm[level],
12921                                                 &sw_plane_wm->wm[level]))
12922                                 continue;
12923
12924                         DRM_ERROR("mismatch in WM pipe %c plane %d level %d (expected e=%d b=%u l=%u, got e=%d b=%u l=%u)\n",
12925                                   pipe_name(pipe), plane + 1, level,
12926                                   sw_plane_wm->wm[level].plane_en,
12927                                   sw_plane_wm->wm[level].plane_res_b,
12928                                   sw_plane_wm->wm[level].plane_res_l,
12929                                   hw_plane_wm->wm[level].plane_en,
12930                                   hw_plane_wm->wm[level].plane_res_b,
12931                                   hw_plane_wm->wm[level].plane_res_l);
12932                 }
12933
12934                 if (!skl_wm_level_equals(&hw_plane_wm->trans_wm,
12935                                          &sw_plane_wm->trans_wm)) {
12936                         DRM_ERROR("mismatch in trans WM pipe %c plane %d (expected e=%d b=%u l=%u, got e=%d b=%u l=%u)\n",
12937                                   pipe_name(pipe), plane + 1,
12938                                   sw_plane_wm->trans_wm.plane_en,
12939                                   sw_plane_wm->trans_wm.plane_res_b,
12940                                   sw_plane_wm->trans_wm.plane_res_l,
12941                                   hw_plane_wm->trans_wm.plane_en,
12942                                   hw_plane_wm->trans_wm.plane_res_b,
12943                                   hw_plane_wm->trans_wm.plane_res_l);
12944                 }
12945
12946                 /* DDB */
12947                 hw_ddb_entry = &hw->ddb_y[plane];
12948                 sw_ddb_entry = &new_crtc_state->wm.skl.plane_ddb_y[plane];
12949
12950                 if (!skl_ddb_entry_equal(hw_ddb_entry, sw_ddb_entry)) {
12951                         DRM_ERROR("mismatch in DDB state pipe %c plane %d (expected (%u,%u), found (%u,%u))\n",
12952                                   pipe_name(pipe), plane + 1,
12953                                   sw_ddb_entry->start, sw_ddb_entry->end,
12954                                   hw_ddb_entry->start, hw_ddb_entry->end);
12955                 }
12956         }
12957
12958         /*
12959          * cursor
12960          * If the cursor plane isn't active, we may not have updated it's ddb
12961          * allocation. In that case since the ddb allocation will be updated
12962          * once the plane becomes visible, we can skip this check
12963          */
12964         if (1) {
12965                 struct skl_plane_wm *hw_plane_wm, *sw_plane_wm;
12966
12967                 hw_plane_wm = &hw->wm.planes[PLANE_CURSOR];
12968                 sw_plane_wm = &sw_wm->planes[PLANE_CURSOR];
12969
12970                 /* Watermarks */
12971                 for (level = 0; level <= max_level; level++) {
12972                         if (skl_wm_level_equals(&hw_plane_wm->wm[level],
12973                                                 &sw_plane_wm->wm[level]))
12974                                 continue;
12975
12976                         DRM_ERROR("mismatch in WM pipe %c cursor level %d (expected e=%d b=%u l=%u, got e=%d b=%u l=%u)\n",
12977                                   pipe_name(pipe), level,
12978                                   sw_plane_wm->wm[level].plane_en,
12979                                   sw_plane_wm->wm[level].plane_res_b,
12980                                   sw_plane_wm->wm[level].plane_res_l,
12981                                   hw_plane_wm->wm[level].plane_en,
12982                                   hw_plane_wm->wm[level].plane_res_b,
12983                                   hw_plane_wm->wm[level].plane_res_l);
12984                 }
12985
12986                 if (!skl_wm_level_equals(&hw_plane_wm->trans_wm,
12987                                          &sw_plane_wm->trans_wm)) {
12988                         DRM_ERROR("mismatch in trans WM pipe %c cursor (expected e=%d b=%u l=%u, got e=%d b=%u l=%u)\n",
12989                                   pipe_name(pipe),
12990                                   sw_plane_wm->trans_wm.plane_en,
12991                                   sw_plane_wm->trans_wm.plane_res_b,
12992                                   sw_plane_wm->trans_wm.plane_res_l,
12993                                   hw_plane_wm->trans_wm.plane_en,
12994                                   hw_plane_wm->trans_wm.plane_res_b,
12995                                   hw_plane_wm->trans_wm.plane_res_l);
12996                 }
12997
12998                 /* DDB */
12999                 hw_ddb_entry = &hw->ddb_y[PLANE_CURSOR];
13000                 sw_ddb_entry = &new_crtc_state->wm.skl.plane_ddb_y[PLANE_CURSOR];
13001
13002                 if (!skl_ddb_entry_equal(hw_ddb_entry, sw_ddb_entry)) {
13003                         DRM_ERROR("mismatch in DDB state pipe %c cursor (expected (%u,%u), found (%u,%u))\n",
13004                                   pipe_name(pipe),
13005                                   sw_ddb_entry->start, sw_ddb_entry->end,
13006                                   hw_ddb_entry->start, hw_ddb_entry->end);
13007                 }
13008         }
13009
13010         kfree(hw);
13011 }
13012
13013 static void
13014 verify_connector_state(struct intel_atomic_state *state,
13015                        struct intel_crtc *crtc)
13016 {
13017         struct drm_connector *connector;
13018         struct drm_connector_state *new_conn_state;
13019         int i;
13020
13021         for_each_new_connector_in_state(&state->base, connector, new_conn_state, i) {
13022                 struct drm_encoder *encoder = connector->encoder;
13023                 struct intel_crtc_state *crtc_state = NULL;
13024
13025                 if (new_conn_state->crtc != &crtc->base)
13026                         continue;
13027
13028                 if (crtc)
13029                         crtc_state = intel_atomic_get_new_crtc_state(state, crtc);
13030
13031                 intel_connector_verify_state(crtc_state, new_conn_state);
13032
13033                 I915_STATE_WARN(new_conn_state->best_encoder != encoder,
13034                      "connector's atomic encoder doesn't match legacy encoder\n");
13035         }
13036 }
13037
13038 static void
13039 verify_encoder_state(struct drm_i915_private *dev_priv, struct intel_atomic_state *state)
13040 {
13041         struct intel_encoder *encoder;
13042         struct drm_connector *connector;
13043         struct drm_connector_state *old_conn_state, *new_conn_state;
13044         int i;
13045
13046         for_each_intel_encoder(&dev_priv->drm, encoder) {
13047                 bool enabled = false, found = false;
13048                 enum pipe pipe;
13049
13050                 DRM_DEBUG_KMS("[ENCODER:%d:%s]\n",
13051                               encoder->base.base.id,
13052                               encoder->base.name);
13053
13054                 for_each_oldnew_connector_in_state(&state->base, connector, old_conn_state,
13055                                                    new_conn_state, i) {
13056                         if (old_conn_state->best_encoder == &encoder->base)
13057                                 found = true;
13058
13059                         if (new_conn_state->best_encoder != &encoder->base)
13060                                 continue;
13061                         found = enabled = true;
13062
13063                         I915_STATE_WARN(new_conn_state->crtc !=
13064                                         encoder->base.crtc,
13065                              "connector's crtc doesn't match encoder crtc\n");
13066                 }
13067
13068                 if (!found)
13069                         continue;
13070
13071                 I915_STATE_WARN(!!encoder->base.crtc != enabled,
13072                      "encoder's enabled state mismatch "
13073                      "(expected %i, found %i)\n",
13074                      !!encoder->base.crtc, enabled);
13075
13076                 if (!encoder->base.crtc) {
13077                         bool active;
13078
13079                         active = encoder->get_hw_state(encoder, &pipe);
13080                         I915_STATE_WARN(active,
13081                              "encoder detached but still enabled on pipe %c.\n",
13082                              pipe_name(pipe));
13083                 }
13084         }
13085 }
13086
13087 static void
13088 verify_crtc_state(struct intel_crtc *crtc,
13089                   struct intel_crtc_state *old_crtc_state,
13090                   struct intel_crtc_state *new_crtc_state)
13091 {
13092         struct drm_device *dev = crtc->base.dev;
13093         struct drm_i915_private *dev_priv = to_i915(dev);
13094         struct intel_encoder *encoder;
13095         struct intel_crtc_state *pipe_config;
13096         struct drm_atomic_state *state;
13097         bool active;
13098
13099         state = old_crtc_state->base.state;
13100         __drm_atomic_helper_crtc_destroy_state(&old_crtc_state->base);
13101         pipe_config = old_crtc_state;
13102         memset(pipe_config, 0, sizeof(*pipe_config));
13103         pipe_config->base.crtc = &crtc->base;
13104         pipe_config->base.state = state;
13105
13106         DRM_DEBUG_KMS("[CRTC:%d:%s]\n", crtc->base.base.id, crtc->base.name);
13107
13108         active = dev_priv->display.get_pipe_config(crtc, pipe_config);
13109
13110         /* we keep both pipes enabled on 830 */
13111         if (IS_I830(dev_priv))
13112                 active = new_crtc_state->base.active;
13113
13114         I915_STATE_WARN(new_crtc_state->base.active != active,
13115              "crtc active state doesn't match with hw state "
13116              "(expected %i, found %i)\n", new_crtc_state->base.active, active);
13117
13118         I915_STATE_WARN(crtc->active != new_crtc_state->base.active,
13119              "transitional active state does not match atomic hw state "
13120              "(expected %i, found %i)\n", new_crtc_state->base.active, crtc->active);
13121
13122         for_each_encoder_on_crtc(dev, &crtc->base, encoder) {
13123                 enum pipe pipe;
13124
13125                 active = encoder->get_hw_state(encoder, &pipe);
13126                 I915_STATE_WARN(active != new_crtc_state->base.active,
13127                         "[ENCODER:%i] active %i with crtc active %i\n",
13128                         encoder->base.base.id, active, new_crtc_state->base.active);
13129
13130                 I915_STATE_WARN(active && crtc->pipe != pipe,
13131                                 "Encoder connected to wrong pipe %c\n",
13132                                 pipe_name(pipe));
13133
13134                 if (active)
13135                         encoder->get_config(encoder, pipe_config);
13136         }
13137
13138         intel_crtc_compute_pixel_rate(pipe_config);
13139
13140         if (!new_crtc_state->base.active)
13141                 return;
13142
13143         intel_pipe_config_sanity_check(dev_priv, pipe_config);
13144
13145         if (!intel_pipe_config_compare(new_crtc_state,
13146                                        pipe_config, false)) {
13147                 I915_STATE_WARN(1, "pipe state doesn't match!\n");
13148                 intel_dump_pipe_config(pipe_config, NULL, "[hw state]");
13149                 intel_dump_pipe_config(new_crtc_state, NULL, "[sw state]");
13150         }
13151 }
13152
13153 static void
13154 intel_verify_planes(struct intel_atomic_state *state)
13155 {
13156         struct intel_plane *plane;
13157         const struct intel_plane_state *plane_state;
13158         int i;
13159
13160         for_each_new_intel_plane_in_state(state, plane,
13161                                           plane_state, i)
13162                 assert_plane(plane, plane_state->slave ||
13163                              plane_state->base.visible);
13164 }
13165
13166 static void
13167 verify_single_dpll_state(struct drm_i915_private *dev_priv,
13168                          struct intel_shared_dpll *pll,
13169                          struct intel_crtc *crtc,
13170                          struct intel_crtc_state *new_crtc_state)
13171 {
13172         struct intel_dpll_hw_state dpll_hw_state;
13173         unsigned int crtc_mask;
13174         bool active;
13175
13176         memset(&dpll_hw_state, 0, sizeof(dpll_hw_state));
13177
13178         DRM_DEBUG_KMS("%s\n", pll->info->name);
13179
13180         active = pll->info->funcs->get_hw_state(dev_priv, pll, &dpll_hw_state);
13181
13182         if (!(pll->info->flags & INTEL_DPLL_ALWAYS_ON)) {
13183                 I915_STATE_WARN(!pll->on && pll->active_mask,
13184                      "pll in active use but not on in sw tracking\n");
13185                 I915_STATE_WARN(pll->on && !pll->active_mask,
13186                      "pll is on but not used by any active crtc\n");
13187                 I915_STATE_WARN(pll->on != active,
13188                      "pll on state mismatch (expected %i, found %i)\n",
13189                      pll->on, active);
13190         }
13191
13192         if (!crtc) {
13193                 I915_STATE_WARN(pll->active_mask & ~pll->state.crtc_mask,
13194                                 "more active pll users than references: %x vs %x\n",
13195                                 pll->active_mask, pll->state.crtc_mask);
13196
13197                 return;
13198         }
13199
13200         crtc_mask = drm_crtc_mask(&crtc->base);
13201
13202         if (new_crtc_state->base.active)
13203                 I915_STATE_WARN(!(pll->active_mask & crtc_mask),
13204                                 "pll active mismatch (expected pipe %c in active mask 0x%02x)\n",
13205                                 pipe_name(drm_crtc_index(&crtc->base)), pll->active_mask);
13206         else
13207                 I915_STATE_WARN(pll->active_mask & crtc_mask,
13208                                 "pll active mismatch (didn't expect pipe %c in active mask 0x%02x)\n",
13209                                 pipe_name(drm_crtc_index(&crtc->base)), pll->active_mask);
13210
13211         I915_STATE_WARN(!(pll->state.crtc_mask & crtc_mask),
13212                         "pll enabled crtcs mismatch (expected 0x%x in 0x%02x)\n",
13213                         crtc_mask, pll->state.crtc_mask);
13214
13215         I915_STATE_WARN(pll->on && memcmp(&pll->state.hw_state,
13216                                           &dpll_hw_state,
13217                                           sizeof(dpll_hw_state)),
13218                         "pll hw state mismatch\n");
13219 }
13220
13221 static void
13222 verify_shared_dpll_state(struct intel_crtc *crtc,
13223                          struct intel_crtc_state *old_crtc_state,
13224                          struct intel_crtc_state *new_crtc_state)
13225 {
13226         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
13227
13228         if (new_crtc_state->shared_dpll)
13229                 verify_single_dpll_state(dev_priv, new_crtc_state->shared_dpll, crtc, new_crtc_state);
13230
13231         if (old_crtc_state->shared_dpll &&
13232             old_crtc_state->shared_dpll != new_crtc_state->shared_dpll) {
13233                 unsigned int crtc_mask = drm_crtc_mask(&crtc->base);
13234                 struct intel_shared_dpll *pll = old_crtc_state->shared_dpll;
13235
13236                 I915_STATE_WARN(pll->active_mask & crtc_mask,
13237                                 "pll active mismatch (didn't expect pipe %c in active mask)\n",
13238                                 pipe_name(drm_crtc_index(&crtc->base)));
13239                 I915_STATE_WARN(pll->state.crtc_mask & crtc_mask,
13240                                 "pll enabled crtcs mismatch (found %x in enabled mask)\n",
13241                                 pipe_name(drm_crtc_index(&crtc->base)));
13242         }
13243 }
13244
13245 static void
13246 intel_modeset_verify_crtc(struct intel_crtc *crtc,
13247                           struct intel_atomic_state *state,
13248                           struct intel_crtc_state *old_crtc_state,
13249                           struct intel_crtc_state *new_crtc_state)
13250 {
13251         if (!needs_modeset(new_crtc_state) && !new_crtc_state->update_pipe)
13252                 return;
13253
13254         verify_wm_state(crtc, new_crtc_state);
13255         verify_connector_state(state, crtc);
13256         verify_crtc_state(crtc, old_crtc_state, new_crtc_state);
13257         verify_shared_dpll_state(crtc, old_crtc_state, new_crtc_state);
13258 }
13259
13260 static void
13261 verify_disabled_dpll_state(struct drm_i915_private *dev_priv)
13262 {
13263         int i;
13264
13265         for (i = 0; i < dev_priv->num_shared_dpll; i++)
13266                 verify_single_dpll_state(dev_priv, &dev_priv->shared_dplls[i], NULL, NULL);
13267 }
13268
13269 static void
13270 intel_modeset_verify_disabled(struct drm_i915_private *dev_priv,
13271                               struct intel_atomic_state *state)
13272 {
13273         verify_encoder_state(dev_priv, state);
13274         verify_connector_state(state, NULL);
13275         verify_disabled_dpll_state(dev_priv);
13276 }
13277
13278 static void update_scanline_offset(const struct intel_crtc_state *crtc_state)
13279 {
13280         struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
13281         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
13282
13283         /*
13284          * The scanline counter increments at the leading edge of hsync.
13285          *
13286          * On most platforms it starts counting from vtotal-1 on the
13287          * first active line. That means the scanline counter value is
13288          * always one less than what we would expect. Ie. just after
13289          * start of vblank, which also occurs at start of hsync (on the
13290          * last active line), the scanline counter will read vblank_start-1.
13291          *
13292          * On gen2 the scanline counter starts counting from 1 instead
13293          * of vtotal-1, so we have to subtract one (or rather add vtotal-1
13294          * to keep the value positive), instead of adding one.
13295          *
13296          * On HSW+ the behaviour of the scanline counter depends on the output
13297          * type. For DP ports it behaves like most other platforms, but on HDMI
13298          * there's an extra 1 line difference. So we need to add two instead of
13299          * one to the value.
13300          *
13301          * On VLV/CHV DSI the scanline counter would appear to increment
13302          * approx. 1/3 of a scanline before start of vblank. Unfortunately
13303          * that means we can't tell whether we're in vblank or not while
13304          * we're on that particular line. We must still set scanline_offset
13305          * to 1 so that the vblank timestamps come out correct when we query
13306          * the scanline counter from within the vblank interrupt handler.
13307          * However if queried just before the start of vblank we'll get an
13308          * answer that's slightly in the future.
13309          */
13310         if (IS_GEN(dev_priv, 2)) {
13311                 const struct drm_display_mode *adjusted_mode = &crtc_state->base.adjusted_mode;
13312                 int vtotal;
13313
13314                 vtotal = adjusted_mode->crtc_vtotal;
13315                 if (adjusted_mode->flags & DRM_MODE_FLAG_INTERLACE)
13316                         vtotal /= 2;
13317
13318                 crtc->scanline_offset = vtotal - 1;
13319         } else if (HAS_DDI(dev_priv) &&
13320                    intel_crtc_has_type(crtc_state, INTEL_OUTPUT_HDMI)) {
13321                 crtc->scanline_offset = 2;
13322         } else
13323                 crtc->scanline_offset = 1;
13324 }
13325
13326 static void intel_modeset_clear_plls(struct intel_atomic_state *state)
13327 {
13328         struct drm_i915_private *dev_priv = to_i915(state->base.dev);
13329         struct intel_crtc_state *new_crtc_state;
13330         struct intel_crtc *crtc;
13331         int i;
13332
13333         if (!dev_priv->display.crtc_compute_clock)
13334                 return;
13335
13336         for_each_new_intel_crtc_in_state(state, crtc, new_crtc_state, i) {
13337                 if (!needs_modeset(new_crtc_state))
13338                         continue;
13339
13340                 intel_release_shared_dplls(state, crtc);
13341         }
13342 }
13343
13344 /*
13345  * This implements the workaround described in the "notes" section of the mode
13346  * set sequence documentation. When going from no pipes or single pipe to
13347  * multiple pipes, and planes are enabled after the pipe, we need to wait at
13348  * least 2 vblanks on the first pipe before enabling planes on the second pipe.
13349  */
13350 static int haswell_mode_set_planes_workaround(struct intel_atomic_state *state)
13351 {
13352         struct intel_crtc_state *crtc_state;
13353         struct intel_crtc *crtc;
13354         struct intel_crtc_state *first_crtc_state = NULL;
13355         struct intel_crtc_state *other_crtc_state = NULL;
13356         enum pipe first_pipe = INVALID_PIPE, enabled_pipe = INVALID_PIPE;
13357         int i;
13358
13359         /* look at all crtc's that are going to be enabled in during modeset */
13360         for_each_new_intel_crtc_in_state(state, crtc, crtc_state, i) {
13361                 if (!crtc_state->base.active ||
13362                     !needs_modeset(crtc_state))
13363                         continue;
13364
13365                 if (first_crtc_state) {
13366                         other_crtc_state = crtc_state;
13367                         break;
13368                 } else {
13369                         first_crtc_state = crtc_state;
13370                         first_pipe = crtc->pipe;
13371                 }
13372         }
13373
13374         /* No workaround needed? */
13375         if (!first_crtc_state)
13376                 return 0;
13377
13378         /* w/a possibly needed, check how many crtc's are already enabled. */
13379         for_each_intel_crtc(state->base.dev, crtc) {
13380                 crtc_state = intel_atomic_get_crtc_state(&state->base, crtc);
13381                 if (IS_ERR(crtc_state))
13382                         return PTR_ERR(crtc_state);
13383
13384                 crtc_state->hsw_workaround_pipe = INVALID_PIPE;
13385
13386                 if (!crtc_state->base.active ||
13387                     needs_modeset(crtc_state))
13388                         continue;
13389
13390                 /* 2 or more enabled crtcs means no need for w/a */
13391                 if (enabled_pipe != INVALID_PIPE)
13392                         return 0;
13393
13394                 enabled_pipe = crtc->pipe;
13395         }
13396
13397         if (enabled_pipe != INVALID_PIPE)
13398                 first_crtc_state->hsw_workaround_pipe = enabled_pipe;
13399         else if (other_crtc_state)
13400                 other_crtc_state->hsw_workaround_pipe = first_pipe;
13401
13402         return 0;
13403 }
13404
13405 static int intel_lock_all_pipes(struct drm_atomic_state *state)
13406 {
13407         struct drm_crtc *crtc;
13408
13409         /* Add all pipes to the state */
13410         for_each_crtc(state->dev, crtc) {
13411                 struct drm_crtc_state *crtc_state;
13412
13413                 crtc_state = drm_atomic_get_crtc_state(state, crtc);
13414                 if (IS_ERR(crtc_state))
13415                         return PTR_ERR(crtc_state);
13416         }
13417
13418         return 0;
13419 }
13420
13421 static int intel_modeset_all_pipes(struct drm_atomic_state *state)
13422 {
13423         struct drm_crtc *crtc;
13424
13425         /*
13426          * Add all pipes to the state, and force
13427          * a modeset on all the active ones.
13428          */
13429         for_each_crtc(state->dev, crtc) {
13430                 struct drm_crtc_state *crtc_state;
13431                 int ret;
13432
13433                 crtc_state = drm_atomic_get_crtc_state(state, crtc);
13434                 if (IS_ERR(crtc_state))
13435                         return PTR_ERR(crtc_state);
13436
13437                 if (!crtc_state->active || needs_modeset(to_intel_crtc_state(crtc_state)))
13438                         continue;
13439
13440                 crtc_state->mode_changed = true;
13441
13442                 ret = drm_atomic_add_affected_connectors(state, crtc);
13443                 if (ret)
13444                         return ret;
13445
13446                 ret = drm_atomic_add_affected_planes(state, crtc);
13447                 if (ret)
13448                         return ret;
13449         }
13450
13451         return 0;
13452 }
13453
13454 static int intel_modeset_checks(struct intel_atomic_state *state)
13455 {
13456         struct drm_i915_private *dev_priv = to_i915(state->base.dev);
13457         struct intel_crtc_state *old_crtc_state, *new_crtc_state;
13458         struct intel_crtc *crtc;
13459         int ret = 0, i;
13460
13461         if (!check_digital_port_conflicts(state)) {
13462                 DRM_DEBUG_KMS("rejecting conflicting digital port configuration\n");
13463                 return -EINVAL;
13464         }
13465
13466         /* keep the current setting */
13467         if (!state->cdclk.force_min_cdclk_changed)
13468                 state->cdclk.force_min_cdclk = dev_priv->cdclk.force_min_cdclk;
13469
13470         state->modeset = true;
13471         state->active_crtcs = dev_priv->active_crtcs;
13472         state->cdclk.logical = dev_priv->cdclk.logical;
13473         state->cdclk.actual = dev_priv->cdclk.actual;
13474         state->cdclk.pipe = INVALID_PIPE;
13475
13476         for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state,
13477                                             new_crtc_state, i) {
13478                 if (new_crtc_state->base.active)
13479                         state->active_crtcs |= 1 << i;
13480                 else
13481                         state->active_crtcs &= ~(1 << i);
13482
13483                 if (old_crtc_state->base.active != new_crtc_state->base.active)
13484                         state->active_pipe_changes |= drm_crtc_mask(&crtc->base);
13485         }
13486
13487         /*
13488          * See if the config requires any additional preparation, e.g.
13489          * to adjust global state with pipes off.  We need to do this
13490          * here so we can get the modeset_pipe updated config for the new
13491          * mode set on this crtc.  For other crtcs we need to use the
13492          * adjusted_mode bits in the crtc directly.
13493          */
13494         if (dev_priv->display.modeset_calc_cdclk) {
13495                 enum pipe pipe;
13496
13497                 ret = dev_priv->display.modeset_calc_cdclk(state);
13498                 if (ret < 0)
13499                         return ret;
13500
13501                 /*
13502                  * Writes to dev_priv->cdclk.logical must protected by
13503                  * holding all the crtc locks, even if we don't end up
13504                  * touching the hardware
13505                  */
13506                 if (intel_cdclk_changed(&dev_priv->cdclk.logical,
13507                                         &state->cdclk.logical)) {
13508                         ret = intel_lock_all_pipes(&state->base);
13509                         if (ret < 0)
13510                                 return ret;
13511                 }
13512
13513                 if (is_power_of_2(state->active_crtcs)) {
13514                         struct intel_crtc *crtc;
13515                         struct intel_crtc_state *crtc_state;
13516
13517                         pipe = ilog2(state->active_crtcs);
13518                         crtc = intel_get_crtc_for_pipe(dev_priv, pipe);
13519                         crtc_state = intel_atomic_get_new_crtc_state(state, crtc);
13520                         if (crtc_state && needs_modeset(crtc_state))
13521                                 pipe = INVALID_PIPE;
13522                 } else {
13523                         pipe = INVALID_PIPE;
13524                 }
13525
13526                 /* All pipes must be switched off while we change the cdclk. */
13527                 if (pipe != INVALID_PIPE &&
13528                     intel_cdclk_needs_cd2x_update(dev_priv,
13529                                                   &dev_priv->cdclk.actual,
13530                                                   &state->cdclk.actual)) {
13531                         ret = intel_lock_all_pipes(&state->base);
13532                         if (ret < 0)
13533                                 return ret;
13534
13535                         state->cdclk.pipe = pipe;
13536                 } else if (intel_cdclk_needs_modeset(&dev_priv->cdclk.actual,
13537                                                      &state->cdclk.actual)) {
13538                         ret = intel_modeset_all_pipes(&state->base);
13539                         if (ret < 0)
13540                                 return ret;
13541
13542                         state->cdclk.pipe = INVALID_PIPE;
13543                 }
13544
13545                 DRM_DEBUG_KMS("New cdclk calculated to be logical %u kHz, actual %u kHz\n",
13546                               state->cdclk.logical.cdclk,
13547                               state->cdclk.actual.cdclk);
13548                 DRM_DEBUG_KMS("New voltage level calculated to be logical %u, actual %u\n",
13549                               state->cdclk.logical.voltage_level,
13550                               state->cdclk.actual.voltage_level);
13551         }
13552
13553         intel_modeset_clear_plls(state);
13554
13555         if (IS_HASWELL(dev_priv))
13556                 return haswell_mode_set_planes_workaround(state);
13557
13558         return 0;
13559 }
13560
13561 /*
13562  * Handle calculation of various watermark data at the end of the atomic check
13563  * phase.  The code here should be run after the per-crtc and per-plane 'check'
13564  * handlers to ensure that all derived state has been updated.
13565  */
13566 static int calc_watermark_data(struct intel_atomic_state *state)
13567 {
13568         struct drm_device *dev = state->base.dev;
13569         struct drm_i915_private *dev_priv = to_i915(dev);
13570
13571         /* Is there platform-specific watermark information to calculate? */
13572         if (dev_priv->display.compute_global_watermarks)
13573                 return dev_priv->display.compute_global_watermarks(state);
13574
13575         return 0;
13576 }
13577
13578 static void intel_crtc_check_fastset(const struct intel_crtc_state *old_crtc_state,
13579                                      struct intel_crtc_state *new_crtc_state)
13580 {
13581         if (!intel_pipe_config_compare(old_crtc_state, new_crtc_state, true))
13582                 return;
13583
13584         new_crtc_state->base.mode_changed = false;
13585         new_crtc_state->update_pipe = true;
13586
13587         /*
13588          * If we're not doing the full modeset we want to
13589          * keep the current M/N values as they may be
13590          * sufficiently different to the computed values
13591          * to cause problems.
13592          *
13593          * FIXME: should really copy more fuzzy state here
13594          */
13595         new_crtc_state->fdi_m_n = old_crtc_state->fdi_m_n;
13596         new_crtc_state->dp_m_n = old_crtc_state->dp_m_n;
13597         new_crtc_state->dp_m2_n2 = old_crtc_state->dp_m2_n2;
13598         new_crtc_state->has_drrs = old_crtc_state->has_drrs;
13599 }
13600
13601 /**
13602  * intel_atomic_check - validate state object
13603  * @dev: drm device
13604  * @_state: state to validate
13605  */
13606 static int intel_atomic_check(struct drm_device *dev,
13607                               struct drm_atomic_state *_state)
13608 {
13609         struct drm_i915_private *dev_priv = to_i915(dev);
13610         struct intel_atomic_state *state = to_intel_atomic_state(_state);
13611         struct intel_crtc_state *old_crtc_state, *new_crtc_state;
13612         struct intel_crtc *crtc;
13613         int ret, i;
13614         bool any_ms = state->cdclk.force_min_cdclk_changed;
13615
13616         /* Catch I915_MODE_FLAG_INHERITED */
13617         for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state,
13618                                             new_crtc_state, i) {
13619                 if (new_crtc_state->base.mode.private_flags !=
13620                     old_crtc_state->base.mode.private_flags)
13621                         new_crtc_state->base.mode_changed = true;
13622         }
13623
13624         ret = drm_atomic_helper_check_modeset(dev, &state->base);
13625         if (ret)
13626                 goto fail;
13627
13628         for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state,
13629                                             new_crtc_state, i) {
13630                 if (!needs_modeset(new_crtc_state))
13631                         continue;
13632
13633                 if (!new_crtc_state->base.enable) {
13634                         any_ms = true;
13635                         continue;
13636                 }
13637
13638                 ret = intel_modeset_pipe_config(new_crtc_state);
13639                 if (ret)
13640                         goto fail;
13641
13642                 intel_crtc_check_fastset(old_crtc_state, new_crtc_state);
13643
13644                 if (needs_modeset(new_crtc_state))
13645                         any_ms = true;
13646         }
13647
13648         ret = drm_dp_mst_atomic_check(&state->base);
13649         if (ret)
13650                 goto fail;
13651
13652         if (any_ms) {
13653                 ret = intel_modeset_checks(state);
13654                 if (ret)
13655                         goto fail;
13656         } else {
13657                 state->cdclk.logical = dev_priv->cdclk.logical;
13658         }
13659
13660         ret = icl_add_linked_planes(state);
13661         if (ret)
13662                 goto fail;
13663
13664         ret = drm_atomic_helper_check_planes(dev, &state->base);
13665         if (ret)
13666                 goto fail;
13667
13668         intel_fbc_choose_crtc(dev_priv, state);
13669         ret = calc_watermark_data(state);
13670         if (ret)
13671                 goto fail;
13672
13673         ret = intel_bw_atomic_check(state);
13674         if (ret)
13675                 goto fail;
13676
13677         for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state,
13678                                             new_crtc_state, i) {
13679                 if (!needs_modeset(new_crtc_state) &&
13680                     !new_crtc_state->update_pipe)
13681                         continue;
13682
13683                 intel_dump_pipe_config(new_crtc_state, state,
13684                                        needs_modeset(new_crtc_state) ?
13685                                        "[modeset]" : "[fastset]");
13686         }
13687
13688         return 0;
13689
13690  fail:
13691         if (ret == -EDEADLK)
13692                 return ret;
13693
13694         /*
13695          * FIXME would probably be nice to know which crtc specifically
13696          * caused the failure, in cases where we can pinpoint it.
13697          */
13698         for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state,
13699                                             new_crtc_state, i)
13700                 intel_dump_pipe_config(new_crtc_state, state, "[failed]");
13701
13702         return ret;
13703 }
13704
13705 static int intel_atomic_prepare_commit(struct drm_device *dev,
13706                                        struct drm_atomic_state *state)
13707 {
13708         return drm_atomic_helper_prepare_planes(dev, state);
13709 }
13710
13711 u32 intel_crtc_get_vblank_counter(struct intel_crtc *crtc)
13712 {
13713         struct drm_device *dev = crtc->base.dev;
13714         struct drm_vblank_crtc *vblank = &dev->vblank[drm_crtc_index(&crtc->base)];
13715
13716         if (!vblank->max_vblank_count)
13717                 return (u32)drm_crtc_accurate_vblank_count(&crtc->base);
13718
13719         return crtc->base.funcs->get_vblank_counter(&crtc->base);
13720 }
13721
13722 static void intel_update_crtc(struct intel_crtc *crtc,
13723                               struct intel_atomic_state *state,
13724                               struct intel_crtc_state *old_crtc_state,
13725                               struct intel_crtc_state *new_crtc_state)
13726 {
13727         struct drm_device *dev = state->base.dev;
13728         struct drm_i915_private *dev_priv = to_i915(dev);
13729         bool modeset = needs_modeset(new_crtc_state);
13730         struct intel_plane_state *new_plane_state =
13731                 intel_atomic_get_new_plane_state(state,
13732                                                  to_intel_plane(crtc->base.primary));
13733
13734         if (modeset) {
13735                 update_scanline_offset(new_crtc_state);
13736                 dev_priv->display.crtc_enable(new_crtc_state, state);
13737
13738                 /* vblanks work again, re-enable pipe CRC. */
13739                 intel_crtc_enable_pipe_crc(crtc);
13740         } else {
13741                 intel_pre_plane_update(old_crtc_state, new_crtc_state);
13742
13743                 if (new_crtc_state->update_pipe)
13744                         intel_encoders_update_pipe(crtc, new_crtc_state, state);
13745         }
13746
13747         if (new_crtc_state->update_pipe && !new_crtc_state->enable_fbc)
13748                 intel_fbc_disable(crtc);
13749         else if (new_plane_state)
13750                 intel_fbc_enable(crtc, new_crtc_state, new_plane_state);
13751
13752         intel_begin_crtc_commit(state, crtc);
13753
13754         if (INTEL_GEN(dev_priv) >= 9)
13755                 skl_update_planes_on_crtc(state, crtc);
13756         else
13757                 i9xx_update_planes_on_crtc(state, crtc);
13758
13759         intel_finish_crtc_commit(state, crtc);
13760 }
13761
13762 static void intel_update_crtcs(struct intel_atomic_state *state)
13763 {
13764         struct intel_crtc *crtc;
13765         struct intel_crtc_state *old_crtc_state, *new_crtc_state;
13766         int i;
13767
13768         for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state, new_crtc_state, i) {
13769                 if (!new_crtc_state->base.active)
13770                         continue;
13771
13772                 intel_update_crtc(crtc, state, old_crtc_state,
13773                                   new_crtc_state);
13774         }
13775 }
13776
13777 static void skl_update_crtcs(struct intel_atomic_state *state)
13778 {
13779         struct drm_i915_private *dev_priv = to_i915(state->base.dev);
13780         struct intel_crtc *crtc;
13781         struct intel_crtc_state *old_crtc_state, *new_crtc_state;
13782         unsigned int updated = 0;
13783         bool progress;
13784         enum pipe pipe;
13785         int i;
13786         u8 hw_enabled_slices = dev_priv->wm.skl_hw.ddb.enabled_slices;
13787         u8 required_slices = state->wm_results.ddb.enabled_slices;
13788         struct skl_ddb_entry entries[I915_MAX_PIPES] = {};
13789
13790         for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state, new_crtc_state, i)
13791                 /* ignore allocations for crtc's that have been turned off. */
13792                 if (new_crtc_state->base.active)
13793                         entries[i] = old_crtc_state->wm.skl.ddb;
13794
13795         /* If 2nd DBuf slice required, enable it here */
13796         if (INTEL_GEN(dev_priv) >= 11 && required_slices > hw_enabled_slices)
13797                 icl_dbuf_slices_update(dev_priv, required_slices);
13798
13799         /*
13800          * Whenever the number of active pipes changes, we need to make sure we
13801          * update the pipes in the right order so that their ddb allocations
13802          * never overlap with eachother inbetween CRTC updates. Otherwise we'll
13803          * cause pipe underruns and other bad stuff.
13804          */
13805         do {
13806                 progress = false;
13807
13808                 for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state, new_crtc_state, i) {
13809                         bool vbl_wait = false;
13810                         unsigned int cmask = drm_crtc_mask(&crtc->base);
13811
13812                         pipe = crtc->pipe;
13813
13814                         if (updated & cmask || !new_crtc_state->base.active)
13815                                 continue;
13816
13817                         if (skl_ddb_allocation_overlaps(&new_crtc_state->wm.skl.ddb,
13818                                                         entries,
13819                                                         INTEL_INFO(dev_priv)->num_pipes, i))
13820                                 continue;
13821
13822                         updated |= cmask;
13823                         entries[i] = new_crtc_state->wm.skl.ddb;
13824
13825                         /*
13826                          * If this is an already active pipe, it's DDB changed,
13827                          * and this isn't the last pipe that needs updating
13828                          * then we need to wait for a vblank to pass for the
13829                          * new ddb allocation to take effect.
13830                          */
13831                         if (!skl_ddb_entry_equal(&new_crtc_state->wm.skl.ddb,
13832                                                  &old_crtc_state->wm.skl.ddb) &&
13833                             !new_crtc_state->base.active_changed &&
13834                             state->wm_results.dirty_pipes != updated)
13835                                 vbl_wait = true;
13836
13837                         intel_update_crtc(crtc, state, old_crtc_state,
13838                                           new_crtc_state);
13839
13840                         if (vbl_wait)
13841                                 intel_wait_for_vblank(dev_priv, pipe);
13842
13843                         progress = true;
13844                 }
13845         } while (progress);
13846
13847         /* If 2nd DBuf slice is no more required disable it */
13848         if (INTEL_GEN(dev_priv) >= 11 && required_slices < hw_enabled_slices)
13849                 icl_dbuf_slices_update(dev_priv, required_slices);
13850 }
13851
13852 static void intel_atomic_helper_free_state(struct drm_i915_private *dev_priv)
13853 {
13854         struct intel_atomic_state *state, *next;
13855         struct llist_node *freed;
13856
13857         freed = llist_del_all(&dev_priv->atomic_helper.free_list);
13858         llist_for_each_entry_safe(state, next, freed, freed)
13859                 drm_atomic_state_put(&state->base);
13860 }
13861
13862 static void intel_atomic_helper_free_state_worker(struct work_struct *work)
13863 {
13864         struct drm_i915_private *dev_priv =
13865                 container_of(work, typeof(*dev_priv), atomic_helper.free_work);
13866
13867         intel_atomic_helper_free_state(dev_priv);
13868 }
13869
13870 static void intel_atomic_commit_fence_wait(struct intel_atomic_state *intel_state)
13871 {
13872         struct wait_queue_entry wait_fence, wait_reset;
13873         struct drm_i915_private *dev_priv = to_i915(intel_state->base.dev);
13874
13875         init_wait_entry(&wait_fence, 0);
13876         init_wait_entry(&wait_reset, 0);
13877         for (;;) {
13878                 prepare_to_wait(&intel_state->commit_ready.wait,
13879                                 &wait_fence, TASK_UNINTERRUPTIBLE);
13880                 prepare_to_wait(&dev_priv->gpu_error.wait_queue,
13881                                 &wait_reset, TASK_UNINTERRUPTIBLE);
13882
13883
13884                 if (i915_sw_fence_done(&intel_state->commit_ready)
13885                     || test_bit(I915_RESET_MODESET, &dev_priv->gpu_error.flags))
13886                         break;
13887
13888                 schedule();
13889         }
13890         finish_wait(&intel_state->commit_ready.wait, &wait_fence);
13891         finish_wait(&dev_priv->gpu_error.wait_queue, &wait_reset);
13892 }
13893
13894 static void intel_atomic_cleanup_work(struct work_struct *work)
13895 {
13896         struct drm_atomic_state *state =
13897                 container_of(work, struct drm_atomic_state, commit_work);
13898         struct drm_i915_private *i915 = to_i915(state->dev);
13899
13900         drm_atomic_helper_cleanup_planes(&i915->drm, state);
13901         drm_atomic_helper_commit_cleanup_done(state);
13902         drm_atomic_state_put(state);
13903
13904         intel_atomic_helper_free_state(i915);
13905 }
13906
13907 static void intel_atomic_commit_tail(struct intel_atomic_state *state)
13908 {
13909         struct drm_device *dev = state->base.dev;
13910         struct drm_i915_private *dev_priv = to_i915(dev);
13911         struct intel_crtc_state *new_crtc_state, *old_crtc_state;
13912         struct intel_crtc *crtc;
13913         u64 put_domains[I915_MAX_PIPES] = {};
13914         intel_wakeref_t wakeref = 0;
13915         int i;
13916
13917         intel_atomic_commit_fence_wait(state);
13918
13919         drm_atomic_helper_wait_for_dependencies(&state->base);
13920
13921         if (state->modeset)
13922                 wakeref = intel_display_power_get(dev_priv, POWER_DOMAIN_MODESET);
13923
13924         for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state, new_crtc_state, i) {
13925                 if (needs_modeset(new_crtc_state) ||
13926                     new_crtc_state->update_pipe) {
13927
13928                         put_domains[crtc->pipe] =
13929                                 modeset_get_crtc_power_domains(crtc,
13930                                         new_crtc_state);
13931                 }
13932
13933                 if (!needs_modeset(new_crtc_state))
13934                         continue;
13935
13936                 intel_pre_plane_update(old_crtc_state, new_crtc_state);
13937
13938                 if (old_crtc_state->base.active) {
13939                         intel_crtc_disable_planes(state, crtc);
13940
13941                         /*
13942                          * We need to disable pipe CRC before disabling the pipe,
13943                          * or we race against vblank off.
13944                          */
13945                         intel_crtc_disable_pipe_crc(crtc);
13946
13947                         dev_priv->display.crtc_disable(old_crtc_state, state);
13948                         crtc->active = false;
13949                         intel_fbc_disable(crtc);
13950                         intel_disable_shared_dpll(old_crtc_state);
13951
13952                         /*
13953                          * Underruns don't always raise
13954                          * interrupts, so check manually.
13955                          */
13956                         intel_check_cpu_fifo_underruns(dev_priv);
13957                         intel_check_pch_fifo_underruns(dev_priv);
13958
13959                         /* FIXME unify this for all platforms */
13960                         if (!new_crtc_state->base.active &&
13961                             !HAS_GMCH(dev_priv) &&
13962                             dev_priv->display.initial_watermarks)
13963                                 dev_priv->display.initial_watermarks(state,
13964                                                                      new_crtc_state);
13965                 }
13966         }
13967
13968         /* FIXME: Eventually get rid of our crtc->config pointer */
13969         for_each_new_intel_crtc_in_state(state, crtc, new_crtc_state, i)
13970                 crtc->config = new_crtc_state;
13971
13972         if (state->modeset) {
13973                 drm_atomic_helper_update_legacy_modeset_state(dev, &state->base);
13974
13975                 intel_set_cdclk_pre_plane_update(dev_priv,
13976                                                  &state->cdclk.actual,
13977                                                  &dev_priv->cdclk.actual,
13978                                                  state->cdclk.pipe);
13979
13980                 /*
13981                  * SKL workaround: bspec recommends we disable the SAGV when we
13982                  * have more then one pipe enabled
13983                  */
13984                 if (!intel_can_enable_sagv(state))
13985                         intel_disable_sagv(dev_priv);
13986
13987                 intel_modeset_verify_disabled(dev_priv, state);
13988         }
13989
13990         /* Complete the events for pipes that have now been disabled */
13991         for_each_new_intel_crtc_in_state(state, crtc, new_crtc_state, i) {
13992                 bool modeset = needs_modeset(new_crtc_state);
13993
13994                 /* Complete events for now disable pipes here. */
13995                 if (modeset && !new_crtc_state->base.active && new_crtc_state->base.event) {
13996                         spin_lock_irq(&dev->event_lock);
13997                         drm_crtc_send_vblank_event(&crtc->base, new_crtc_state->base.event);
13998                         spin_unlock_irq(&dev->event_lock);
13999
14000                         new_crtc_state->base.event = NULL;
14001                 }
14002         }
14003
14004         if (state->modeset)
14005                 intel_encoders_update_prepare(state);
14006
14007         /* Now enable the clocks, plane, pipe, and connectors that we set up. */
14008         dev_priv->display.update_crtcs(state);
14009
14010         if (state->modeset) {
14011                 intel_encoders_update_complete(state);
14012
14013                 intel_set_cdclk_post_plane_update(dev_priv,
14014                                                   &state->cdclk.actual,
14015                                                   &dev_priv->cdclk.actual,
14016                                                   state->cdclk.pipe);
14017         }
14018
14019         /* FIXME: We should call drm_atomic_helper_commit_hw_done() here
14020          * already, but still need the state for the delayed optimization. To
14021          * fix this:
14022          * - wrap the optimization/post_plane_update stuff into a per-crtc work.
14023          * - schedule that vblank worker _before_ calling hw_done
14024          * - at the start of commit_tail, cancel it _synchrously
14025          * - switch over to the vblank wait helper in the core after that since
14026          *   we don't need out special handling any more.
14027          */
14028         drm_atomic_helper_wait_for_flip_done(dev, &state->base);
14029
14030         for_each_new_intel_crtc_in_state(state, crtc, new_crtc_state, i) {
14031                 if (new_crtc_state->base.active &&
14032                     !needs_modeset(new_crtc_state) &&
14033                     (new_crtc_state->base.color_mgmt_changed ||
14034                      new_crtc_state->update_pipe))
14035                         intel_color_load_luts(new_crtc_state);
14036         }
14037
14038         /*
14039          * Now that the vblank has passed, we can go ahead and program the
14040          * optimal watermarks on platforms that need two-step watermark
14041          * programming.
14042          *
14043          * TODO: Move this (and other cleanup) to an async worker eventually.
14044          */
14045         for_each_new_intel_crtc_in_state(state, crtc, new_crtc_state, i) {
14046                 if (dev_priv->display.optimize_watermarks)
14047                         dev_priv->display.optimize_watermarks(state,
14048                                                               new_crtc_state);
14049         }
14050
14051         for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state, new_crtc_state, i) {
14052                 intel_post_plane_update(old_crtc_state);
14053
14054                 if (put_domains[i])
14055                         modeset_put_power_domains(dev_priv, put_domains[i]);
14056
14057                 intel_modeset_verify_crtc(crtc, state, old_crtc_state, new_crtc_state);
14058         }
14059
14060         if (state->modeset)
14061                 intel_verify_planes(state);
14062
14063         if (state->modeset && intel_can_enable_sagv(state))
14064                 intel_enable_sagv(dev_priv);
14065
14066         drm_atomic_helper_commit_hw_done(&state->base);
14067
14068         if (state->modeset) {
14069                 /* As one of the primary mmio accessors, KMS has a high
14070                  * likelihood of triggering bugs in unclaimed access. After we
14071                  * finish modesetting, see if an error has been flagged, and if
14072                  * so enable debugging for the next modeset - and hope we catch
14073                  * the culprit.
14074                  */
14075                 intel_uncore_arm_unclaimed_mmio_detection(&dev_priv->uncore);
14076                 intel_display_power_put(dev_priv, POWER_DOMAIN_MODESET, wakeref);
14077         }
14078         intel_runtime_pm_put(&dev_priv->runtime_pm, state->wakeref);
14079
14080         /*
14081          * Defer the cleanup of the old state to a separate worker to not
14082          * impede the current task (userspace for blocking modesets) that
14083          * are executed inline. For out-of-line asynchronous modesets/flips,
14084          * deferring to a new worker seems overkill, but we would place a
14085          * schedule point (cond_resched()) here anyway to keep latencies
14086          * down.
14087          */
14088         INIT_WORK(&state->base.commit_work, intel_atomic_cleanup_work);
14089         queue_work(system_highpri_wq, &state->base.commit_work);
14090 }
14091
14092 static void intel_atomic_commit_work(struct work_struct *work)
14093 {
14094         struct intel_atomic_state *state =
14095                 container_of(work, struct intel_atomic_state, base.commit_work);
14096
14097         intel_atomic_commit_tail(state);
14098 }
14099
14100 static int __i915_sw_fence_call
14101 intel_atomic_commit_ready(struct i915_sw_fence *fence,
14102                           enum i915_sw_fence_notify notify)
14103 {
14104         struct intel_atomic_state *state =
14105                 container_of(fence, struct intel_atomic_state, commit_ready);
14106
14107         switch (notify) {
14108         case FENCE_COMPLETE:
14109                 /* we do blocking waits in the worker, nothing to do here */
14110                 break;
14111         case FENCE_FREE:
14112                 {
14113                         struct intel_atomic_helper *helper =
14114                                 &to_i915(state->base.dev)->atomic_helper;
14115
14116                         if (llist_add(&state->freed, &helper->free_list))
14117                                 schedule_work(&helper->free_work);
14118                         break;
14119                 }
14120         }
14121
14122         return NOTIFY_DONE;
14123 }
14124
14125 static void intel_atomic_track_fbs(struct drm_atomic_state *state)
14126 {
14127         struct drm_plane_state *old_plane_state, *new_plane_state;
14128         struct drm_plane *plane;
14129         int i;
14130
14131         for_each_oldnew_plane_in_state(state, plane, old_plane_state, new_plane_state, i)
14132                 i915_gem_track_fb(intel_fb_obj(old_plane_state->fb),
14133                                   intel_fb_obj(new_plane_state->fb),
14134                                   to_intel_plane(plane)->frontbuffer_bit);
14135 }
14136
14137 /**
14138  * intel_atomic_commit - commit validated state object
14139  * @dev: DRM device
14140  * @state: the top-level driver state object
14141  * @nonblock: nonblocking commit
14142  *
14143  * This function commits a top-level state object that has been validated
14144  * with drm_atomic_helper_check().
14145  *
14146  * RETURNS
14147  * Zero for success or -errno.
14148  */
14149 static int intel_atomic_commit(struct drm_device *dev,
14150                                struct drm_atomic_state *state,
14151                                bool nonblock)
14152 {
14153         struct intel_atomic_state *intel_state = to_intel_atomic_state(state);
14154         struct drm_i915_private *dev_priv = to_i915(dev);
14155         int ret = 0;
14156
14157         intel_state->wakeref = intel_runtime_pm_get(&dev_priv->runtime_pm);
14158
14159         drm_atomic_state_get(state);
14160         i915_sw_fence_init(&intel_state->commit_ready,
14161                            intel_atomic_commit_ready);
14162
14163         /*
14164          * The intel_legacy_cursor_update() fast path takes care
14165          * of avoiding the vblank waits for simple cursor
14166          * movement and flips. For cursor on/off and size changes,
14167          * we want to perform the vblank waits so that watermark
14168          * updates happen during the correct frames. Gen9+ have
14169          * double buffered watermarks and so shouldn't need this.
14170          *
14171          * Unset state->legacy_cursor_update before the call to
14172          * drm_atomic_helper_setup_commit() because otherwise
14173          * drm_atomic_helper_wait_for_flip_done() is a noop and
14174          * we get FIFO underruns because we didn't wait
14175          * for vblank.
14176          *
14177          * FIXME doing watermarks and fb cleanup from a vblank worker
14178          * (assuming we had any) would solve these problems.
14179          */
14180         if (INTEL_GEN(dev_priv) < 9 && state->legacy_cursor_update) {
14181                 struct intel_crtc_state *new_crtc_state;
14182                 struct intel_crtc *crtc;
14183                 int i;
14184
14185                 for_each_new_intel_crtc_in_state(intel_state, crtc, new_crtc_state, i)
14186                         if (new_crtc_state->wm.need_postvbl_update ||
14187                             new_crtc_state->update_wm_post)
14188                                 state->legacy_cursor_update = false;
14189         }
14190
14191         ret = intel_atomic_prepare_commit(dev, state);
14192         if (ret) {
14193                 DRM_DEBUG_ATOMIC("Preparing state failed with %i\n", ret);
14194                 i915_sw_fence_commit(&intel_state->commit_ready);
14195                 intel_runtime_pm_put(&dev_priv->runtime_pm, intel_state->wakeref);
14196                 return ret;
14197         }
14198
14199         ret = drm_atomic_helper_setup_commit(state, nonblock);
14200         if (!ret)
14201                 ret = drm_atomic_helper_swap_state(state, true);
14202
14203         if (ret) {
14204                 i915_sw_fence_commit(&intel_state->commit_ready);
14205
14206                 drm_atomic_helper_cleanup_planes(dev, state);
14207                 intel_runtime_pm_put(&dev_priv->runtime_pm, intel_state->wakeref);
14208                 return ret;
14209         }
14210         dev_priv->wm.distrust_bios_wm = false;
14211         intel_shared_dpll_swap_state(state);
14212         intel_atomic_track_fbs(state);
14213
14214         if (intel_state->modeset) {
14215                 memcpy(dev_priv->min_cdclk, intel_state->min_cdclk,
14216                        sizeof(intel_state->min_cdclk));
14217                 memcpy(dev_priv->min_voltage_level,
14218                        intel_state->min_voltage_level,
14219                        sizeof(intel_state->min_voltage_level));
14220                 dev_priv->active_crtcs = intel_state->active_crtcs;
14221                 dev_priv->cdclk.force_min_cdclk =
14222                         intel_state->cdclk.force_min_cdclk;
14223
14224                 intel_cdclk_swap_state(intel_state);
14225         }
14226
14227         drm_atomic_state_get(state);
14228         INIT_WORK(&state->commit_work, intel_atomic_commit_work);
14229
14230         i915_sw_fence_commit(&intel_state->commit_ready);
14231         if (nonblock && intel_state->modeset) {
14232                 queue_work(dev_priv->modeset_wq, &state->commit_work);
14233         } else if (nonblock) {
14234                 queue_work(system_unbound_wq, &state->commit_work);
14235         } else {
14236                 if (intel_state->modeset)
14237                         flush_workqueue(dev_priv->modeset_wq);
14238                 intel_atomic_commit_tail(intel_state);
14239         }
14240
14241         return 0;
14242 }
14243
14244 struct wait_rps_boost {
14245         struct wait_queue_entry wait;
14246
14247         struct drm_crtc *crtc;
14248         struct i915_request *request;
14249 };
14250
14251 static int do_rps_boost(struct wait_queue_entry *_wait,
14252                         unsigned mode, int sync, void *key)
14253 {
14254         struct wait_rps_boost *wait = container_of(_wait, typeof(*wait), wait);
14255         struct i915_request *rq = wait->request;
14256
14257         /*
14258          * If we missed the vblank, but the request is already running it
14259          * is reasonable to assume that it will complete before the next
14260          * vblank without our intervention, so leave RPS alone.
14261          */
14262         if (!i915_request_started(rq))
14263                 gen6_rps_boost(rq);
14264         i915_request_put(rq);
14265
14266         drm_crtc_vblank_put(wait->crtc);
14267
14268         list_del(&wait->wait.entry);
14269         kfree(wait);
14270         return 1;
14271 }
14272
14273 static void add_rps_boost_after_vblank(struct drm_crtc *crtc,
14274                                        struct dma_fence *fence)
14275 {
14276         struct wait_rps_boost *wait;
14277
14278         if (!dma_fence_is_i915(fence))
14279                 return;
14280
14281         if (INTEL_GEN(to_i915(crtc->dev)) < 6)
14282                 return;
14283
14284         if (drm_crtc_vblank_get(crtc))
14285                 return;
14286
14287         wait = kmalloc(sizeof(*wait), GFP_KERNEL);
14288         if (!wait) {
14289                 drm_crtc_vblank_put(crtc);
14290                 return;
14291         }
14292
14293         wait->request = to_request(dma_fence_get(fence));
14294         wait->crtc = crtc;
14295
14296         wait->wait.func = do_rps_boost;
14297         wait->wait.flags = 0;
14298
14299         add_wait_queue(drm_crtc_vblank_waitqueue(crtc), &wait->wait);
14300 }
14301
14302 static int intel_plane_pin_fb(struct intel_plane_state *plane_state)
14303 {
14304         struct intel_plane *plane = to_intel_plane(plane_state->base.plane);
14305         struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
14306         struct drm_framebuffer *fb = plane_state->base.fb;
14307         struct i915_vma *vma;
14308
14309         if (plane->id == PLANE_CURSOR &&
14310             INTEL_INFO(dev_priv)->display.cursor_needs_physical) {
14311                 struct drm_i915_gem_object *obj = intel_fb_obj(fb);
14312                 const int align = intel_cursor_alignment(dev_priv);
14313                 int err;
14314
14315                 err = i915_gem_object_attach_phys(obj, align);
14316                 if (err)
14317                         return err;
14318         }
14319
14320         vma = intel_pin_and_fence_fb_obj(fb,
14321                                          &plane_state->view,
14322                                          intel_plane_uses_fence(plane_state),
14323                                          &plane_state->flags);
14324         if (IS_ERR(vma))
14325                 return PTR_ERR(vma);
14326
14327         plane_state->vma = vma;
14328
14329         return 0;
14330 }
14331
14332 static void intel_plane_unpin_fb(struct intel_plane_state *old_plane_state)
14333 {
14334         struct i915_vma *vma;
14335
14336         vma = fetch_and_zero(&old_plane_state->vma);
14337         if (vma)
14338                 intel_unpin_fb_vma(vma, old_plane_state->flags);
14339 }
14340
14341 static void fb_obj_bump_render_priority(struct drm_i915_gem_object *obj)
14342 {
14343         struct i915_sched_attr attr = {
14344                 .priority = I915_PRIORITY_DISPLAY,
14345         };
14346
14347         i915_gem_object_wait_priority(obj, 0, &attr);
14348 }
14349
14350 /**
14351  * intel_prepare_plane_fb - Prepare fb for usage on plane
14352  * @plane: drm plane to prepare for
14353  * @new_state: the plane state being prepared
14354  *
14355  * Prepares a framebuffer for usage on a display plane.  Generally this
14356  * involves pinning the underlying object and updating the frontbuffer tracking
14357  * bits.  Some older platforms need special physical address handling for
14358  * cursor planes.
14359  *
14360  * Must be called with struct_mutex held.
14361  *
14362  * Returns 0 on success, negative error code on failure.
14363  */
14364 int
14365 intel_prepare_plane_fb(struct drm_plane *plane,
14366                        struct drm_plane_state *new_state)
14367 {
14368         struct intel_atomic_state *intel_state =
14369                 to_intel_atomic_state(new_state->state);
14370         struct drm_i915_private *dev_priv = to_i915(plane->dev);
14371         struct drm_framebuffer *fb = new_state->fb;
14372         struct drm_i915_gem_object *obj = intel_fb_obj(fb);
14373         struct drm_i915_gem_object *old_obj = intel_fb_obj(plane->state->fb);
14374         int ret;
14375
14376         if (old_obj) {
14377                 struct intel_crtc_state *crtc_state =
14378                         intel_atomic_get_new_crtc_state(intel_state,
14379                                                         to_intel_crtc(plane->state->crtc));
14380
14381                 /* Big Hammer, we also need to ensure that any pending
14382                  * MI_WAIT_FOR_EVENT inside a user batch buffer on the
14383                  * current scanout is retired before unpinning the old
14384                  * framebuffer. Note that we rely on userspace rendering
14385                  * into the buffer attached to the pipe they are waiting
14386                  * on. If not, userspace generates a GPU hang with IPEHR
14387                  * point to the MI_WAIT_FOR_EVENT.
14388                  *
14389                  * This should only fail upon a hung GPU, in which case we
14390                  * can safely continue.
14391                  */
14392                 if (needs_modeset(crtc_state)) {
14393                         ret = i915_sw_fence_await_reservation(&intel_state->commit_ready,
14394                                                               old_obj->base.resv, NULL,
14395                                                               false, 0,
14396                                                               GFP_KERNEL);
14397                         if (ret < 0)
14398                                 return ret;
14399                 }
14400         }
14401
14402         if (new_state->fence) { /* explicit fencing */
14403                 ret = i915_sw_fence_await_dma_fence(&intel_state->commit_ready,
14404                                                     new_state->fence,
14405                                                     I915_FENCE_TIMEOUT,
14406                                                     GFP_KERNEL);
14407                 if (ret < 0)
14408                         return ret;
14409         }
14410
14411         if (!obj)
14412                 return 0;
14413
14414         ret = i915_gem_object_pin_pages(obj);
14415         if (ret)
14416                 return ret;
14417
14418         ret = mutex_lock_interruptible(&dev_priv->drm.struct_mutex);
14419         if (ret) {
14420                 i915_gem_object_unpin_pages(obj);
14421                 return ret;
14422         }
14423
14424         ret = intel_plane_pin_fb(to_intel_plane_state(new_state));
14425
14426         mutex_unlock(&dev_priv->drm.struct_mutex);
14427         i915_gem_object_unpin_pages(obj);
14428         if (ret)
14429                 return ret;
14430
14431         fb_obj_bump_render_priority(obj);
14432         intel_fb_obj_flush(obj, ORIGIN_DIRTYFB);
14433
14434         if (!new_state->fence) { /* implicit fencing */
14435                 struct dma_fence *fence;
14436
14437                 ret = i915_sw_fence_await_reservation(&intel_state->commit_ready,
14438                                                       obj->base.resv, NULL,
14439                                                       false, I915_FENCE_TIMEOUT,
14440                                                       GFP_KERNEL);
14441                 if (ret < 0)
14442                         return ret;
14443
14444                 fence = reservation_object_get_excl_rcu(obj->base.resv);
14445                 if (fence) {
14446                         add_rps_boost_after_vblank(new_state->crtc, fence);
14447                         dma_fence_put(fence);
14448                 }
14449         } else {
14450                 add_rps_boost_after_vblank(new_state->crtc, new_state->fence);
14451         }
14452
14453         /*
14454          * We declare pageflips to be interactive and so merit a small bias
14455          * towards upclocking to deliver the frame on time. By only changing
14456          * the RPS thresholds to sample more regularly and aim for higher
14457          * clocks we can hopefully deliver low power workloads (like kodi)
14458          * that are not quite steady state without resorting to forcing
14459          * maximum clocks following a vblank miss (see do_rps_boost()).
14460          */
14461         if (!intel_state->rps_interactive) {
14462                 intel_rps_mark_interactive(dev_priv, true);
14463                 intel_state->rps_interactive = true;
14464         }
14465
14466         return 0;
14467 }
14468
14469 /**
14470  * intel_cleanup_plane_fb - Cleans up an fb after plane use
14471  * @plane: drm plane to clean up for
14472  * @old_state: the state from the previous modeset
14473  *
14474  * Cleans up a framebuffer that has just been removed from a plane.
14475  *
14476  * Must be called with struct_mutex held.
14477  */
14478 void
14479 intel_cleanup_plane_fb(struct drm_plane *plane,
14480                        struct drm_plane_state *old_state)
14481 {
14482         struct intel_atomic_state *intel_state =
14483                 to_intel_atomic_state(old_state->state);
14484         struct drm_i915_private *dev_priv = to_i915(plane->dev);
14485
14486         if (intel_state->rps_interactive) {
14487                 intel_rps_mark_interactive(dev_priv, false);
14488                 intel_state->rps_interactive = false;
14489         }
14490
14491         /* Should only be called after a successful intel_prepare_plane_fb()! */
14492         mutex_lock(&dev_priv->drm.struct_mutex);
14493         intel_plane_unpin_fb(to_intel_plane_state(old_state));
14494         mutex_unlock(&dev_priv->drm.struct_mutex);
14495 }
14496
14497 int
14498 skl_max_scale(const struct intel_crtc_state *crtc_state,
14499               u32 pixel_format)
14500 {
14501         struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
14502         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
14503         int max_scale, mult;
14504         int crtc_clock, max_dotclk, tmpclk1, tmpclk2;
14505
14506         if (!crtc_state->base.enable)
14507                 return DRM_PLANE_HELPER_NO_SCALING;
14508
14509         crtc_clock = crtc_state->base.adjusted_mode.crtc_clock;
14510         max_dotclk = to_intel_atomic_state(crtc_state->base.state)->cdclk.logical.cdclk;
14511
14512         if (IS_GEMINILAKE(dev_priv) || INTEL_GEN(dev_priv) >= 10)
14513                 max_dotclk *= 2;
14514
14515         if (WARN_ON_ONCE(!crtc_clock || max_dotclk < crtc_clock))
14516                 return DRM_PLANE_HELPER_NO_SCALING;
14517
14518         /*
14519          * skl max scale is lower of:
14520          *    close to 3 but not 3, -1 is for that purpose
14521          *            or
14522          *    cdclk/crtc_clock
14523          */
14524         mult = is_planar_yuv_format(pixel_format) ? 2 : 3;
14525         tmpclk1 = (1 << 16) * mult - 1;
14526         tmpclk2 = (1 << 8) * ((max_dotclk << 8) / crtc_clock);
14527         max_scale = min(tmpclk1, tmpclk2);
14528
14529         return max_scale;
14530 }
14531
14532 static void intel_begin_crtc_commit(struct intel_atomic_state *state,
14533                                     struct intel_crtc *crtc)
14534 {
14535         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
14536         struct intel_crtc_state *old_crtc_state =
14537                 intel_atomic_get_old_crtc_state(state, crtc);
14538         struct intel_crtc_state *new_crtc_state =
14539                 intel_atomic_get_new_crtc_state(state, crtc);
14540         bool modeset = needs_modeset(new_crtc_state);
14541
14542         /* Perform vblank evasion around commit operation */
14543         intel_pipe_update_start(new_crtc_state);
14544
14545         if (modeset)
14546                 goto out;
14547
14548         if (new_crtc_state->base.color_mgmt_changed ||
14549             new_crtc_state->update_pipe)
14550                 intel_color_commit(new_crtc_state);
14551
14552         if (new_crtc_state->update_pipe)
14553                 intel_update_pipe_config(old_crtc_state, new_crtc_state);
14554         else if (INTEL_GEN(dev_priv) >= 9)
14555                 skl_detach_scalers(new_crtc_state);
14556
14557         if (INTEL_GEN(dev_priv) >= 9 || IS_BROADWELL(dev_priv))
14558                 bdw_set_pipemisc(new_crtc_state);
14559
14560 out:
14561         if (dev_priv->display.atomic_update_watermarks)
14562                 dev_priv->display.atomic_update_watermarks(state,
14563                                                            new_crtc_state);
14564 }
14565
14566 void intel_crtc_arm_fifo_underrun(struct intel_crtc *crtc,
14567                                   struct intel_crtc_state *crtc_state)
14568 {
14569         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
14570
14571         if (!IS_GEN(dev_priv, 2))
14572                 intel_set_cpu_fifo_underrun_reporting(dev_priv, crtc->pipe, true);
14573
14574         if (crtc_state->has_pch_encoder) {
14575                 enum pipe pch_transcoder =
14576                         intel_crtc_pch_transcoder(crtc);
14577
14578                 intel_set_pch_fifo_underrun_reporting(dev_priv, pch_transcoder, true);
14579         }
14580 }
14581
14582 static void intel_finish_crtc_commit(struct intel_atomic_state *state,
14583                                      struct intel_crtc *crtc)
14584 {
14585         struct intel_crtc_state *old_crtc_state =
14586                 intel_atomic_get_old_crtc_state(state, crtc);
14587         struct intel_crtc_state *new_crtc_state =
14588                 intel_atomic_get_new_crtc_state(state, crtc);
14589
14590         intel_pipe_update_end(new_crtc_state);
14591
14592         if (new_crtc_state->update_pipe &&
14593             !needs_modeset(new_crtc_state) &&
14594             old_crtc_state->base.mode.private_flags & I915_MODE_FLAG_INHERITED)
14595                 intel_crtc_arm_fifo_underrun(crtc, new_crtc_state);
14596 }
14597
14598 /**
14599  * intel_plane_destroy - destroy a plane
14600  * @plane: plane to destroy
14601  *
14602  * Common destruction function for all types of planes (primary, cursor,
14603  * sprite).
14604  */
14605 void intel_plane_destroy(struct drm_plane *plane)
14606 {
14607         drm_plane_cleanup(plane);
14608         kfree(to_intel_plane(plane));
14609 }
14610
14611 static bool i8xx_plane_format_mod_supported(struct drm_plane *_plane,
14612                                             u32 format, u64 modifier)
14613 {
14614         switch (modifier) {
14615         case DRM_FORMAT_MOD_LINEAR:
14616         case I915_FORMAT_MOD_X_TILED:
14617                 break;
14618         default:
14619                 return false;
14620         }
14621
14622         switch (format) {
14623         case DRM_FORMAT_C8:
14624         case DRM_FORMAT_RGB565:
14625         case DRM_FORMAT_XRGB1555:
14626         case DRM_FORMAT_XRGB8888:
14627                 return modifier == DRM_FORMAT_MOD_LINEAR ||
14628                         modifier == I915_FORMAT_MOD_X_TILED;
14629         default:
14630                 return false;
14631         }
14632 }
14633
14634 static bool i965_plane_format_mod_supported(struct drm_plane *_plane,
14635                                             u32 format, u64 modifier)
14636 {
14637         switch (modifier) {
14638         case DRM_FORMAT_MOD_LINEAR:
14639         case I915_FORMAT_MOD_X_TILED:
14640                 break;
14641         default:
14642                 return false;
14643         }
14644
14645         switch (format) {
14646         case DRM_FORMAT_C8:
14647         case DRM_FORMAT_RGB565:
14648         case DRM_FORMAT_XRGB8888:
14649         case DRM_FORMAT_XBGR8888:
14650         case DRM_FORMAT_XRGB2101010:
14651         case DRM_FORMAT_XBGR2101010:
14652                 return modifier == DRM_FORMAT_MOD_LINEAR ||
14653                         modifier == I915_FORMAT_MOD_X_TILED;
14654         default:
14655                 return false;
14656         }
14657 }
14658
14659 static bool intel_cursor_format_mod_supported(struct drm_plane *_plane,
14660                                               u32 format, u64 modifier)
14661 {
14662         return modifier == DRM_FORMAT_MOD_LINEAR &&
14663                 format == DRM_FORMAT_ARGB8888;
14664 }
14665
14666 static const struct drm_plane_funcs i965_plane_funcs = {
14667         .update_plane = drm_atomic_helper_update_plane,
14668         .disable_plane = drm_atomic_helper_disable_plane,
14669         .destroy = intel_plane_destroy,
14670         .atomic_duplicate_state = intel_plane_duplicate_state,
14671         .atomic_destroy_state = intel_plane_destroy_state,
14672         .format_mod_supported = i965_plane_format_mod_supported,
14673 };
14674
14675 static const struct drm_plane_funcs i8xx_plane_funcs = {
14676         .update_plane = drm_atomic_helper_update_plane,
14677         .disable_plane = drm_atomic_helper_disable_plane,
14678         .destroy = intel_plane_destroy,
14679         .atomic_duplicate_state = intel_plane_duplicate_state,
14680         .atomic_destroy_state = intel_plane_destroy_state,
14681         .format_mod_supported = i8xx_plane_format_mod_supported,
14682 };
14683
14684 static int
14685 intel_legacy_cursor_update(struct drm_plane *plane,
14686                            struct drm_crtc *crtc,
14687                            struct drm_framebuffer *fb,
14688                            int crtc_x, int crtc_y,
14689                            unsigned int crtc_w, unsigned int crtc_h,
14690                            u32 src_x, u32 src_y,
14691                            u32 src_w, u32 src_h,
14692                            struct drm_modeset_acquire_ctx *ctx)
14693 {
14694         struct drm_i915_private *dev_priv = to_i915(crtc->dev);
14695         int ret;
14696         struct drm_plane_state *old_plane_state, *new_plane_state;
14697         struct intel_plane *intel_plane = to_intel_plane(plane);
14698         struct drm_framebuffer *old_fb;
14699         struct intel_crtc_state *crtc_state =
14700                 to_intel_crtc_state(crtc->state);
14701         struct intel_crtc_state *new_crtc_state;
14702
14703         /*
14704          * When crtc is inactive or there is a modeset pending,
14705          * wait for it to complete in the slowpath
14706          */
14707         if (!crtc_state->base.active || needs_modeset(crtc_state) ||
14708             crtc_state->update_pipe)
14709                 goto slow;
14710
14711         old_plane_state = plane->state;
14712         /*
14713          * Don't do an async update if there is an outstanding commit modifying
14714          * the plane.  This prevents our async update's changes from getting
14715          * overridden by a previous synchronous update's state.
14716          */
14717         if (old_plane_state->commit &&
14718             !try_wait_for_completion(&old_plane_state->commit->hw_done))
14719                 goto slow;
14720
14721         /*
14722          * If any parameters change that may affect watermarks,
14723          * take the slowpath. Only changing fb or position should be
14724          * in the fastpath.
14725          */
14726         if (old_plane_state->crtc != crtc ||
14727             old_plane_state->src_w != src_w ||
14728             old_plane_state->src_h != src_h ||
14729             old_plane_state->crtc_w != crtc_w ||
14730             old_plane_state->crtc_h != crtc_h ||
14731             !old_plane_state->fb != !fb)
14732                 goto slow;
14733
14734         new_plane_state = intel_plane_duplicate_state(plane);
14735         if (!new_plane_state)
14736                 return -ENOMEM;
14737
14738         new_crtc_state = to_intel_crtc_state(intel_crtc_duplicate_state(crtc));
14739         if (!new_crtc_state) {
14740                 ret = -ENOMEM;
14741                 goto out_free;
14742         }
14743
14744         drm_atomic_set_fb_for_plane(new_plane_state, fb);
14745
14746         new_plane_state->src_x = src_x;
14747         new_plane_state->src_y = src_y;
14748         new_plane_state->src_w = src_w;
14749         new_plane_state->src_h = src_h;
14750         new_plane_state->crtc_x = crtc_x;
14751         new_plane_state->crtc_y = crtc_y;
14752         new_plane_state->crtc_w = crtc_w;
14753         new_plane_state->crtc_h = crtc_h;
14754
14755         ret = intel_plane_atomic_check_with_state(crtc_state, new_crtc_state,
14756                                                   to_intel_plane_state(old_plane_state),
14757                                                   to_intel_plane_state(new_plane_state));
14758         if (ret)
14759                 goto out_free;
14760
14761         ret = mutex_lock_interruptible(&dev_priv->drm.struct_mutex);
14762         if (ret)
14763                 goto out_free;
14764
14765         ret = intel_plane_pin_fb(to_intel_plane_state(new_plane_state));
14766         if (ret)
14767                 goto out_unlock;
14768
14769         intel_fb_obj_flush(intel_fb_obj(fb), ORIGIN_FLIP);
14770
14771         old_fb = old_plane_state->fb;
14772         i915_gem_track_fb(intel_fb_obj(old_fb), intel_fb_obj(fb),
14773                           intel_plane->frontbuffer_bit);
14774
14775         /* Swap plane state */
14776         plane->state = new_plane_state;
14777
14778         /*
14779          * We cannot swap crtc_state as it may be in use by an atomic commit or
14780          * page flip that's running simultaneously. If we swap crtc_state and
14781          * destroy the old state, we will cause a use-after-free there.
14782          *
14783          * Only update active_planes, which is needed for our internal
14784          * bookkeeping. Either value will do the right thing when updating
14785          * planes atomically. If the cursor was part of the atomic update then
14786          * we would have taken the slowpath.
14787          */
14788         crtc_state->active_planes = new_crtc_state->active_planes;
14789
14790         if (plane->state->visible)
14791                 intel_update_plane(intel_plane, crtc_state,
14792                                    to_intel_plane_state(plane->state));
14793         else
14794                 intel_disable_plane(intel_plane, crtc_state);
14795
14796         intel_plane_unpin_fb(to_intel_plane_state(old_plane_state));
14797
14798 out_unlock:
14799         mutex_unlock(&dev_priv->drm.struct_mutex);
14800 out_free:
14801         if (new_crtc_state)
14802                 intel_crtc_destroy_state(crtc, &new_crtc_state->base);
14803         if (ret)
14804                 intel_plane_destroy_state(plane, new_plane_state);
14805         else
14806                 intel_plane_destroy_state(plane, old_plane_state);
14807         return ret;
14808
14809 slow:
14810         return drm_atomic_helper_update_plane(plane, crtc, fb,
14811                                               crtc_x, crtc_y, crtc_w, crtc_h,
14812                                               src_x, src_y, src_w, src_h, ctx);
14813 }
14814
14815 static const struct drm_plane_funcs intel_cursor_plane_funcs = {
14816         .update_plane = intel_legacy_cursor_update,
14817         .disable_plane = drm_atomic_helper_disable_plane,
14818         .destroy = intel_plane_destroy,
14819         .atomic_duplicate_state = intel_plane_duplicate_state,
14820         .atomic_destroy_state = intel_plane_destroy_state,
14821         .format_mod_supported = intel_cursor_format_mod_supported,
14822 };
14823
14824 static bool i9xx_plane_has_fbc(struct drm_i915_private *dev_priv,
14825                                enum i9xx_plane_id i9xx_plane)
14826 {
14827         if (!HAS_FBC(dev_priv))
14828                 return false;
14829
14830         if (IS_BROADWELL(dev_priv) || IS_HASWELL(dev_priv))
14831                 return i9xx_plane == PLANE_A; /* tied to pipe A */
14832         else if (IS_IVYBRIDGE(dev_priv))
14833                 return i9xx_plane == PLANE_A || i9xx_plane == PLANE_B ||
14834                         i9xx_plane == PLANE_C;
14835         else if (INTEL_GEN(dev_priv) >= 4)
14836                 return i9xx_plane == PLANE_A || i9xx_plane == PLANE_B;
14837         else
14838                 return i9xx_plane == PLANE_A;
14839 }
14840
14841 static struct intel_plane *
14842 intel_primary_plane_create(struct drm_i915_private *dev_priv, enum pipe pipe)
14843 {
14844         struct intel_plane *plane;
14845         const struct drm_plane_funcs *plane_funcs;
14846         unsigned int supported_rotations;
14847         unsigned int possible_crtcs;
14848         const u64 *modifiers;
14849         const u32 *formats;
14850         int num_formats;
14851         int ret;
14852
14853         if (INTEL_GEN(dev_priv) >= 9)
14854                 return skl_universal_plane_create(dev_priv, pipe,
14855                                                   PLANE_PRIMARY);
14856
14857         plane = intel_plane_alloc();
14858         if (IS_ERR(plane))
14859                 return plane;
14860
14861         plane->pipe = pipe;
14862         /*
14863          * On gen2/3 only plane A can do FBC, but the panel fitter and LVDS
14864          * port is hooked to pipe B. Hence we want plane A feeding pipe B.
14865          */
14866         if (HAS_FBC(dev_priv) && INTEL_GEN(dev_priv) < 4)
14867                 plane->i9xx_plane = (enum i9xx_plane_id) !pipe;
14868         else
14869                 plane->i9xx_plane = (enum i9xx_plane_id) pipe;
14870         plane->id = PLANE_PRIMARY;
14871         plane->frontbuffer_bit = INTEL_FRONTBUFFER(pipe, plane->id);
14872
14873         plane->has_fbc = i9xx_plane_has_fbc(dev_priv, plane->i9xx_plane);
14874         if (plane->has_fbc) {
14875                 struct intel_fbc *fbc = &dev_priv->fbc;
14876
14877                 fbc->possible_framebuffer_bits |= plane->frontbuffer_bit;
14878         }
14879
14880         if (INTEL_GEN(dev_priv) >= 4) {
14881                 formats = i965_primary_formats;
14882                 num_formats = ARRAY_SIZE(i965_primary_formats);
14883                 modifiers = i9xx_format_modifiers;
14884
14885                 plane->max_stride = i9xx_plane_max_stride;
14886                 plane->update_plane = i9xx_update_plane;
14887                 plane->disable_plane = i9xx_disable_plane;
14888                 plane->get_hw_state = i9xx_plane_get_hw_state;
14889                 plane->check_plane = i9xx_plane_check;
14890
14891                 plane_funcs = &i965_plane_funcs;
14892         } else {
14893                 formats = i8xx_primary_formats;
14894                 num_formats = ARRAY_SIZE(i8xx_primary_formats);
14895                 modifiers = i9xx_format_modifiers;
14896
14897                 plane->max_stride = i9xx_plane_max_stride;
14898                 plane->update_plane = i9xx_update_plane;
14899                 plane->disable_plane = i9xx_disable_plane;
14900                 plane->get_hw_state = i9xx_plane_get_hw_state;
14901                 plane->check_plane = i9xx_plane_check;
14902
14903                 plane_funcs = &i8xx_plane_funcs;
14904         }
14905
14906         possible_crtcs = BIT(pipe);
14907
14908         if (INTEL_GEN(dev_priv) >= 5 || IS_G4X(dev_priv))
14909                 ret = drm_universal_plane_init(&dev_priv->drm, &plane->base,
14910                                                possible_crtcs, plane_funcs,
14911                                                formats, num_formats, modifiers,
14912                                                DRM_PLANE_TYPE_PRIMARY,
14913                                                "primary %c", pipe_name(pipe));
14914         else
14915                 ret = drm_universal_plane_init(&dev_priv->drm, &plane->base,
14916                                                possible_crtcs, plane_funcs,
14917                                                formats, num_formats, modifiers,
14918                                                DRM_PLANE_TYPE_PRIMARY,
14919                                                "plane %c",
14920                                                plane_name(plane->i9xx_plane));
14921         if (ret)
14922                 goto fail;
14923
14924         if (IS_CHERRYVIEW(dev_priv) && pipe == PIPE_B) {
14925                 supported_rotations =
14926                         DRM_MODE_ROTATE_0 | DRM_MODE_ROTATE_180 |
14927                         DRM_MODE_REFLECT_X;
14928         } else if (INTEL_GEN(dev_priv) >= 4) {
14929                 supported_rotations =
14930                         DRM_MODE_ROTATE_0 | DRM_MODE_ROTATE_180;
14931         } else {
14932                 supported_rotations = DRM_MODE_ROTATE_0;
14933         }
14934
14935         if (INTEL_GEN(dev_priv) >= 4)
14936                 drm_plane_create_rotation_property(&plane->base,
14937                                                    DRM_MODE_ROTATE_0,
14938                                                    supported_rotations);
14939
14940         drm_plane_helper_add(&plane->base, &intel_plane_helper_funcs);
14941
14942         return plane;
14943
14944 fail:
14945         intel_plane_free(plane);
14946
14947         return ERR_PTR(ret);
14948 }
14949
14950 static struct intel_plane *
14951 intel_cursor_plane_create(struct drm_i915_private *dev_priv,
14952                           enum pipe pipe)
14953 {
14954         unsigned int possible_crtcs;
14955         struct intel_plane *cursor;
14956         int ret;
14957
14958         cursor = intel_plane_alloc();
14959         if (IS_ERR(cursor))
14960                 return cursor;
14961
14962         cursor->pipe = pipe;
14963         cursor->i9xx_plane = (enum i9xx_plane_id) pipe;
14964         cursor->id = PLANE_CURSOR;
14965         cursor->frontbuffer_bit = INTEL_FRONTBUFFER(pipe, cursor->id);
14966
14967         if (IS_I845G(dev_priv) || IS_I865G(dev_priv)) {
14968                 cursor->max_stride = i845_cursor_max_stride;
14969                 cursor->update_plane = i845_update_cursor;
14970                 cursor->disable_plane = i845_disable_cursor;
14971                 cursor->get_hw_state = i845_cursor_get_hw_state;
14972                 cursor->check_plane = i845_check_cursor;
14973         } else {
14974                 cursor->max_stride = i9xx_cursor_max_stride;
14975                 cursor->update_plane = i9xx_update_cursor;
14976                 cursor->disable_plane = i9xx_disable_cursor;
14977                 cursor->get_hw_state = i9xx_cursor_get_hw_state;
14978                 cursor->check_plane = i9xx_check_cursor;
14979         }
14980
14981         cursor->cursor.base = ~0;
14982         cursor->cursor.cntl = ~0;
14983
14984         if (IS_I845G(dev_priv) || IS_I865G(dev_priv) || HAS_CUR_FBC(dev_priv))
14985                 cursor->cursor.size = ~0;
14986
14987         possible_crtcs = BIT(pipe);
14988
14989         ret = drm_universal_plane_init(&dev_priv->drm, &cursor->base,
14990                                        possible_crtcs, &intel_cursor_plane_funcs,
14991                                        intel_cursor_formats,
14992                                        ARRAY_SIZE(intel_cursor_formats),
14993                                        cursor_format_modifiers,
14994                                        DRM_PLANE_TYPE_CURSOR,
14995                                        "cursor %c", pipe_name(pipe));
14996         if (ret)
14997                 goto fail;
14998
14999         if (INTEL_GEN(dev_priv) >= 4)
15000                 drm_plane_create_rotation_property(&cursor->base,
15001                                                    DRM_MODE_ROTATE_0,
15002                                                    DRM_MODE_ROTATE_0 |
15003                                                    DRM_MODE_ROTATE_180);
15004
15005         drm_plane_helper_add(&cursor->base, &intel_plane_helper_funcs);
15006
15007         return cursor;
15008
15009 fail:
15010         intel_plane_free(cursor);
15011
15012         return ERR_PTR(ret);
15013 }
15014
15015 static void intel_crtc_init_scalers(struct intel_crtc *crtc,
15016                                     struct intel_crtc_state *crtc_state)
15017 {
15018         struct intel_crtc_scaler_state *scaler_state =
15019                 &crtc_state->scaler_state;
15020         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
15021         int i;
15022
15023         crtc->num_scalers = RUNTIME_INFO(dev_priv)->num_scalers[crtc->pipe];
15024         if (!crtc->num_scalers)
15025                 return;
15026
15027         for (i = 0; i < crtc->num_scalers; i++) {
15028                 struct intel_scaler *scaler = &scaler_state->scalers[i];
15029
15030                 scaler->in_use = 0;
15031                 scaler->mode = 0;
15032         }
15033
15034         scaler_state->scaler_id = -1;
15035 }
15036
15037 #define INTEL_CRTC_FUNCS \
15038         .gamma_set = drm_atomic_helper_legacy_gamma_set, \
15039         .set_config = drm_atomic_helper_set_config, \
15040         .destroy = intel_crtc_destroy, \
15041         .page_flip = drm_atomic_helper_page_flip, \
15042         .atomic_duplicate_state = intel_crtc_duplicate_state, \
15043         .atomic_destroy_state = intel_crtc_destroy_state, \
15044         .set_crc_source = intel_crtc_set_crc_source, \
15045         .verify_crc_source = intel_crtc_verify_crc_source, \
15046         .get_crc_sources = intel_crtc_get_crc_sources
15047
15048 static const struct drm_crtc_funcs bdw_crtc_funcs = {
15049         INTEL_CRTC_FUNCS,
15050
15051         .get_vblank_counter = g4x_get_vblank_counter,
15052         .enable_vblank = bdw_enable_vblank,
15053         .disable_vblank = bdw_disable_vblank,
15054 };
15055
15056 static const struct drm_crtc_funcs ilk_crtc_funcs = {
15057         INTEL_CRTC_FUNCS,
15058
15059         .get_vblank_counter = g4x_get_vblank_counter,
15060         .enable_vblank = ilk_enable_vblank,
15061         .disable_vblank = ilk_disable_vblank,
15062 };
15063
15064 static const struct drm_crtc_funcs g4x_crtc_funcs = {
15065         INTEL_CRTC_FUNCS,
15066
15067         .get_vblank_counter = g4x_get_vblank_counter,
15068         .enable_vblank = i965_enable_vblank,
15069         .disable_vblank = i965_disable_vblank,
15070 };
15071
15072 static const struct drm_crtc_funcs i965_crtc_funcs = {
15073         INTEL_CRTC_FUNCS,
15074
15075         .get_vblank_counter = i915_get_vblank_counter,
15076         .enable_vblank = i965_enable_vblank,
15077         .disable_vblank = i965_disable_vblank,
15078 };
15079
15080 static const struct drm_crtc_funcs i945gm_crtc_funcs = {
15081         INTEL_CRTC_FUNCS,
15082
15083         .get_vblank_counter = i915_get_vblank_counter,
15084         .enable_vblank = i945gm_enable_vblank,
15085         .disable_vblank = i945gm_disable_vblank,
15086 };
15087
15088 static const struct drm_crtc_funcs i915_crtc_funcs = {
15089         INTEL_CRTC_FUNCS,
15090
15091         .get_vblank_counter = i915_get_vblank_counter,
15092         .enable_vblank = i8xx_enable_vblank,
15093         .disable_vblank = i8xx_disable_vblank,
15094 };
15095
15096 static const struct drm_crtc_funcs i8xx_crtc_funcs = {
15097         INTEL_CRTC_FUNCS,
15098
15099         /* no hw vblank counter */
15100         .enable_vblank = i8xx_enable_vblank,
15101         .disable_vblank = i8xx_disable_vblank,
15102 };
15103
15104 static int intel_crtc_init(struct drm_i915_private *dev_priv, enum pipe pipe)
15105 {
15106         const struct drm_crtc_funcs *funcs;
15107         struct intel_crtc *intel_crtc;
15108         struct intel_crtc_state *crtc_state = NULL;
15109         struct intel_plane *primary = NULL;
15110         struct intel_plane *cursor = NULL;
15111         int sprite, ret;
15112
15113         intel_crtc = kzalloc(sizeof(*intel_crtc), GFP_KERNEL);
15114         if (!intel_crtc)
15115                 return -ENOMEM;
15116
15117         crtc_state = kzalloc(sizeof(*crtc_state), GFP_KERNEL);
15118         if (!crtc_state) {
15119                 ret = -ENOMEM;
15120                 goto fail;
15121         }
15122         __drm_atomic_helper_crtc_reset(&intel_crtc->base, &crtc_state->base);
15123         intel_crtc->config = crtc_state;
15124
15125         primary = intel_primary_plane_create(dev_priv, pipe);
15126         if (IS_ERR(primary)) {
15127                 ret = PTR_ERR(primary);
15128                 goto fail;
15129         }
15130         intel_crtc->plane_ids_mask |= BIT(primary->id);
15131
15132         for_each_sprite(dev_priv, pipe, sprite) {
15133                 struct intel_plane *plane;
15134
15135                 plane = intel_sprite_plane_create(dev_priv, pipe, sprite);
15136                 if (IS_ERR(plane)) {
15137                         ret = PTR_ERR(plane);
15138                         goto fail;
15139                 }
15140                 intel_crtc->plane_ids_mask |= BIT(plane->id);
15141         }
15142
15143         cursor = intel_cursor_plane_create(dev_priv, pipe);
15144         if (IS_ERR(cursor)) {
15145                 ret = PTR_ERR(cursor);
15146                 goto fail;
15147         }
15148         intel_crtc->plane_ids_mask |= BIT(cursor->id);
15149
15150         if (HAS_GMCH(dev_priv)) {
15151                 if (IS_CHERRYVIEW(dev_priv) ||
15152                     IS_VALLEYVIEW(dev_priv) || IS_G4X(dev_priv))
15153                         funcs = &g4x_crtc_funcs;
15154                 else if (IS_GEN(dev_priv, 4))
15155                         funcs = &i965_crtc_funcs;
15156                 else if (IS_I945GM(dev_priv))
15157                         funcs = &i945gm_crtc_funcs;
15158                 else if (IS_GEN(dev_priv, 3))
15159                         funcs = &i915_crtc_funcs;
15160                 else
15161                         funcs = &i8xx_crtc_funcs;
15162         } else {
15163                 if (INTEL_GEN(dev_priv) >= 8)
15164                         funcs = &bdw_crtc_funcs;
15165                 else
15166                         funcs = &ilk_crtc_funcs;
15167         }
15168
15169         ret = drm_crtc_init_with_planes(&dev_priv->drm, &intel_crtc->base,
15170                                         &primary->base, &cursor->base,
15171                                         funcs, "pipe %c", pipe_name(pipe));
15172         if (ret)
15173                 goto fail;
15174
15175         intel_crtc->pipe = pipe;
15176
15177         /* initialize shared scalers */
15178         intel_crtc_init_scalers(intel_crtc, crtc_state);
15179
15180         BUG_ON(pipe >= ARRAY_SIZE(dev_priv->pipe_to_crtc_mapping) ||
15181                dev_priv->pipe_to_crtc_mapping[pipe] != NULL);
15182         dev_priv->pipe_to_crtc_mapping[pipe] = intel_crtc;
15183
15184         if (INTEL_GEN(dev_priv) < 9) {
15185                 enum i9xx_plane_id i9xx_plane = primary->i9xx_plane;
15186
15187                 BUG_ON(i9xx_plane >= ARRAY_SIZE(dev_priv->plane_to_crtc_mapping) ||
15188                        dev_priv->plane_to_crtc_mapping[i9xx_plane] != NULL);
15189                 dev_priv->plane_to_crtc_mapping[i9xx_plane] = intel_crtc;
15190         }
15191
15192         drm_crtc_helper_add(&intel_crtc->base, &intel_helper_funcs);
15193
15194         intel_color_init(intel_crtc);
15195
15196         WARN_ON(drm_crtc_index(&intel_crtc->base) != intel_crtc->pipe);
15197
15198         return 0;
15199
15200 fail:
15201         /*
15202          * drm_mode_config_cleanup() will free up any
15203          * crtcs/planes already initialized.
15204          */
15205         kfree(crtc_state);
15206         kfree(intel_crtc);
15207
15208         return ret;
15209 }
15210
15211 int intel_get_pipe_from_crtc_id_ioctl(struct drm_device *dev, void *data,
15212                                       struct drm_file *file)
15213 {
15214         struct drm_i915_get_pipe_from_crtc_id *pipe_from_crtc_id = data;
15215         struct drm_crtc *drmmode_crtc;
15216         struct intel_crtc *crtc;
15217
15218         drmmode_crtc = drm_crtc_find(dev, file, pipe_from_crtc_id->crtc_id);
15219         if (!drmmode_crtc)
15220                 return -ENOENT;
15221
15222         crtc = to_intel_crtc(drmmode_crtc);
15223         pipe_from_crtc_id->pipe = crtc->pipe;
15224
15225         return 0;
15226 }
15227
15228 static int intel_encoder_clones(struct intel_encoder *encoder)
15229 {
15230         struct drm_device *dev = encoder->base.dev;
15231         struct intel_encoder *source_encoder;
15232         int index_mask = 0;
15233         int entry = 0;
15234
15235         for_each_intel_encoder(dev, source_encoder) {
15236                 if (encoders_cloneable(encoder, source_encoder))
15237                         index_mask |= (1 << entry);
15238
15239                 entry++;
15240         }
15241
15242         return index_mask;
15243 }
15244
15245 static bool ilk_has_edp_a(struct drm_i915_private *dev_priv)
15246 {
15247         if (!IS_MOBILE(dev_priv))
15248                 return false;
15249
15250         if ((I915_READ(DP_A) & DP_DETECTED) == 0)
15251                 return false;
15252
15253         if (IS_GEN(dev_priv, 5) && (I915_READ(FUSE_STRAP) & ILK_eDP_A_DISABLE))
15254                 return false;
15255
15256         return true;
15257 }
15258
15259 static bool intel_ddi_crt_present(struct drm_i915_private *dev_priv)
15260 {
15261         if (INTEL_GEN(dev_priv) >= 9)
15262                 return false;
15263
15264         if (IS_HSW_ULT(dev_priv) || IS_BDW_ULT(dev_priv))
15265                 return false;
15266
15267         if (HAS_PCH_LPT_H(dev_priv) &&
15268             I915_READ(SFUSE_STRAP) & SFUSE_STRAP_CRT_DISABLED)
15269                 return false;
15270
15271         /* DDI E can't be used if DDI A requires 4 lanes */
15272         if (I915_READ(DDI_BUF_CTL(PORT_A)) & DDI_A_4_LANES)
15273                 return false;
15274
15275         if (!dev_priv->vbt.int_crt_support)
15276                 return false;
15277
15278         return true;
15279 }
15280
15281 void intel_pps_unlock_regs_wa(struct drm_i915_private *dev_priv)
15282 {
15283         int pps_num;
15284         int pps_idx;
15285
15286         if (HAS_DDI(dev_priv))
15287                 return;
15288         /*
15289          * This w/a is needed at least on CPT/PPT, but to be sure apply it
15290          * everywhere where registers can be write protected.
15291          */
15292         if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
15293                 pps_num = 2;
15294         else
15295                 pps_num = 1;
15296
15297         for (pps_idx = 0; pps_idx < pps_num; pps_idx++) {
15298                 u32 val = I915_READ(PP_CONTROL(pps_idx));
15299
15300                 val = (val & ~PANEL_UNLOCK_MASK) | PANEL_UNLOCK_REGS;
15301                 I915_WRITE(PP_CONTROL(pps_idx), val);
15302         }
15303 }
15304
15305 static void intel_pps_init(struct drm_i915_private *dev_priv)
15306 {
15307         if (HAS_PCH_SPLIT(dev_priv) || IS_GEN9_LP(dev_priv))
15308                 dev_priv->pps_mmio_base = PCH_PPS_BASE;
15309         else if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
15310                 dev_priv->pps_mmio_base = VLV_PPS_BASE;
15311         else
15312                 dev_priv->pps_mmio_base = PPS_BASE;
15313
15314         intel_pps_unlock_regs_wa(dev_priv);
15315 }
15316
15317 static void intel_setup_outputs(struct drm_i915_private *dev_priv)
15318 {
15319         struct intel_encoder *encoder;
15320         bool dpd_is_edp = false;
15321
15322         intel_pps_init(dev_priv);
15323
15324         if (!HAS_DISPLAY(dev_priv))
15325                 return;
15326
15327         if (IS_ELKHARTLAKE(dev_priv)) {
15328                 intel_ddi_init(dev_priv, PORT_A);
15329                 intel_ddi_init(dev_priv, PORT_B);
15330                 intel_ddi_init(dev_priv, PORT_C);
15331                 icl_dsi_init(dev_priv);
15332         } else if (INTEL_GEN(dev_priv) >= 11) {
15333                 intel_ddi_init(dev_priv, PORT_A);
15334                 intel_ddi_init(dev_priv, PORT_B);
15335                 intel_ddi_init(dev_priv, PORT_C);
15336                 intel_ddi_init(dev_priv, PORT_D);
15337                 intel_ddi_init(dev_priv, PORT_E);
15338                 /*
15339                  * On some ICL SKUs port F is not present. No strap bits for
15340                  * this, so rely on VBT.
15341                  * Work around broken VBTs on SKUs known to have no port F.
15342                  */
15343                 if (IS_ICL_WITH_PORT_F(dev_priv) &&
15344                     intel_bios_is_port_present(dev_priv, PORT_F))
15345                         intel_ddi_init(dev_priv, PORT_F);
15346
15347                 icl_dsi_init(dev_priv);
15348         } else if (IS_GEN9_LP(dev_priv)) {
15349                 /*
15350                  * FIXME: Broxton doesn't support port detection via the
15351                  * DDI_BUF_CTL_A or SFUSE_STRAP registers, find another way to
15352                  * detect the ports.
15353                  */
15354                 intel_ddi_init(dev_priv, PORT_A);
15355                 intel_ddi_init(dev_priv, PORT_B);
15356                 intel_ddi_init(dev_priv, PORT_C);
15357
15358                 vlv_dsi_init(dev_priv);
15359         } else if (HAS_DDI(dev_priv)) {
15360                 int found;
15361
15362                 if (intel_ddi_crt_present(dev_priv))
15363                         intel_crt_init(dev_priv);
15364
15365                 /*
15366                  * Haswell uses DDI functions to detect digital outputs.
15367                  * On SKL pre-D0 the strap isn't connected, so we assume
15368                  * it's there.
15369                  */
15370                 found = I915_READ(DDI_BUF_CTL(PORT_A)) & DDI_INIT_DISPLAY_DETECTED;
15371                 /* WaIgnoreDDIAStrap: skl */
15372                 if (found || IS_GEN9_BC(dev_priv))
15373                         intel_ddi_init(dev_priv, PORT_A);
15374
15375                 /* DDI B, C, D, and F detection is indicated by the SFUSE_STRAP
15376                  * register */
15377                 found = I915_READ(SFUSE_STRAP);
15378
15379                 if (found & SFUSE_STRAP_DDIB_DETECTED)
15380                         intel_ddi_init(dev_priv, PORT_B);
15381                 if (found & SFUSE_STRAP_DDIC_DETECTED)
15382                         intel_ddi_init(dev_priv, PORT_C);
15383                 if (found & SFUSE_STRAP_DDID_DETECTED)
15384                         intel_ddi_init(dev_priv, PORT_D);
15385                 if (found & SFUSE_STRAP_DDIF_DETECTED)
15386                         intel_ddi_init(dev_priv, PORT_F);
15387                 /*
15388                  * On SKL we don't have a way to detect DDI-E so we rely on VBT.
15389                  */
15390                 if (IS_GEN9_BC(dev_priv) &&
15391                     intel_bios_is_port_present(dev_priv, PORT_E))
15392                         intel_ddi_init(dev_priv, PORT_E);
15393
15394         } else if (HAS_PCH_SPLIT(dev_priv)) {
15395                 int found;
15396
15397                 /*
15398                  * intel_edp_init_connector() depends on this completing first,
15399                  * to prevent the registration of both eDP and LVDS and the
15400                  * incorrect sharing of the PPS.
15401                  */
15402                 intel_lvds_init(dev_priv);
15403                 intel_crt_init(dev_priv);
15404
15405                 dpd_is_edp = intel_dp_is_port_edp(dev_priv, PORT_D);
15406
15407                 if (ilk_has_edp_a(dev_priv))
15408                         intel_dp_init(dev_priv, DP_A, PORT_A);
15409
15410                 if (I915_READ(PCH_HDMIB) & SDVO_DETECTED) {
15411                         /* PCH SDVOB multiplex with HDMIB */
15412                         found = intel_sdvo_init(dev_priv, PCH_SDVOB, PORT_B);
15413                         if (!found)
15414                                 intel_hdmi_init(dev_priv, PCH_HDMIB, PORT_B);
15415                         if (!found && (I915_READ(PCH_DP_B) & DP_DETECTED))
15416                                 intel_dp_init(dev_priv, PCH_DP_B, PORT_B);
15417                 }
15418
15419                 if (I915_READ(PCH_HDMIC) & SDVO_DETECTED)
15420                         intel_hdmi_init(dev_priv, PCH_HDMIC, PORT_C);
15421
15422                 if (!dpd_is_edp && I915_READ(PCH_HDMID) & SDVO_DETECTED)
15423                         intel_hdmi_init(dev_priv, PCH_HDMID, PORT_D);
15424
15425                 if (I915_READ(PCH_DP_C) & DP_DETECTED)
15426                         intel_dp_init(dev_priv, PCH_DP_C, PORT_C);
15427
15428                 if (I915_READ(PCH_DP_D) & DP_DETECTED)
15429                         intel_dp_init(dev_priv, PCH_DP_D, PORT_D);
15430         } else if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) {
15431                 bool has_edp, has_port;
15432
15433                 if (IS_VALLEYVIEW(dev_priv) && dev_priv->vbt.int_crt_support)
15434                         intel_crt_init(dev_priv);
15435
15436                 /*
15437                  * The DP_DETECTED bit is the latched state of the DDC
15438                  * SDA pin at boot. However since eDP doesn't require DDC
15439                  * (no way to plug in a DP->HDMI dongle) the DDC pins for
15440                  * eDP ports may have been muxed to an alternate function.
15441                  * Thus we can't rely on the DP_DETECTED bit alone to detect
15442                  * eDP ports. Consult the VBT as well as DP_DETECTED to
15443                  * detect eDP ports.
15444                  *
15445                  * Sadly the straps seem to be missing sometimes even for HDMI
15446                  * ports (eg. on Voyo V3 - CHT x7-Z8700), so check both strap
15447                  * and VBT for the presence of the port. Additionally we can't
15448                  * trust the port type the VBT declares as we've seen at least
15449                  * HDMI ports that the VBT claim are DP or eDP.
15450                  */
15451                 has_edp = intel_dp_is_port_edp(dev_priv, PORT_B);
15452                 has_port = intel_bios_is_port_present(dev_priv, PORT_B);
15453                 if (I915_READ(VLV_DP_B) & DP_DETECTED || has_port)
15454                         has_edp &= intel_dp_init(dev_priv, VLV_DP_B, PORT_B);
15455                 if ((I915_READ(VLV_HDMIB) & SDVO_DETECTED || has_port) && !has_edp)
15456                         intel_hdmi_init(dev_priv, VLV_HDMIB, PORT_B);
15457
15458                 has_edp = intel_dp_is_port_edp(dev_priv, PORT_C);
15459                 has_port = intel_bios_is_port_present(dev_priv, PORT_C);
15460                 if (I915_READ(VLV_DP_C) & DP_DETECTED || has_port)
15461                         has_edp &= intel_dp_init(dev_priv, VLV_DP_C, PORT_C);
15462                 if ((I915_READ(VLV_HDMIC) & SDVO_DETECTED || has_port) && !has_edp)
15463                         intel_hdmi_init(dev_priv, VLV_HDMIC, PORT_C);
15464
15465                 if (IS_CHERRYVIEW(dev_priv)) {
15466                         /*
15467                          * eDP not supported on port D,
15468                          * so no need to worry about it
15469                          */
15470                         has_port = intel_bios_is_port_present(dev_priv, PORT_D);
15471                         if (I915_READ(CHV_DP_D) & DP_DETECTED || has_port)
15472                                 intel_dp_init(dev_priv, CHV_DP_D, PORT_D);
15473                         if (I915_READ(CHV_HDMID) & SDVO_DETECTED || has_port)
15474                                 intel_hdmi_init(dev_priv, CHV_HDMID, PORT_D);
15475                 }
15476
15477                 vlv_dsi_init(dev_priv);
15478         } else if (IS_PINEVIEW(dev_priv)) {
15479                 intel_lvds_init(dev_priv);
15480                 intel_crt_init(dev_priv);
15481         } else if (IS_GEN_RANGE(dev_priv, 3, 4)) {
15482                 bool found = false;
15483
15484                 if (IS_MOBILE(dev_priv))
15485                         intel_lvds_init(dev_priv);
15486
15487                 intel_crt_init(dev_priv);
15488
15489                 if (I915_READ(GEN3_SDVOB) & SDVO_DETECTED) {
15490                         DRM_DEBUG_KMS("probing SDVOB\n");
15491                         found = intel_sdvo_init(dev_priv, GEN3_SDVOB, PORT_B);
15492                         if (!found && IS_G4X(dev_priv)) {
15493                                 DRM_DEBUG_KMS("probing HDMI on SDVOB\n");
15494                                 intel_hdmi_init(dev_priv, GEN4_HDMIB, PORT_B);
15495                         }
15496
15497                         if (!found && IS_G4X(dev_priv))
15498                                 intel_dp_init(dev_priv, DP_B, PORT_B);
15499                 }
15500
15501                 /* Before G4X SDVOC doesn't have its own detect register */
15502
15503                 if (I915_READ(GEN3_SDVOB) & SDVO_DETECTED) {
15504                         DRM_DEBUG_KMS("probing SDVOC\n");
15505                         found = intel_sdvo_init(dev_priv, GEN3_SDVOC, PORT_C);
15506                 }
15507
15508                 if (!found && (I915_READ(GEN3_SDVOC) & SDVO_DETECTED)) {
15509
15510                         if (IS_G4X(dev_priv)) {
15511                                 DRM_DEBUG_KMS("probing HDMI on SDVOC\n");
15512                                 intel_hdmi_init(dev_priv, GEN4_HDMIC, PORT_C);
15513                         }
15514                         if (IS_G4X(dev_priv))
15515                                 intel_dp_init(dev_priv, DP_C, PORT_C);
15516                 }
15517
15518                 if (IS_G4X(dev_priv) && (I915_READ(DP_D) & DP_DETECTED))
15519                         intel_dp_init(dev_priv, DP_D, PORT_D);
15520
15521                 if (SUPPORTS_TV(dev_priv))
15522                         intel_tv_init(dev_priv);
15523         } else if (IS_GEN(dev_priv, 2)) {
15524                 if (IS_I85X(dev_priv))
15525                         intel_lvds_init(dev_priv);
15526
15527                 intel_crt_init(dev_priv);
15528                 intel_dvo_init(dev_priv);
15529         }
15530
15531         intel_psr_init(dev_priv);
15532
15533         for_each_intel_encoder(&dev_priv->drm, encoder) {
15534                 encoder->base.possible_crtcs = encoder->crtc_mask;
15535                 encoder->base.possible_clones =
15536                         intel_encoder_clones(encoder);
15537         }
15538
15539         intel_init_pch_refclk(dev_priv);
15540
15541         drm_helper_move_panel_connectors_to_head(&dev_priv->drm);
15542 }
15543
15544 static void intel_user_framebuffer_destroy(struct drm_framebuffer *fb)
15545 {
15546         struct intel_framebuffer *intel_fb = to_intel_framebuffer(fb);
15547         struct drm_i915_gem_object *obj = intel_fb_obj(fb);
15548
15549         drm_framebuffer_cleanup(fb);
15550
15551         i915_gem_object_lock(obj);
15552         WARN_ON(!obj->framebuffer_references--);
15553         i915_gem_object_unlock(obj);
15554
15555         i915_gem_object_put(obj);
15556
15557         kfree(intel_fb);
15558 }
15559
15560 static int intel_user_framebuffer_create_handle(struct drm_framebuffer *fb,
15561                                                 struct drm_file *file,
15562                                                 unsigned int *handle)
15563 {
15564         struct drm_i915_gem_object *obj = intel_fb_obj(fb);
15565
15566         if (obj->userptr.mm) {
15567                 DRM_DEBUG("attempting to use a userptr for a framebuffer, denied\n");
15568                 return -EINVAL;
15569         }
15570
15571         return drm_gem_handle_create(file, &obj->base, handle);
15572 }
15573
15574 static int intel_user_framebuffer_dirty(struct drm_framebuffer *fb,
15575                                         struct drm_file *file,
15576                                         unsigned flags, unsigned color,
15577                                         struct drm_clip_rect *clips,
15578                                         unsigned num_clips)
15579 {
15580         struct drm_i915_gem_object *obj = intel_fb_obj(fb);
15581
15582         i915_gem_object_flush_if_display(obj);
15583         intel_fb_obj_flush(obj, ORIGIN_DIRTYFB);
15584
15585         return 0;
15586 }
15587
15588 static const struct drm_framebuffer_funcs intel_fb_funcs = {
15589         .destroy = intel_user_framebuffer_destroy,
15590         .create_handle = intel_user_framebuffer_create_handle,
15591         .dirty = intel_user_framebuffer_dirty,
15592 };
15593
15594 static int intel_framebuffer_init(struct intel_framebuffer *intel_fb,
15595                                   struct drm_i915_gem_object *obj,
15596                                   struct drm_mode_fb_cmd2 *mode_cmd)
15597 {
15598         struct drm_i915_private *dev_priv = to_i915(obj->base.dev);
15599         struct drm_framebuffer *fb = &intel_fb->base;
15600         u32 max_stride;
15601         unsigned int tiling, stride;
15602         int ret = -EINVAL;
15603         int i;
15604
15605         i915_gem_object_lock(obj);
15606         obj->framebuffer_references++;
15607         tiling = i915_gem_object_get_tiling(obj);
15608         stride = i915_gem_object_get_stride(obj);
15609         i915_gem_object_unlock(obj);
15610
15611         if (mode_cmd->flags & DRM_MODE_FB_MODIFIERS) {
15612                 /*
15613                  * If there's a fence, enforce that
15614                  * the fb modifier and tiling mode match.
15615                  */
15616                 if (tiling != I915_TILING_NONE &&
15617                     tiling != intel_fb_modifier_to_tiling(mode_cmd->modifier[0])) {
15618                         DRM_DEBUG_KMS("tiling_mode doesn't match fb modifier\n");
15619                         goto err;
15620                 }
15621         } else {
15622                 if (tiling == I915_TILING_X) {
15623                         mode_cmd->modifier[0] = I915_FORMAT_MOD_X_TILED;
15624                 } else if (tiling == I915_TILING_Y) {
15625                         DRM_DEBUG_KMS("No Y tiling for legacy addfb\n");
15626                         goto err;
15627                 }
15628         }
15629
15630         if (!drm_any_plane_has_format(&dev_priv->drm,
15631                                       mode_cmd->pixel_format,
15632                                       mode_cmd->modifier[0])) {
15633                 struct drm_format_name_buf format_name;
15634
15635                 DRM_DEBUG_KMS("unsupported pixel format %s / modifier 0x%llx\n",
15636                               drm_get_format_name(mode_cmd->pixel_format,
15637                                                   &format_name),
15638                               mode_cmd->modifier[0]);
15639                 goto err;
15640         }
15641
15642         /*
15643          * gen2/3 display engine uses the fence if present,
15644          * so the tiling mode must match the fb modifier exactly.
15645          */
15646         if (INTEL_GEN(dev_priv) < 4 &&
15647             tiling != intel_fb_modifier_to_tiling(mode_cmd->modifier[0])) {
15648                 DRM_DEBUG_KMS("tiling_mode must match fb modifier exactly on gen2/3\n");
15649                 goto err;
15650         }
15651
15652         max_stride = intel_fb_max_stride(dev_priv, mode_cmd->pixel_format,
15653                                          mode_cmd->modifier[0]);
15654         if (mode_cmd->pitches[0] > max_stride) {
15655                 DRM_DEBUG_KMS("%s pitch (%u) must be at most %d\n",
15656                               mode_cmd->modifier[0] != DRM_FORMAT_MOD_LINEAR ?
15657                               "tiled" : "linear",
15658                               mode_cmd->pitches[0], max_stride);
15659                 goto err;
15660         }
15661
15662         /*
15663          * If there's a fence, enforce that
15664          * the fb pitch and fence stride match.
15665          */
15666         if (tiling != I915_TILING_NONE && mode_cmd->pitches[0] != stride) {
15667                 DRM_DEBUG_KMS("pitch (%d) must match tiling stride (%d)\n",
15668                               mode_cmd->pitches[0], stride);
15669                 goto err;
15670         }
15671
15672         /* FIXME need to adjust LINOFF/TILEOFF accordingly. */
15673         if (mode_cmd->offsets[0] != 0)
15674                 goto err;
15675
15676         drm_helper_mode_fill_fb_struct(&dev_priv->drm, fb, mode_cmd);
15677
15678         for (i = 0; i < fb->format->num_planes; i++) {
15679                 u32 stride_alignment;
15680
15681                 if (mode_cmd->handles[i] != mode_cmd->handles[0]) {
15682                         DRM_DEBUG_KMS("bad plane %d handle\n", i);
15683                         goto err;
15684                 }
15685
15686                 stride_alignment = intel_fb_stride_alignment(fb, i);
15687
15688                 /*
15689                  * Display WA #0531: skl,bxt,kbl,glk
15690                  *
15691                  * Render decompression and plane width > 3840
15692                  * combined with horizontal panning requires the
15693                  * plane stride to be a multiple of 4. We'll just
15694                  * require the entire fb to accommodate that to avoid
15695                  * potential runtime errors at plane configuration time.
15696                  */
15697                 if (IS_GEN(dev_priv, 9) && i == 0 && fb->width > 3840 &&
15698                     is_ccs_modifier(fb->modifier))
15699                         stride_alignment *= 4;
15700
15701                 if (fb->pitches[i] & (stride_alignment - 1)) {
15702                         DRM_DEBUG_KMS("plane %d pitch (%d) must be at least %u byte aligned\n",
15703                                       i, fb->pitches[i], stride_alignment);
15704                         goto err;
15705                 }
15706
15707                 fb->obj[i] = &obj->base;
15708         }
15709
15710         ret = intel_fill_fb_info(dev_priv, fb);
15711         if (ret)
15712                 goto err;
15713
15714         ret = drm_framebuffer_init(&dev_priv->drm, fb, &intel_fb_funcs);
15715         if (ret) {
15716                 DRM_ERROR("framebuffer init failed %d\n", ret);
15717                 goto err;
15718         }
15719
15720         return 0;
15721
15722 err:
15723         i915_gem_object_lock(obj);
15724         obj->framebuffer_references--;
15725         i915_gem_object_unlock(obj);
15726         return ret;
15727 }
15728
15729 static struct drm_framebuffer *
15730 intel_user_framebuffer_create(struct drm_device *dev,
15731                               struct drm_file *filp,
15732                               const struct drm_mode_fb_cmd2 *user_mode_cmd)
15733 {
15734         struct drm_framebuffer *fb;
15735         struct drm_i915_gem_object *obj;
15736         struct drm_mode_fb_cmd2 mode_cmd = *user_mode_cmd;
15737
15738         obj = i915_gem_object_lookup(filp, mode_cmd.handles[0]);
15739         if (!obj)
15740                 return ERR_PTR(-ENOENT);
15741
15742         fb = intel_framebuffer_create(obj, &mode_cmd);
15743         if (IS_ERR(fb))
15744                 i915_gem_object_put(obj);
15745
15746         return fb;
15747 }
15748
15749 static void intel_atomic_state_free(struct drm_atomic_state *state)
15750 {
15751         struct intel_atomic_state *intel_state = to_intel_atomic_state(state);
15752
15753         drm_atomic_state_default_release(state);
15754
15755         i915_sw_fence_fini(&intel_state->commit_ready);
15756
15757         kfree(state);
15758 }
15759
15760 static enum drm_mode_status
15761 intel_mode_valid(struct drm_device *dev,
15762                  const struct drm_display_mode *mode)
15763 {
15764         struct drm_i915_private *dev_priv = to_i915(dev);
15765         int hdisplay_max, htotal_max;
15766         int vdisplay_max, vtotal_max;
15767
15768         /*
15769          * Can't reject DBLSCAN here because Xorg ddxen can add piles
15770          * of DBLSCAN modes to the output's mode list when they detect
15771          * the scaling mode property on the connector. And they don't
15772          * ask the kernel to validate those modes in any way until
15773          * modeset time at which point the client gets a protocol error.
15774          * So in order to not upset those clients we silently ignore the
15775          * DBLSCAN flag on such connectors. For other connectors we will
15776          * reject modes with the DBLSCAN flag in encoder->compute_config().
15777          * And we always reject DBLSCAN modes in connector->mode_valid()
15778          * as we never want such modes on the connector's mode list.
15779          */
15780
15781         if (mode->vscan > 1)
15782                 return MODE_NO_VSCAN;
15783
15784         if (mode->flags & DRM_MODE_FLAG_HSKEW)
15785                 return MODE_H_ILLEGAL;
15786
15787         if (mode->flags & (DRM_MODE_FLAG_CSYNC |
15788                            DRM_MODE_FLAG_NCSYNC |
15789                            DRM_MODE_FLAG_PCSYNC))
15790                 return MODE_HSYNC;
15791
15792         if (mode->flags & (DRM_MODE_FLAG_BCAST |
15793                            DRM_MODE_FLAG_PIXMUX |
15794                            DRM_MODE_FLAG_CLKDIV2))
15795                 return MODE_BAD;
15796
15797         if (INTEL_GEN(dev_priv) >= 9 ||
15798             IS_BROADWELL(dev_priv) || IS_HASWELL(dev_priv)) {
15799                 hdisplay_max = 8192; /* FDI max 4096 handled elsewhere */
15800                 vdisplay_max = 4096;
15801                 htotal_max = 8192;
15802                 vtotal_max = 8192;
15803         } else if (INTEL_GEN(dev_priv) >= 3) {
15804                 hdisplay_max = 4096;
15805                 vdisplay_max = 4096;
15806                 htotal_max = 8192;
15807                 vtotal_max = 8192;
15808         } else {
15809                 hdisplay_max = 2048;
15810                 vdisplay_max = 2048;
15811                 htotal_max = 4096;
15812                 vtotal_max = 4096;
15813         }
15814
15815         if (mode->hdisplay > hdisplay_max ||
15816             mode->hsync_start > htotal_max ||
15817             mode->hsync_end > htotal_max ||
15818             mode->htotal > htotal_max)
15819                 return MODE_H_ILLEGAL;
15820
15821         if (mode->vdisplay > vdisplay_max ||
15822             mode->vsync_start > vtotal_max ||
15823             mode->vsync_end > vtotal_max ||
15824             mode->vtotal > vtotal_max)
15825                 return MODE_V_ILLEGAL;
15826
15827         return MODE_OK;
15828 }
15829
15830 static const struct drm_mode_config_funcs intel_mode_funcs = {
15831         .fb_create = intel_user_framebuffer_create,
15832         .get_format_info = intel_get_format_info,
15833         .output_poll_changed = intel_fbdev_output_poll_changed,
15834         .mode_valid = intel_mode_valid,
15835         .atomic_check = intel_atomic_check,
15836         .atomic_commit = intel_atomic_commit,
15837         .atomic_state_alloc = intel_atomic_state_alloc,
15838         .atomic_state_clear = intel_atomic_state_clear,
15839         .atomic_state_free = intel_atomic_state_free,
15840 };
15841
15842 /**
15843  * intel_init_display_hooks - initialize the display modesetting hooks
15844  * @dev_priv: device private
15845  */
15846 void intel_init_display_hooks(struct drm_i915_private *dev_priv)
15847 {
15848         intel_init_cdclk_hooks(dev_priv);
15849
15850         if (INTEL_GEN(dev_priv) >= 9) {
15851                 dev_priv->display.get_pipe_config = haswell_get_pipe_config;
15852                 dev_priv->display.get_initial_plane_config =
15853                         skylake_get_initial_plane_config;
15854                 dev_priv->display.crtc_compute_clock =
15855                         haswell_crtc_compute_clock;
15856                 dev_priv->display.crtc_enable = haswell_crtc_enable;
15857                 dev_priv->display.crtc_disable = haswell_crtc_disable;
15858         } else if (HAS_DDI(dev_priv)) {
15859                 dev_priv->display.get_pipe_config = haswell_get_pipe_config;
15860                 dev_priv->display.get_initial_plane_config =
15861                         i9xx_get_initial_plane_config;
15862                 dev_priv->display.crtc_compute_clock =
15863                         haswell_crtc_compute_clock;
15864                 dev_priv->display.crtc_enable = haswell_crtc_enable;
15865                 dev_priv->display.crtc_disable = haswell_crtc_disable;
15866         } else if (HAS_PCH_SPLIT(dev_priv)) {
15867                 dev_priv->display.get_pipe_config = ironlake_get_pipe_config;
15868                 dev_priv->display.get_initial_plane_config =
15869                         i9xx_get_initial_plane_config;
15870                 dev_priv->display.crtc_compute_clock =
15871                         ironlake_crtc_compute_clock;
15872                 dev_priv->display.crtc_enable = ironlake_crtc_enable;
15873                 dev_priv->display.crtc_disable = ironlake_crtc_disable;
15874         } else if (IS_CHERRYVIEW(dev_priv)) {
15875                 dev_priv->display.get_pipe_config = i9xx_get_pipe_config;
15876                 dev_priv->display.get_initial_plane_config =
15877                         i9xx_get_initial_plane_config;
15878                 dev_priv->display.crtc_compute_clock = chv_crtc_compute_clock;
15879                 dev_priv->display.crtc_enable = valleyview_crtc_enable;
15880                 dev_priv->display.crtc_disable = i9xx_crtc_disable;
15881         } else if (IS_VALLEYVIEW(dev_priv)) {
15882                 dev_priv->display.get_pipe_config = i9xx_get_pipe_config;
15883                 dev_priv->display.get_initial_plane_config =
15884                         i9xx_get_initial_plane_config;
15885                 dev_priv->display.crtc_compute_clock = vlv_crtc_compute_clock;
15886                 dev_priv->display.crtc_enable = valleyview_crtc_enable;
15887                 dev_priv->display.crtc_disable = i9xx_crtc_disable;
15888         } else if (IS_G4X(dev_priv)) {
15889                 dev_priv->display.get_pipe_config = i9xx_get_pipe_config;
15890                 dev_priv->display.get_initial_plane_config =
15891                         i9xx_get_initial_plane_config;
15892                 dev_priv->display.crtc_compute_clock = g4x_crtc_compute_clock;
15893                 dev_priv->display.crtc_enable = i9xx_crtc_enable;
15894                 dev_priv->display.crtc_disable = i9xx_crtc_disable;
15895         } else if (IS_PINEVIEW(dev_priv)) {
15896                 dev_priv->display.get_pipe_config = i9xx_get_pipe_config;
15897                 dev_priv->display.get_initial_plane_config =
15898                         i9xx_get_initial_plane_config;
15899                 dev_priv->display.crtc_compute_clock = pnv_crtc_compute_clock;
15900                 dev_priv->display.crtc_enable = i9xx_crtc_enable;
15901                 dev_priv->display.crtc_disable = i9xx_crtc_disable;
15902         } else if (!IS_GEN(dev_priv, 2)) {
15903                 dev_priv->display.get_pipe_config = i9xx_get_pipe_config;
15904                 dev_priv->display.get_initial_plane_config =
15905                         i9xx_get_initial_plane_config;
15906                 dev_priv->display.crtc_compute_clock = i9xx_crtc_compute_clock;
15907                 dev_priv->display.crtc_enable = i9xx_crtc_enable;
15908                 dev_priv->display.crtc_disable = i9xx_crtc_disable;
15909         } else {
15910                 dev_priv->display.get_pipe_config = i9xx_get_pipe_config;
15911                 dev_priv->display.get_initial_plane_config =
15912                         i9xx_get_initial_plane_config;
15913                 dev_priv->display.crtc_compute_clock = i8xx_crtc_compute_clock;
15914                 dev_priv->display.crtc_enable = i9xx_crtc_enable;
15915                 dev_priv->display.crtc_disable = i9xx_crtc_disable;
15916         }
15917
15918         if (IS_GEN(dev_priv, 5)) {
15919                 dev_priv->display.fdi_link_train = ironlake_fdi_link_train;
15920         } else if (IS_GEN(dev_priv, 6)) {
15921                 dev_priv->display.fdi_link_train = gen6_fdi_link_train;
15922         } else if (IS_IVYBRIDGE(dev_priv)) {
15923                 /* FIXME: detect B0+ stepping and use auto training */
15924                 dev_priv->display.fdi_link_train = ivb_manual_fdi_link_train;
15925         } else if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv)) {
15926                 dev_priv->display.fdi_link_train = hsw_fdi_link_train;
15927         }
15928
15929         if (INTEL_GEN(dev_priv) >= 9)
15930                 dev_priv->display.update_crtcs = skl_update_crtcs;
15931         else
15932                 dev_priv->display.update_crtcs = intel_update_crtcs;
15933 }
15934
15935 static i915_reg_t i915_vgacntrl_reg(struct drm_i915_private *dev_priv)
15936 {
15937         if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
15938                 return VLV_VGACNTRL;
15939         else if (INTEL_GEN(dev_priv) >= 5)
15940                 return CPU_VGACNTRL;
15941         else
15942                 return VGACNTRL;
15943 }
15944
15945 /* Disable the VGA plane that we never use */
15946 static void i915_disable_vga(struct drm_i915_private *dev_priv)
15947 {
15948         struct pci_dev *pdev = dev_priv->drm.pdev;
15949         u8 sr1;
15950         i915_reg_t vga_reg = i915_vgacntrl_reg(dev_priv);
15951
15952         /* WaEnableVGAAccessThroughIOPort:ctg,elk,ilk,snb,ivb,vlv,hsw */
15953         vga_get_uninterruptible(pdev, VGA_RSRC_LEGACY_IO);
15954         outb(SR01, VGA_SR_INDEX);
15955         sr1 = inb(VGA_SR_DATA);
15956         outb(sr1 | 1<<5, VGA_SR_DATA);
15957         vga_put(pdev, VGA_RSRC_LEGACY_IO);
15958         udelay(300);
15959
15960         I915_WRITE(vga_reg, VGA_DISP_DISABLE);
15961         POSTING_READ(vga_reg);
15962 }
15963
15964 void intel_modeset_init_hw(struct drm_device *dev)
15965 {
15966         struct drm_i915_private *dev_priv = to_i915(dev);
15967
15968         intel_update_cdclk(dev_priv);
15969         intel_dump_cdclk_state(&dev_priv->cdclk.hw, "Current CDCLK");
15970         dev_priv->cdclk.logical = dev_priv->cdclk.actual = dev_priv->cdclk.hw;
15971 }
15972
15973 /*
15974  * Calculate what we think the watermarks should be for the state we've read
15975  * out of the hardware and then immediately program those watermarks so that
15976  * we ensure the hardware settings match our internal state.
15977  *
15978  * We can calculate what we think WM's should be by creating a duplicate of the
15979  * current state (which was constructed during hardware readout) and running it
15980  * through the atomic check code to calculate new watermark values in the
15981  * state object.
15982  */
15983 static void sanitize_watermarks(struct drm_device *dev)
15984 {
15985         struct drm_i915_private *dev_priv = to_i915(dev);
15986         struct drm_atomic_state *state;
15987         struct intel_atomic_state *intel_state;
15988         struct intel_crtc *crtc;
15989         struct intel_crtc_state *crtc_state;
15990         struct drm_modeset_acquire_ctx ctx;
15991         int ret;
15992         int i;
15993
15994         /* Only supported on platforms that use atomic watermark design */
15995         if (!dev_priv->display.optimize_watermarks)
15996                 return;
15997
15998         /*
15999          * We need to hold connection_mutex before calling duplicate_state so
16000          * that the connector loop is protected.
16001          */
16002         drm_modeset_acquire_init(&ctx, 0);
16003 retry:
16004         ret = drm_modeset_lock_all_ctx(dev, &ctx);
16005         if (ret == -EDEADLK) {
16006                 drm_modeset_backoff(&ctx);
16007                 goto retry;
16008         } else if (WARN_ON(ret)) {
16009                 goto fail;
16010         }
16011
16012         state = drm_atomic_helper_duplicate_state(dev, &ctx);
16013         if (WARN_ON(IS_ERR(state)))
16014                 goto fail;
16015
16016         intel_state = to_intel_atomic_state(state);
16017
16018         /*
16019          * Hardware readout is the only time we don't want to calculate
16020          * intermediate watermarks (since we don't trust the current
16021          * watermarks).
16022          */
16023         if (!HAS_GMCH(dev_priv))
16024                 intel_state->skip_intermediate_wm = true;
16025
16026         ret = intel_atomic_check(dev, state);
16027         if (ret) {
16028                 /*
16029                  * If we fail here, it means that the hardware appears to be
16030                  * programmed in a way that shouldn't be possible, given our
16031                  * understanding of watermark requirements.  This might mean a
16032                  * mistake in the hardware readout code or a mistake in the
16033                  * watermark calculations for a given platform.  Raise a WARN
16034                  * so that this is noticeable.
16035                  *
16036                  * If this actually happens, we'll have to just leave the
16037                  * BIOS-programmed watermarks untouched and hope for the best.
16038                  */
16039                 WARN(true, "Could not determine valid watermarks for inherited state\n");
16040                 goto put_state;
16041         }
16042
16043         /* Write calculated watermark values back */
16044         for_each_new_intel_crtc_in_state(intel_state, crtc, crtc_state, i) {
16045                 crtc_state->wm.need_postvbl_update = true;
16046                 dev_priv->display.optimize_watermarks(intel_state, crtc_state);
16047
16048                 to_intel_crtc_state(crtc->base.state)->wm = crtc_state->wm;
16049         }
16050
16051 put_state:
16052         drm_atomic_state_put(state);
16053 fail:
16054         drm_modeset_drop_locks(&ctx);
16055         drm_modeset_acquire_fini(&ctx);
16056 }
16057
16058 static void intel_update_fdi_pll_freq(struct drm_i915_private *dev_priv)
16059 {
16060         if (IS_GEN(dev_priv, 5)) {
16061                 u32 fdi_pll_clk =
16062                         I915_READ(FDI_PLL_BIOS_0) & FDI_PLL_FB_CLOCK_MASK;
16063
16064                 dev_priv->fdi_pll_freq = (fdi_pll_clk + 2) * 10000;
16065         } else if (IS_GEN(dev_priv, 6) || IS_IVYBRIDGE(dev_priv)) {
16066                 dev_priv->fdi_pll_freq = 270000;
16067         } else {
16068                 return;
16069         }
16070
16071         DRM_DEBUG_DRIVER("FDI PLL freq=%d\n", dev_priv->fdi_pll_freq);
16072 }
16073
16074 static int intel_initial_commit(struct drm_device *dev)
16075 {
16076         struct drm_atomic_state *state = NULL;
16077         struct drm_modeset_acquire_ctx ctx;
16078         struct drm_crtc *crtc;
16079         struct drm_crtc_state *crtc_state;
16080         int ret = 0;
16081
16082         state = drm_atomic_state_alloc(dev);
16083         if (!state)
16084                 return -ENOMEM;
16085
16086         drm_modeset_acquire_init(&ctx, 0);
16087
16088 retry:
16089         state->acquire_ctx = &ctx;
16090
16091         drm_for_each_crtc(crtc, dev) {
16092                 crtc_state = drm_atomic_get_crtc_state(state, crtc);
16093                 if (IS_ERR(crtc_state)) {
16094                         ret = PTR_ERR(crtc_state);
16095                         goto out;
16096                 }
16097
16098                 if (crtc_state->active) {
16099                         ret = drm_atomic_add_affected_planes(state, crtc);
16100                         if (ret)
16101                                 goto out;
16102
16103                         /*
16104                          * FIXME hack to force a LUT update to avoid the
16105                          * plane update forcing the pipe gamma on without
16106                          * having a proper LUT loaded. Remove once we
16107                          * have readout for pipe gamma enable.
16108                          */
16109                         crtc_state->color_mgmt_changed = true;
16110                 }
16111         }
16112
16113         ret = drm_atomic_commit(state);
16114
16115 out:
16116         if (ret == -EDEADLK) {
16117                 drm_atomic_state_clear(state);
16118                 drm_modeset_backoff(&ctx);
16119                 goto retry;
16120         }
16121
16122         drm_atomic_state_put(state);
16123
16124         drm_modeset_drop_locks(&ctx);
16125         drm_modeset_acquire_fini(&ctx);
16126
16127         return ret;
16128 }
16129
16130 int intel_modeset_init(struct drm_device *dev)
16131 {
16132         struct drm_i915_private *dev_priv = to_i915(dev);
16133         struct i915_ggtt *ggtt = &dev_priv->ggtt;
16134         enum pipe pipe;
16135         struct intel_crtc *crtc;
16136         int ret;
16137
16138         dev_priv->modeset_wq = alloc_ordered_workqueue("i915_modeset", 0);
16139
16140         drm_mode_config_init(dev);
16141
16142         ret = intel_bw_init(dev_priv);
16143         if (ret)
16144                 return ret;
16145
16146         dev->mode_config.min_width = 0;
16147         dev->mode_config.min_height = 0;
16148
16149         dev->mode_config.preferred_depth = 24;
16150         dev->mode_config.prefer_shadow = 1;
16151
16152         dev->mode_config.allow_fb_modifiers = true;
16153
16154         dev->mode_config.funcs = &intel_mode_funcs;
16155
16156         init_llist_head(&dev_priv->atomic_helper.free_list);
16157         INIT_WORK(&dev_priv->atomic_helper.free_work,
16158                   intel_atomic_helper_free_state_worker);
16159
16160         intel_init_quirks(dev_priv);
16161
16162         intel_fbc_init(dev_priv);
16163
16164         intel_init_pm(dev_priv);
16165
16166         /*
16167          * There may be no VBT; and if the BIOS enabled SSC we can
16168          * just keep using it to avoid unnecessary flicker.  Whereas if the
16169          * BIOS isn't using it, don't assume it will work even if the VBT
16170          * indicates as much.
16171          */
16172         if (HAS_PCH_IBX(dev_priv) || HAS_PCH_CPT(dev_priv)) {
16173                 bool bios_lvds_use_ssc = !!(I915_READ(PCH_DREF_CONTROL) &
16174                                             DREF_SSC1_ENABLE);
16175
16176                 if (dev_priv->vbt.lvds_use_ssc != bios_lvds_use_ssc) {
16177                         DRM_DEBUG_KMS("SSC %sabled by BIOS, overriding VBT which says %sabled\n",
16178                                      bios_lvds_use_ssc ? "en" : "dis",
16179                                      dev_priv->vbt.lvds_use_ssc ? "en" : "dis");
16180                         dev_priv->vbt.lvds_use_ssc = bios_lvds_use_ssc;
16181                 }
16182         }
16183
16184         /*
16185          * Maximum framebuffer dimensions, chosen to match
16186          * the maximum render engine surface size on gen4+.
16187          */
16188         if (INTEL_GEN(dev_priv) >= 7) {
16189                 dev->mode_config.max_width = 16384;
16190                 dev->mode_config.max_height = 16384;
16191         } else if (INTEL_GEN(dev_priv) >= 4) {
16192                 dev->mode_config.max_width = 8192;
16193                 dev->mode_config.max_height = 8192;
16194         } else if (IS_GEN(dev_priv, 3)) {
16195                 dev->mode_config.max_width = 4096;
16196                 dev->mode_config.max_height = 4096;
16197         } else {
16198                 dev->mode_config.max_width = 2048;
16199                 dev->mode_config.max_height = 2048;
16200         }
16201
16202         if (IS_I845G(dev_priv) || IS_I865G(dev_priv)) {
16203                 dev->mode_config.cursor_width = IS_I845G(dev_priv) ? 64 : 512;
16204                 dev->mode_config.cursor_height = 1023;
16205         } else if (IS_GEN(dev_priv, 2)) {
16206                 dev->mode_config.cursor_width = 64;
16207                 dev->mode_config.cursor_height = 64;
16208         } else {
16209                 dev->mode_config.cursor_width = 256;
16210                 dev->mode_config.cursor_height = 256;
16211         }
16212
16213         dev->mode_config.fb_base = ggtt->gmadr.start;
16214
16215         DRM_DEBUG_KMS("%d display pipe%s available.\n",
16216                       INTEL_INFO(dev_priv)->num_pipes,
16217                       INTEL_INFO(dev_priv)->num_pipes > 1 ? "s" : "");
16218
16219         for_each_pipe(dev_priv, pipe) {
16220                 ret = intel_crtc_init(dev_priv, pipe);
16221                 if (ret) {
16222                         drm_mode_config_cleanup(dev);
16223                         return ret;
16224                 }
16225         }
16226
16227         intel_shared_dpll_init(dev);
16228         intel_update_fdi_pll_freq(dev_priv);
16229
16230         intel_update_czclk(dev_priv);
16231         intel_modeset_init_hw(dev);
16232
16233         intel_hdcp_component_init(dev_priv);
16234
16235         if (dev_priv->max_cdclk_freq == 0)
16236                 intel_update_max_cdclk(dev_priv);
16237
16238         /* Just disable it once at startup */
16239         i915_disable_vga(dev_priv);
16240         intel_setup_outputs(dev_priv);
16241
16242         drm_modeset_lock_all(dev);
16243         intel_modeset_setup_hw_state(dev, dev->mode_config.acquire_ctx);
16244         drm_modeset_unlock_all(dev);
16245
16246         for_each_intel_crtc(dev, crtc) {
16247                 struct intel_initial_plane_config plane_config = {};
16248
16249                 if (!crtc->active)
16250                         continue;
16251
16252                 /*
16253                  * Note that reserving the BIOS fb up front prevents us
16254                  * from stuffing other stolen allocations like the ring
16255                  * on top.  This prevents some ugliness at boot time, and
16256                  * can even allow for smooth boot transitions if the BIOS
16257                  * fb is large enough for the active pipe configuration.
16258                  */
16259                 dev_priv->display.get_initial_plane_config(crtc,
16260                                                            &plane_config);
16261
16262                 /*
16263                  * If the fb is shared between multiple heads, we'll
16264                  * just get the first one.
16265                  */
16266                 intel_find_initial_plane_obj(crtc, &plane_config);
16267         }
16268
16269         /*
16270          * Make sure hardware watermarks really match the state we read out.
16271          * Note that we need to do this after reconstructing the BIOS fb's
16272          * since the watermark calculation done here will use pstate->fb.
16273          */
16274         if (!HAS_GMCH(dev_priv))
16275                 sanitize_watermarks(dev);
16276
16277         /*
16278          * Force all active planes to recompute their states. So that on
16279          * mode_setcrtc after probe, all the intel_plane_state variables
16280          * are already calculated and there is no assert_plane warnings
16281          * during bootup.
16282          */
16283         ret = intel_initial_commit(dev);
16284         if (ret)
16285                 DRM_DEBUG_KMS("Initial commit in probe failed.\n");
16286
16287         return 0;
16288 }
16289
16290 void i830_enable_pipe(struct drm_i915_private *dev_priv, enum pipe pipe)
16291 {
16292         struct intel_crtc *crtc = intel_get_crtc_for_pipe(dev_priv, pipe);
16293         /* 640x480@60Hz, ~25175 kHz */
16294         struct dpll clock = {
16295                 .m1 = 18,
16296                 .m2 = 7,
16297                 .p1 = 13,
16298                 .p2 = 4,
16299                 .n = 2,
16300         };
16301         u32 dpll, fp;
16302         int i;
16303
16304         WARN_ON(i9xx_calc_dpll_params(48000, &clock) != 25154);
16305
16306         DRM_DEBUG_KMS("enabling pipe %c due to force quirk (vco=%d dot=%d)\n",
16307                       pipe_name(pipe), clock.vco, clock.dot);
16308
16309         fp = i9xx_dpll_compute_fp(&clock);
16310         dpll = DPLL_DVO_2X_MODE |
16311                 DPLL_VGA_MODE_DIS |
16312                 ((clock.p1 - 2) << DPLL_FPA01_P1_POST_DIV_SHIFT) |
16313                 PLL_P2_DIVIDE_BY_4 |
16314                 PLL_REF_INPUT_DREFCLK |
16315                 DPLL_VCO_ENABLE;
16316
16317         I915_WRITE(FP0(pipe), fp);
16318         I915_WRITE(FP1(pipe), fp);
16319
16320         I915_WRITE(HTOTAL(pipe), (640 - 1) | ((800 - 1) << 16));
16321         I915_WRITE(HBLANK(pipe), (640 - 1) | ((800 - 1) << 16));
16322         I915_WRITE(HSYNC(pipe), (656 - 1) | ((752 - 1) << 16));
16323         I915_WRITE(VTOTAL(pipe), (480 - 1) | ((525 - 1) << 16));
16324         I915_WRITE(VBLANK(pipe), (480 - 1) | ((525 - 1) << 16));
16325         I915_WRITE(VSYNC(pipe), (490 - 1) | ((492 - 1) << 16));
16326         I915_WRITE(PIPESRC(pipe), ((640 - 1) << 16) | (480 - 1));
16327
16328         /*
16329          * Apparently we need to have VGA mode enabled prior to changing
16330          * the P1/P2 dividers. Otherwise the DPLL will keep using the old
16331          * dividers, even though the register value does change.
16332          */
16333         I915_WRITE(DPLL(pipe), dpll & ~DPLL_VGA_MODE_DIS);
16334         I915_WRITE(DPLL(pipe), dpll);
16335
16336         /* Wait for the clocks to stabilize. */
16337         POSTING_READ(DPLL(pipe));
16338         udelay(150);
16339
16340         /* The pixel multiplier can only be updated once the
16341          * DPLL is enabled and the clocks are stable.
16342          *
16343          * So write it again.
16344          */
16345         I915_WRITE(DPLL(pipe), dpll);
16346
16347         /* We do this three times for luck */
16348         for (i = 0; i < 3 ; i++) {
16349                 I915_WRITE(DPLL(pipe), dpll);
16350                 POSTING_READ(DPLL(pipe));
16351                 udelay(150); /* wait for warmup */
16352         }
16353
16354         I915_WRITE(PIPECONF(pipe), PIPECONF_ENABLE | PIPECONF_PROGRESSIVE);
16355         POSTING_READ(PIPECONF(pipe));
16356
16357         intel_wait_for_pipe_scanline_moving(crtc);
16358 }
16359
16360 void i830_disable_pipe(struct drm_i915_private *dev_priv, enum pipe pipe)
16361 {
16362         struct intel_crtc *crtc = intel_get_crtc_for_pipe(dev_priv, pipe);
16363
16364         DRM_DEBUG_KMS("disabling pipe %c due to force quirk\n",
16365                       pipe_name(pipe));
16366
16367         WARN_ON(I915_READ(DSPCNTR(PLANE_A)) & DISPLAY_PLANE_ENABLE);
16368         WARN_ON(I915_READ(DSPCNTR(PLANE_B)) & DISPLAY_PLANE_ENABLE);
16369         WARN_ON(I915_READ(DSPCNTR(PLANE_C)) & DISPLAY_PLANE_ENABLE);
16370         WARN_ON(I915_READ(CURCNTR(PIPE_A)) & MCURSOR_MODE);
16371         WARN_ON(I915_READ(CURCNTR(PIPE_B)) & MCURSOR_MODE);
16372
16373         I915_WRITE(PIPECONF(pipe), 0);
16374         POSTING_READ(PIPECONF(pipe));
16375
16376         intel_wait_for_pipe_scanline_stopped(crtc);
16377
16378         I915_WRITE(DPLL(pipe), DPLL_VGA_MODE_DIS);
16379         POSTING_READ(DPLL(pipe));
16380 }
16381
16382 static void
16383 intel_sanitize_plane_mapping(struct drm_i915_private *dev_priv)
16384 {
16385         struct intel_crtc *crtc;
16386
16387         if (INTEL_GEN(dev_priv) >= 4)
16388                 return;
16389
16390         for_each_intel_crtc(&dev_priv->drm, crtc) {
16391                 struct intel_plane *plane =
16392                         to_intel_plane(crtc->base.primary);
16393                 struct intel_crtc *plane_crtc;
16394                 enum pipe pipe;
16395
16396                 if (!plane->get_hw_state(plane, &pipe))
16397                         continue;
16398
16399                 if (pipe == crtc->pipe)
16400                         continue;
16401
16402                 DRM_DEBUG_KMS("[PLANE:%d:%s] attached to the wrong pipe, disabling plane\n",
16403                               plane->base.base.id, plane->base.name);
16404
16405                 plane_crtc = intel_get_crtc_for_pipe(dev_priv, pipe);
16406                 intel_plane_disable_noatomic(plane_crtc, plane);
16407         }
16408 }
16409
16410 static bool intel_crtc_has_encoders(struct intel_crtc *crtc)
16411 {
16412         struct drm_device *dev = crtc->base.dev;
16413         struct intel_encoder *encoder;
16414
16415         for_each_encoder_on_crtc(dev, &crtc->base, encoder)
16416                 return true;
16417
16418         return false;
16419 }
16420
16421 static struct intel_connector *intel_encoder_find_connector(struct intel_encoder *encoder)
16422 {
16423         struct drm_device *dev = encoder->base.dev;
16424         struct intel_connector *connector;
16425
16426         for_each_connector_on_encoder(dev, &encoder->base, connector)
16427                 return connector;
16428
16429         return NULL;
16430 }
16431
16432 static bool has_pch_trancoder(struct drm_i915_private *dev_priv,
16433                               enum pipe pch_transcoder)
16434 {
16435         return HAS_PCH_IBX(dev_priv) || HAS_PCH_CPT(dev_priv) ||
16436                 (HAS_PCH_LPT_H(dev_priv) && pch_transcoder == PIPE_A);
16437 }
16438
16439 static void intel_sanitize_crtc(struct intel_crtc *crtc,
16440                                 struct drm_modeset_acquire_ctx *ctx)
16441 {
16442         struct drm_device *dev = crtc->base.dev;
16443         struct drm_i915_private *dev_priv = to_i915(dev);
16444         struct intel_crtc_state *crtc_state = to_intel_crtc_state(crtc->base.state);
16445         enum transcoder cpu_transcoder = crtc_state->cpu_transcoder;
16446
16447         /* Clear any frame start delays used for debugging left by the BIOS */
16448         if (crtc->active && !transcoder_is_dsi(cpu_transcoder)) {
16449                 i915_reg_t reg = PIPECONF(cpu_transcoder);
16450
16451                 I915_WRITE(reg,
16452                            I915_READ(reg) & ~PIPECONF_FRAME_START_DELAY_MASK);
16453         }
16454
16455         if (crtc_state->base.active) {
16456                 struct intel_plane *plane;
16457
16458                 /* Disable everything but the primary plane */
16459                 for_each_intel_plane_on_crtc(dev, crtc, plane) {
16460                         const struct intel_plane_state *plane_state =
16461                                 to_intel_plane_state(plane->base.state);
16462
16463                         if (plane_state->base.visible &&
16464                             plane->base.type != DRM_PLANE_TYPE_PRIMARY)
16465                                 intel_plane_disable_noatomic(crtc, plane);
16466                 }
16467
16468                 /*
16469                  * Disable any background color set by the BIOS, but enable the
16470                  * gamma and CSC to match how we program our planes.
16471                  */
16472                 if (INTEL_GEN(dev_priv) >= 9)
16473                         I915_WRITE(SKL_BOTTOM_COLOR(crtc->pipe),
16474                                    SKL_BOTTOM_COLOR_GAMMA_ENABLE |
16475                                    SKL_BOTTOM_COLOR_CSC_ENABLE);
16476         }
16477
16478         /* Adjust the state of the output pipe according to whether we
16479          * have active connectors/encoders. */
16480         if (crtc_state->base.active && !intel_crtc_has_encoders(crtc))
16481                 intel_crtc_disable_noatomic(&crtc->base, ctx);
16482
16483         if (crtc_state->base.active || HAS_GMCH(dev_priv)) {
16484                 /*
16485                  * We start out with underrun reporting disabled to avoid races.
16486                  * For correct bookkeeping mark this on active crtcs.
16487                  *
16488                  * Also on gmch platforms we dont have any hardware bits to
16489                  * disable the underrun reporting. Which means we need to start
16490                  * out with underrun reporting disabled also on inactive pipes,
16491                  * since otherwise we'll complain about the garbage we read when
16492                  * e.g. coming up after runtime pm.
16493                  *
16494                  * No protection against concurrent access is required - at
16495                  * worst a fifo underrun happens which also sets this to false.
16496                  */
16497                 crtc->cpu_fifo_underrun_disabled = true;
16498                 /*
16499                  * We track the PCH trancoder underrun reporting state
16500                  * within the crtc. With crtc for pipe A housing the underrun
16501                  * reporting state for PCH transcoder A, crtc for pipe B housing
16502                  * it for PCH transcoder B, etc. LPT-H has only PCH transcoder A,
16503                  * and marking underrun reporting as disabled for the non-existing
16504                  * PCH transcoders B and C would prevent enabling the south
16505                  * error interrupt (see cpt_can_enable_serr_int()).
16506                  */
16507                 if (has_pch_trancoder(dev_priv, crtc->pipe))
16508                         crtc->pch_fifo_underrun_disabled = true;
16509         }
16510 }
16511
16512 static bool has_bogus_dpll_config(const struct intel_crtc_state *crtc_state)
16513 {
16514         struct drm_i915_private *dev_priv = to_i915(crtc_state->base.crtc->dev);
16515
16516         /*
16517          * Some SNB BIOSen (eg. ASUS K53SV) are known to misprogram
16518          * the hardware when a high res displays plugged in. DPLL P
16519          * divider is zero, and the pipe timings are bonkers. We'll
16520          * try to disable everything in that case.
16521          *
16522          * FIXME would be nice to be able to sanitize this state
16523          * without several WARNs, but for now let's take the easy
16524          * road.
16525          */
16526         return IS_GEN(dev_priv, 6) &&
16527                 crtc_state->base.active &&
16528                 crtc_state->shared_dpll &&
16529                 crtc_state->port_clock == 0;
16530 }
16531
16532 static void intel_sanitize_encoder(struct intel_encoder *encoder)
16533 {
16534         struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
16535         struct intel_connector *connector;
16536         struct intel_crtc *crtc = to_intel_crtc(encoder->base.crtc);
16537         struct intel_crtc_state *crtc_state = crtc ?
16538                 to_intel_crtc_state(crtc->base.state) : NULL;
16539
16540         /* We need to check both for a crtc link (meaning that the
16541          * encoder is active and trying to read from a pipe) and the
16542          * pipe itself being active. */
16543         bool has_active_crtc = crtc_state &&
16544                 crtc_state->base.active;
16545
16546         if (crtc_state && has_bogus_dpll_config(crtc_state)) {
16547                 DRM_DEBUG_KMS("BIOS has misprogrammed the hardware. Disabling pipe %c\n",
16548                               pipe_name(crtc->pipe));
16549                 has_active_crtc = false;
16550         }
16551
16552         connector = intel_encoder_find_connector(encoder);
16553         if (connector && !has_active_crtc) {
16554                 DRM_DEBUG_KMS("[ENCODER:%d:%s] has active connectors but no active pipe!\n",
16555                               encoder->base.base.id,
16556                               encoder->base.name);
16557
16558                 /* Connector is active, but has no active pipe. This is
16559                  * fallout from our resume register restoring. Disable
16560                  * the encoder manually again. */
16561                 if (crtc_state) {
16562                         struct drm_encoder *best_encoder;
16563
16564                         DRM_DEBUG_KMS("[ENCODER:%d:%s] manually disabled\n",
16565                                       encoder->base.base.id,
16566                                       encoder->base.name);
16567
16568                         /* avoid oopsing in case the hooks consult best_encoder */
16569                         best_encoder = connector->base.state->best_encoder;
16570                         connector->base.state->best_encoder = &encoder->base;
16571
16572                         if (encoder->disable)
16573                                 encoder->disable(encoder, crtc_state,
16574                                                  connector->base.state);
16575                         if (encoder->post_disable)
16576                                 encoder->post_disable(encoder, crtc_state,
16577                                                       connector->base.state);
16578
16579                         connector->base.state->best_encoder = best_encoder;
16580                 }
16581                 encoder->base.crtc = NULL;
16582
16583                 /* Inconsistent output/port/pipe state happens presumably due to
16584                  * a bug in one of the get_hw_state functions. Or someplace else
16585                  * in our code, like the register restore mess on resume. Clamp
16586                  * things to off as a safer default. */
16587
16588                 connector->base.dpms = DRM_MODE_DPMS_OFF;
16589                 connector->base.encoder = NULL;
16590         }
16591
16592         /* notify opregion of the sanitized encoder state */
16593         intel_opregion_notify_encoder(encoder, connector && has_active_crtc);
16594
16595         if (INTEL_GEN(dev_priv) >= 11)
16596                 icl_sanitize_encoder_pll_mapping(encoder);
16597 }
16598
16599 void i915_redisable_vga_power_on(struct drm_i915_private *dev_priv)
16600 {
16601         i915_reg_t vga_reg = i915_vgacntrl_reg(dev_priv);
16602
16603         if (!(I915_READ(vga_reg) & VGA_DISP_DISABLE)) {
16604                 DRM_DEBUG_KMS("Something enabled VGA plane, disabling it\n");
16605                 i915_disable_vga(dev_priv);
16606         }
16607 }
16608
16609 void i915_redisable_vga(struct drm_i915_private *dev_priv)
16610 {
16611         intel_wakeref_t wakeref;
16612
16613         /*
16614          * This function can be called both from intel_modeset_setup_hw_state or
16615          * at a very early point in our resume sequence, where the power well
16616          * structures are not yet restored. Since this function is at a very
16617          * paranoid "someone might have enabled VGA while we were not looking"
16618          * level, just check if the power well is enabled instead of trying to
16619          * follow the "don't touch the power well if we don't need it" policy
16620          * the rest of the driver uses.
16621          */
16622         wakeref = intel_display_power_get_if_enabled(dev_priv,
16623                                                      POWER_DOMAIN_VGA);
16624         if (!wakeref)
16625                 return;
16626
16627         i915_redisable_vga_power_on(dev_priv);
16628
16629         intel_display_power_put(dev_priv, POWER_DOMAIN_VGA, wakeref);
16630 }
16631
16632 /* FIXME read out full plane state for all planes */
16633 static void readout_plane_state(struct drm_i915_private *dev_priv)
16634 {
16635         struct intel_plane *plane;
16636         struct intel_crtc *crtc;
16637
16638         for_each_intel_plane(&dev_priv->drm, plane) {
16639                 struct intel_plane_state *plane_state =
16640                         to_intel_plane_state(plane->base.state);
16641                 struct intel_crtc_state *crtc_state;
16642                 enum pipe pipe = PIPE_A;
16643                 bool visible;
16644
16645                 visible = plane->get_hw_state(plane, &pipe);
16646
16647                 crtc = intel_get_crtc_for_pipe(dev_priv, pipe);
16648                 crtc_state = to_intel_crtc_state(crtc->base.state);
16649
16650                 intel_set_plane_visible(crtc_state, plane_state, visible);
16651
16652                 DRM_DEBUG_KMS("[PLANE:%d:%s] hw state readout: %s, pipe %c\n",
16653                               plane->base.base.id, plane->base.name,
16654                               enableddisabled(visible), pipe_name(pipe));
16655         }
16656
16657         for_each_intel_crtc(&dev_priv->drm, crtc) {
16658                 struct intel_crtc_state *crtc_state =
16659                         to_intel_crtc_state(crtc->base.state);
16660
16661                 fixup_active_planes(crtc_state);
16662         }
16663 }
16664
16665 static void intel_modeset_readout_hw_state(struct drm_device *dev)
16666 {
16667         struct drm_i915_private *dev_priv = to_i915(dev);
16668         enum pipe pipe;
16669         struct intel_crtc *crtc;
16670         struct intel_encoder *encoder;
16671         struct intel_connector *connector;
16672         struct drm_connector_list_iter conn_iter;
16673         int i;
16674
16675         dev_priv->active_crtcs = 0;
16676
16677         for_each_intel_crtc(dev, crtc) {
16678                 struct intel_crtc_state *crtc_state =
16679                         to_intel_crtc_state(crtc->base.state);
16680
16681                 __drm_atomic_helper_crtc_destroy_state(&crtc_state->base);
16682                 memset(crtc_state, 0, sizeof(*crtc_state));
16683                 __drm_atomic_helper_crtc_reset(&crtc->base, &crtc_state->base);
16684
16685                 crtc_state->base.active = crtc_state->base.enable =
16686                         dev_priv->display.get_pipe_config(crtc, crtc_state);
16687
16688                 crtc->base.enabled = crtc_state->base.enable;
16689                 crtc->active = crtc_state->base.active;
16690
16691                 if (crtc_state->base.active)
16692                         dev_priv->active_crtcs |= 1 << crtc->pipe;
16693
16694                 DRM_DEBUG_KMS("[CRTC:%d:%s] hw state readout: %s\n",
16695                               crtc->base.base.id, crtc->base.name,
16696                               enableddisabled(crtc_state->base.active));
16697         }
16698
16699         readout_plane_state(dev_priv);
16700
16701         for (i = 0; i < dev_priv->num_shared_dpll; i++) {
16702                 struct intel_shared_dpll *pll = &dev_priv->shared_dplls[i];
16703
16704                 pll->on = pll->info->funcs->get_hw_state(dev_priv, pll,
16705                                                         &pll->state.hw_state);
16706
16707                 if (IS_ELKHARTLAKE(dev_priv) && pll->on &&
16708                     pll->info->id == DPLL_ID_EHL_DPLL4) {
16709                         pll->wakeref = intel_display_power_get(dev_priv,
16710                                                                POWER_DOMAIN_DPLL_DC_OFF);
16711                 }
16712
16713                 pll->state.crtc_mask = 0;
16714                 for_each_intel_crtc(dev, crtc) {
16715                         struct intel_crtc_state *crtc_state =
16716                                 to_intel_crtc_state(crtc->base.state);
16717
16718                         if (crtc_state->base.active &&
16719                             crtc_state->shared_dpll == pll)
16720                                 pll->state.crtc_mask |= 1 << crtc->pipe;
16721                 }
16722                 pll->active_mask = pll->state.crtc_mask;
16723
16724                 DRM_DEBUG_KMS("%s hw state readout: crtc_mask 0x%08x, on %i\n",
16725                               pll->info->name, pll->state.crtc_mask, pll->on);
16726         }
16727
16728         for_each_intel_encoder(dev, encoder) {
16729                 pipe = 0;
16730
16731                 if (encoder->get_hw_state(encoder, &pipe)) {
16732                         struct intel_crtc_state *crtc_state;
16733
16734                         crtc = intel_get_crtc_for_pipe(dev_priv, pipe);
16735                         crtc_state = to_intel_crtc_state(crtc->base.state);
16736
16737                         encoder->base.crtc = &crtc->base;
16738                         encoder->get_config(encoder, crtc_state);
16739                 } else {
16740                         encoder->base.crtc = NULL;
16741                 }
16742
16743                 DRM_DEBUG_KMS("[ENCODER:%d:%s] hw state readout: %s, pipe %c\n",
16744                               encoder->base.base.id, encoder->base.name,
16745                               enableddisabled(encoder->base.crtc),
16746                               pipe_name(pipe));
16747         }
16748
16749         drm_connector_list_iter_begin(dev, &conn_iter);
16750         for_each_intel_connector_iter(connector, &conn_iter) {
16751                 if (connector->get_hw_state(connector)) {
16752                         connector->base.dpms = DRM_MODE_DPMS_ON;
16753
16754                         encoder = connector->encoder;
16755                         connector->base.encoder = &encoder->base;
16756
16757                         if (encoder->base.crtc &&
16758                             encoder->base.crtc->state->active) {
16759                                 /*
16760                                  * This has to be done during hardware readout
16761                                  * because anything calling .crtc_disable may
16762                                  * rely on the connector_mask being accurate.
16763                                  */
16764                                 encoder->base.crtc->state->connector_mask |=
16765                                         drm_connector_mask(&connector->base);
16766                                 encoder->base.crtc->state->encoder_mask |=
16767                                         drm_encoder_mask(&encoder->base);
16768                         }
16769
16770                 } else {
16771                         connector->base.dpms = DRM_MODE_DPMS_OFF;
16772                         connector->base.encoder = NULL;
16773                 }
16774                 DRM_DEBUG_KMS("[CONNECTOR:%d:%s] hw state readout: %s\n",
16775                               connector->base.base.id, connector->base.name,
16776                               enableddisabled(connector->base.encoder));
16777         }
16778         drm_connector_list_iter_end(&conn_iter);
16779
16780         for_each_intel_crtc(dev, crtc) {
16781                 struct intel_bw_state *bw_state =
16782                         to_intel_bw_state(dev_priv->bw_obj.state);
16783                 struct intel_crtc_state *crtc_state =
16784                         to_intel_crtc_state(crtc->base.state);
16785                 struct intel_plane *plane;
16786                 int min_cdclk = 0;
16787
16788                 memset(&crtc->base.mode, 0, sizeof(crtc->base.mode));
16789                 if (crtc_state->base.active) {
16790                         intel_mode_from_pipe_config(&crtc->base.mode, crtc_state);
16791                         crtc->base.mode.hdisplay = crtc_state->pipe_src_w;
16792                         crtc->base.mode.vdisplay = crtc_state->pipe_src_h;
16793                         intel_mode_from_pipe_config(&crtc_state->base.adjusted_mode, crtc_state);
16794                         WARN_ON(drm_atomic_set_mode_for_crtc(crtc->base.state, &crtc->base.mode));
16795
16796                         /*
16797                          * The initial mode needs to be set in order to keep
16798                          * the atomic core happy. It wants a valid mode if the
16799                          * crtc's enabled, so we do the above call.
16800                          *
16801                          * But we don't set all the derived state fully, hence
16802                          * set a flag to indicate that a full recalculation is
16803                          * needed on the next commit.
16804                          */
16805                         crtc_state->base.mode.private_flags = I915_MODE_FLAG_INHERITED;
16806
16807                         intel_crtc_compute_pixel_rate(crtc_state);
16808
16809                         if (dev_priv->display.modeset_calc_cdclk) {
16810                                 min_cdclk = intel_crtc_compute_min_cdclk(crtc_state);
16811                                 if (WARN_ON(min_cdclk < 0))
16812                                         min_cdclk = 0;
16813                         }
16814
16815                         drm_calc_timestamping_constants(&crtc->base,
16816                                                         &crtc_state->base.adjusted_mode);
16817                         update_scanline_offset(crtc_state);
16818                 }
16819
16820                 dev_priv->min_cdclk[crtc->pipe] = min_cdclk;
16821                 dev_priv->min_voltage_level[crtc->pipe] =
16822                         crtc_state->min_voltage_level;
16823
16824                 for_each_intel_plane_on_crtc(&dev_priv->drm, crtc, plane) {
16825                         const struct intel_plane_state *plane_state =
16826                                 to_intel_plane_state(plane->base.state);
16827
16828                         /*
16829                          * FIXME don't have the fb yet, so can't
16830                          * use intel_plane_data_rate() :(
16831                          */
16832                         if (plane_state->base.visible)
16833                                 crtc_state->data_rate[plane->id] =
16834                                         4 * crtc_state->pixel_rate;
16835                 }
16836
16837                 intel_bw_crtc_update(bw_state, crtc_state);
16838
16839                 intel_pipe_config_sanity_check(dev_priv, crtc_state);
16840         }
16841 }
16842
16843 static void
16844 get_encoder_power_domains(struct drm_i915_private *dev_priv)
16845 {
16846         struct intel_encoder *encoder;
16847
16848         for_each_intel_encoder(&dev_priv->drm, encoder) {
16849                 struct intel_crtc_state *crtc_state;
16850
16851                 if (!encoder->get_power_domains)
16852                         continue;
16853
16854                 /*
16855                  * MST-primary and inactive encoders don't have a crtc state
16856                  * and neither of these require any power domain references.
16857                  */
16858                 if (!encoder->base.crtc)
16859                         continue;
16860
16861                 crtc_state = to_intel_crtc_state(encoder->base.crtc->state);
16862                 encoder->get_power_domains(encoder, crtc_state);
16863         }
16864 }
16865
16866 static void intel_early_display_was(struct drm_i915_private *dev_priv)
16867 {
16868         /* Display WA #1185 WaDisableDARBFClkGating:cnl,glk */
16869         if (IS_CANNONLAKE(dev_priv) || IS_GEMINILAKE(dev_priv))
16870                 I915_WRITE(GEN9_CLKGATE_DIS_0, I915_READ(GEN9_CLKGATE_DIS_0) |
16871                            DARBF_GATING_DIS);
16872
16873         if (IS_HASWELL(dev_priv)) {
16874                 /*
16875                  * WaRsPkgCStateDisplayPMReq:hsw
16876                  * System hang if this isn't done before disabling all planes!
16877                  */
16878                 I915_WRITE(CHICKEN_PAR1_1,
16879                            I915_READ(CHICKEN_PAR1_1) | FORCE_ARB_IDLE_PLANES);
16880         }
16881 }
16882
16883 static void ibx_sanitize_pch_hdmi_port(struct drm_i915_private *dev_priv,
16884                                        enum port port, i915_reg_t hdmi_reg)
16885 {
16886         u32 val = I915_READ(hdmi_reg);
16887
16888         if (val & SDVO_ENABLE ||
16889             (val & SDVO_PIPE_SEL_MASK) == SDVO_PIPE_SEL(PIPE_A))
16890                 return;
16891
16892         DRM_DEBUG_KMS("Sanitizing transcoder select for HDMI %c\n",
16893                       port_name(port));
16894
16895         val &= ~SDVO_PIPE_SEL_MASK;
16896         val |= SDVO_PIPE_SEL(PIPE_A);
16897
16898         I915_WRITE(hdmi_reg, val);
16899 }
16900
16901 static void ibx_sanitize_pch_dp_port(struct drm_i915_private *dev_priv,
16902                                      enum port port, i915_reg_t dp_reg)
16903 {
16904         u32 val = I915_READ(dp_reg);
16905
16906         if (val & DP_PORT_EN ||
16907             (val & DP_PIPE_SEL_MASK) == DP_PIPE_SEL(PIPE_A))
16908                 return;
16909
16910         DRM_DEBUG_KMS("Sanitizing transcoder select for DP %c\n",
16911                       port_name(port));
16912
16913         val &= ~DP_PIPE_SEL_MASK;
16914         val |= DP_PIPE_SEL(PIPE_A);
16915
16916         I915_WRITE(dp_reg, val);
16917 }
16918
16919 static void ibx_sanitize_pch_ports(struct drm_i915_private *dev_priv)
16920 {
16921         /*
16922          * The BIOS may select transcoder B on some of the PCH
16923          * ports even it doesn't enable the port. This would trip
16924          * assert_pch_dp_disabled() and assert_pch_hdmi_disabled().
16925          * Sanitize the transcoder select bits to prevent that. We
16926          * assume that the BIOS never actually enabled the port,
16927          * because if it did we'd actually have to toggle the port
16928          * on and back off to make the transcoder A select stick
16929          * (see. intel_dp_link_down(), intel_disable_hdmi(),
16930          * intel_disable_sdvo()).
16931          */
16932         ibx_sanitize_pch_dp_port(dev_priv, PORT_B, PCH_DP_B);
16933         ibx_sanitize_pch_dp_port(dev_priv, PORT_C, PCH_DP_C);
16934         ibx_sanitize_pch_dp_port(dev_priv, PORT_D, PCH_DP_D);
16935
16936         /* PCH SDVOB multiplex with HDMIB */
16937         ibx_sanitize_pch_hdmi_port(dev_priv, PORT_B, PCH_HDMIB);
16938         ibx_sanitize_pch_hdmi_port(dev_priv, PORT_C, PCH_HDMIC);
16939         ibx_sanitize_pch_hdmi_port(dev_priv, PORT_D, PCH_HDMID);
16940 }
16941
16942 /* Scan out the current hw modeset state,
16943  * and sanitizes it to the current state
16944  */
16945 static void
16946 intel_modeset_setup_hw_state(struct drm_device *dev,
16947                              struct drm_modeset_acquire_ctx *ctx)
16948 {
16949         struct drm_i915_private *dev_priv = to_i915(dev);
16950         struct intel_crtc_state *crtc_state;
16951         struct intel_encoder *encoder;
16952         struct intel_crtc *crtc;
16953         intel_wakeref_t wakeref;
16954         int i;
16955
16956         wakeref = intel_display_power_get(dev_priv, POWER_DOMAIN_INIT);
16957
16958         intel_early_display_was(dev_priv);
16959         intel_modeset_readout_hw_state(dev);
16960
16961         /* HW state is read out, now we need to sanitize this mess. */
16962
16963         /* Sanitize the TypeC port mode upfront, encoders depend on this */
16964         for_each_intel_encoder(dev, encoder) {
16965                 /* We need to sanitize only the MST primary port. */
16966                 if (encoder->type != INTEL_OUTPUT_DP_MST &&
16967                     intel_port_is_tc(dev_priv, encoder->port))
16968                         intel_tc_port_sanitize(enc_to_dig_port(&encoder->base));
16969         }
16970
16971         get_encoder_power_domains(dev_priv);
16972
16973         if (HAS_PCH_IBX(dev_priv))
16974                 ibx_sanitize_pch_ports(dev_priv);
16975
16976         /*
16977          * intel_sanitize_plane_mapping() may need to do vblank
16978          * waits, so we need vblank interrupts restored beforehand.
16979          */
16980         for_each_intel_crtc(&dev_priv->drm, crtc) {
16981                 crtc_state = to_intel_crtc_state(crtc->base.state);
16982
16983                 drm_crtc_vblank_reset(&crtc->base);
16984
16985                 if (crtc_state->base.active)
16986                         intel_crtc_vblank_on(crtc_state);
16987         }
16988
16989         intel_sanitize_plane_mapping(dev_priv);
16990
16991         for_each_intel_encoder(dev, encoder)
16992                 intel_sanitize_encoder(encoder);
16993
16994         for_each_intel_crtc(&dev_priv->drm, crtc) {
16995                 crtc_state = to_intel_crtc_state(crtc->base.state);
16996                 intel_sanitize_crtc(crtc, ctx);
16997                 intel_dump_pipe_config(crtc_state, NULL, "[setup_hw_state]");
16998         }
16999
17000         intel_modeset_update_connector_atomic_state(dev);
17001
17002         for (i = 0; i < dev_priv->num_shared_dpll; i++) {
17003                 struct intel_shared_dpll *pll = &dev_priv->shared_dplls[i];
17004
17005                 if (!pll->on || pll->active_mask)
17006                         continue;
17007
17008                 DRM_DEBUG_KMS("%s enabled but not in use, disabling\n",
17009                               pll->info->name);
17010
17011                 pll->info->funcs->disable(dev_priv, pll);
17012                 pll->on = false;
17013         }
17014
17015         if (IS_G4X(dev_priv)) {
17016                 g4x_wm_get_hw_state(dev_priv);
17017                 g4x_wm_sanitize(dev_priv);
17018         } else if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) {
17019                 vlv_wm_get_hw_state(dev_priv);
17020                 vlv_wm_sanitize(dev_priv);
17021         } else if (INTEL_GEN(dev_priv) >= 9) {
17022                 skl_wm_get_hw_state(dev_priv);
17023         } else if (HAS_PCH_SPLIT(dev_priv)) {
17024                 ilk_wm_get_hw_state(dev_priv);
17025         }
17026
17027         for_each_intel_crtc(dev, crtc) {
17028                 u64 put_domains;
17029
17030                 crtc_state = to_intel_crtc_state(crtc->base.state);
17031                 put_domains = modeset_get_crtc_power_domains(crtc, crtc_state);
17032                 if (WARN_ON(put_domains))
17033                         modeset_put_power_domains(dev_priv, put_domains);
17034         }
17035
17036         intel_display_power_put(dev_priv, POWER_DOMAIN_INIT, wakeref);
17037
17038         intel_fbc_init_pipe_state(dev_priv);
17039 }
17040
17041 void intel_display_resume(struct drm_device *dev)
17042 {
17043         struct drm_i915_private *dev_priv = to_i915(dev);
17044         struct drm_atomic_state *state = dev_priv->modeset_restore_state;
17045         struct drm_modeset_acquire_ctx ctx;
17046         int ret;
17047
17048         dev_priv->modeset_restore_state = NULL;
17049         if (state)
17050                 state->acquire_ctx = &ctx;
17051
17052         drm_modeset_acquire_init(&ctx, 0);
17053
17054         while (1) {
17055                 ret = drm_modeset_lock_all_ctx(dev, &ctx);
17056                 if (ret != -EDEADLK)
17057                         break;
17058
17059                 drm_modeset_backoff(&ctx);
17060         }
17061
17062         if (!ret)
17063                 ret = __intel_display_resume(dev, state, &ctx);
17064
17065         intel_enable_ipc(dev_priv);
17066         drm_modeset_drop_locks(&ctx);
17067         drm_modeset_acquire_fini(&ctx);
17068
17069         if (ret)
17070                 DRM_ERROR("Restoring old state failed with %i\n", ret);
17071         if (state)
17072                 drm_atomic_state_put(state);
17073 }
17074
17075 static void intel_hpd_poll_fini(struct drm_device *dev)
17076 {
17077         struct intel_connector *connector;
17078         struct drm_connector_list_iter conn_iter;
17079
17080         /* Kill all the work that may have been queued by hpd. */
17081         drm_connector_list_iter_begin(dev, &conn_iter);
17082         for_each_intel_connector_iter(connector, &conn_iter) {
17083                 if (connector->modeset_retry_work.func)
17084                         cancel_work_sync(&connector->modeset_retry_work);
17085                 if (connector->hdcp.shim) {
17086                         cancel_delayed_work_sync(&connector->hdcp.check_work);
17087                         cancel_work_sync(&connector->hdcp.prop_work);
17088                 }
17089         }
17090         drm_connector_list_iter_end(&conn_iter);
17091 }
17092
17093 void intel_modeset_cleanup(struct drm_device *dev)
17094 {
17095         struct drm_i915_private *dev_priv = to_i915(dev);
17096
17097         flush_workqueue(dev_priv->modeset_wq);
17098
17099         flush_work(&dev_priv->atomic_helper.free_work);
17100         WARN_ON(!llist_empty(&dev_priv->atomic_helper.free_list));
17101
17102         /*
17103          * Interrupts and polling as the first thing to avoid creating havoc.
17104          * Too much stuff here (turning of connectors, ...) would
17105          * experience fancy races otherwise.
17106          */
17107         intel_irq_uninstall(dev_priv);
17108
17109         /*
17110          * Due to the hpd irq storm handling the hotplug work can re-arm the
17111          * poll handlers. Hence disable polling after hpd handling is shut down.
17112          */
17113         intel_hpd_poll_fini(dev);
17114
17115         /* poll work can call into fbdev, hence clean that up afterwards */
17116         intel_fbdev_fini(dev_priv);
17117
17118         intel_unregister_dsm_handler();
17119
17120         intel_fbc_global_disable(dev_priv);
17121
17122         /* flush any delayed tasks or pending work */
17123         flush_scheduled_work();
17124
17125         intel_hdcp_component_fini(dev_priv);
17126
17127         drm_mode_config_cleanup(dev);
17128
17129         intel_overlay_cleanup(dev_priv);
17130
17131         intel_gmbus_teardown(dev_priv);
17132
17133         destroy_workqueue(dev_priv->modeset_wq);
17134
17135         intel_fbc_cleanup_cfb(dev_priv);
17136 }
17137
17138 /*
17139  * set vga decode state - true == enable VGA decode
17140  */
17141 int intel_modeset_vga_set_state(struct drm_i915_private *dev_priv, bool state)
17142 {
17143         unsigned reg = INTEL_GEN(dev_priv) >= 6 ? SNB_GMCH_CTRL : INTEL_GMCH_CTRL;
17144         u16 gmch_ctrl;
17145
17146         if (pci_read_config_word(dev_priv->bridge_dev, reg, &gmch_ctrl)) {
17147                 DRM_ERROR("failed to read control word\n");
17148                 return -EIO;
17149         }
17150
17151         if (!!(gmch_ctrl & INTEL_GMCH_VGA_DISABLE) == !state)
17152                 return 0;
17153
17154         if (state)
17155                 gmch_ctrl &= ~INTEL_GMCH_VGA_DISABLE;
17156         else
17157                 gmch_ctrl |= INTEL_GMCH_VGA_DISABLE;
17158
17159         if (pci_write_config_word(dev_priv->bridge_dev, reg, gmch_ctrl)) {
17160                 DRM_ERROR("failed to write control word\n");
17161                 return -EIO;
17162         }
17163
17164         return 0;
17165 }
17166
17167 #if IS_ENABLED(CONFIG_DRM_I915_CAPTURE_ERROR)
17168
17169 struct intel_display_error_state {
17170
17171         u32 power_well_driver;
17172
17173         struct intel_cursor_error_state {
17174                 u32 control;
17175                 u32 position;
17176                 u32 base;
17177                 u32 size;
17178         } cursor[I915_MAX_PIPES];
17179
17180         struct intel_pipe_error_state {
17181                 bool power_domain_on;
17182                 u32 source;
17183                 u32 stat;
17184         } pipe[I915_MAX_PIPES];
17185
17186         struct intel_plane_error_state {
17187                 u32 control;
17188                 u32 stride;
17189                 u32 size;
17190                 u32 pos;
17191                 u32 addr;
17192                 u32 surface;
17193                 u32 tile_offset;
17194         } plane[I915_MAX_PIPES];
17195
17196         struct intel_transcoder_error_state {
17197                 bool available;
17198                 bool power_domain_on;
17199                 enum transcoder cpu_transcoder;
17200
17201                 u32 conf;
17202
17203                 u32 htotal;
17204                 u32 hblank;
17205                 u32 hsync;
17206                 u32 vtotal;
17207                 u32 vblank;
17208                 u32 vsync;
17209         } transcoder[4];
17210 };
17211
17212 struct intel_display_error_state *
17213 intel_display_capture_error_state(struct drm_i915_private *dev_priv)
17214 {
17215         struct intel_display_error_state *error;
17216         int transcoders[] = {
17217                 TRANSCODER_A,
17218                 TRANSCODER_B,
17219                 TRANSCODER_C,
17220                 TRANSCODER_EDP,
17221         };
17222         int i;
17223
17224         BUILD_BUG_ON(ARRAY_SIZE(transcoders) != ARRAY_SIZE(error->transcoder));
17225
17226         if (!HAS_DISPLAY(dev_priv))
17227                 return NULL;
17228
17229         error = kzalloc(sizeof(*error), GFP_ATOMIC);
17230         if (error == NULL)
17231                 return NULL;
17232
17233         if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
17234                 error->power_well_driver = I915_READ(HSW_PWR_WELL_CTL2);
17235
17236         for_each_pipe(dev_priv, i) {
17237                 error->pipe[i].power_domain_on =
17238                         __intel_display_power_is_enabled(dev_priv,
17239                                                          POWER_DOMAIN_PIPE(i));
17240                 if (!error->pipe[i].power_domain_on)
17241                         continue;
17242
17243                 error->cursor[i].control = I915_READ(CURCNTR(i));
17244                 error->cursor[i].position = I915_READ(CURPOS(i));
17245                 error->cursor[i].base = I915_READ(CURBASE(i));
17246
17247                 error->plane[i].control = I915_READ(DSPCNTR(i));
17248                 error->plane[i].stride = I915_READ(DSPSTRIDE(i));
17249                 if (INTEL_GEN(dev_priv) <= 3) {
17250                         error->plane[i].size = I915_READ(DSPSIZE(i));
17251                         error->plane[i].pos = I915_READ(DSPPOS(i));
17252                 }
17253                 if (INTEL_GEN(dev_priv) <= 7 && !IS_HASWELL(dev_priv))
17254                         error->plane[i].addr = I915_READ(DSPADDR(i));
17255                 if (INTEL_GEN(dev_priv) >= 4) {
17256                         error->plane[i].surface = I915_READ(DSPSURF(i));
17257                         error->plane[i].tile_offset = I915_READ(DSPTILEOFF(i));
17258                 }
17259
17260                 error->pipe[i].source = I915_READ(PIPESRC(i));
17261
17262                 if (HAS_GMCH(dev_priv))
17263                         error->pipe[i].stat = I915_READ(PIPESTAT(i));
17264         }
17265
17266         for (i = 0; i < ARRAY_SIZE(error->transcoder); i++) {
17267                 enum transcoder cpu_transcoder = transcoders[i];
17268
17269                 if (!INTEL_INFO(dev_priv)->trans_offsets[cpu_transcoder])
17270                         continue;
17271
17272                 error->transcoder[i].available = true;
17273                 error->transcoder[i].power_domain_on =
17274                         __intel_display_power_is_enabled(dev_priv,
17275                                 POWER_DOMAIN_TRANSCODER(cpu_transcoder));
17276                 if (!error->transcoder[i].power_domain_on)
17277                         continue;
17278
17279                 error->transcoder[i].cpu_transcoder = cpu_transcoder;
17280
17281                 error->transcoder[i].conf = I915_READ(PIPECONF(cpu_transcoder));
17282                 error->transcoder[i].htotal = I915_READ(HTOTAL(cpu_transcoder));
17283                 error->transcoder[i].hblank = I915_READ(HBLANK(cpu_transcoder));
17284                 error->transcoder[i].hsync = I915_READ(HSYNC(cpu_transcoder));
17285                 error->transcoder[i].vtotal = I915_READ(VTOTAL(cpu_transcoder));
17286                 error->transcoder[i].vblank = I915_READ(VBLANK(cpu_transcoder));
17287                 error->transcoder[i].vsync = I915_READ(VSYNC(cpu_transcoder));
17288         }
17289
17290         return error;
17291 }
17292
17293 #define err_printf(e, ...) i915_error_printf(e, __VA_ARGS__)
17294
17295 void
17296 intel_display_print_error_state(struct drm_i915_error_state_buf *m,
17297                                 struct intel_display_error_state *error)
17298 {
17299         struct drm_i915_private *dev_priv = m->i915;
17300         int i;
17301
17302         if (!error)
17303                 return;
17304
17305         err_printf(m, "Num Pipes: %d\n", INTEL_INFO(dev_priv)->num_pipes);
17306         if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
17307                 err_printf(m, "PWR_WELL_CTL2: %08x\n",
17308                            error->power_well_driver);
17309         for_each_pipe(dev_priv, i) {
17310                 err_printf(m, "Pipe [%d]:\n", i);
17311                 err_printf(m, "  Power: %s\n",
17312                            onoff(error->pipe[i].power_domain_on));
17313                 err_printf(m, "  SRC: %08x\n", error->pipe[i].source);
17314                 err_printf(m, "  STAT: %08x\n", error->pipe[i].stat);
17315
17316                 err_printf(m, "Plane [%d]:\n", i);
17317                 err_printf(m, "  CNTR: %08x\n", error->plane[i].control);
17318                 err_printf(m, "  STRIDE: %08x\n", error->plane[i].stride);
17319                 if (INTEL_GEN(dev_priv) <= 3) {
17320                         err_printf(m, "  SIZE: %08x\n", error->plane[i].size);
17321                         err_printf(m, "  POS: %08x\n", error->plane[i].pos);
17322                 }
17323                 if (INTEL_GEN(dev_priv) <= 7 && !IS_HASWELL(dev_priv))
17324                         err_printf(m, "  ADDR: %08x\n", error->plane[i].addr);
17325                 if (INTEL_GEN(dev_priv) >= 4) {
17326                         err_printf(m, "  SURF: %08x\n", error->plane[i].surface);
17327                         err_printf(m, "  TILEOFF: %08x\n", error->plane[i].tile_offset);
17328                 }
17329
17330                 err_printf(m, "Cursor [%d]:\n", i);
17331                 err_printf(m, "  CNTR: %08x\n", error->cursor[i].control);
17332                 err_printf(m, "  POS: %08x\n", error->cursor[i].position);
17333                 err_printf(m, "  BASE: %08x\n", error->cursor[i].base);
17334         }
17335
17336         for (i = 0; i < ARRAY_SIZE(error->transcoder); i++) {
17337                 if (!error->transcoder[i].available)
17338                         continue;
17339
17340                 err_printf(m, "CPU transcoder: %s\n",
17341                            transcoder_name(error->transcoder[i].cpu_transcoder));
17342                 err_printf(m, "  Power: %s\n",
17343                            onoff(error->transcoder[i].power_domain_on));
17344                 err_printf(m, "  CONF: %08x\n", error->transcoder[i].conf);
17345                 err_printf(m, "  HTOTAL: %08x\n", error->transcoder[i].htotal);
17346                 err_printf(m, "  HBLANK: %08x\n", error->transcoder[i].hblank);
17347                 err_printf(m, "  HSYNC: %08x\n", error->transcoder[i].hsync);
17348                 err_printf(m, "  VTOTAL: %08x\n", error->transcoder[i].vtotal);
17349                 err_printf(m, "  VBLANK: %08x\n", error->transcoder[i].vblank);
17350                 err_printf(m, "  VSYNC: %08x\n", error->transcoder[i].vsync);
17351         }
17352 }
17353
17354 #endif