drivers/gpu/drm/i915/i915_drv.c

   1 /* i915_drv.c -- i830,i845,i855,i865,i915 driver -*- linux-c -*-
   2  */
   3 /*
   4  *
   5  * Copyright 2003 Tungsten Graphics, Inc., Cedar Park, Texas.
   6  * All Rights Reserved.
   7  *
   8  * Permission is hereby granted, free of charge, to any person obtaining a
   9  * copy of this software and associated documentation files (the
  10  * "Software"), to deal in the Software without restriction, including
  11  * without limitation the rights to use, copy, modify, merge, publish,
  12  * distribute, sub license, and/or sell copies of the Software, and to
  13  * permit persons to whom the Software is furnished to do so, subject to
  14  * the following conditions:
  15  *
  16  * The above copyright notice and this permission notice (including the
  17  * next paragraph) shall be included in all copies or substantial portions
  18  * of the Software.
  19  *
  20  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
  21  * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  22  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT.
  23  * IN NO EVENT SHALL TUNGSTEN GRAPHICS AND/OR ITS SUPPLIERS BE LIABLE FOR
  24  * ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
  25  * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
  26  * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  27  *
  28  */
  29
  30 #include <linux/acpi.h>
  31 #include <linux/device.h>
  32 #include <linux/oom.h>
  33 #include <linux/module.h>
  34 #include <linux/pci.h>
  35 #include <linux/pm.h>
  36 #include <linux/pm_runtime.h>
  37 #include <linux/pnp.h>
  38 #include <linux/slab.h>
  39 #include <linux/vga_switcheroo.h>
  40 #include <linux/vt.h>
  41 #include <acpi/video.h>
  42
  43 #include <drm/drm_atomic_helper.h>
  44 #include <drm/drm_ioctl.h>
  45 #include <drm/drm_irq.h>
  46 #include <drm/drm_probe_helper.h>
  47 #include <drm/i915_drm.h>
  48
  49 #include "display/intel_acpi.h"
  50 #include "display/intel_audio.h"
  51 #include "display/intel_bw.h"
  52 #include "display/intel_cdclk.h"
  53 #include "display/intel_display_types.h"
  54 #include "display/intel_dp.h"
  55 #include "display/intel_fbdev.h"
  56 #include "display/intel_hotplug.h"
  57 #include "display/intel_overlay.h"
  58 #include "display/intel_pipe_crc.h"
  59 #include "display/intel_sprite.h"
  60 #include "display/intel_vga.h"
  61
  62 #include "gem/i915_gem_context.h"
  63 #include "gem/i915_gem_ioctls.h"
  64 #include "gt/intel_gt.h"
  65 #include "gt/intel_gt_pm.h"
  66 #include "gt/intel_rc6.h"
  67
  68 #include "i915_debugfs.h"
  69 #include "i915_drv.h"
  70 #include "i915_irq.h"
  71 #include "i915_memcpy.h"
  72 #include "i915_perf.h"
  73 #include "i915_query.h"
  74 #include "i915_suspend.h"
  75 #include "i915_switcheroo.h"
  76 #include "i915_sysfs.h"
  77 #include "i915_trace.h"
  78 #include "i915_vgpu.h"
  79 #include "intel_csr.h"
  80 #include "intel_memory_region.h"
  81 #include "intel_pm.h"
  82
  83 static struct drm_driver driver;
  84
  85 struct vlv_s0ix_state {
  86         /* GAM */
  87         u32 wr_watermark;
  88         u32 gfx_prio_ctrl;
  89         u32 arb_mode;
  90         u32 gfx_pend_tlb0;
  91         u32 gfx_pend_tlb1;
  92         u32 lra_limits[GEN7_LRA_LIMITS_REG_NUM];
  93         u32 media_max_req_count;
  94         u32 gfx_max_req_count;
  95         u32 render_hwsp;
  96         u32 ecochk;
  97         u32 bsd_hwsp;
  98         u32 blt_hwsp;
  99         u32 tlb_rd_addr;
 100
 101         /* MBC */
 102         u32 g3dctl;
 103         u32 gsckgctl;
 104         u32 mbctl;
 105
 106         /* GCP */
 107         u32 ucgctl1;
 108         u32 ucgctl3;
 109         u32 rcgctl1;
 110         u32 rcgctl2;
 111         u32 rstctl;
 112         u32 misccpctl;
 113
 114         /* GPM */
 115         u32 gfxpause;
 116         u32 rpdeuhwtc;
 117         u32 rpdeuc;
 118         u32 ecobus;
 119         u32 pwrdwnupctl;
 120         u32 rp_down_timeout;
 121         u32 rp_deucsw;
 122         u32 rcubmabdtmr;
 123         u32 rcedata;
 124         u32 spare2gh;
 125
 126         /* Display 1 CZ domain */
 127         u32 gt_imr;
 128         u32 gt_ier;
 129         u32 pm_imr;
 130         u32 pm_ier;
 131         u32 gt_scratch[GEN7_GT_SCRATCH_REG_NUM];
 132
 133         /* GT SA CZ domain */
 134         u32 tilectl;
 135         u32 gt_fifoctl;
 136         u32 gtlc_wake_ctrl;
 137         u32 gtlc_survive;
 138         u32 pmwgicz;
 139
 140         /* Display 2 CZ domain */
 141         u32 gu_ctl0;
 142         u32 gu_ctl1;
 143         u32 pcbr;
 144         u32 clock_gate_dis2;
 145 };
 146
 147 static int i915_get_bridge_dev(struct drm_i915_private *dev_priv)
 148 {
 149         int domain = pci_domain_nr(dev_priv->drm.pdev->bus);
 150
 151         dev_priv->bridge_dev =
 152                 pci_get_domain_bus_and_slot(domain, 0, PCI_DEVFN(0, 0));
 153         if (!dev_priv->bridge_dev) {
 154                 DRM_ERROR("bridge device not found\n");
 155                 return -1;
 156         }
 157         return 0;
 158 }
 159
 160 /* Allocate space for the MCH regs if needed, return nonzero on error */
 161 static int
 162 intel_alloc_mchbar_resource(struct drm_i915_private *dev_priv)
 163 {
 164         int reg = INTEL_GEN(dev_priv) >= 4 ? MCHBAR_I965 : MCHBAR_I915;
 165         u32 temp_lo, temp_hi = 0;
 166         u64 mchbar_addr;
 167         int ret;
 168
 169         if (INTEL_GEN(dev_priv) >= 4)
 170                 pci_read_config_dword(dev_priv->bridge_dev, reg + 4, &temp_hi);
 171         pci_read_config_dword(dev_priv->bridge_dev, reg, &temp_lo);
 172         mchbar_addr = ((u64)temp_hi << 32) | temp_lo;
 173
 174         /* If ACPI doesn't have it, assume we need to allocate it ourselves */
 175 #ifdef CONFIG_PNP
 176         if (mchbar_addr &&
 177             pnp_range_reserved(mchbar_addr, mchbar_addr + MCHBAR_SIZE))
 178                 return 0;
 179 #endif
 180
 181         /* Get some space for it */
 182         dev_priv->mch_res.name = "i915 MCHBAR";
 183         dev_priv->mch_res.flags = IORESOURCE_MEM;
 184         ret = pci_bus_alloc_resource(dev_priv->bridge_dev->bus,
 185                                      &dev_priv->mch_res,
 186                                      MCHBAR_SIZE, MCHBAR_SIZE,
 187                                      PCIBIOS_MIN_MEM,
 188                                      0, pcibios_align_resource,
 189                                      dev_priv->bridge_dev);
 190         if (ret) {
 191                 DRM_DEBUG_DRIVER("failed bus alloc: %d\n", ret);
 192                 dev_priv->mch_res.start = 0;
 193                 return ret;
 194         }
 195
 196         if (INTEL_GEN(dev_priv) >= 4)
 197                 pci_write_config_dword(dev_priv->bridge_dev, reg + 4,
 198                                        upper_32_bits(dev_priv->mch_res.start));
 199
 200         pci_write_config_dword(dev_priv->bridge_dev, reg,
 201                                lower_32_bits(dev_priv->mch_res.start));
 202         return 0;
 203 }
 204
 205 /* Setup MCHBAR if possible, return true if we should disable it again */
 206 static void
 207 intel_setup_mchbar(struct drm_i915_private *dev_priv)
 208 {
 209         int mchbar_reg = INTEL_GEN(dev_priv) >= 4 ? MCHBAR_I965 : MCHBAR_I915;
 210         u32 temp;
 211         bool enabled;
 212
 213         if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
 214                 return;
 215
 216         dev_priv->mchbar_need_disable = false;
 217
 218         if (IS_I915G(dev_priv) || IS_I915GM(dev_priv)) {
 219                 pci_read_config_dword(dev_priv->bridge_dev, DEVEN, &temp);
 220                 enabled = !!(temp & DEVEN_MCHBAR_EN);
 221         } else {
 222                 pci_read_config_dword(dev_priv->bridge_dev, mchbar_reg, &temp);
 223                 enabled = temp & 1;
 224         }
 225
 226         /* If it's already enabled, don't have to do anything */
 227         if (enabled)
 228                 return;
 229
 230         if (intel_alloc_mchbar_resource(dev_priv))
 231                 return;
 232
 233         dev_priv->mchbar_need_disable = true;
 234
 235         /* Space is allocated or reserved, so enable it. */
 236         if (IS_I915G(dev_priv) || IS_I915GM(dev_priv)) {
 237                 pci_write_config_dword(dev_priv->bridge_dev, DEVEN,
 238                                        temp | DEVEN_MCHBAR_EN);
 239         } else {
 240                 pci_read_config_dword(dev_priv->bridge_dev, mchbar_reg, &temp);
 241                 pci_write_config_dword(dev_priv->bridge_dev, mchbar_reg, temp | 1);
 242         }
 243 }
 244
 245 static void
 246 intel_teardown_mchbar(struct drm_i915_private *dev_priv)
 247 {
 248         int mchbar_reg = INTEL_GEN(dev_priv) >= 4 ? MCHBAR_I965 : MCHBAR_I915;
 249
 250         if (dev_priv->mchbar_need_disable) {
 251                 if (IS_I915G(dev_priv) || IS_I915GM(dev_priv)) {
 252                         u32 deven_val;
 253
 254                         pci_read_config_dword(dev_priv->bridge_dev, DEVEN,
 255                                               &deven_val);
 256                         deven_val &= ~DEVEN_MCHBAR_EN;
 257                         pci_write_config_dword(dev_priv->bridge_dev, DEVEN,
 258                                                deven_val);
 259                 } else {
 260                         u32 mchbar_val;
 261
 262                         pci_read_config_dword(dev_priv->bridge_dev, mchbar_reg,
 263                                               &mchbar_val);
 264                         mchbar_val &= ~1;
 265                         pci_write_config_dword(dev_priv->bridge_dev, mchbar_reg,
 266                                                mchbar_val);
 267                 }
 268         }
 269
 270         if (dev_priv->mch_res.start)
 271                 release_resource(&dev_priv->mch_res);
 272 }
 273
 274 static int i915_driver_modeset_probe(struct drm_i915_private *i915)
 275 {
 276         int ret;
 277
 278         if (i915_inject_probe_failure(i915))
 279                 return -ENODEV;
 280
 281         if (HAS_DISPLAY(i915) && INTEL_DISPLAY_ENABLED(i915)) {
 282                 ret = drm_vblank_init(&i915->drm,
 283                                       INTEL_NUM_PIPES(i915));
 284                 if (ret)
 285                         goto out;
 286         }
 287
 288         intel_bios_init(i915);
 289
 290         ret = intel_vga_register(i915);
 291         if (ret)
 292                 goto out;
 293
 294         intel_register_dsm_handler();
 295
 296         ret = i915_switcheroo_register(i915);
 297         if (ret)
 298                 goto cleanup_vga_client;
 299
 300         intel_power_domains_init_hw(i915, false);
 301
 302         intel_csr_ucode_init(i915);
 303
 304         ret = intel_irq_install(i915);
 305         if (ret)
 306                 goto cleanup_csr;
 307
 308         /* Important: The output setup functions called by modeset_init need
 309          * working irqs for e.g. gmbus and dp aux transfers. */
 310         ret = intel_modeset_init(i915);
 311         if (ret)
 312                 goto cleanup_irq;
 313
 314         ret = i915_gem_init(i915);
 315         if (ret)
 316                 goto cleanup_modeset;
 317
 318         intel_overlay_setup(i915);
 319
 320         if (!HAS_DISPLAY(i915) || !INTEL_DISPLAY_ENABLED(i915))
 321                 return 0;
 322
 323         ret = intel_fbdev_init(&i915->drm);
 324         if (ret)
 325                 goto cleanup_gem;
 326
 327         /* Only enable hotplug handling once the fbdev is fully set up. */
 328         intel_hpd_init(i915);
 329
 330         intel_init_ipc(i915);
 331
 332         return 0;
 333
 334 cleanup_gem:
 335         i915_gem_suspend(i915);
 336         i915_gem_driver_remove(i915);
 337         i915_gem_driver_release(i915);
 338 cleanup_modeset:
 339         intel_modeset_driver_remove(i915);
 340 cleanup_irq:
 341         intel_irq_uninstall(i915);
 342 cleanup_csr:
 343         intel_csr_ucode_fini(i915);
 344         intel_power_domains_driver_remove(i915);
 345         i915_switcheroo_unregister(i915);
 346 cleanup_vga_client:
 347         intel_vga_unregister(i915);
 348 out:
 349         return ret;
 350 }
 351
 352 static void i915_driver_modeset_remove(struct drm_i915_private *i915)
 353 {
 354         intel_modeset_driver_remove(i915);
 355
 356         intel_irq_uninstall(i915);
 357
 358         intel_bios_driver_remove(i915);
 359
 360         i915_switcheroo_unregister(i915);
 361
 362         intel_vga_unregister(i915);
 363
 364         intel_csr_ucode_fini(i915);
 365 }
 366
 367 static void intel_init_dpio(struct drm_i915_private *dev_priv)
 368 {
 369         /*
 370          * IOSF_PORT_DPIO is used for VLV x2 PHY (DP/HDMI B and C),
 371          * CHV x1 PHY (DP/HDMI D)
 372          * IOSF_PORT_DPIO_2 is used for CHV x2 PHY (DP/HDMI B and C)
 373          */
 374         if (IS_CHERRYVIEW(dev_priv)) {
 375                 DPIO_PHY_IOSF_PORT(DPIO_PHY0) = IOSF_PORT_DPIO_2;
 376                 DPIO_PHY_IOSF_PORT(DPIO_PHY1) = IOSF_PORT_DPIO;
 377         } else if (IS_VALLEYVIEW(dev_priv)) {
 378                 DPIO_PHY_IOSF_PORT(DPIO_PHY0) = IOSF_PORT_DPIO;
 379         }
 380 }
 381
 382 static int i915_workqueues_init(struct drm_i915_private *dev_priv)
 383 {
 384         /*
 385          * The i915 workqueue is primarily used for batched retirement of
 386          * requests (and thus managing bo) once the task has been completed
 387          * by the GPU. i915_retire_requests() is called directly when we
 388          * need high-priority retirement, such as waiting for an explicit
 389          * bo.
 390          *
 391          * It is also used for periodic low-priority events, such as
 392          * idle-timers and recording error state.
 393          *
 394          * All tasks on the workqueue are expected to acquire the dev mutex
 395          * so there is no point in running more than one instance of the
 396          * workqueue at any time.  Use an ordered one.
 397          */
 398         dev_priv->wq = alloc_ordered_workqueue("i915", 0);
 399         if (dev_priv->wq == NULL)
 400                 goto out_err;
 401
 402         dev_priv->hotplug.dp_wq = alloc_ordered_workqueue("i915-dp", 0);
 403         if (dev_priv->hotplug.dp_wq == NULL)
 404                 goto out_free_wq;
 405
 406         return 0;
 407
 408 out_free_wq:
 409         destroy_workqueue(dev_priv->wq);
 410 out_err:
 411         DRM_ERROR("Failed to allocate workqueues.\n");
 412
 413         return -ENOMEM;
 414 }
 415
 416 static void i915_workqueues_cleanup(struct drm_i915_private *dev_priv)
 417 {
 418         destroy_workqueue(dev_priv->hotplug.dp_wq);
 419         destroy_workqueue(dev_priv->wq);
 420 }
 421
 422 /*
 423  * We don't keep the workarounds for pre-production hardware, so we expect our
 424  * driver to fail on these machines in one way or another. A little warning on
 425  * dmesg may help both the user and the bug triagers.
 426  *
 427  * Our policy for removing pre-production workarounds is to keep the
 428  * current gen workarounds as a guide to the bring-up of the next gen
 429  * (workarounds have a habit of persisting!). Anything older than that
 430  * should be removed along with the complications they introduce.
 431  */
 432 static void intel_detect_preproduction_hw(struct drm_i915_private *dev_priv)
 433 {
 434         bool pre = false;
 435
 436         pre |= IS_HSW_EARLY_SDV(dev_priv);
 437         pre |= IS_SKL_REVID(dev_priv, 0, SKL_REVID_F0);
 438         pre |= IS_BXT_REVID(dev_priv, 0, BXT_REVID_B_LAST);
 439         pre |= IS_KBL_REVID(dev_priv, 0, KBL_REVID_A0);
 440
 441         if (pre) {
 442                 DRM_ERROR("This is a pre-production stepping. "
 443                           "It may not be fully functional.\n");
 444                 add_taint(TAINT_MACHINE_CHECK, LOCKDEP_STILL_OK);
 445         }
 446 }
 447
 448 static int vlv_alloc_s0ix_state(struct drm_i915_private *i915)
 449 {
 450         if (!IS_VALLEYVIEW(i915))
 451                 return 0;
 452
 453         /* we write all the values in the struct, so no need to zero it out */
 454         i915->vlv_s0ix_state = kmalloc(sizeof(*i915->vlv_s0ix_state),
 455                                        GFP_KERNEL);
 456         if (!i915->vlv_s0ix_state)
 457                 return -ENOMEM;
 458
 459         return 0;
 460 }
 461
 462 static void vlv_free_s0ix_state(struct drm_i915_private *i915)
 463 {
 464         if (!i915->vlv_s0ix_state)
 465                 return;
 466
 467         kfree(i915->vlv_s0ix_state);
 468         i915->vlv_s0ix_state = NULL;
 469 }
 470
 471 /**
 472  * i915_driver_early_probe - setup state not requiring device access
 473  * @dev_priv: device private
 474  *
 475  * Initialize everything that is a "SW-only" state, that is state not
 476  * requiring accessing the device or exposing the driver via kernel internal
 477  * or userspace interfaces. Example steps belonging here: lock initialization,
 478  * system memory allocation, setting up device specific attributes and
 479  * function hooks not requiring accessing the device.
 480  */
 481 static int i915_driver_early_probe(struct drm_i915_private *dev_priv)
 482 {
 483         int ret = 0;
 484
 485         if (i915_inject_probe_failure(dev_priv))
 486                 return -ENODEV;
 487
 488         intel_device_info_subplatform_init(dev_priv);
 489
 490         intel_uncore_mmio_debug_init_early(&dev_priv->mmio_debug);
 491         intel_uncore_init_early(&dev_priv->uncore, dev_priv);
 492
 493         spin_lock_init(&dev_priv->irq_lock);
 494         spin_lock_init(&dev_priv->gpu_error.lock);
 495         mutex_init(&dev_priv->backlight_lock);
 496
 497         mutex_init(&dev_priv->sb_lock);
 498         pm_qos_add_request(&dev_priv->sb_qos,
 499                            PM_QOS_CPU_DMA_LATENCY, PM_QOS_DEFAULT_VALUE);
 500
 501         mutex_init(&dev_priv->av_mutex);
 502         mutex_init(&dev_priv->wm.wm_mutex);
 503         mutex_init(&dev_priv->pps_mutex);
 504         mutex_init(&dev_priv->hdcp_comp_mutex);
 505
 506         i915_memcpy_init_early(dev_priv);
 507         intel_runtime_pm_init_early(&dev_priv->runtime_pm);
 508
 509         ret = i915_workqueues_init(dev_priv);
 510         if (ret < 0)
 511                 return ret;
 512
 513         ret = vlv_alloc_s0ix_state(dev_priv);
 514         if (ret < 0)
 515                 goto err_workqueues;
 516
 517         intel_wopcm_init_early(&dev_priv->wopcm);
 518
 519         intel_gt_init_early(&dev_priv->gt, dev_priv);
 520
 521         i915_gem_init_early(dev_priv);
 522
 523         /* This must be called before any calls to HAS_PCH_* */
 524         intel_detect_pch(dev_priv);
 525
 526         intel_pm_setup(dev_priv);
 527         intel_init_dpio(dev_priv);
 528         ret = intel_power_domains_init(dev_priv);
 529         if (ret < 0)
 530                 goto err_gem;
 531         intel_irq_init(dev_priv);
 532         intel_init_display_hooks(dev_priv);
 533         intel_init_clock_gating_hooks(dev_priv);
 534         intel_init_audio_hooks(dev_priv);
 535         intel_display_crc_init(dev_priv);
 536
 537         intel_detect_preproduction_hw(dev_priv);
 538
 539         return 0;
 540
 541 err_gem:
 542         i915_gem_cleanup_early(dev_priv);
 543         intel_gt_driver_late_release(&dev_priv->gt);
 544         vlv_free_s0ix_state(dev_priv);
 545 err_workqueues:
 546         i915_workqueues_cleanup(dev_priv);
 547         return ret;
 548 }
 549
 550 /**
 551  * i915_driver_late_release - cleanup the setup done in
 552  *                             i915_driver_early_probe()
 553  * @dev_priv: device private
 554  */
 555 static void i915_driver_late_release(struct drm_i915_private *dev_priv)
 556 {
 557         intel_irq_fini(dev_priv);
 558         intel_power_domains_cleanup(dev_priv);
 559         i915_gem_cleanup_early(dev_priv);
 560         intel_gt_driver_late_release(&dev_priv->gt);
 561         vlv_free_s0ix_state(dev_priv);
 562         i915_workqueues_cleanup(dev_priv);
 563
 564         pm_qos_remove_request(&dev_priv->sb_qos);
 565         mutex_destroy(&dev_priv->sb_lock);
 566 }
 567
 568 /**
 569  * i915_driver_mmio_probe - setup device MMIO
 570  * @dev_priv: device private
 571  *
 572  * Setup minimal device state necessary for MMIO accesses later in the
 573  * initialization sequence. The setup here should avoid any other device-wide
 574  * side effects or exposing the driver via kernel internal or user space
 575  * interfaces.
 576  */
 577 static int i915_driver_mmio_probe(struct drm_i915_private *dev_priv)
 578 {
 579         int ret;
 580
 581         if (i915_inject_probe_failure(dev_priv))
 582                 return -ENODEV;
 583
 584         if (i915_get_bridge_dev(dev_priv))
 585                 return -EIO;
 586
 587         ret = intel_uncore_init_mmio(&dev_priv->uncore);
 588         if (ret < 0)
 589                 goto err_bridge;
 590
 591         /* Try to make sure MCHBAR is enabled before poking at it */
 592         intel_setup_mchbar(dev_priv);
 593
 594         intel_device_info_init_mmio(dev_priv);
 595
 596         intel_uncore_prune_mmio_domains(&dev_priv->uncore);
 597
 598         intel_uc_init_mmio(&dev_priv->gt.uc);
 599
 600         ret = intel_engines_init_mmio(&dev_priv->gt);
 601         if (ret)
 602                 goto err_uncore;
 603
 604         return 0;
 605
 606 err_uncore:
 607         intel_teardown_mchbar(dev_priv);
 608         intel_uncore_fini_mmio(&dev_priv->uncore);
 609 err_bridge:
 610         pci_dev_put(dev_priv->bridge_dev);
 611
 612         return ret;
 613 }
 614
 615 /**
 616  * i915_driver_mmio_release - cleanup the setup done in i915_driver_mmio_probe()
 617  * @dev_priv: device private
 618  */
 619 static void i915_driver_mmio_release(struct drm_i915_private *dev_priv)
 620 {
 621         intel_engines_cleanup(&dev_priv->gt);
 622         intel_teardown_mchbar(dev_priv);
 623         intel_uncore_fini_mmio(&dev_priv->uncore);
 624         pci_dev_put(dev_priv->bridge_dev);
 625 }
 626
 627 static void intel_sanitize_options(struct drm_i915_private *dev_priv)
 628 {
 629         intel_gvt_sanitize_options(dev_priv);
 630 }
 631
 632 #define DRAM_TYPE_STR(type) [INTEL_DRAM_ ## type] = #type
 633
 634 static const char *intel_dram_type_str(enum intel_dram_type type)
 635 {
 636         static const char * const str[] = {
 637                 DRAM_TYPE_STR(UNKNOWN),
 638                 DRAM_TYPE_STR(DDR3),
 639                 DRAM_TYPE_STR(DDR4),
 640                 DRAM_TYPE_STR(LPDDR3),
 641                 DRAM_TYPE_STR(LPDDR4),
 642         };
 643
 644         if (type >= ARRAY_SIZE(str))
 645                 type = INTEL_DRAM_UNKNOWN;
 646
 647         return str[type];
 648 }
 649
 650 #undef DRAM_TYPE_STR
 651
 652 static int intel_dimm_num_devices(const struct dram_dimm_info *dimm)
 653 {
 654         return dimm->ranks * 64 / (dimm->width ?: 1);
 655 }
 656
 657 /* Returns total GB for the whole DIMM */
 658 static int skl_get_dimm_size(u16 val)
 659 {
 660         return val & SKL_DRAM_SIZE_MASK;
 661 }
 662
 663 static int skl_get_dimm_width(u16 val)
 664 {
 665         if (skl_get_dimm_size(val) == 0)
 666                 return 0;
 667
 668         switch (val & SKL_DRAM_WIDTH_MASK) {
 669         case SKL_DRAM_WIDTH_X8:
 670         case SKL_DRAM_WIDTH_X16:
 671         case SKL_DRAM_WIDTH_X32:
 672                 val = (val & SKL_DRAM_WIDTH_MASK) >> SKL_DRAM_WIDTH_SHIFT;
 673                 return 8 << val;
 674         default:
 675                 MISSING_CASE(val);
 676                 return 0;
 677         }
 678 }
 679
 680 static int skl_get_dimm_ranks(u16 val)
 681 {
 682         if (skl_get_dimm_size(val) == 0)
 683                 return 0;
 684
 685         val = (val & SKL_DRAM_RANK_MASK) >> SKL_DRAM_RANK_SHIFT;
 686
 687         return val + 1;
 688 }
 689
 690 /* Returns total GB for the whole DIMM */
 691 static int cnl_get_dimm_size(u16 val)
 692 {
 693         return (val & CNL_DRAM_SIZE_MASK) / 2;
 694 }
 695
 696 static int cnl_get_dimm_width(u16 val)
 697 {
 698         if (cnl_get_dimm_size(val) == 0)
 699                 return 0;
 700
 701         switch (val & CNL_DRAM_WIDTH_MASK) {
 702         case CNL_DRAM_WIDTH_X8:
 703         case CNL_DRAM_WIDTH_X16:
 704         case CNL_DRAM_WIDTH_X32:
 705                 val = (val & CNL_DRAM_WIDTH_MASK) >> CNL_DRAM_WIDTH_SHIFT;
 706                 return 8 << val;
 707         default:
 708                 MISSING_CASE(val);
 709                 return 0;
 710         }
 711 }
 712
 713 static int cnl_get_dimm_ranks(u16 val)
 714 {
 715         if (cnl_get_dimm_size(val) == 0)
 716                 return 0;
 717
 718         val = (val & CNL_DRAM_RANK_MASK) >> CNL_DRAM_RANK_SHIFT;
 719
 720         return val + 1;
 721 }
 722
 723 static bool
 724 skl_is_16gb_dimm(const struct dram_dimm_info *dimm)
 725 {
 726         /* Convert total GB to Gb per DRAM device */
 727         return 8 * dimm->size / (intel_dimm_num_devices(dimm) ?: 1) == 16;
 728 }
 729
 730 static void
 731 skl_dram_get_dimm_info(struct drm_i915_private *dev_priv,
 732                        struct dram_dimm_info *dimm,
 733                        int channel, char dimm_name, u16 val)
 734 {
 735         if (INTEL_GEN(dev_priv) >= 10) {
 736                 dimm->size = cnl_get_dimm_size(val);
 737                 dimm->width = cnl_get_dimm_width(val);
 738                 dimm->ranks = cnl_get_dimm_ranks(val);
 739         } else {
 740                 dimm->size = skl_get_dimm_size(val);
 741                 dimm->width = skl_get_dimm_width(val);
 742                 dimm->ranks = skl_get_dimm_ranks(val);
 743         }
 744
 745         DRM_DEBUG_KMS("CH%u DIMM %c size: %u GB, width: X%u, ranks: %u, 16Gb DIMMs: %s\n",
 746                       channel, dimm_name, dimm->size, dimm->width, dimm->ranks,
 747                       yesno(skl_is_16gb_dimm(dimm)));
 748 }
 749
 750 static int
 751 skl_dram_get_channel_info(struct drm_i915_private *dev_priv,
 752                           struct dram_channel_info *ch,
 753                           int channel, u32 val)
 754 {
 755         skl_dram_get_dimm_info(dev_priv, &ch->dimm_l,
 756                                channel, 'L', val & 0xffff);
 757         skl_dram_get_dimm_info(dev_priv, &ch->dimm_s,
 758                                channel, 'S', val >> 16);
 759
 760         if (ch->dimm_l.size == 0 && ch->dimm_s.size == 0) {
 761                 DRM_DEBUG_KMS("CH%u not populated\n", channel);
 762                 return -EINVAL;
 763         }
 764
 765         if (ch->dimm_l.ranks == 2 || ch->dimm_s.ranks == 2)
 766                 ch->ranks = 2;
 767         else if (ch->dimm_l.ranks == 1 && ch->dimm_s.ranks == 1)
 768                 ch->ranks = 2;
 769         else
 770                 ch->ranks = 1;
 771
 772         ch->is_16gb_dimm =
 773                 skl_is_16gb_dimm(&ch->dimm_l) ||
 774                 skl_is_16gb_dimm(&ch->dimm_s);
 775
 776         DRM_DEBUG_KMS("CH%u ranks: %u, 16Gb DIMMs: %s\n",
 777                       channel, ch->ranks, yesno(ch->is_16gb_dimm));
 778
 779         return 0;
 780 }
 781
 782 static bool
 783 intel_is_dram_symmetric(const struct dram_channel_info *ch0,
 784                         const struct dram_channel_info *ch1)
 785 {
 786         return !memcmp(ch0, ch1, sizeof(*ch0)) &&
 787                 (ch0->dimm_s.size == 0 ||
 788                  !memcmp(&ch0->dimm_l, &ch0->dimm_s, sizeof(ch0->dimm_l)));
 789 }
 790
 791 static int
 792 skl_dram_get_channels_info(struct drm_i915_private *dev_priv)
 793 {
 794         struct dram_info *dram_info = &dev_priv->dram_info;
 795         struct dram_channel_info ch0 = {}, ch1 = {};
 796         u32 val;
 797         int ret;
 798
 799         val = I915_READ(SKL_MAD_DIMM_CH0_0_0_0_MCHBAR_MCMAIN);
 800         ret = skl_dram_get_channel_info(dev_priv, &ch0, 0, val);
 801         if (ret == 0)
 802                 dram_info->num_channels++;
 803
 804         val = I915_READ(SKL_MAD_DIMM_CH1_0_0_0_MCHBAR_MCMAIN);
 805         ret = skl_dram_get_channel_info(dev_priv, &ch1, 1, val);
 806         if (ret == 0)
 807                 dram_info->num_channels++;
 808
 809         if (dram_info->num_channels == 0) {
 810                 DRM_INFO("Number of memory channels is zero\n");
 811                 return -EINVAL;
 812         }
 813
 814         /*
 815          * If any of the channel is single rank channel, worst case output
 816          * will be same as if single rank memory, so consider single rank
 817          * memory.
 818          */
 819         if (ch0.ranks == 1 || ch1.ranks == 1)
 820                 dram_info->ranks = 1;
 821         else
 822                 dram_info->ranks = max(ch0.ranks, ch1.ranks);
 823
 824         if (dram_info->ranks == 0) {
 825                 DRM_INFO("couldn't get memory rank information\n");
 826                 return -EINVAL;
 827         }
 828
 829         dram_info->is_16gb_dimm = ch0.is_16gb_dimm || ch1.is_16gb_dimm;
 830
 831         dram_info->symmetric_memory = intel_is_dram_symmetric(&ch0, &ch1);
 832
 833         DRM_DEBUG_KMS("Memory configuration is symmetric? %s\n",
 834                       yesno(dram_info->symmetric_memory));
 835         return 0;
 836 }
 837
 838 static enum intel_dram_type
 839 skl_get_dram_type(struct drm_i915_private *dev_priv)
 840 {
 841         u32 val;
 842
 843         val = I915_READ(SKL_MAD_INTER_CHANNEL_0_0_0_MCHBAR_MCMAIN);
 844
 845         switch (val & SKL_DRAM_DDR_TYPE_MASK) {
 846         case SKL_DRAM_DDR_TYPE_DDR3:
 847                 return INTEL_DRAM_DDR3;
 848         case SKL_DRAM_DDR_TYPE_DDR4:
 849                 return INTEL_DRAM_DDR4;
 850         case SKL_DRAM_DDR_TYPE_LPDDR3:
 851                 return INTEL_DRAM_LPDDR3;
 852         case SKL_DRAM_DDR_TYPE_LPDDR4:
 853                 return INTEL_DRAM_LPDDR4;
 854         default:
 855                 MISSING_CASE(val);
 856                 return INTEL_DRAM_UNKNOWN;
 857         }
 858 }
 859
 860 static int
 861 skl_get_dram_info(struct drm_i915_private *dev_priv)
 862 {
 863         struct dram_info *dram_info = &dev_priv->dram_info;
 864         u32 mem_freq_khz, val;
 865         int ret;
 866
 867         dram_info->type = skl_get_dram_type(dev_priv);
 868         DRM_DEBUG_KMS("DRAM type: %s\n", intel_dram_type_str(dram_info->type));
 869
 870         ret = skl_dram_get_channels_info(dev_priv);
 871         if (ret)
 872                 return ret;
 873
 874         val = I915_READ(SKL_MC_BIOS_DATA_0_0_0_MCHBAR_PCU);
 875         mem_freq_khz = DIV_ROUND_UP((val & SKL_REQ_DATA_MASK) *
 876                                     SKL_MEMORY_FREQ_MULTIPLIER_HZ, 1000);
 877
 878         dram_info->bandwidth_kbps = dram_info->num_channels *
 879                                                         mem_freq_khz * 8;
 880
 881         if (dram_info->bandwidth_kbps == 0) {
 882                 DRM_INFO("Couldn't get system memory bandwidth\n");
 883                 return -EINVAL;
 884         }
 885
 886         dram_info->valid = true;
 887         return 0;
 888 }
 889
 890 /* Returns Gb per DRAM device */
 891 static int bxt_get_dimm_size(u32 val)
 892 {
 893         switch (val & BXT_DRAM_SIZE_MASK) {
 894         case BXT_DRAM_SIZE_4GBIT:
 895                 return 4;
 896         case BXT_DRAM_SIZE_6GBIT:
 897                 return 6;
 898         case BXT_DRAM_SIZE_8GBIT:
 899                 return 8;
 900         case BXT_DRAM_SIZE_12GBIT:
 901                 return 12;
 902         case BXT_DRAM_SIZE_16GBIT:
 903                 return 16;
 904         default:
 905                 MISSING_CASE(val);
 906                 return 0;
 907         }
 908 }
 909
 910 static int bxt_get_dimm_width(u32 val)
 911 {
 912         if (!bxt_get_dimm_size(val))
 913                 return 0;
 914
 915         val = (val & BXT_DRAM_WIDTH_MASK) >> BXT_DRAM_WIDTH_SHIFT;
 916
 917         return 8 << val;
 918 }
 919
 920 static int bxt_get_dimm_ranks(u32 val)
 921 {
 922         if (!bxt_get_dimm_size(val))
 923                 return 0;
 924
 925         switch (val & BXT_DRAM_RANK_MASK) {
 926         case BXT_DRAM_RANK_SINGLE:
 927                 return 1;
 928         case BXT_DRAM_RANK_DUAL:
 929                 return 2;
 930         default:
 931                 MISSING_CASE(val);
 932                 return 0;
 933         }
 934 }
 935
 936 static enum intel_dram_type bxt_get_dimm_type(u32 val)
 937 {
 938         if (!bxt_get_dimm_size(val))
 939                 return INTEL_DRAM_UNKNOWN;
 940
 941         switch (val & BXT_DRAM_TYPE_MASK) {
 942         case BXT_DRAM_TYPE_DDR3:
 943                 return INTEL_DRAM_DDR3;
 944         case BXT_DRAM_TYPE_LPDDR3:
 945                 return INTEL_DRAM_LPDDR3;
 946         case BXT_DRAM_TYPE_DDR4:
 947                 return INTEL_DRAM_DDR4;
 948         case BXT_DRAM_TYPE_LPDDR4:
 949                 return INTEL_DRAM_LPDDR4;
 950         default:
 951                 MISSING_CASE(val);
 952                 return INTEL_DRAM_UNKNOWN;
 953         }
 954 }
 955
 956 static void bxt_get_dimm_info(struct dram_dimm_info *dimm,
 957                               u32 val)
 958 {
 959         dimm->width = bxt_get_dimm_width(val);
 960         dimm->ranks = bxt_get_dimm_ranks(val);
 961
 962         /*
 963          * Size in register is Gb per DRAM device. Convert to total
 964          * GB to match the way we report this for non-LP platforms.
 965          */
 966         dimm->size = bxt_get_dimm_size(val) * intel_dimm_num_devices(dimm) / 8;
 967 }
 968
 969 static int
 970 bxt_get_dram_info(struct drm_i915_private *dev_priv)
 971 {
 972         struct dram_info *dram_info = &dev_priv->dram_info;
 973         u32 dram_channels;
 974         u32 mem_freq_khz, val;
 975         u8 num_active_channels;
 976         int i;
 977
 978         val = I915_READ(BXT_P_CR_MC_BIOS_REQ_0_0_0);
 979         mem_freq_khz = DIV_ROUND_UP((val & BXT_REQ_DATA_MASK) *
 980                                     BXT_MEMORY_FREQ_MULTIPLIER_HZ, 1000);
 981
 982         dram_channels = val & BXT_DRAM_CHANNEL_ACTIVE_MASK;
 983         num_active_channels = hweight32(dram_channels);
 984
 985         /* Each active bit represents 4-byte channel */
 986         dram_info->bandwidth_kbps = (mem_freq_khz * num_active_channels * 4);
 987
 988         if (dram_info->bandwidth_kbps == 0) {
 989                 DRM_INFO("Couldn't get system memory bandwidth\n");
 990                 return -EINVAL;
 991         }
 992
 993         /*
 994          * Now read each DUNIT8/9/10/11 to check the rank of each dimms.
 995          */
 996         for (i = BXT_D_CR_DRP0_DUNIT_START; i <= BXT_D_CR_DRP0_DUNIT_END; i++) {
 997                 struct dram_dimm_info dimm;
 998                 enum intel_dram_type type;
 999
1000                 val = I915_READ(BXT_D_CR_DRP0_DUNIT(i));
1001                 if (val == 0xFFFFFFFF)
1002                         continue;
1003
1004                 dram_info->num_channels++;
1005
1006                 bxt_get_dimm_info(&dimm, val);
1007                 type = bxt_get_dimm_type(val);
1008
1009                 WARN_ON(type != INTEL_DRAM_UNKNOWN &&
1010                         dram_info->type != INTEL_DRAM_UNKNOWN &&
1011                         dram_info->type != type);
1012
1013                 DRM_DEBUG_KMS("CH%u DIMM size: %u GB, width: X%u, ranks: %u, type: %s\n",
1014                               i - BXT_D_CR_DRP0_DUNIT_START,
1015                               dimm.size, dimm.width, dimm.ranks,
1016                               intel_dram_type_str(type));
1017
1018                 /*
1019                  * If any of the channel is single rank channel,
1020                  * worst case output will be same as if single rank
1021                  * memory, so consider single rank memory.
1022                  */
1023                 if (dram_info->ranks == 0)
1024                         dram_info->ranks = dimm.ranks;
1025                 else if (dimm.ranks == 1)
1026                         dram_info->ranks = 1;
1027
1028                 if (type != INTEL_DRAM_UNKNOWN)
1029                         dram_info->type = type;
1030         }
1031
1032         if (dram_info->type == INTEL_DRAM_UNKNOWN ||
1033             dram_info->ranks == 0) {
1034                 DRM_INFO("couldn't get memory information\n");
1035                 return -EINVAL;
1036         }
1037
1038         dram_info->valid = true;
1039         return 0;
1040 }
1041
1042 static void
1043 intel_get_dram_info(struct drm_i915_private *dev_priv)
1044 {
1045         struct dram_info *dram_info = &dev_priv->dram_info;
1046         int ret;
1047
1048         /*
1049          * Assume 16Gb DIMMs are present until proven otherwise.
1050          * This is only used for the level 0 watermark latency
1051          * w/a which does not apply to bxt/glk.
1052          */
1053         dram_info->is_16gb_dimm = !IS_GEN9_LP(dev_priv);
1054
1055         if (INTEL_GEN(dev_priv) < 9)
1056                 return;
1057
1058         if (IS_GEN9_LP(dev_priv))
1059                 ret = bxt_get_dram_info(dev_priv);
1060         else
1061                 ret = skl_get_dram_info(dev_priv);
1062         if (ret)
1063                 return;
1064
1065         DRM_DEBUG_KMS("DRAM bandwidth: %u kBps, channels: %u\n",
1066                       dram_info->bandwidth_kbps,
1067                       dram_info->num_channels);
1068
1069         DRM_DEBUG_KMS("DRAM ranks: %u, 16Gb DIMMs: %s\n",
1070                       dram_info->ranks, yesno(dram_info->is_16gb_dimm));
1071 }
1072
1073 static u32 gen9_edram_size_mb(struct drm_i915_private *dev_priv, u32 cap)
1074 {
1075         static const u8 ways[8] = { 4, 8, 12, 16, 16, 16, 16, 16 };
1076         static const u8 sets[4] = { 1, 1, 2, 2 };
1077
1078         return EDRAM_NUM_BANKS(cap) *
1079                 ways[EDRAM_WAYS_IDX(cap)] *
1080                 sets[EDRAM_SETS_IDX(cap)];
1081 }
1082
1083 static void edram_detect(struct drm_i915_private *dev_priv)
1084 {
1085         u32 edram_cap = 0;
1086
1087         if (!(IS_HASWELL(dev_priv) ||
1088               IS_BROADWELL(dev_priv) ||
1089               INTEL_GEN(dev_priv) >= 9))
1090                 return;
1091
1092         edram_cap = __raw_uncore_read32(&dev_priv->uncore, HSW_EDRAM_CAP);
1093
1094         /* NB: We can't write IDICR yet because we don't have gt funcs set up */
1095
1096         if (!(edram_cap & EDRAM_ENABLED))
1097                 return;
1098
1099         /*
1100          * The needed capability bits for size calculation are not there with
1101          * pre gen9 so return 128MB always.
1102          */
1103         if (INTEL_GEN(dev_priv) < 9)
1104                 dev_priv->edram_size_mb = 128;
1105         else
1106                 dev_priv->edram_size_mb =
1107                         gen9_edram_size_mb(dev_priv, edram_cap);
1108
1109         dev_info(dev_priv->drm.dev,
1110                  "Found %uMB of eDRAM\n", dev_priv->edram_size_mb);
1111 }
1112
1113 /**
1114  * i915_driver_hw_probe - setup state requiring device access
1115  * @dev_priv: device private
1116  *
1117  * Setup state that requires accessing the device, but doesn't require
1118  * exposing the driver via kernel internal or userspace interfaces.
1119  */
1120 static int i915_driver_hw_probe(struct drm_i915_private *dev_priv)
1121 {
1122         struct pci_dev *pdev = dev_priv->drm.pdev;
1123         int ret;
1124
1125         if (i915_inject_probe_failure(dev_priv))
1126                 return -ENODEV;
1127
1128         intel_device_info_runtime_init(dev_priv);
1129
1130         if (HAS_PPGTT(dev_priv)) {
1131                 if (intel_vgpu_active(dev_priv) &&
1132                     !intel_vgpu_has_full_ppgtt(dev_priv)) {
1133                         i915_report_error(dev_priv,
1134                                           "incompatible vGPU found, support for isolated ppGTT required\n");
1135                         return -ENXIO;
1136                 }
1137         }
1138
1139         if (HAS_EXECLISTS(dev_priv)) {
1140                 /*
1141                  * Older GVT emulation depends upon intercepting CSB mmio,
1142                  * which we no longer use, preferring to use the HWSP cache
1143                  * instead.
1144                  */
1145                 if (intel_vgpu_active(dev_priv) &&
1146                     !intel_vgpu_has_hwsp_emulation(dev_priv)) {
1147                         i915_report_error(dev_priv,
1148                                           "old vGPU host found, support for HWSP emulation required\n");
1149                         return -ENXIO;
1150                 }
1151         }
1152
1153         intel_sanitize_options(dev_priv);
1154
1155         /* needs to be done before ggtt probe */
1156         edram_detect(dev_priv);
1157
1158         i915_perf_init(dev_priv);
1159
1160         ret = i915_ggtt_probe_hw(dev_priv);
1161         if (ret)
1162                 goto err_perf;
1163
1164         ret = drm_fb_helper_remove_conflicting_pci_framebuffers(pdev, "inteldrmfb");
1165         if (ret)
1166                 goto err_ggtt;
1167
1168         ret = i915_ggtt_init_hw(dev_priv);
1169         if (ret)
1170                 goto err_ggtt;
1171
1172         ret = intel_memory_regions_hw_probe(dev_priv);
1173         if (ret)
1174                 goto err_ggtt;
1175
1176         intel_gt_init_hw_early(&dev_priv->gt, &dev_priv->ggtt);
1177
1178         ret = i915_ggtt_enable_hw(dev_priv);
1179         if (ret) {
1180                 DRM_ERROR("failed to enable GGTT\n");
1181                 goto err_mem_regions;
1182         }
1183
1184         pci_set_master(pdev);
1185
1186         /*
1187          * We don't have a max segment size, so set it to the max so sg's
1188          * debugging layer doesn't complain
1189          */
1190         dma_set_max_seg_size(&pdev->dev, UINT_MAX);
1191
1192         /* overlay on gen2 is broken and can't address above 1G */
1193         if (IS_GEN(dev_priv, 2)) {
1194                 ret = dma_set_coherent_mask(&pdev->dev, DMA_BIT_MASK(30));
1195                 if (ret) {
1196                         DRM_ERROR("failed to set DMA mask\n");
1197
1198                         goto err_mem_regions;
1199                 }
1200         }
1201
1202         /* 965GM sometimes incorrectly writes to hardware status page (HWS)
1203          * using 32bit addressing, overwriting memory if HWS is located
1204          * above 4GB.
1205          *
1206          * The documentation also mentions an issue with undefined
1207          * behaviour if any general state is accessed within a page above 4GB,
1208          * which also needs to be handled carefully.
1209          */
1210         if (IS_I965G(dev_priv) || IS_I965GM(dev_priv)) {
1211                 ret = dma_set_coherent_mask(&pdev->dev, DMA_BIT_MASK(32));
1212
1213                 if (ret) {
1214                         DRM_ERROR("failed to set DMA mask\n");
1215
1216                         goto err_mem_regions;
1217                 }
1218         }
1219
1220         pm_qos_add_request(&dev_priv->pm_qos, PM_QOS_CPU_DMA_LATENCY,
1221                            PM_QOS_DEFAULT_VALUE);
1222
1223         intel_gt_init_workarounds(dev_priv);
1224
1225         /* On the 945G/GM, the chipset reports the MSI capability on the
1226          * integrated graphics even though the support isn't actually there
1227          * according to the published specs.  It doesn't appear to function
1228          * correctly in testing on 945G.
1229          * This may be a side effect of MSI having been made available for PEG
1230          * and the registers being closely associated.
1231          *
1232          * According to chipset errata, on the 965GM, MSI interrupts may
1233          * be lost or delayed, and was defeatured. MSI interrupts seem to
1234          * get lost on g4x as well, and interrupt delivery seems to stay
1235          * properly dead afterwards. So we'll just disable them for all
1236          * pre-gen5 chipsets.
1237          *
1238          * dp aux and gmbus irq on gen4 seems to be able to generate legacy
1239          * interrupts even when in MSI mode. This results in spurious
1240          * interrupt warnings if the legacy irq no. is shared with another
1241          * device. The kernel then disables that interrupt source and so
1242          * prevents the other device from working properly.
1243          */
1244         if (INTEL_GEN(dev_priv) >= 5) {
1245                 if (pci_enable_msi(pdev) < 0)
1246                         DRM_DEBUG_DRIVER("can't enable MSI");
1247         }
1248
1249         ret = intel_gvt_init(dev_priv);
1250         if (ret)
1251                 goto err_msi;
1252
1253         intel_opregion_setup(dev_priv);
1254         /*
1255          * Fill the dram structure to get the system raw bandwidth and
1256          * dram info. This will be used for memory latency calculation.
1257          */
1258         intel_get_dram_info(dev_priv);
1259
1260         intel_bw_init_hw(dev_priv);
1261
1262         return 0;
1263
1264 err_msi:
1265         if (pdev->msi_enabled)
1266                 pci_disable_msi(pdev);
1267         pm_qos_remove_request(&dev_priv->pm_qos);
1268 err_mem_regions:
1269         intel_memory_regions_driver_release(dev_priv);
1270 err_ggtt:
1271         i915_ggtt_driver_release(dev_priv);
1272 err_perf:
1273         i915_perf_fini(dev_priv);
1274         return ret;
1275 }
1276
1277 /**
1278  * i915_driver_hw_remove - cleanup the setup done in i915_driver_hw_probe()
1279  * @dev_priv: device private
1280  */
1281 static void i915_driver_hw_remove(struct drm_i915_private *dev_priv)
1282 {
1283         struct pci_dev *pdev = dev_priv->drm.pdev;
1284
1285         i915_perf_fini(dev_priv);
1286
1287         if (pdev->msi_enabled)
1288                 pci_disable_msi(pdev);
1289
1290         pm_qos_remove_request(&dev_priv->pm_qos);
1291 }
1292
1293 /**
1294  * i915_driver_register - register the driver with the rest of the system
1295  * @dev_priv: device private
1296  *
1297  * Perform any steps necessary to make the driver available via kernel
1298  * internal or userspace interfaces.
1299  */
1300 static void i915_driver_register(struct drm_i915_private *dev_priv)
1301 {
1302         struct drm_device *dev = &dev_priv->drm;
1303
1304         i915_gem_driver_register(dev_priv);
1305         i915_pmu_register(dev_priv);
1306
1307         /*
1308          * Notify a valid surface after modesetting,
1309          * when running inside a VM.
1310          */
1311         if (intel_vgpu_active(dev_priv))
1312                 I915_WRITE(vgtif_reg(display_ready), VGT_DRV_DISPLAY_READY);
1313
1314         /* Reveal our presence to userspace */
1315         if (drm_dev_register(dev, 0) == 0) {
1316                 i915_debugfs_register(dev_priv);
1317                 i915_setup_sysfs(dev_priv);
1318
1319                 /* Depends on sysfs having been initialized */
1320                 i915_perf_register(dev_priv);
1321         } else
1322                 DRM_ERROR("Failed to register driver for userspace access!\n");
1323
1324         if (HAS_DISPLAY(dev_priv) && INTEL_DISPLAY_ENABLED(dev_priv)) {
1325                 /* Must be done after probing outputs */
1326                 intel_opregion_register(dev_priv);
1327                 acpi_video_register();
1328         }
1329
1330         intel_gt_driver_register(&dev_priv->gt);
1331
1332         intel_audio_init(dev_priv);
1333
1334         /*
1335          * Some ports require correctly set-up hpd registers for detection to
1336          * work properly (leading to ghost connected connector status), e.g. VGA
1337          * on gm45.  Hence we can only set up the initial fbdev config after hpd
1338          * irqs are fully enabled. We do it last so that the async config
1339          * cannot run before the connectors are registered.
1340          */
1341         intel_fbdev_initial_config_async(dev);
1342
1343         /*
1344          * We need to coordinate the hotplugs with the asynchronous fbdev
1345          * configuration, for which we use the fbdev->async_cookie.
1346          */
1347         if (HAS_DISPLAY(dev_priv) && INTEL_DISPLAY_ENABLED(dev_priv))
1348                 drm_kms_helper_poll_init(dev);
1349
1350         intel_power_domains_enable(dev_priv);
1351         intel_runtime_pm_enable(&dev_priv->runtime_pm);
1352 }
1353
1354 /**
1355  * i915_driver_unregister - cleanup the registration done in i915_driver_regiser()
1356  * @dev_priv: device private
1357  */
1358 static void i915_driver_unregister(struct drm_i915_private *dev_priv)
1359 {
1360         intel_runtime_pm_disable(&dev_priv->runtime_pm);
1361         intel_power_domains_disable(dev_priv);
1362
1363         intel_fbdev_unregister(dev_priv);
1364         intel_audio_deinit(dev_priv);
1365
1366         /*
1367          * After flushing the fbdev (incl. a late async config which will
1368          * have delayed queuing of a hotplug event), then flush the hotplug
1369          * events.
1370          */
1371         drm_kms_helper_poll_fini(&dev_priv->drm);
1372
1373         intel_gt_driver_unregister(&dev_priv->gt);
1374         acpi_video_unregister();
1375         intel_opregion_unregister(dev_priv);
1376
1377         i915_perf_unregister(dev_priv);
1378         i915_pmu_unregister(dev_priv);
1379
1380         i915_teardown_sysfs(dev_priv);
1381         drm_dev_unplug(&dev_priv->drm);
1382
1383         i915_gem_driver_unregister(dev_priv);
1384 }
1385
1386 static void i915_welcome_messages(struct drm_i915_private *dev_priv)
1387 {
1388         if (drm_debug_enabled(DRM_UT_DRIVER)) {
1389                 struct drm_printer p = drm_debug_printer("i915 device info:");
1390
1391                 drm_printf(&p, "pciid=0x%04x rev=0x%02x platform=%s (subplatform=0x%x) gen=%i\n",
1392                            INTEL_DEVID(dev_priv),
1393                            INTEL_REVID(dev_priv),
1394                            intel_platform_name(INTEL_INFO(dev_priv)->platform),
1395                            intel_subplatform(RUNTIME_INFO(dev_priv),
1396                                              INTEL_INFO(dev_priv)->platform),
1397                            INTEL_GEN(dev_priv));
1398
1399                 intel_device_info_dump_flags(INTEL_INFO(dev_priv), &p);
1400                 intel_device_info_dump_runtime(RUNTIME_INFO(dev_priv), &p);
1401         }
1402
1403         if (IS_ENABLED(CONFIG_DRM_I915_DEBUG))
1404                 DRM_INFO("DRM_I915_DEBUG enabled\n");
1405         if (IS_ENABLED(CONFIG_DRM_I915_DEBUG_GEM))
1406                 DRM_INFO("DRM_I915_DEBUG_GEM enabled\n");
1407         if (IS_ENABLED(CONFIG_DRM_I915_DEBUG_RUNTIME_PM))
1408                 DRM_INFO("DRM_I915_DEBUG_RUNTIME_PM enabled\n");
1409 }
1410
1411 static struct drm_i915_private *
1412 i915_driver_create(struct pci_dev *pdev, const struct pci_device_id *ent)
1413 {
1414         const struct intel_device_info *match_info =
1415                 (struct intel_device_info *)ent->driver_data;
1416         struct intel_device_info *device_info;
1417         struct drm_i915_private *i915;
1418         int err;
1419
1420         i915 = kzalloc(sizeof(*i915), GFP_KERNEL);
1421         if (!i915)
1422                 return ERR_PTR(-ENOMEM);
1423
1424         err = drm_dev_init(&i915->drm, &driver, &pdev->dev);
1425         if (err) {
1426                 kfree(i915);
1427                 return ERR_PTR(err);
1428         }
1429
1430         i915->drm.dev_private = i915;
1431
1432         i915->drm.pdev = pdev;
1433         pci_set_drvdata(pdev, i915);
1434
1435         /* Setup the write-once "constant" device info */
1436         device_info = mkwrite_device_info(i915);
1437         memcpy(device_info, match_info, sizeof(*device_info));
1438         RUNTIME_INFO(i915)->device_id = pdev->device;
1439
1440         BUG_ON(device_info->gen > BITS_PER_TYPE(device_info->gen_mask));
1441
1442         return i915;
1443 }
1444
1445 static void i915_driver_destroy(struct drm_i915_private *i915)
1446 {
1447         struct pci_dev *pdev = i915->drm.pdev;
1448
1449         drm_dev_fini(&i915->drm);
1450         kfree(i915);
1451
1452         /* And make sure we never chase our dangling pointer from pci_dev */
1453         pci_set_drvdata(pdev, NULL);
1454 }
1455
1456 /**
1457  * i915_driver_probe - setup chip and create an initial config
1458  * @pdev: PCI device
1459  * @ent: matching PCI ID entry
1460  *
1461  * The driver probe routine has to do several things:
1462  *   - drive output discovery via intel_modeset_init()
1463  *   - initialize the memory manager
1464  *   - allocate initial config memory
1465  *   - setup the DRM framebuffer with the allocated memory
1466  */
1467 int i915_driver_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
1468 {
1469         const struct intel_device_info *match_info =
1470                 (struct intel_device_info *)ent->driver_data;
1471         struct drm_i915_private *dev_priv;
1472         int ret;
1473
1474         dev_priv = i915_driver_create(pdev, ent);
1475         if (IS_ERR(dev_priv))
1476                 return PTR_ERR(dev_priv);
1477
1478         /* Disable nuclear pageflip by default on pre-ILK */
1479         if (!i915_modparams.nuclear_pageflip && match_info->gen < 5)
1480                 dev_priv->drm.driver_features &= ~DRIVER_ATOMIC;
1481
1482         /*
1483          * Check if we support fake LMEM -- for now we only unleash this for
1484          * the live selftests(test-and-exit).
1485          */
1486 #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
1487         if (IS_ENABLED(CONFIG_DRM_I915_UNSTABLE_FAKE_LMEM)) {
1488                 if (INTEL_GEN(dev_priv) >= 9 && i915_selftest.live < 0 &&
1489                     i915_modparams.fake_lmem_start) {
1490                         mkwrite_device_info(dev_priv)->memory_regions =
1491                                 REGION_SMEM | REGION_LMEM | REGION_STOLEN;
1492                         mkwrite_device_info(dev_priv)->is_dgfx = true;
1493                         GEM_BUG_ON(!HAS_LMEM(dev_priv));
1494                         GEM_BUG_ON(!IS_DGFX(dev_priv));
1495                 }
1496         }
1497 #endif
1498
1499         ret = pci_enable_device(pdev);
1500         if (ret)
1501                 goto out_fini;
1502
1503         ret = i915_driver_early_probe(dev_priv);
1504         if (ret < 0)
1505                 goto out_pci_disable;
1506
1507         disable_rpm_wakeref_asserts(&dev_priv->runtime_pm);
1508
1509         i915_detect_vgpu(dev_priv);
1510
1511         ret = i915_driver_mmio_probe(dev_priv);
1512         if (ret < 0)
1513                 goto out_runtime_pm_put;
1514
1515         ret = i915_driver_hw_probe(dev_priv);
1516         if (ret < 0)
1517                 goto out_cleanup_mmio;
1518
1519         ret = i915_driver_modeset_probe(dev_priv);
1520         if (ret < 0)
1521                 goto out_cleanup_hw;
1522
1523         i915_driver_register(dev_priv);
1524
1525         enable_rpm_wakeref_asserts(&dev_priv->runtime_pm);
1526
1527         i915_welcome_messages(dev_priv);
1528
1529         return 0;
1530
1531 out_cleanup_hw:
1532         i915_driver_hw_remove(dev_priv);
1533         intel_memory_regions_driver_release(dev_priv);
1534         i915_ggtt_driver_release(dev_priv);
1535 out_cleanup_mmio:
1536         i915_driver_mmio_release(dev_priv);
1537 out_runtime_pm_put:
1538         enable_rpm_wakeref_asserts(&dev_priv->runtime_pm);
1539         i915_driver_late_release(dev_priv);
1540 out_pci_disable:
1541         pci_disable_device(pdev);
1542 out_fini:
1543         i915_probe_error(dev_priv, "Device initialization failed (%d)\n", ret);
1544         i915_driver_destroy(dev_priv);
1545         return ret;
1546 }
1547
1548 void i915_driver_remove(struct drm_i915_private *i915)
1549 {
1550         disable_rpm_wakeref_asserts(&i915->runtime_pm);
1551
1552         i915_driver_unregister(i915);
1553
1554         /*
1555          * After unregistering the device to prevent any new users, cancel
1556          * all in-flight requests so that we can quickly unbind the active
1557          * resources.
1558          */
1559         intel_gt_set_wedged(&i915->gt);
1560
1561         /* Flush any external code that still may be under the RCU lock */
1562         synchronize_rcu();
1563
1564         i915_gem_suspend(i915);
1565
1566         drm_atomic_helper_shutdown(&i915->drm);
1567
1568         intel_gvt_driver_remove(i915);
1569
1570         i915_driver_modeset_remove(i915);
1571
1572         i915_reset_error_state(i915);
1573         i915_gem_driver_remove(i915);
1574
1575         intel_power_domains_driver_remove(i915);
1576
1577         i915_driver_hw_remove(i915);
1578
1579         enable_rpm_wakeref_asserts(&i915->runtime_pm);
1580 }
1581
1582 static void i915_driver_release(struct drm_device *dev)
1583 {
1584         struct drm_i915_private *dev_priv = to_i915(dev);
1585         struct intel_runtime_pm *rpm = &dev_priv->runtime_pm;
1586
1587         disable_rpm_wakeref_asserts(rpm);
1588
1589         i915_gem_driver_release(dev_priv);
1590
1591         intel_memory_regions_driver_release(dev_priv);
1592         i915_ggtt_driver_release(dev_priv);
1593
1594         i915_driver_mmio_release(dev_priv);
1595
1596         enable_rpm_wakeref_asserts(rpm);
1597         intel_runtime_pm_driver_release(rpm);
1598
1599         i915_driver_late_release(dev_priv);
1600         i915_driver_destroy(dev_priv);
1601 }
1602
1603 static int i915_driver_open(struct drm_device *dev, struct drm_file *file)
1604 {
1605         struct drm_i915_private *i915 = to_i915(dev);
1606         int ret;
1607
1608         ret = i915_gem_open(i915, file);
1609         if (ret)
1610                 return ret;
1611
1612         return 0;
1613 }
1614
1615 /**
1616  * i915_driver_lastclose - clean up after all DRM clients have exited
1617  * @dev: DRM device
1618  *
1619  * Take care of cleaning up after all DRM clients have exited.  In the
1620  * mode setting case, we want to restore the kernel's initial mode (just
1621  * in case the last client left us in a bad state).
1622  *
1623  * Additionally, in the non-mode setting case, we'll tear down the GTT
1624  * and DMA structures, since the kernel won't be using them, and clea
1625  * up any GEM state.
1626  */
1627 static void i915_driver_lastclose(struct drm_device *dev)
1628 {
1629         intel_fbdev_restore_mode(dev);
1630         vga_switcheroo_process_delayed_switch();
1631 }
1632
1633 static void i915_driver_postclose(struct drm_device *dev, struct drm_file *file)
1634 {
1635         struct drm_i915_file_private *file_priv = file->driver_priv;
1636
1637         i915_gem_context_close(file);
1638         i915_gem_release(dev, file);
1639
1640         kfree_rcu(file_priv, rcu);
1641
1642         /* Catch up with all the deferred frees from "this" client */
1643         i915_gem_flush_free_objects(to_i915(dev));
1644 }
1645
1646 static void intel_suspend_encoders(struct drm_i915_private *dev_priv)
1647 {
1648         struct drm_device *dev = &dev_priv->drm;
1649         struct intel_encoder *encoder;
1650
1651         drm_modeset_lock_all(dev);
1652         for_each_intel_encoder(dev, encoder)
1653                 if (encoder->suspend)
1654                         encoder->suspend(encoder);
1655         drm_modeset_unlock_all(dev);
1656 }
1657
1658 static int vlv_resume_prepare(struct drm_i915_private *dev_priv,
1659                               bool rpm_resume);
1660 static int vlv_suspend_complete(struct drm_i915_private *dev_priv);
1661
1662 static bool suspend_to_idle(struct drm_i915_private *dev_priv)
1663 {
1664 #if IS_ENABLED(CONFIG_ACPI_SLEEP)
1665         if (acpi_target_system_state() < ACPI_STATE_S3)
1666                 return true;
1667 #endif
1668         return false;
1669 }
1670
1671 static int i915_drm_prepare(struct drm_device *dev)
1672 {
1673         struct drm_i915_private *i915 = to_i915(dev);
1674
1675         /*
1676          * NB intel_display_suspend() may issue new requests after we've
1677          * ostensibly marked the GPU as ready-to-sleep here. We need to
1678          * split out that work and pull it forward so that after point,
1679          * the GPU is not woken again.
1680          */
1681         i915_gem_suspend(i915);
1682
1683         return 0;
1684 }
1685
1686 static int i915_drm_suspend(struct drm_device *dev)
1687 {
1688         struct drm_i915_private *dev_priv = to_i915(dev);
1689         struct pci_dev *pdev = dev_priv->drm.pdev;
1690         pci_power_t opregion_target_state;
1691
1692         disable_rpm_wakeref_asserts(&dev_priv->runtime_pm);
1693
1694         /* We do a lot of poking in a lot of registers, make sure they work
1695          * properly. */
1696         intel_power_domains_disable(dev_priv);
1697
1698         drm_kms_helper_poll_disable(dev);
1699
1700         pci_save_state(pdev);
1701
1702         intel_display_suspend(dev);
1703
1704         intel_dp_mst_suspend(dev_priv);
1705
1706         intel_runtime_pm_disable_interrupts(dev_priv);
1707         intel_hpd_cancel_work(dev_priv);
1708
1709         intel_suspend_encoders(dev_priv);
1710
1711         intel_suspend_hw(dev_priv);
1712
1713         i915_gem_suspend_gtt_mappings(dev_priv);
1714
1715         i915_save_state(dev_priv);
1716
1717         opregion_target_state = suspend_to_idle(dev_priv) ? PCI_D1 : PCI_D3cold;
1718         intel_opregion_suspend(dev_priv, opregion_target_state);
1719
1720         intel_fbdev_set_suspend(dev, FBINFO_STATE_SUSPENDED, true);
1721
1722         dev_priv->suspend_count++;
1723
1724         intel_csr_ucode_suspend(dev_priv);
1725
1726         enable_rpm_wakeref_asserts(&dev_priv->runtime_pm);
1727
1728         return 0;
1729 }
1730
1731 static enum i915_drm_suspend_mode
1732 get_suspend_mode(struct drm_i915_private *dev_priv, bool hibernate)
1733 {
1734         if (hibernate)
1735                 return I915_DRM_SUSPEND_HIBERNATE;
1736
1737         if (suspend_to_idle(dev_priv))
1738                 return I915_DRM_SUSPEND_IDLE;
1739
1740         return I915_DRM_SUSPEND_MEM;
1741 }
1742
1743 static int i915_drm_suspend_late(struct drm_device *dev, bool hibernation)
1744 {
1745         struct drm_i915_private *dev_priv = to_i915(dev);
1746         struct pci_dev *pdev = dev_priv->drm.pdev;
1747         struct intel_runtime_pm *rpm = &dev_priv->runtime_pm;
1748         int ret = 0;
1749
1750         disable_rpm_wakeref_asserts(rpm);
1751
1752         i915_gem_suspend_late(dev_priv);
1753
1754         intel_uncore_suspend(&dev_priv->uncore);
1755
1756         intel_power_domains_suspend(dev_priv,
1757                                     get_suspend_mode(dev_priv, hibernation));
1758
1759         intel_display_power_suspend_late(dev_priv);
1760
1761         if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
1762                 ret = vlv_suspend_complete(dev_priv);
1763
1764         if (ret) {
1765                 DRM_ERROR("Suspend complete failed: %d\n", ret);
1766                 intel_power_domains_resume(dev_priv);
1767
1768                 goto out;
1769         }
1770
1771         pci_disable_device(pdev);
1772         /*
1773          * During hibernation on some platforms the BIOS may try to access
1774          * the device even though it's already in D3 and hang the machine. So
1775          * leave the device in D0 on those platforms and hope the BIOS will
1776          * power down the device properly. The issue was seen on multiple old
1777          * GENs with different BIOS vendors, so having an explicit blacklist
1778          * is inpractical; apply the workaround on everything pre GEN6. The
1779          * platforms where the issue was seen:
1780          * Lenovo Thinkpad X301, X61s, X60, T60, X41
1781          * Fujitsu FSC S7110
1782          * Acer Aspire 1830T
1783          */
1784         if (!(hibernation && INTEL_GEN(dev_priv) < 6))
1785                 pci_set_power_state(pdev, PCI_D3hot);
1786
1787 out:
1788         enable_rpm_wakeref_asserts(rpm);
1789         if (!dev_priv->uncore.user_forcewake_count)
1790                 intel_runtime_pm_driver_release(rpm);
1791
1792         return ret;
1793 }
1794
1795 int i915_suspend_switcheroo(struct drm_i915_private *i915, pm_message_t state)
1796 {
1797         int error;
1798
1799         if (WARN_ON_ONCE(state.event != PM_EVENT_SUSPEND &&
1800                          state.event != PM_EVENT_FREEZE))
1801                 return -EINVAL;
1802
1803         if (i915->drm.switch_power_state == DRM_SWITCH_POWER_OFF)
1804                 return 0;
1805
1806         error = i915_drm_suspend(&i915->drm);
1807         if (error)
1808                 return error;
1809
1810         return i915_drm_suspend_late(&i915->drm, false);
1811 }
1812
1813 static int i915_drm_resume(struct drm_device *dev)
1814 {
1815         struct drm_i915_private *dev_priv = to_i915(dev);
1816         int ret;
1817
1818         disable_rpm_wakeref_asserts(&dev_priv->runtime_pm);
1819
1820         intel_rc6_ctx_wa_resume(&dev_priv->gt.rc6);
1821
1822         intel_gt_sanitize(&dev_priv->gt, true);
1823
1824         ret = i915_ggtt_enable_hw(dev_priv);
1825         if (ret)
1826                 DRM_ERROR("failed to re-enable GGTT\n");
1827
1828         i915_gem_restore_gtt_mappings(dev_priv);
1829         i915_gem_restore_fences(&dev_priv->ggtt);
1830
1831         intel_csr_ucode_resume(dev_priv);
1832
1833         i915_restore_state(dev_priv);
1834         intel_pps_unlock_regs_wa(dev_priv);
1835
1836         intel_init_pch_refclk(dev_priv);
1837
1838         /*
1839          * Interrupts have to be enabled before any batches are run. If not the
1840          * GPU will hang. i915_gem_init_hw() will initiate batches to
1841          * update/restore the context.
1842          *
1843          * drm_mode_config_reset() needs AUX interrupts.
1844          *
1845          * Modeset enabling in intel_modeset_init_hw() also needs working
1846          * interrupts.
1847          */
1848         intel_runtime_pm_enable_interrupts(dev_priv);
1849
1850         drm_mode_config_reset(dev);
1851
1852         i915_gem_resume(dev_priv);
1853
1854         intel_modeset_init_hw(dev_priv);
1855         intel_init_clock_gating(dev_priv);
1856
1857         spin_lock_irq(&dev_priv->irq_lock);
1858         if (dev_priv->display.hpd_irq_setup)
1859                 dev_priv->display.hpd_irq_setup(dev_priv);
1860         spin_unlock_irq(&dev_priv->irq_lock);
1861
1862         intel_dp_mst_resume(dev_priv);
1863
1864         intel_display_resume(dev);
1865
1866         drm_kms_helper_poll_enable(dev);
1867
1868         /*
1869          * ... but also need to make sure that hotplug processing
1870          * doesn't cause havoc. Like in the driver load code we don't
1871          * bother with the tiny race here where we might lose hotplug
1872          * notifications.
1873          * */
1874         intel_hpd_init(dev_priv);
1875
1876         intel_opregion_resume(dev_priv);
1877
1878         intel_fbdev_set_suspend(dev, FBINFO_STATE_RUNNING, false);
1879
1880         intel_power_domains_enable(dev_priv);
1881
1882         enable_rpm_wakeref_asserts(&dev_priv->runtime_pm);
1883
1884         return 0;
1885 }
1886
1887 static int i915_drm_resume_early(struct drm_device *dev)
1888 {
1889         struct drm_i915_private *dev_priv = to_i915(dev);
1890         struct pci_dev *pdev = dev_priv->drm.pdev;
1891         int ret;
1892
1893         /*
1894          * We have a resume ordering issue with the snd-hda driver also
1895          * requiring our device to be power up. Due to the lack of a
1896          * parent/child relationship we currently solve this with an early
1897          * resume hook.
1898          *
1899          * FIXME: This should be solved with a special hdmi sink device or
1900          * similar so that power domains can be employed.
1901          */
1902
1903         /*
1904          * Note that we need to set the power state explicitly, since we
1905          * powered off the device during freeze and the PCI core won't power
1906          * it back up for us during thaw. Powering off the device during
1907          * freeze is not a hard requirement though, and during the
1908          * suspend/resume phases the PCI core makes sure we get here with the
1909          * device powered on. So in case we change our freeze logic and keep
1910          * the device powered we can also remove the following set power state
1911          * call.
1912          */
1913         ret = pci_set_power_state(pdev, PCI_D0);
1914         if (ret) {
1915                 DRM_ERROR("failed to set PCI D0 power state (%d)\n", ret);
1916                 return ret;
1917         }
1918
1919         /*
1920          * Note that pci_enable_device() first enables any parent bridge
1921          * device and only then sets the power state for this device. The
1922          * bridge enabling is a nop though, since bridge devices are resumed
1923          * first. The order of enabling power and enabling the device is
1924          * imposed by the PCI core as described above, so here we preserve the
1925          * same order for the freeze/thaw phases.
1926          *
1927          * TODO: eventually we should remove pci_disable_device() /
1928          * pci_enable_enable_device() from suspend/resume. Due to how they
1929          * depend on the device enable refcount we can't anyway depend on them
1930          * disabling/enabling the device.
1931          */
1932         if (pci_enable_device(pdev))
1933                 return -EIO;
1934
1935         pci_set_master(pdev);
1936
1937         disable_rpm_wakeref_asserts(&dev_priv->runtime_pm);
1938
1939         if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
1940                 ret = vlv_resume_prepare(dev_priv, false);
1941         if (ret)
1942                 DRM_ERROR("Resume prepare failed: %d, continuing anyway\n",
1943                           ret);
1944
1945         intel_uncore_resume_early(&dev_priv->uncore);
1946
1947         intel_gt_check_and_clear_faults(&dev_priv->gt);
1948
1949         intel_display_power_resume_early(dev_priv);
1950
1951         intel_power_domains_resume(dev_priv);
1952
1953         enable_rpm_wakeref_asserts(&dev_priv->runtime_pm);
1954
1955         return ret;
1956 }
1957
1958 int i915_resume_switcheroo(struct drm_i915_private *i915)
1959 {
1960         int ret;
1961
1962         if (i915->drm.switch_power_state == DRM_SWITCH_POWER_OFF)
1963                 return 0;
1964
1965         ret = i915_drm_resume_early(&i915->drm);
1966         if (ret)
1967                 return ret;
1968
1969         return i915_drm_resume(&i915->drm);
1970 }
1971
1972 static int i915_pm_prepare(struct device *kdev)
1973 {
1974         struct drm_i915_private *i915 = kdev_to_i915(kdev);
1975
1976         if (!i915) {
1977                 dev_err(kdev, "DRM not initialized, aborting suspend.\n");
1978                 return -ENODEV;
1979         }
1980
1981         if (i915->drm.switch_power_state == DRM_SWITCH_POWER_OFF)
1982                 return 0;
1983
1984         return i915_drm_prepare(&i915->drm);
1985 }
1986
1987 static int i915_pm_suspend(struct device *kdev)
1988 {
1989         struct drm_i915_private *i915 = kdev_to_i915(kdev);
1990
1991         if (!i915) {
1992                 dev_err(kdev, "DRM not initialized, aborting suspend.\n");
1993                 return -ENODEV;
1994         }
1995
1996         if (i915->drm.switch_power_state == DRM_SWITCH_POWER_OFF)
1997                 return 0;
1998
1999         return i915_drm_suspend(&i915->drm);
2000 }
2001
2002 static int i915_pm_suspend_late(struct device *kdev)
2003 {
2004         struct drm_i915_private *i915 = kdev_to_i915(kdev);
2005
2006         /*
2007          * We have a suspend ordering issue with the snd-hda driver also
2008          * requiring our device to be power up. Due to the lack of a
2009          * parent/child relationship we currently solve this with an late
2010          * suspend hook.
2011          *
2012          * FIXME: This should be solved with a special hdmi sink device or
2013          * similar so that power domains can be employed.
2014          */
2015         if (i915->drm.switch_power_state == DRM_SWITCH_POWER_OFF)
2016                 return 0;
2017
2018         return i915_drm_suspend_late(&i915->drm, false);
2019 }
2020
2021 static int i915_pm_poweroff_late(struct device *kdev)
2022 {
2023         struct drm_i915_private *i915 = kdev_to_i915(kdev);
2024
2025         if (i915->drm.switch_power_state == DRM_SWITCH_POWER_OFF)
2026                 return 0;
2027
2028         return i915_drm_suspend_late(&i915->drm, true);
2029 }
2030
2031 static int i915_pm_resume_early(struct device *kdev)
2032 {
2033         struct drm_i915_private *i915 = kdev_to_i915(kdev);
2034
2035         if (i915->drm.switch_power_state == DRM_SWITCH_POWER_OFF)
2036                 return 0;
2037
2038         return i915_drm_resume_early(&i915->drm);
2039 }
2040
2041 static int i915_pm_resume(struct device *kdev)
2042 {
2043         struct drm_i915_private *i915 = kdev_to_i915(kdev);
2044
2045         if (i915->drm.switch_power_state == DRM_SWITCH_POWER_OFF)
2046                 return 0;
2047
2048         return i915_drm_resume(&i915->drm);
2049 }
2050
2051 /* freeze: before creating the hibernation_image */
2052 static int i915_pm_freeze(struct device *kdev)
2053 {
2054         struct drm_i915_private *i915 = kdev_to_i915(kdev);
2055         int ret;
2056
2057         if (i915->drm.switch_power_state != DRM_SWITCH_POWER_OFF) {
2058                 ret = i915_drm_suspend(&i915->drm);
2059                 if (ret)
2060                         return ret;
2061         }
2062
2063         ret = i915_gem_freeze(i915);
2064         if (ret)
2065                 return ret;
2066
2067         return 0;
2068 }
2069
2070 static int i915_pm_freeze_late(struct device *kdev)
2071 {
2072         struct drm_i915_private *i915 = kdev_to_i915(kdev);
2073         int ret;
2074
2075         if (i915->drm.switch_power_state != DRM_SWITCH_POWER_OFF) {
2076                 ret = i915_drm_suspend_late(&i915->drm, true);
2077                 if (ret)
2078                         return ret;
2079         }
2080
2081         ret = i915_gem_freeze_late(i915);
2082         if (ret)
2083                 return ret;
2084
2085         return 0;
2086 }
2087
2088 /* thaw: called after creating the hibernation image, but before turning off. */
2089 static int i915_pm_thaw_early(struct device *kdev)
2090 {
2091         return i915_pm_resume_early(kdev);
2092 }
2093
2094 static int i915_pm_thaw(struct device *kdev)
2095 {
2096         return i915_pm_resume(kdev);
2097 }
2098
2099 /* restore: called after loading the hibernation image. */
2100 static int i915_pm_restore_early(struct device *kdev)
2101 {
2102         return i915_pm_resume_early(kdev);
2103 }
2104
2105 static int i915_pm_restore(struct device *kdev)
2106 {
2107         return i915_pm_resume(kdev);
2108 }
2109
2110 /*
2111  * Save all Gunit registers that may be lost after a D3 and a subsequent
2112  * S0i[R123] transition. The list of registers needing a save/restore is
2113  * defined in the VLV2_S0IXRegs document. This documents marks all Gunit
2114  * registers in the following way:
2115  * - Driver: saved/restored by the driver
2116  * - Punit : saved/restored by the Punit firmware
2117  * - No, w/o marking: no need to save/restore, since the register is R/O or
2118  *                    used internally by the HW in a way that doesn't depend
2119  *                    keeping the content across a suspend/resume.
2120  * - Debug : used for debugging
2121  *
2122  * We save/restore all registers marked with 'Driver', with the following
2123  * exceptions:
2124  * - Registers out of use, including also registers marked with 'Debug'.
2125  *   These have no effect on the driver's operation, so we don't save/restore
2126  *   them to reduce the overhead.
2127  * - Registers that are fully setup by an initialization function called from
2128  *   the resume path. For example many clock gating and RPS/RC6 registers.
2129  * - Registers that provide the right functionality with their reset defaults.
2130  *
2131  * TODO: Except for registers that based on the above 3 criteria can be safely
2132  * ignored, we save/restore all others, practically treating the HW context as
2133  * a black-box for the driver. Further investigation is needed to reduce the
2134  * saved/restored registers even further, by following the same 3 criteria.
2135  */
2136 static void vlv_save_gunit_s0ix_state(struct drm_i915_private *dev_priv)
2137 {
2138         struct vlv_s0ix_state *s = dev_priv->vlv_s0ix_state;
2139         int i;
2140
2141         if (!s)
2142                 return;
2143
2144         /* GAM 0x4000-0x4770 */
2145         s->wr_watermark         = I915_READ(GEN7_WR_WATERMARK);
2146         s->gfx_prio_ctrl        = I915_READ(GEN7_GFX_PRIO_CTRL);
2147         s->arb_mode             = I915_READ(ARB_MODE);
2148         s->gfx_pend_tlb0        = I915_READ(GEN7_GFX_PEND_TLB0);
2149         s->gfx_pend_tlb1        = I915_READ(GEN7_GFX_PEND_TLB1);
2150
2151         for (i = 0; i < ARRAY_SIZE(s->lra_limits); i++)
2152                 s->lra_limits[i] = I915_READ(GEN7_LRA_LIMITS(i));
2153
2154         s->media_max_req_count  = I915_READ(GEN7_MEDIA_MAX_REQ_COUNT);
2155         s->gfx_max_req_count    = I915_READ(GEN7_GFX_MAX_REQ_COUNT);
2156
2157         s->render_hwsp          = I915_READ(RENDER_HWS_PGA_GEN7);
2158         s->ecochk               = I915_READ(GAM_ECOCHK);
2159         s->bsd_hwsp             = I915_READ(BSD_HWS_PGA_GEN7);
2160         s->blt_hwsp             = I915_READ(BLT_HWS_PGA_GEN7);
2161
2162         s->tlb_rd_addr          = I915_READ(GEN7_TLB_RD_ADDR);
2163
2164         /* MBC 0x9024-0x91D0, 0x8500 */
2165         s->g3dctl               = I915_READ(VLV_G3DCTL);
2166         s->gsckgctl             = I915_READ(VLV_GSCKGCTL);
2167         s->mbctl                = I915_READ(GEN6_MBCTL);
2168
2169         /* GCP 0x9400-0x9424, 0x8100-0x810C */
2170         s->ucgctl1              = I915_READ(GEN6_UCGCTL1);
2171         s->ucgctl3              = I915_READ(GEN6_UCGCTL3);
2172         s->rcgctl1              = I915_READ(GEN6_RCGCTL1);
2173         s->rcgctl2              = I915_READ(GEN6_RCGCTL2);
2174         s->rstctl               = I915_READ(GEN6_RSTCTL);
2175         s->misccpctl            = I915_READ(GEN7_MISCCPCTL);
2176
2177         /* GPM 0xA000-0xAA84, 0x8000-0x80FC */
2178         s->gfxpause             = I915_READ(GEN6_GFXPAUSE);
2179         s->rpdeuhwtc            = I915_READ(GEN6_RPDEUHWTC);
2180         s->rpdeuc               = I915_READ(GEN6_RPDEUC);
2181         s->ecobus               = I915_READ(ECOBUS);
2182         s->pwrdwnupctl          = I915_READ(VLV_PWRDWNUPCTL);
2183         s->rp_down_timeout      = I915_READ(GEN6_RP_DOWN_TIMEOUT);
2184         s->rp_deucsw            = I915_READ(GEN6_RPDEUCSW);
2185         s->rcubmabdtmr          = I915_READ(GEN6_RCUBMABDTMR);
2186         s->rcedata              = I915_READ(VLV_RCEDATA);
2187         s->spare2gh             = I915_READ(VLV_SPAREG2H);
2188
2189         /* Display CZ domain, 0x4400C-0x4402C, 0x4F000-0x4F11F */
2190         s->gt_imr               = I915_READ(GTIMR);
2191         s->gt_ier               = I915_READ(GTIER);
2192         s->pm_imr               = I915_READ(GEN6_PMIMR);
2193         s->pm_ier               = I915_READ(GEN6_PMIER);
2194
2195         for (i = 0; i < ARRAY_SIZE(s->gt_scratch); i++)
2196                 s->gt_scratch[i] = I915_READ(GEN7_GT_SCRATCH(i));
2197
2198         /* GT SA CZ domain, 0x100000-0x138124 */
2199         s->tilectl              = I915_READ(TILECTL);
2200         s->gt_fifoctl           = I915_READ(GTFIFOCTL);
2201         s->gtlc_wake_ctrl       = I915_READ(VLV_GTLC_WAKE_CTRL);
2202         s->gtlc_survive         = I915_READ(VLV_GTLC_SURVIVABILITY_REG);
2203         s->pmwgicz              = I915_READ(VLV_PMWGICZ);
2204
2205         /* Gunit-Display CZ domain, 0x182028-0x1821CF */
2206         s->gu_ctl0              = I915_READ(VLV_GU_CTL0);
2207         s->gu_ctl1              = I915_READ(VLV_GU_CTL1);
2208         s->pcbr                 = I915_READ(VLV_PCBR);
2209         s->clock_gate_dis2      = I915_READ(VLV_GUNIT_CLOCK_GATE2);
2210
2211         /*
2212          * Not saving any of:
2213          * DFT,         0x9800-0x9EC0
2214          * SARB,        0xB000-0xB1FC
2215          * GAC,         0x5208-0x524C, 0x14000-0x14C000
2216          * PCI CFG
2217          */
2218 }
2219
2220 static void vlv_restore_gunit_s0ix_state(struct drm_i915_private *dev_priv)
2221 {
2222         struct vlv_s0ix_state *s = dev_priv->vlv_s0ix_state;
2223         u32 val;
2224         int i;
2225
2226         if (!s)
2227                 return;
2228
2229         /* GAM 0x4000-0x4770 */
2230         I915_WRITE(GEN7_WR_WATERMARK,   s->wr_watermark);
2231         I915_WRITE(GEN7_GFX_PRIO_CTRL,  s->gfx_prio_ctrl);
2232         I915_WRITE(ARB_MODE,            s->arb_mode | (0xffff << 16));
2233         I915_WRITE(GEN7_GFX_PEND_TLB0,  s->gfx_pend_tlb0);
2234         I915_WRITE(GEN7_GFX_PEND_TLB1,  s->gfx_pend_tlb1);
2235
2236         for (i = 0; i < ARRAY_SIZE(s->lra_limits); i++)
2237                 I915_WRITE(GEN7_LRA_LIMITS(i), s->lra_limits[i]);
2238
2239         I915_WRITE(GEN7_MEDIA_MAX_REQ_COUNT, s->media_max_req_count);
2240         I915_WRITE(GEN7_GFX_MAX_REQ_COUNT, s->gfx_max_req_count);
2241
2242         I915_WRITE(RENDER_HWS_PGA_GEN7, s->render_hwsp);
2243         I915_WRITE(GAM_ECOCHK,          s->ecochk);
2244         I915_WRITE(BSD_HWS_PGA_GEN7,    s->bsd_hwsp);
2245         I915_WRITE(BLT_HWS_PGA_GEN7,    s->blt_hwsp);
2246
2247         I915_WRITE(GEN7_TLB_RD_ADDR,    s->tlb_rd_addr);
2248
2249         /* MBC 0x9024-0x91D0, 0x8500 */
2250         I915_WRITE(VLV_G3DCTL,          s->g3dctl);
2251         I915_WRITE(VLV_GSCKGCTL,        s->gsckgctl);
2252         I915_WRITE(GEN6_MBCTL,          s->mbctl);
2253
2254         /* GCP 0x9400-0x9424, 0x8100-0x810C */
2255         I915_WRITE(GEN6_UCGCTL1,        s->ucgctl1);
2256         I915_WRITE(GEN6_UCGCTL3,        s->ucgctl3);
2257         I915_WRITE(GEN6_RCGCTL1,        s->rcgctl1);
2258         I915_WRITE(GEN6_RCGCTL2,        s->rcgctl2);
2259         I915_WRITE(GEN6_RSTCTL,         s->rstctl);
2260         I915_WRITE(GEN7_MISCCPCTL,      s->misccpctl);
2261
2262         /* GPM 0xA000-0xAA84, 0x8000-0x80FC */
2263         I915_WRITE(GEN6_GFXPAUSE,       s->gfxpause);
2264         I915_WRITE(GEN6_RPDEUHWTC,      s->rpdeuhwtc);
2265         I915_WRITE(GEN6_RPDEUC,         s->rpdeuc);
2266         I915_WRITE(ECOBUS,              s->ecobus);
2267         I915_WRITE(VLV_PWRDWNUPCTL,     s->pwrdwnupctl);
2268         I915_WRITE(GEN6_RP_DOWN_TIMEOUT,s->rp_down_timeout);
2269         I915_WRITE(GEN6_RPDEUCSW,       s->rp_deucsw);
2270         I915_WRITE(GEN6_RCUBMABDTMR,    s->rcubmabdtmr);
2271         I915_WRITE(VLV_RCEDATA,         s->rcedata);
2272         I915_WRITE(VLV_SPAREG2H,        s->spare2gh);
2273
2274         /* Display CZ domain, 0x4400C-0x4402C, 0x4F000-0x4F11F */
2275         I915_WRITE(GTIMR,               s->gt_imr);
2276         I915_WRITE(GTIER,               s->gt_ier);
2277         I915_WRITE(GEN6_PMIMR,          s->pm_imr);
2278         I915_WRITE(GEN6_PMIER,          s->pm_ier);
2279
2280         for (i = 0; i < ARRAY_SIZE(s->gt_scratch); i++)
2281                 I915_WRITE(GEN7_GT_SCRATCH(i), s->gt_scratch[i]);
2282
2283         /* GT SA CZ domain, 0x100000-0x138124 */
2284         I915_WRITE(TILECTL,                     s->tilectl);
2285         I915_WRITE(GTFIFOCTL,                   s->gt_fifoctl);
2286         /*
2287          * Preserve the GT allow wake and GFX force clock bit, they are not
2288          * be restored, as they are used to control the s0ix suspend/resume
2289          * sequence by the caller.
2290          */
2291         val = I915_READ(VLV_GTLC_WAKE_CTRL);
2292         val &= VLV_GTLC_ALLOWWAKEREQ;
2293         val |= s->gtlc_wake_ctrl & ~VLV_GTLC_ALLOWWAKEREQ;
2294         I915_WRITE(VLV_GTLC_WAKE_CTRL, val);
2295
2296         val = I915_READ(VLV_GTLC_SURVIVABILITY_REG);
2297         val &= VLV_GFX_CLK_FORCE_ON_BIT;
2298         val |= s->gtlc_survive & ~VLV_GFX_CLK_FORCE_ON_BIT;
2299         I915_WRITE(VLV_GTLC_SURVIVABILITY_REG, val);
2300
2301         I915_WRITE(VLV_PMWGICZ,                 s->pmwgicz);
2302
2303         /* Gunit-Display CZ domain, 0x182028-0x1821CF */
2304         I915_WRITE(VLV_GU_CTL0,                 s->gu_ctl0);
2305         I915_WRITE(VLV_GU_CTL1,                 s->gu_ctl1);
2306         I915_WRITE(VLV_PCBR,                    s->pcbr);
2307         I915_WRITE(VLV_GUNIT_CLOCK_GATE2,       s->clock_gate_dis2);
2308 }
2309
2310 static int vlv_wait_for_pw_status(struct drm_i915_private *i915,
2311                                   u32 mask, u32 val)
2312 {
2313         i915_reg_t reg = VLV_GTLC_PW_STATUS;
2314         u32 reg_value;
2315         int ret;
2316
2317         /* The HW does not like us polling for PW_STATUS frequently, so
2318          * use the sleeping loop rather than risk the busy spin within
2319          * intel_wait_for_register().
2320          *
2321          * Transitioning between RC6 states should be at most 2ms (see
2322          * valleyview_enable_rps) so use a 3ms timeout.
2323          */
2324         ret = wait_for(((reg_value =
2325                          intel_uncore_read_notrace(&i915->uncore, reg)) & mask)
2326                        == val, 3);
2327
2328         /* just trace the final value */
2329         trace_i915_reg_rw(false, reg, reg_value, sizeof(reg_value), true);
2330
2331         return ret;
2332 }
2333
2334 int vlv_force_gfx_clock(struct drm_i915_private *dev_priv, bool force_on)
2335 {
2336         u32 val;
2337         int err;
2338
2339         val = I915_READ(VLV_GTLC_SURVIVABILITY_REG);
2340         val &= ~VLV_GFX_CLK_FORCE_ON_BIT;
2341         if (force_on)
2342                 val |= VLV_GFX_CLK_FORCE_ON_BIT;
2343         I915_WRITE(VLV_GTLC_SURVIVABILITY_REG, val);
2344
2345         if (!force_on)
2346                 return 0;
2347
2348         err = intel_wait_for_register(&dev_priv->uncore,
2349                                       VLV_GTLC_SURVIVABILITY_REG,
2350                                       VLV_GFX_CLK_STATUS_BIT,
2351                                       VLV_GFX_CLK_STATUS_BIT,
2352                                       20);
2353         if (err)
2354                 DRM_ERROR("timeout waiting for GFX clock force-on (%08x)\n",
2355                           I915_READ(VLV_GTLC_SURVIVABILITY_REG));
2356
2357         return err;
2358 }
2359
2360 static int vlv_allow_gt_wake(struct drm_i915_private *dev_priv, bool allow)
2361 {
2362         u32 mask;
2363         u32 val;
2364         int err;
2365
2366         val = I915_READ(VLV_GTLC_WAKE_CTRL);
2367         val &= ~VLV_GTLC_ALLOWWAKEREQ;
2368         if (allow)
2369                 val |= VLV_GTLC_ALLOWWAKEREQ;
2370         I915_WRITE(VLV_GTLC_WAKE_CTRL, val);
2371         POSTING_READ(VLV_GTLC_WAKE_CTRL);
2372
2373         mask = VLV_GTLC_ALLOWWAKEACK;
2374         val = allow ? mask : 0;
2375
2376         err = vlv_wait_for_pw_status(dev_priv, mask, val);
2377         if (err)
2378                 DRM_ERROR("timeout disabling GT waking\n");
2379
2380         return err;
2381 }
2382
2383 static void vlv_wait_for_gt_wells(struct drm_i915_private *dev_priv,
2384                                   bool wait_for_on)
2385 {
2386         u32 mask;
2387         u32 val;
2388
2389         mask = VLV_GTLC_PW_MEDIA_STATUS_MASK | VLV_GTLC_PW_RENDER_STATUS_MASK;
2390         val = wait_for_on ? mask : 0;
2391
2392         /*
2393          * RC6 transitioning can be delayed up to 2 msec (see
2394          * valleyview_enable_rps), use 3 msec for safety.
2395          *
2396          * This can fail to turn off the rc6 if the GPU is stuck after a failed
2397          * reset and we are trying to force the machine to sleep.
2398          */
2399         if (vlv_wait_for_pw_status(dev_priv, mask, val))
2400                 DRM_DEBUG_DRIVER("timeout waiting for GT wells to go %s\n",
2401                                  onoff(wait_for_on));
2402 }
2403
2404 static void vlv_check_no_gt_access(struct drm_i915_private *dev_priv)
2405 {
2406         if (!(I915_READ(VLV_GTLC_PW_STATUS) & VLV_GTLC_ALLOWWAKEERR))
2407                 return;
2408
2409         DRM_DEBUG_DRIVER("GT register access while GT waking disabled\n");
2410         I915_WRITE(VLV_GTLC_PW_STATUS, VLV_GTLC_ALLOWWAKEERR);
2411 }
2412
2413 static int vlv_suspend_complete(struct drm_i915_private *dev_priv)
2414 {
2415         u32 mask;
2416         int err;
2417
2418         /*
2419          * Bspec defines the following GT well on flags as debug only, so
2420          * don't treat them as hard failures.
2421          */
2422         vlv_wait_for_gt_wells(dev_priv, false);
2423
2424         mask = VLV_GTLC_RENDER_CTX_EXISTS | VLV_GTLC_MEDIA_CTX_EXISTS;
2425         WARN_ON((I915_READ(VLV_GTLC_WAKE_CTRL) & mask) != mask);
2426
2427         vlv_check_no_gt_access(dev_priv);
2428
2429         err = vlv_force_gfx_clock(dev_priv, true);
2430         if (err)
2431                 goto err1;
2432
2433         err = vlv_allow_gt_wake(dev_priv, false);
2434         if (err)
2435                 goto err2;
2436
2437         vlv_save_gunit_s0ix_state(dev_priv);
2438
2439         err = vlv_force_gfx_clock(dev_priv, false);
2440         if (err)
2441                 goto err2;
2442
2443         return 0;
2444
2445 err2:
2446         /* For safety always re-enable waking and disable gfx clock forcing */
2447         vlv_allow_gt_wake(dev_priv, true);
2448 err1:
2449         vlv_force_gfx_clock(dev_priv, false);
2450
2451         return err;
2452 }
2453
2454 static int vlv_resume_prepare(struct drm_i915_private *dev_priv,
2455                                 bool rpm_resume)
2456 {
2457         int err;
2458         int ret;
2459
2460         /*
2461          * If any of the steps fail just try to continue, that's the best we
2462          * can do at this point. Return the first error code (which will also
2463          * leave RPM permanently disabled).
2464          */
2465         ret = vlv_force_gfx_clock(dev_priv, true);
2466
2467         vlv_restore_gunit_s0ix_state(dev_priv);
2468
2469         err = vlv_allow_gt_wake(dev_priv, true);
2470         if (!ret)
2471                 ret = err;
2472
2473         err = vlv_force_gfx_clock(dev_priv, false);
2474         if (!ret)
2475                 ret = err;
2476
2477         vlv_check_no_gt_access(dev_priv);
2478
2479         if (rpm_resume)
2480                 intel_init_clock_gating(dev_priv);
2481
2482         return ret;
2483 }
2484
2485 static int intel_runtime_suspend(struct device *kdev)
2486 {
2487         struct drm_i915_private *dev_priv = kdev_to_i915(kdev);
2488         struct intel_runtime_pm *rpm = &dev_priv->runtime_pm;
2489         int ret = 0;
2490
2491         if (WARN_ON_ONCE(!HAS_RUNTIME_PM(dev_priv)))
2492                 return -ENODEV;
2493
2494         DRM_DEBUG_KMS("Suspending device\n");
2495
2496         disable_rpm_wakeref_asserts(rpm);
2497
2498         /*
2499          * We are safe here against re-faults, since the fault handler takes
2500          * an RPM reference.
2501          */
2502         i915_gem_runtime_suspend(dev_priv);
2503
2504         intel_gt_runtime_suspend(&dev_priv->gt);
2505
2506         intel_runtime_pm_disable_interrupts(dev_priv);
2507
2508         intel_uncore_suspend(&dev_priv->uncore);
2509
2510         intel_display_power_suspend(dev_priv);
2511
2512         if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
2513                 ret = vlv_suspend_complete(dev_priv);
2514
2515         if (ret) {
2516                 DRM_ERROR("Runtime suspend failed, disabling it (%d)\n", ret);
2517                 intel_uncore_runtime_resume(&dev_priv->uncore);
2518
2519                 intel_runtime_pm_enable_interrupts(dev_priv);
2520
2521                 intel_gt_runtime_resume(&dev_priv->gt);
2522
2523                 i915_gem_restore_fences(&dev_priv->ggtt);
2524
2525                 enable_rpm_wakeref_asserts(rpm);
2526
2527                 return ret;
2528         }
2529
2530         enable_rpm_wakeref_asserts(rpm);
2531         intel_runtime_pm_driver_release(rpm);
2532
2533         if (intel_uncore_arm_unclaimed_mmio_detection(&dev_priv->uncore))
2534                 DRM_ERROR("Unclaimed access detected prior to suspending\n");
2535
2536         rpm->suspended = true;
2537
2538         /*
2539          * FIXME: We really should find a document that references the arguments
2540          * used below!
2541          */
2542         if (IS_BROADWELL(dev_priv)) {
2543                 /*
2544                  * On Broadwell, if we use PCI_D1 the PCH DDI ports will stop
2545                  * being detected, and the call we do at intel_runtime_resume()
2546                  * won't be able to restore them. Since PCI_D3hot matches the
2547                  * actual specification and appears to be working, use it.
2548                  */
2549                 intel_opregion_notify_adapter(dev_priv, PCI_D3hot);
2550         } else {
2551                 /*
2552                  * current versions of firmware which depend on this opregion
2553                  * notification have repurposed the D1 definition to mean
2554                  * "runtime suspended" vs. what you would normally expect (D3)
2555                  * to distinguish it from notifications that might be sent via
2556                  * the suspend path.
2557                  */
2558                 intel_opregion_notify_adapter(dev_priv, PCI_D1);
2559         }
2560
2561         assert_forcewakes_inactive(&dev_priv->uncore);
2562
2563         if (!IS_VALLEYVIEW(dev_priv) && !IS_CHERRYVIEW(dev_priv))
2564                 intel_hpd_poll_init(dev_priv);
2565
2566         DRM_DEBUG_KMS("Device suspended\n");
2567         return 0;
2568 }
2569
2570 static int intel_runtime_resume(struct device *kdev)
2571 {
2572         struct drm_i915_private *dev_priv = kdev_to_i915(kdev);
2573         struct intel_runtime_pm *rpm = &dev_priv->runtime_pm;
2574         int ret = 0;
2575
2576         if (WARN_ON_ONCE(!HAS_RUNTIME_PM(dev_priv)))
2577                 return -ENODEV;
2578
2579         DRM_DEBUG_KMS("Resuming device\n");
2580
2581         WARN_ON_ONCE(atomic_read(&rpm->wakeref_count));
2582         disable_rpm_wakeref_asserts(rpm);
2583
2584         intel_opregion_notify_adapter(dev_priv, PCI_D0);
2585         rpm->suspended = false;
2586         if (intel_uncore_unclaimed_mmio(&dev_priv->uncore))
2587                 DRM_DEBUG_DRIVER("Unclaimed access during suspend, bios?\n");
2588
2589         intel_display_power_resume(dev_priv);
2590
2591         if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
2592                 ret = vlv_resume_prepare(dev_priv, true);
2593
2594         intel_uncore_runtime_resume(&dev_priv->uncore);
2595
2596         intel_runtime_pm_enable_interrupts(dev_priv);
2597
2598         /*
2599          * No point of rolling back things in case of an error, as the best
2600          * we can do is to hope that things will still work (and disable RPM).
2601          */
2602         intel_gt_runtime_resume(&dev_priv->gt);
2603         i915_gem_restore_fences(&dev_priv->ggtt);
2604
2605         /*
2606          * On VLV/CHV display interrupts are part of the display
2607          * power well, so hpd is reinitialized from there. For
2608          * everyone else do it here.
2609          */
2610         if (!IS_VALLEYVIEW(dev_priv) && !IS_CHERRYVIEW(dev_priv))
2611                 intel_hpd_init(dev_priv);
2612
2613         intel_enable_ipc(dev_priv);
2614
2615         enable_rpm_wakeref_asserts(rpm);
2616
2617         if (ret)
2618                 DRM_ERROR("Runtime resume failed, disabling it (%d)\n", ret);
2619         else
2620                 DRM_DEBUG_KMS("Device resumed\n");
2621
2622         return ret;
2623 }
2624
2625 const struct dev_pm_ops i915_pm_ops = {
2626         /*
2627          * S0ix (via system suspend) and S3 event handlers [PMSG_SUSPEND,
2628          * PMSG_RESUME]
2629          */
2630         .prepare = i915_pm_prepare,
2631         .suspend = i915_pm_suspend,
2632         .suspend_late = i915_pm_suspend_late,
2633         .resume_early = i915_pm_resume_early,
2634         .resume = i915_pm_resume,
2635
2636         /*
2637          * S4 event handlers
2638          * @freeze, @freeze_late    : called (1) before creating the
2639          *                            hibernation image [PMSG_FREEZE] and
2640          *                            (2) after rebooting, before restoring
2641          *                            the image [PMSG_QUIESCE]
2642          * @thaw, @thaw_early       : called (1) after creating the hibernation
2643          *                            image, before writing it [PMSG_THAW]
2644          *                            and (2) after failing to create or
2645          *                            restore the image [PMSG_RECOVER]
2646          * @poweroff, @poweroff_late: called after writing the hibernation
2647          *                            image, before rebooting [PMSG_HIBERNATE]
2648          * @restore, @restore_early : called after rebooting and restoring the
2649          *                            hibernation image [PMSG_RESTORE]
2650          */
2651         .freeze = i915_pm_freeze,
2652         .freeze_late = i915_pm_freeze_late,
2653         .thaw_early = i915_pm_thaw_early,
2654         .thaw = i915_pm_thaw,
2655         .poweroff = i915_pm_suspend,
2656         .poweroff_late = i915_pm_poweroff_late,
2657         .restore_early = i915_pm_restore_early,
2658         .restore = i915_pm_restore,
2659
2660         /* S0ix (via runtime suspend) event handlers */
2661         .runtime_suspend = intel_runtime_suspend,
2662         .runtime_resume = intel_runtime_resume,
2663 };
2664
2665 static const struct vm_operations_struct i915_gem_vm_ops = {
2666         .fault = i915_gem_fault,
2667         .open = drm_gem_vm_open,
2668         .close = drm_gem_vm_close,
2669 };
2670
2671 static const struct file_operations i915_driver_fops = {
2672         .owner = THIS_MODULE,
2673         .open = drm_open,
2674         .release = drm_release,
2675         .unlocked_ioctl = drm_ioctl,
2676         .mmap = drm_gem_mmap,
2677         .poll = drm_poll,
2678         .read = drm_read,
2679         .compat_ioctl = i915_compat_ioctl,
2680         .llseek = noop_llseek,
2681 };
2682
2683 static int
2684 i915_gem_reject_pin_ioctl(struct drm_device *dev, void *data,
2685                           struct drm_file *file)
2686 {
2687         return -ENODEV;
2688 }
2689
2690 static const struct drm_ioctl_desc i915_ioctls[] = {
2691         DRM_IOCTL_DEF_DRV(I915_INIT, drm_noop, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY),
2692         DRM_IOCTL_DEF_DRV(I915_FLUSH, drm_noop, DRM_AUTH),
2693         DRM_IOCTL_DEF_DRV(I915_FLIP, drm_noop, DRM_AUTH),
2694         DRM_IOCTL_DEF_DRV(I915_BATCHBUFFER, drm_noop, DRM_AUTH),
2695         DRM_IOCTL_DEF_DRV(I915_IRQ_EMIT, drm_noop, DRM_AUTH),
2696         DRM_IOCTL_DEF_DRV(I915_IRQ_WAIT, drm_noop, DRM_AUTH),
2697         DRM_IOCTL_DEF_DRV(I915_GETPARAM, i915_getparam_ioctl, DRM_RENDER_ALLOW),
2698         DRM_IOCTL_DEF_DRV(I915_SETPARAM, drm_noop, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY),
2699         DRM_IOCTL_DEF_DRV(I915_ALLOC, drm_noop, DRM_AUTH),
2700         DRM_IOCTL_DEF_DRV(I915_FREE, drm_noop, DRM_AUTH),
2701         DRM_IOCTL_DEF_DRV(I915_INIT_HEAP, drm_noop, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY),
2702         DRM_IOCTL_DEF_DRV(I915_CMDBUFFER, drm_noop, DRM_AUTH),
2703         DRM_IOCTL_DEF_DRV(I915_DESTROY_HEAP,  drm_noop, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY),
2704         DRM_IOCTL_DEF_DRV(I915_SET_VBLANK_PIPE,  drm_noop, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY),
2705         DRM_IOCTL_DEF_DRV(I915_GET_VBLANK_PIPE,  drm_noop, DRM_AUTH),
2706         DRM_IOCTL_DEF_DRV(I915_VBLANK_SWAP, drm_noop, DRM_AUTH),
2707         DRM_IOCTL_DEF_DRV(I915_HWS_ADDR, drm_noop, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY),
2708         DRM_IOCTL_DEF_DRV(I915_GEM_INIT, drm_noop, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY),
2709         DRM_IOCTL_DEF_DRV(I915_GEM_EXECBUFFER, i915_gem_execbuffer_ioctl, DRM_AUTH),
2710         DRM_IOCTL_DEF_DRV(I915_GEM_EXECBUFFER2_WR, i915_gem_execbuffer2_ioctl, DRM_RENDER_ALLOW),
2711         DRM_IOCTL_DEF_DRV(I915_GEM_PIN, i915_gem_reject_pin_ioctl, DRM_AUTH|DRM_ROOT_ONLY),
2712         DRM_IOCTL_DEF_DRV(I915_GEM_UNPIN, i915_gem_reject_pin_ioctl, DRM_AUTH|DRM_ROOT_ONLY),
2713         DRM_IOCTL_DEF_DRV(I915_GEM_BUSY, i915_gem_busy_ioctl, DRM_RENDER_ALLOW),
2714         DRM_IOCTL_DEF_DRV(I915_GEM_SET_CACHING, i915_gem_set_caching_ioctl, DRM_RENDER_ALLOW),
2715         DRM_IOCTL_DEF_DRV(I915_GEM_GET_CACHING, i915_gem_get_caching_ioctl, DRM_RENDER_ALLOW),
2716         DRM_IOCTL_DEF_DRV(I915_GEM_THROTTLE, i915_gem_throttle_ioctl, DRM_RENDER_ALLOW),
2717         DRM_IOCTL_DEF_DRV(I915_GEM_ENTERVT, drm_noop, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY),
2718         DRM_IOCTL_DEF_DRV(I915_GEM_LEAVEVT, drm_noop, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY),
2719         DRM_IOCTL_DEF_DRV(I915_GEM_CREATE, i915_gem_create_ioctl, DRM_RENDER_ALLOW),
2720         DRM_IOCTL_DEF_DRV(I915_GEM_PREAD, i915_gem_pread_ioctl, DRM_RENDER_ALLOW),
2721         DRM_IOCTL_DEF_DRV(I915_GEM_PWRITE, i915_gem_pwrite_ioctl, DRM_RENDER_ALLOW),
2722         DRM_IOCTL_DEF_DRV(I915_GEM_MMAP, i915_gem_mmap_ioctl, DRM_RENDER_ALLOW),
2723         DRM_IOCTL_DEF_DRV(I915_GEM_MMAP_GTT, i915_gem_mmap_gtt_ioctl, DRM_RENDER_ALLOW),
2724         DRM_IOCTL_DEF_DRV(I915_GEM_SET_DOMAIN, i915_gem_set_domain_ioctl, DRM_RENDER_ALLOW),
2725         DRM_IOCTL_DEF_DRV(I915_GEM_SW_FINISH, i915_gem_sw_finish_ioctl, DRM_RENDER_ALLOW),
2726         DRM_IOCTL_DEF_DRV(I915_GEM_SET_TILING, i915_gem_set_tiling_ioctl, DRM_RENDER_ALLOW),
2727         DRM_IOCTL_DEF_DRV(I915_GEM_GET_TILING, i915_gem_get_tiling_ioctl, DRM_RENDER_ALLOW),
2728         DRM_IOCTL_DEF_DRV(I915_GEM_GET_APERTURE, i915_gem_get_aperture_ioctl, DRM_RENDER_ALLOW),
2729         DRM_IOCTL_DEF_DRV(I915_GET_PIPE_FROM_CRTC_ID, intel_get_pipe_from_crtc_id_ioctl, 0),
2730         DRM_IOCTL_DEF_DRV(I915_GEM_MADVISE, i915_gem_madvise_ioctl, DRM_RENDER_ALLOW),
2731         DRM_IOCTL_DEF_DRV(I915_OVERLAY_PUT_IMAGE, intel_overlay_put_image_ioctl, DRM_MASTER),
2732         DRM_IOCTL_DEF_DRV(I915_OVERLAY_ATTRS, intel_overlay_attrs_ioctl, DRM_MASTER),
2733         DRM_IOCTL_DEF_DRV(I915_SET_SPRITE_COLORKEY, intel_sprite_set_colorkey_ioctl, DRM_MASTER),
2734         DRM_IOCTL_DEF_DRV(I915_GET_SPRITE_COLORKEY, drm_noop, DRM_MASTER),
2735         DRM_IOCTL_DEF_DRV(I915_GEM_WAIT, i915_gem_wait_ioctl, DRM_RENDER_ALLOW),
2736         DRM_IOCTL_DEF_DRV(I915_GEM_CONTEXT_CREATE_EXT, i915_gem_context_create_ioctl, DRM_RENDER_ALLOW),
2737         DRM_IOCTL_DEF_DRV(I915_GEM_CONTEXT_DESTROY, i915_gem_context_destroy_ioctl, DRM_RENDER_ALLOW),
2738         DRM_IOCTL_DEF_DRV(I915_REG_READ, i915_reg_read_ioctl, DRM_RENDER_ALLOW),
2739         DRM_IOCTL_DEF_DRV(I915_GET_RESET_STATS, i915_gem_context_reset_stats_ioctl, DRM_RENDER_ALLOW),
2740         DRM_IOCTL_DEF_DRV(I915_GEM_USERPTR, i915_gem_userptr_ioctl, DRM_RENDER_ALLOW),
2741         DRM_IOCTL_DEF_DRV(I915_GEM_CONTEXT_GETPARAM, i915_gem_context_getparam_ioctl, DRM_RENDER_ALLOW),
2742         DRM_IOCTL_DEF_DRV(I915_GEM_CONTEXT_SETPARAM, i915_gem_context_setparam_ioctl, DRM_RENDER_ALLOW),
2743         DRM_IOCTL_DEF_DRV(I915_PERF_OPEN, i915_perf_open_ioctl, DRM_RENDER_ALLOW),
2744         DRM_IOCTL_DEF_DRV(I915_PERF_ADD_CONFIG, i915_perf_add_config_ioctl, DRM_RENDER_ALLOW),
2745         DRM_IOCTL_DEF_DRV(I915_PERF_REMOVE_CONFIG, i915_perf_remove_config_ioctl, DRM_RENDER_ALLOW),
2746         DRM_IOCTL_DEF_DRV(I915_QUERY, i915_query_ioctl, DRM_RENDER_ALLOW),
2747         DRM_IOCTL_DEF_DRV(I915_GEM_VM_CREATE, i915_gem_vm_create_ioctl, DRM_RENDER_ALLOW),
2748         DRM_IOCTL_DEF_DRV(I915_GEM_VM_DESTROY, i915_gem_vm_destroy_ioctl, DRM_RENDER_ALLOW),
2749 };
2750
2751 static struct drm_driver driver = {
2752         /* Don't use MTRRs here; the Xserver or userspace app should
2753          * deal with them for Intel hardware.
2754          */
2755         .driver_features =
2756             DRIVER_GEM |
2757             DRIVER_RENDER | DRIVER_MODESET | DRIVER_ATOMIC | DRIVER_SYNCOBJ,
2758         .release = i915_driver_release,
2759         .open = i915_driver_open,
2760         .lastclose = i915_driver_lastclose,
2761         .postclose = i915_driver_postclose,
2762
2763         .gem_close_object = i915_gem_close_object,
2764         .gem_free_object_unlocked = i915_gem_free_object,
2765         .gem_vm_ops = &i915_gem_vm_ops,
2766
2767         .prime_handle_to_fd = drm_gem_prime_handle_to_fd,
2768         .prime_fd_to_handle = drm_gem_prime_fd_to_handle,
2769         .gem_prime_export = i915_gem_prime_export,
2770         .gem_prime_import = i915_gem_prime_import,
2771
2772         .get_vblank_timestamp = drm_calc_vbltimestamp_from_scanoutpos,
2773         .get_scanout_position = i915_get_crtc_scanoutpos,
2774
2775         .dumb_create = i915_gem_dumb_create,
2776         .dumb_map_offset = i915_gem_mmap_gtt,
2777         .ioctls = i915_ioctls,
2778         .num_ioctls = ARRAY_SIZE(i915_ioctls),
2779         .fops = &i915_driver_fops,
2780         .name = DRIVER_NAME,
2781         .desc = DRIVER_DESC,
2782         .date = DRIVER_DATE,
2783         .major = DRIVER_MAJOR,
2784         .minor = DRIVER_MINOR,
2785         .patchlevel = DRIVER_PATCHLEVEL,
2786 };
2787
2788 #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
2789 #include "selftests/mock_drm.c"
2790 #endif