drivers/gpu/drm/i915/gt/intel_workarounds.c

   1 /*
   2  * SPDX-License-Identifier: MIT
   3  *
   4  * Copyright © 2014-2018 Intel Corporation
   5  */
   6
   7 #include "i915_drv.h"
   8 #include "intel_context.h"
   9 #include "intel_engine_pm.h"
  10 #include "intel_gt.h"
  11 #include "intel_ring.h"
  12 #include "intel_workarounds.h"
  13
  14 /**
  15  * DOC: Hardware workarounds
  16  *
  17  * This file is intended as a central place to implement most [1]_ of the
  18  * required workarounds for hardware to work as originally intended. They fall
  19  * in five basic categories depending on how/when they are applied:
  20  *
  21  * - Workarounds that touch registers that are saved/restored to/from the HW
  22  *   context image. The list is emitted (via Load Register Immediate commands)
  23  *   everytime a new context is created.
  24  * - GT workarounds. The list of these WAs is applied whenever these registers
  25  *   revert to default values (on GPU reset, suspend/resume [2]_, etc..).
  26  * - Display workarounds. The list is applied during display clock-gating
  27  *   initialization.
  28  * - Workarounds that whitelist a privileged register, so that UMDs can manage
  29  *   them directly. This is just a special case of a MMMIO workaround (as we
  30  *   write the list of these to/be-whitelisted registers to some special HW
  31  *   registers).
  32  * - Workaround batchbuffers, that get executed automatically by the hardware
  33  *   on every HW context restore.
  34  *
  35  * .. [1] Please notice that there are other WAs that, due to their nature,
  36  *    cannot be applied from a central place. Those are peppered around the rest
  37  *    of the code, as needed.
  38  *
  39  * .. [2] Technically, some registers are powercontext saved & restored, so they
  40  *    survive a suspend/resume. In practice, writing them again is not too
  41  *    costly and simplifies things. We can revisit this in the future.
  42  *
  43  * Layout
  44  * ~~~~~~
  45  *
  46  * Keep things in this file ordered by WA type, as per the above (context, GT,
  47  * display, register whitelist, batchbuffer). Then, inside each type, keep the
  48  * following order:
  49  *
  50  * - Infrastructure functions and macros
  51  * - WAs per platform in standard gen/chrono order
  52  * - Public functions to init or apply the given workaround type.
  53  */
  54
  55 static void wa_init_start(struct i915_wa_list *wal, const char *name, const char *engine_name)
  56 {
  57         wal->name = name;
  58         wal->engine_name = engine_name;
  59 }
  60
  61 #define WA_LIST_CHUNK (1 << 4)
  62
  63 static void wa_init_finish(struct i915_wa_list *wal)
  64 {
  65         /* Trim unused entries. */
  66         if (!IS_ALIGNED(wal->count, WA_LIST_CHUNK)) {
  67                 struct i915_wa *list = kmemdup(wal->list,
  68                                                wal->count * sizeof(*list),
  69                                                GFP_KERNEL);
  70
  71                 if (list) {
  72                         kfree(wal->list);
  73                         wal->list = list;
  74                 }
  75         }
  76
  77         if (!wal->count)
  78                 return;
  79
  80         DRM_DEBUG_DRIVER("Initialized %u %s workarounds on %s\n",
  81                          wal->wa_count, wal->name, wal->engine_name);
  82 }
  83
  84 static void _wa_add(struct i915_wa_list *wal, const struct i915_wa *wa)
  85 {
  86         unsigned int addr = i915_mmio_reg_offset(wa->reg);
  87         unsigned int start = 0, end = wal->count;
  88         const unsigned int grow = WA_LIST_CHUNK;
  89         struct i915_wa *wa_;
  90
  91         GEM_BUG_ON(!is_power_of_2(grow));
  92
  93         if (IS_ALIGNED(wal->count, grow)) { /* Either uninitialized or full. */
  94                 struct i915_wa *list;
  95
  96                 list = kmalloc_array(ALIGN(wal->count + 1, grow), sizeof(*wa),
  97                                      GFP_KERNEL);
  98                 if (!list) {
  99                         DRM_ERROR("No space for workaround init!\n");
 100                         return;
 101                 }
 102
 103                 if (wal->list)
 104                         memcpy(list, wal->list, sizeof(*wa) * wal->count);
 105
 106                 wal->list = list;
 107         }
 108
 109         while (start < end) {
 110                 unsigned int mid = start + (end - start) / 2;
 111
 112                 if (i915_mmio_reg_offset(wal->list[mid].reg) < addr) {
 113                         start = mid + 1;
 114                 } else if (i915_mmio_reg_offset(wal->list[mid].reg) > addr) {
 115                         end = mid;
 116                 } else {
 117                         wa_ = &wal->list[mid];
 118
 119                         if ((wa->clr | wa_->clr) && !(wa->clr & ~wa_->clr)) {
 120                                 DRM_ERROR("Discarding overwritten w/a for reg %04x (clear: %08x, set: %08x)\n",
 121                                           i915_mmio_reg_offset(wa_->reg),
 122                                           wa_->clr, wa_->set);
 123
 124                                 wa_->set &= ~wa->clr;
 125                         }
 126
 127                         wal->wa_count++;
 128                         wa_->set |= wa->set;
 129                         wa_->clr |= wa->clr;
 130                         wa_->read |= wa->read;
 131                         return;
 132                 }
 133         }
 134
 135         wal->wa_count++;
 136         wa_ = &wal->list[wal->count++];
 137         *wa_ = *wa;
 138
 139         while (wa_-- > wal->list) {
 140                 GEM_BUG_ON(i915_mmio_reg_offset(wa_[0].reg) ==
 141                            i915_mmio_reg_offset(wa_[1].reg));
 142                 if (i915_mmio_reg_offset(wa_[1].reg) >
 143                     i915_mmio_reg_offset(wa_[0].reg))
 144                         break;
 145
 146                 swap(wa_[1], wa_[0]);
 147         }
 148 }
 149
 150 static void wa_add(struct i915_wa_list *wal, i915_reg_t reg,
 151                    u32 clear, u32 set, u32 read_mask)
 152 {
 153         struct i915_wa wa = {
 154                 .reg  = reg,
 155                 .clr  = clear,
 156                 .set  = set,
 157                 .read = read_mask,
 158         };
 159
 160         _wa_add(wal, &wa);
 161 }
 162
 163 static void
 164 wa_write_masked_or(struct i915_wa_list *wal, i915_reg_t reg, u32 clear, u32 set)
 165 {
 166         wa_add(wal, reg, clear, set, clear);
 167 }
 168
 169 static void
 170 wa_write(struct i915_wa_list *wal, i915_reg_t reg, u32 set)
 171 {
 172         wa_write_masked_or(wal, reg, ~0, set);
 173 }
 174
 175 static void
 176 wa_write_or(struct i915_wa_list *wal, i915_reg_t reg, u32 set)
 177 {
 178         wa_write_masked_or(wal, reg, set, set);
 179 }
 180
 181 static void
 182 wa_masked_en(struct i915_wa_list *wal, i915_reg_t reg, u32 val)
 183 {
 184         wa_add(wal, reg, 0, _MASKED_BIT_ENABLE(val), val);
 185 }
 186
 187 static void
 188 wa_masked_dis(struct i915_wa_list *wal, i915_reg_t reg, u32 val)
 189 {
 190         wa_add(wal, reg, 0, _MASKED_BIT_DISABLE(val), val);
 191 }
 192
 193 #define WA_SET_BIT_MASKED(addr, mask) \
 194         wa_masked_en(wal, (addr), (mask))
 195
 196 #define WA_CLR_BIT_MASKED(addr, mask) \
 197         wa_masked_dis(wal, (addr), (mask))
 198
 199 #define WA_SET_FIELD_MASKED(addr, mask, value) \
 200         wa_write_masked_or(wal, (addr), 0, _MASKED_FIELD((mask), (value)))
 201
 202 static void gen8_ctx_workarounds_init(struct intel_engine_cs *engine,
 203                                       struct i915_wa_list *wal)
 204 {
 205         WA_SET_BIT_MASKED(INSTPM, INSTPM_FORCE_ORDERING);
 206
 207         /* WaDisableAsyncFlipPerfMode:bdw,chv */
 208         WA_SET_BIT_MASKED(MI_MODE, ASYNC_FLIP_PERF_DISABLE);
 209
 210         /* WaDisablePartialInstShootdown:bdw,chv */
 211         WA_SET_BIT_MASKED(GEN8_ROW_CHICKEN,
 212                           PARTIAL_INSTRUCTION_SHOOTDOWN_DISABLE);
 213
 214         /* Use Force Non-Coherent whenever executing a 3D context. This is a
 215          * workaround for for a possible hang in the unlikely event a TLB
 216          * invalidation occurs during a PSD flush.
 217          */
 218         /* WaForceEnableNonCoherent:bdw,chv */
 219         /* WaHdcDisableFetchWhenMasked:bdw,chv */
 220         WA_SET_BIT_MASKED(HDC_CHICKEN0,
 221                           HDC_DONOT_FETCH_MEM_WHEN_MASKED |
 222                           HDC_FORCE_NON_COHERENT);
 223
 224         /* From the Haswell PRM, Command Reference: Registers, CACHE_MODE_0:
 225          * "The Hierarchical Z RAW Stall Optimization allows non-overlapping
 226          *  polygons in the same 8x4 pixel/sample area to be processed without
 227          *  stalling waiting for the earlier ones to write to Hierarchical Z
 228          *  buffer."
 229          *
 230          * This optimization is off by default for BDW and CHV; turn it on.
 231          */
 232         WA_CLR_BIT_MASKED(CACHE_MODE_0_GEN7, HIZ_RAW_STALL_OPT_DISABLE);
 233
 234         /* Wa4x4STCOptimizationDisable:bdw,chv */
 235         WA_SET_BIT_MASKED(CACHE_MODE_1, GEN8_4x4_STC_OPTIMIZATION_DISABLE);
 236
 237         /*
 238          * BSpec recommends 8x4 when MSAA is used,
 239          * however in practice 16x4 seems fastest.
 240          *
 241          * Note that PS/WM thread counts depend on the WIZ hashing
 242          * disable bit, which we don't touch here, but it's good
 243          * to keep in mind (see 3DSTATE_PS and 3DSTATE_WM).
 244          */
 245         WA_SET_FIELD_MASKED(GEN7_GT_MODE,
 246                             GEN6_WIZ_HASHING_MASK,
 247                             GEN6_WIZ_HASHING_16x4);
 248 }
 249
 250 static void bdw_ctx_workarounds_init(struct intel_engine_cs *engine,
 251                                      struct i915_wa_list *wal)
 252 {
 253         struct drm_i915_private *i915 = engine->i915;
 254
 255         gen8_ctx_workarounds_init(engine, wal);
 256
 257         /* WaDisableThreadStallDopClockGating:bdw (pre-production) */
 258         WA_SET_BIT_MASKED(GEN8_ROW_CHICKEN, STALL_DOP_GATING_DISABLE);
 259
 260         /* WaDisableDopClockGating:bdw
 261          *
 262          * Also see the related UCGTCL1 write in bdw_init_clock_gating()
 263          * to disable EUTC clock gating.
 264          */
 265         WA_SET_BIT_MASKED(GEN7_ROW_CHICKEN2,
 266                           DOP_CLOCK_GATING_DISABLE);
 267
 268         WA_SET_BIT_MASKED(HALF_SLICE_CHICKEN3,
 269                           GEN8_SAMPLER_POWER_BYPASS_DIS);
 270
 271         WA_SET_BIT_MASKED(HDC_CHICKEN0,
 272                           /* WaForceContextSaveRestoreNonCoherent:bdw */
 273                           HDC_FORCE_CONTEXT_SAVE_RESTORE_NON_COHERENT |
 274                           /* WaDisableFenceDestinationToSLM:bdw (pre-prod) */
 275                           (IS_BDW_GT3(i915) ? HDC_FENCE_DEST_SLM_DISABLE : 0));
 276 }
 277
 278 static void chv_ctx_workarounds_init(struct intel_engine_cs *engine,
 279                                      struct i915_wa_list *wal)
 280 {
 281         gen8_ctx_workarounds_init(engine, wal);
 282
 283         /* WaDisableThreadStallDopClockGating:chv */
 284         WA_SET_BIT_MASKED(GEN8_ROW_CHICKEN, STALL_DOP_GATING_DISABLE);
 285
 286         /* Improve HiZ throughput on CHV. */
 287         WA_SET_BIT_MASKED(HIZ_CHICKEN, CHV_HZ_8X8_MODE_IN_1X);
 288 }
 289
 290 static void gen9_ctx_workarounds_init(struct intel_engine_cs *engine,
 291                                       struct i915_wa_list *wal)
 292 {
 293         struct drm_i915_private *i915 = engine->i915;
 294
 295         if (HAS_LLC(i915)) {
 296                 /* WaCompressedResourceSamplerPbeMediaNewHashMode:skl,kbl
 297                  *
 298                  * Must match Display Engine. See
 299                  * WaCompressedResourceDisplayNewHashMode.
 300                  */
 301                 WA_SET_BIT_MASKED(COMMON_SLICE_CHICKEN2,
 302                                   GEN9_PBE_COMPRESSED_HASH_SELECTION);
 303                 WA_SET_BIT_MASKED(GEN9_HALF_SLICE_CHICKEN7,
 304                                   GEN9_SAMPLER_HASH_COMPRESSED_READ_ADDR);
 305         }
 306
 307         /* WaClearFlowControlGpgpuContextSave:skl,bxt,kbl,glk,cfl */
 308         /* WaDisablePartialInstShootdown:skl,bxt,kbl,glk,cfl */
 309         WA_SET_BIT_MASKED(GEN8_ROW_CHICKEN,
 310                           FLOW_CONTROL_ENABLE |
 311                           PARTIAL_INSTRUCTION_SHOOTDOWN_DISABLE);
 312
 313         /* WaEnableYV12BugFixInHalfSliceChicken7:skl,bxt,kbl,glk,cfl */
 314         /* WaEnableSamplerGPGPUPreemptionSupport:skl,bxt,kbl,cfl */
 315         WA_SET_BIT_MASKED(GEN9_HALF_SLICE_CHICKEN7,
 316                           GEN9_ENABLE_YV12_BUGFIX |
 317                           GEN9_ENABLE_GPGPU_PREEMPTION);
 318
 319         /* Wa4x4STCOptimizationDisable:skl,bxt,kbl,glk,cfl */
 320         /* WaDisablePartialResolveInVc:skl,bxt,kbl,cfl */
 321         WA_SET_BIT_MASKED(CACHE_MODE_1,
 322                           GEN8_4x4_STC_OPTIMIZATION_DISABLE |
 323                           GEN9_PARTIAL_RESOLVE_IN_VC_DISABLE);
 324
 325         /* WaCcsTlbPrefetchDisable:skl,bxt,kbl,glk,cfl */
 326         WA_CLR_BIT_MASKED(GEN9_HALF_SLICE_CHICKEN5,
 327                           GEN9_CCS_TLB_PREFETCH_ENABLE);
 328
 329         /* WaForceContextSaveRestoreNonCoherent:skl,bxt,kbl,cfl */
 330         WA_SET_BIT_MASKED(HDC_CHICKEN0,
 331                           HDC_FORCE_CONTEXT_SAVE_RESTORE_NON_COHERENT |
 332                           HDC_FORCE_CSR_NON_COHERENT_OVR_DISABLE);
 333
 334         /* WaForceEnableNonCoherent and WaDisableHDCInvalidation are
 335          * both tied to WaForceContextSaveRestoreNonCoherent
 336          * in some hsds for skl. We keep the tie for all gen9. The
 337          * documentation is a bit hazy and so we want to get common behaviour,
 338          * even though there is no clear evidence we would need both on kbl/bxt.
 339          * This area has been source of system hangs so we play it safe
 340          * and mimic the skl regardless of what bspec says.
 341          *
 342          * Use Force Non-Coherent whenever executing a 3D context. This
 343          * is a workaround for a possible hang in the unlikely event
 344          * a TLB invalidation occurs during a PSD flush.
 345          */
 346
 347         /* WaForceEnableNonCoherent:skl,bxt,kbl,cfl */
 348         WA_SET_BIT_MASKED(HDC_CHICKEN0,
 349                           HDC_FORCE_NON_COHERENT);
 350
 351         /* WaDisableSamplerPowerBypassForSOPingPong:skl,bxt,kbl,cfl */
 352         if (IS_SKYLAKE(i915) || IS_KABYLAKE(i915) || IS_COFFEELAKE(i915))
 353                 WA_SET_BIT_MASKED(HALF_SLICE_CHICKEN3,
 354                                   GEN8_SAMPLER_POWER_BYPASS_DIS);
 355
 356         /* WaDisableSTUnitPowerOptimization:skl,bxt,kbl,glk,cfl */
 357         WA_SET_BIT_MASKED(HALF_SLICE_CHICKEN2, GEN8_ST_PO_DISABLE);
 358
 359         /*
 360          * Supporting preemption with fine-granularity requires changes in the
 361          * batch buffer programming. Since we can't break old userspace, we
 362          * need to set our default preemption level to safe value. Userspace is
 363          * still able to use more fine-grained preemption levels, since in
 364          * WaEnablePreemptionGranularityControlByUMD we're whitelisting the
 365          * per-ctx register. As such, WaDisable{3D,GPGPU}MidCmdPreemption are
 366          * not real HW workarounds, but merely a way to start using preemption
 367          * while maintaining old contract with userspace.
 368          */
 369
 370         /* WaDisable3DMidCmdPreemption:skl,bxt,glk,cfl,[cnl] */
 371         WA_CLR_BIT_MASKED(GEN8_CS_CHICKEN1, GEN9_PREEMPT_3D_OBJECT_LEVEL);
 372
 373         /* WaDisableGPGPUMidCmdPreemption:skl,bxt,blk,cfl,[cnl] */
 374         WA_SET_FIELD_MASKED(GEN8_CS_CHICKEN1,
 375                             GEN9_PREEMPT_GPGPU_LEVEL_MASK,
 376                             GEN9_PREEMPT_GPGPU_COMMAND_LEVEL);
 377
 378         /* WaClearHIZ_WM_CHICKEN3:bxt,glk */
 379         if (IS_GEN9_LP(i915))
 380                 WA_SET_BIT_MASKED(GEN9_WM_CHICKEN3, GEN9_FACTOR_IN_CLR_VAL_HIZ);
 381 }
 382
 383 static void skl_tune_iz_hashing(struct intel_engine_cs *engine,
 384                                 struct i915_wa_list *wal)
 385 {
 386         struct drm_i915_private *i915 = engine->i915;
 387         u8 vals[3] = { 0, 0, 0 };
 388         unsigned int i;
 389
 390         for (i = 0; i < 3; i++) {
 391                 u8 ss;
 392
 393                 /*
 394                  * Only consider slices where one, and only one, subslice has 7
 395                  * EUs
 396                  */
 397                 if (!is_power_of_2(RUNTIME_INFO(i915)->sseu.subslice_7eu[i]))
 398                         continue;
 399
 400                 /*
 401                  * subslice_7eu[i] != 0 (because of the check above) and
 402                  * ss_max == 4 (maximum number of subslices possible per slice)
 403                  *
 404                  * ->    0 <= ss <= 3;
 405                  */
 406                 ss = ffs(RUNTIME_INFO(i915)->sseu.subslice_7eu[i]) - 1;
 407                 vals[i] = 3 - ss;
 408         }
 409
 410         if (vals[0] == 0 && vals[1] == 0 && vals[2] == 0)
 411                 return;
 412
 413         /* Tune IZ hashing. See intel_device_info_runtime_init() */
 414         WA_SET_FIELD_MASKED(GEN7_GT_MODE,
 415                             GEN9_IZ_HASHING_MASK(2) |
 416                             GEN9_IZ_HASHING_MASK(1) |
 417                             GEN9_IZ_HASHING_MASK(0),
 418                             GEN9_IZ_HASHING(2, vals[2]) |
 419                             GEN9_IZ_HASHING(1, vals[1]) |
 420                             GEN9_IZ_HASHING(0, vals[0]));
 421 }
 422
 423 static void skl_ctx_workarounds_init(struct intel_engine_cs *engine,
 424                                      struct i915_wa_list *wal)
 425 {
 426         gen9_ctx_workarounds_init(engine, wal);
 427         skl_tune_iz_hashing(engine, wal);
 428 }
 429
 430 static void bxt_ctx_workarounds_init(struct intel_engine_cs *engine,
 431                                      struct i915_wa_list *wal)
 432 {
 433         gen9_ctx_workarounds_init(engine, wal);
 434
 435         /* WaDisableThreadStallDopClockGating:bxt */
 436         WA_SET_BIT_MASKED(GEN8_ROW_CHICKEN,
 437                           STALL_DOP_GATING_DISABLE);
 438
 439         /* WaToEnableHwFixForPushConstHWBug:bxt */
 440         WA_SET_BIT_MASKED(COMMON_SLICE_CHICKEN2,
 441                           GEN8_SBE_DISABLE_REPLAY_BUF_OPTIMIZATION);
 442 }
 443
 444 static void kbl_ctx_workarounds_init(struct intel_engine_cs *engine,
 445                                      struct i915_wa_list *wal)
 446 {
 447         struct drm_i915_private *i915 = engine->i915;
 448
 449         gen9_ctx_workarounds_init(engine, wal);
 450
 451         /* WaToEnableHwFixForPushConstHWBug:kbl */
 452         if (IS_KBL_REVID(i915, KBL_REVID_C0, REVID_FOREVER))
 453                 WA_SET_BIT_MASKED(COMMON_SLICE_CHICKEN2,
 454                                   GEN8_SBE_DISABLE_REPLAY_BUF_OPTIMIZATION);
 455
 456         /* WaDisableSbeCacheDispatchPortSharing:kbl */
 457         WA_SET_BIT_MASKED(GEN7_HALF_SLICE_CHICKEN1,
 458                           GEN7_SBE_SS_CACHE_DISPATCH_PORT_SHARING_DISABLE);
 459 }
 460
 461 static void glk_ctx_workarounds_init(struct intel_engine_cs *engine,
 462                                      struct i915_wa_list *wal)
 463 {
 464         gen9_ctx_workarounds_init(engine, wal);
 465
 466         /* WaToEnableHwFixForPushConstHWBug:glk */
 467         WA_SET_BIT_MASKED(COMMON_SLICE_CHICKEN2,
 468                           GEN8_SBE_DISABLE_REPLAY_BUF_OPTIMIZATION);
 469 }
 470
 471 static void cfl_ctx_workarounds_init(struct intel_engine_cs *engine,
 472                                      struct i915_wa_list *wal)
 473 {
 474         gen9_ctx_workarounds_init(engine, wal);
 475
 476         /* WaToEnableHwFixForPushConstHWBug:cfl */
 477         WA_SET_BIT_MASKED(COMMON_SLICE_CHICKEN2,
 478                           GEN8_SBE_DISABLE_REPLAY_BUF_OPTIMIZATION);
 479
 480         /* WaDisableSbeCacheDispatchPortSharing:cfl */
 481         WA_SET_BIT_MASKED(GEN7_HALF_SLICE_CHICKEN1,
 482                           GEN7_SBE_SS_CACHE_DISPATCH_PORT_SHARING_DISABLE);
 483 }
 484
 485 static void cnl_ctx_workarounds_init(struct intel_engine_cs *engine,
 486                                      struct i915_wa_list *wal)
 487 {
 488         struct drm_i915_private *i915 = engine->i915;
 489
 490         /* WaForceContextSaveRestoreNonCoherent:cnl */
 491         WA_SET_BIT_MASKED(CNL_HDC_CHICKEN0,
 492                           HDC_FORCE_CONTEXT_SAVE_RESTORE_NON_COHERENT);
 493
 494         /* WaThrottleEUPerfToAvoidTDBackPressure:cnl(pre-prod) */
 495         if (IS_CNL_REVID(i915, CNL_REVID_B0, CNL_REVID_B0))
 496                 WA_SET_BIT_MASKED(GEN8_ROW_CHICKEN, THROTTLE_12_5);
 497
 498         /* WaDisableReplayBufferBankArbitrationOptimization:cnl */
 499         WA_SET_BIT_MASKED(COMMON_SLICE_CHICKEN2,
 500                           GEN8_SBE_DISABLE_REPLAY_BUF_OPTIMIZATION);
 501
 502         /* WaDisableEnhancedSBEVertexCaching:cnl (pre-prod) */
 503         if (IS_CNL_REVID(i915, 0, CNL_REVID_B0))
 504                 WA_SET_BIT_MASKED(COMMON_SLICE_CHICKEN2,
 505                                   GEN8_CSC2_SBE_VUE_CACHE_CONSERVATIVE);
 506
 507         /* WaPushConstantDereferenceHoldDisable:cnl */
 508         WA_SET_BIT_MASKED(GEN7_ROW_CHICKEN2, PUSH_CONSTANT_DEREF_DISABLE);
 509
 510         /* FtrEnableFastAnisoL1BankingFix:cnl */
 511         WA_SET_BIT_MASKED(HALF_SLICE_CHICKEN3, CNL_FAST_ANISO_L1_BANKING_FIX);
 512
 513         /* WaDisable3DMidCmdPreemption:cnl */
 514         WA_CLR_BIT_MASKED(GEN8_CS_CHICKEN1, GEN9_PREEMPT_3D_OBJECT_LEVEL);
 515
 516         /* WaDisableGPGPUMidCmdPreemption:cnl */
 517         WA_SET_FIELD_MASKED(GEN8_CS_CHICKEN1,
 518                             GEN9_PREEMPT_GPGPU_LEVEL_MASK,
 519                             GEN9_PREEMPT_GPGPU_COMMAND_LEVEL);
 520
 521         /* WaDisableEarlyEOT:cnl */
 522         WA_SET_BIT_MASKED(GEN8_ROW_CHICKEN, DISABLE_EARLY_EOT);
 523 }
 524
 525 static void icl_ctx_workarounds_init(struct intel_engine_cs *engine,
 526                                      struct i915_wa_list *wal)
 527 {
 528         struct drm_i915_private *i915 = engine->i915;
 529
 530         /* WaDisableBankHangMode:icl */
 531         wa_write(wal,
 532                  GEN8_L3CNTLREG,
 533                  intel_uncore_read(engine->uncore, GEN8_L3CNTLREG) |
 534                  GEN8_ERRDETBCTRL);
 535
 536         /* Wa_1604370585:icl (pre-prod)
 537          * Formerly known as WaPushConstantDereferenceHoldDisable
 538          */
 539         if (IS_ICL_REVID(i915, ICL_REVID_A0, ICL_REVID_B0))
 540                 WA_SET_BIT_MASKED(GEN7_ROW_CHICKEN2,
 541                                   PUSH_CONSTANT_DEREF_DISABLE);
 542
 543         /* WaForceEnableNonCoherent:icl
 544          * This is not the same workaround as in early Gen9 platforms, where
 545          * lacking this could cause system hangs, but coherency performance
 546          * overhead is high and only a few compute workloads really need it
 547          * (the register is whitelisted in hardware now, so UMDs can opt in
 548          * for coherency if they have a good reason).
 549          */
 550         WA_SET_BIT_MASKED(ICL_HDC_MODE, HDC_FORCE_NON_COHERENT);
 551
 552         /* Wa_2006611047:icl (pre-prod)
 553          * Formerly known as WaDisableImprovedTdlClkGating
 554          */
 555         if (IS_ICL_REVID(i915, ICL_REVID_A0, ICL_REVID_A0))
 556                 WA_SET_BIT_MASKED(GEN7_ROW_CHICKEN2,
 557                                   GEN11_TDL_CLOCK_GATING_FIX_DISABLE);
 558
 559         /* Wa_2006665173:icl (pre-prod) */
 560         if (IS_ICL_REVID(i915, ICL_REVID_A0, ICL_REVID_A0))
 561                 WA_SET_BIT_MASKED(GEN11_COMMON_SLICE_CHICKEN3,
 562                                   GEN11_BLEND_EMB_FIX_DISABLE_IN_RCC);
 563
 564         /* WaEnableFloatBlendOptimization:icl */
 565         wa_write_masked_or(wal,
 566                            GEN10_CACHE_MODE_SS,
 567                            0, /* write-only, so skip validation */
 568                            _MASKED_BIT_ENABLE(FLOAT_BLEND_OPTIMIZATION_ENABLE));
 569
 570         /* WaDisableGPGPUMidThreadPreemption:icl */
 571         WA_SET_FIELD_MASKED(GEN8_CS_CHICKEN1,
 572                             GEN9_PREEMPT_GPGPU_LEVEL_MASK,
 573                             GEN9_PREEMPT_GPGPU_THREAD_GROUP_LEVEL);
 574
 575         /* allow headerless messages for preemptible GPGPU context */
 576         WA_SET_BIT_MASKED(GEN10_SAMPLER_MODE,
 577                           GEN11_SAMPLER_ENABLE_HEADLESS_MSG);
 578 }
 579
 580 static void tgl_ctx_workarounds_init(struct intel_engine_cs *engine,
 581                                      struct i915_wa_list *wal)
 582 {
 583         /*
 584          * Wa_1409142259:tgl
 585          * Wa_1409347922:tgl
 586          * Wa_1409252684:tgl
 587          * Wa_1409217633:tgl
 588          * Wa_1409207793:tgl
 589          * Wa_1409178076:tgl
 590          * Wa_1408979724:tgl
 591          */
 592         WA_SET_BIT_MASKED(GEN11_COMMON_SLICE_CHICKEN3,
 593                           GEN12_DISABLE_CPS_AWARE_COLOR_PIPE);
 594
 595         /*
 596          * Wa_1604555607:gen12 and Wa_1608008084:gen12
 597          * FF_MODE2 register will return the wrong value when read. The default
 598          * value for this register is zero for all fields and there are no bit
 599          * masks. So instead of doing a RMW we should just write the TDS timer
 600          * value for Wa_1604555607.
 601          */
 602         wa_add(wal, FF_MODE2, FF_MODE2_TDS_TIMER_MASK,
 603                FF_MODE2_TDS_TIMER_128, 0);
 604
 605         /* WaDisableGPGPUMidThreadPreemption:tgl */
 606         WA_SET_FIELD_MASKED(GEN8_CS_CHICKEN1,
 607                             GEN9_PREEMPT_GPGPU_LEVEL_MASK,
 608                             GEN9_PREEMPT_GPGPU_THREAD_GROUP_LEVEL);
 609 }
 610
 611 static void
 612 __intel_engine_init_ctx_wa(struct intel_engine_cs *engine,
 613                            struct i915_wa_list *wal,
 614                            const char *name)
 615 {
 616         struct drm_i915_private *i915 = engine->i915;
 617
 618         if (engine->class != RENDER_CLASS)
 619                 return;
 620
 621         wa_init_start(wal, name, engine->name);
 622
 623         if (IS_GEN(i915, 12))
 624                 tgl_ctx_workarounds_init(engine, wal);
 625         else if (IS_GEN(i915, 11))
 626                 icl_ctx_workarounds_init(engine, wal);
 627         else if (IS_CANNONLAKE(i915))
 628                 cnl_ctx_workarounds_init(engine, wal);
 629         else if (IS_COFFEELAKE(i915))
 630                 cfl_ctx_workarounds_init(engine, wal);
 631         else if (IS_GEMINILAKE(i915))
 632                 glk_ctx_workarounds_init(engine, wal);
 633         else if (IS_KABYLAKE(i915))
 634                 kbl_ctx_workarounds_init(engine, wal);
 635         else if (IS_BROXTON(i915))
 636                 bxt_ctx_workarounds_init(engine, wal);
 637         else if (IS_SKYLAKE(i915))
 638                 skl_ctx_workarounds_init(engine, wal);
 639         else if (IS_CHERRYVIEW(i915))
 640                 chv_ctx_workarounds_init(engine, wal);
 641         else if (IS_BROADWELL(i915))
 642                 bdw_ctx_workarounds_init(engine, wal);
 643         else if (INTEL_GEN(i915) < 8)
 644                 return;
 645         else
 646                 MISSING_CASE(INTEL_GEN(i915));
 647
 648         wa_init_finish(wal);
 649 }
 650
 651 void intel_engine_init_ctx_wa(struct intel_engine_cs *engine)
 652 {
 653         __intel_engine_init_ctx_wa(engine, &engine->ctx_wa_list, "context");
 654 }
 655
 656 int intel_engine_emit_ctx_wa(struct i915_request *rq)
 657 {
 658         struct i915_wa_list *wal = &rq->engine->ctx_wa_list;
 659         struct i915_wa *wa;
 660         unsigned int i;
 661         u32 *cs;
 662         int ret;
 663
 664         if (wal->count == 0)
 665                 return 0;
 666
 667         ret = rq->engine->emit_flush(rq, EMIT_BARRIER);
 668         if (ret)
 669                 return ret;
 670
 671         cs = intel_ring_begin(rq, (wal->count * 2 + 2));
 672         if (IS_ERR(cs))
 673                 return PTR_ERR(cs);
 674
 675         *cs++ = MI_LOAD_REGISTER_IMM(wal->count);
 676         for (i = 0, wa = wal->list; i < wal->count; i++, wa++) {
 677                 *cs++ = i915_mmio_reg_offset(wa->reg);
 678                 *cs++ = wa->set;
 679         }
 680         *cs++ = MI_NOOP;
 681
 682         intel_ring_advance(rq, cs);
 683
 684         ret = rq->engine->emit_flush(rq, EMIT_BARRIER);
 685         if (ret)
 686                 return ret;
 687
 688         return 0;
 689 }
 690
 691 static void
 692 gen9_gt_workarounds_init(struct drm_i915_private *i915, struct i915_wa_list *wal)
 693 {
 694         /* WaDisableKillLogic:bxt,skl,kbl */
 695         if (!IS_COFFEELAKE(i915))
 696                 wa_write_or(wal,
 697                             GAM_ECOCHK,
 698                             ECOCHK_DIS_TLB);
 699
 700         if (HAS_LLC(i915)) {
 701                 /* WaCompressedResourceSamplerPbeMediaNewHashMode:skl,kbl
 702                  *
 703                  * Must match Display Engine. See
 704                  * WaCompressedResourceDisplayNewHashMode.
 705                  */
 706                 wa_write_or(wal,
 707                             MMCD_MISC_CTRL,
 708                             MMCD_PCLA | MMCD_HOTSPOT_EN);
 709         }
 710
 711         /* WaDisableHDCInvalidation:skl,bxt,kbl,cfl */
 712         wa_write_or(wal,
 713                     GAM_ECOCHK,
 714                     BDW_DISABLE_HDC_INVALIDATION);
 715 }
 716
 717 static void
 718 skl_gt_workarounds_init(struct drm_i915_private *i915, struct i915_wa_list *wal)
 719 {
 720         gen9_gt_workarounds_init(i915, wal);
 721
 722         /* WaDisableGafsUnitClkGating:skl */
 723         wa_write_or(wal,
 724                     GEN7_UCGCTL4,
 725                     GEN8_EU_GAUNIT_CLOCK_GATE_DISABLE);
 726
 727         /* WaInPlaceDecompressionHang:skl */
 728         if (IS_SKL_REVID(i915, SKL_REVID_H0, REVID_FOREVER))
 729                 wa_write_or(wal,
 730                             GEN9_GAMT_ECO_REG_RW_IA,
 731                             GAMT_ECO_ENABLE_IN_PLACE_DECOMPRESS);
 732 }
 733
 734 static void
 735 bxt_gt_workarounds_init(struct drm_i915_private *i915, struct i915_wa_list *wal)
 736 {
 737         gen9_gt_workarounds_init(i915, wal);
 738
 739         /* WaInPlaceDecompressionHang:bxt */
 740         wa_write_or(wal,
 741                     GEN9_GAMT_ECO_REG_RW_IA,
 742                     GAMT_ECO_ENABLE_IN_PLACE_DECOMPRESS);
 743 }
 744
 745 static void
 746 kbl_gt_workarounds_init(struct drm_i915_private *i915, struct i915_wa_list *wal)
 747 {
 748         gen9_gt_workarounds_init(i915, wal);
 749
 750         /* WaDisableDynamicCreditSharing:kbl */
 751         if (IS_KBL_REVID(i915, 0, KBL_REVID_B0))
 752                 wa_write_or(wal,
 753                             GAMT_CHKN_BIT_REG,
 754                             GAMT_CHKN_DISABLE_DYNAMIC_CREDIT_SHARING);
 755
 756         /* WaDisableGafsUnitClkGating:kbl */
 757         wa_write_or(wal,
 758                     GEN7_UCGCTL4,
 759                     GEN8_EU_GAUNIT_CLOCK_GATE_DISABLE);
 760
 761         /* WaInPlaceDecompressionHang:kbl */
 762         wa_write_or(wal,
 763                     GEN9_GAMT_ECO_REG_RW_IA,
 764                     GAMT_ECO_ENABLE_IN_PLACE_DECOMPRESS);
 765 }
 766
 767 static void
 768 glk_gt_workarounds_init(struct drm_i915_private *i915, struct i915_wa_list *wal)
 769 {
 770         gen9_gt_workarounds_init(i915, wal);
 771 }
 772
 773 static void
 774 cfl_gt_workarounds_init(struct drm_i915_private *i915, struct i915_wa_list *wal)
 775 {
 776         gen9_gt_workarounds_init(i915, wal);
 777
 778         /* WaDisableGafsUnitClkGating:cfl */
 779         wa_write_or(wal,
 780                     GEN7_UCGCTL4,
 781                     GEN8_EU_GAUNIT_CLOCK_GATE_DISABLE);
 782
 783         /* WaInPlaceDecompressionHang:cfl */
 784         wa_write_or(wal,
 785                     GEN9_GAMT_ECO_REG_RW_IA,
 786                     GAMT_ECO_ENABLE_IN_PLACE_DECOMPRESS);
 787 }
 788
 789 static void
 790 wa_init_mcr(struct drm_i915_private *i915, struct i915_wa_list *wal)
 791 {
 792         const struct sseu_dev_info *sseu = &RUNTIME_INFO(i915)->sseu;
 793         unsigned int slice, subslice;
 794         u32 l3_en, mcr, mcr_mask;
 795
 796         GEM_BUG_ON(INTEL_GEN(i915) < 10);
 797
 798         /*
 799          * WaProgramMgsrForL3BankSpecificMmioReads: cnl,icl
 800          * L3Banks could be fused off in single slice scenario. If that is
 801          * the case, we might need to program MCR select to a valid L3Bank
 802          * by default, to make sure we correctly read certain registers
 803          * later on (in the range 0xB100 - 0xB3FF).
 804          *
 805          * WaProgramMgsrForCorrectSliceSpecificMmioReads:cnl,icl
 806          * Before any MMIO read into slice/subslice specific registers, MCR
 807          * packet control register needs to be programmed to point to any
 808          * enabled s/ss pair. Otherwise, incorrect values will be returned.
 809          * This means each subsequent MMIO read will be forwarded to an
 810          * specific s/ss combination, but this is OK since these registers
 811          * are consistent across s/ss in almost all cases. In the rare
 812          * occasions, such as INSTDONE, where this value is dependent
 813          * on s/ss combo, the read should be done with read_subslice_reg.
 814          *
 815          * Since GEN8_MCR_SELECTOR contains dual-purpose bits which select both
 816          * to which subslice, or to which L3 bank, the respective mmio reads
 817          * will go, we have to find a common index which works for both
 818          * accesses.
 819          *
 820          * Case where we cannot find a common index fortunately should not
 821          * happen in production hardware, so we only emit a warning instead of
 822          * implementing something more complex that requires checking the range
 823          * of every MMIO read.
 824          */
 825
 826         if (INTEL_GEN(i915) >= 10 && is_power_of_2(sseu->slice_mask)) {
 827                 u32 l3_fuse =
 828                         intel_uncore_read(&i915->uncore, GEN10_MIRROR_FUSE3) &
 829                         GEN10_L3BANK_MASK;
 830
 831                 DRM_DEBUG_DRIVER("L3 fuse = %x\n", l3_fuse);
 832                 l3_en = ~(l3_fuse << GEN10_L3BANK_PAIR_COUNT | l3_fuse);
 833         } else {
 834                 l3_en = ~0;
 835         }
 836
 837         slice = fls(sseu->slice_mask) - 1;
 838         subslice = fls(l3_en & intel_sseu_get_subslices(sseu, slice));
 839         if (!subslice) {
 840                 DRM_WARN("No common index found between subslice mask %x and L3 bank mask %x!\n",
 841                          intel_sseu_get_subslices(sseu, slice), l3_en);
 842                 subslice = fls(l3_en);
 843                 drm_WARN_ON(&i915->drm, !subslice);
 844         }
 845         subslice--;
 846
 847         if (INTEL_GEN(i915) >= 11) {
 848                 mcr = GEN11_MCR_SLICE(slice) | GEN11_MCR_SUBSLICE(subslice);
 849                 mcr_mask = GEN11_MCR_SLICE_MASK | GEN11_MCR_SUBSLICE_MASK;
 850         } else {
 851                 mcr = GEN8_MCR_SLICE(slice) | GEN8_MCR_SUBSLICE(subslice);
 852                 mcr_mask = GEN8_MCR_SLICE_MASK | GEN8_MCR_SUBSLICE_MASK;
 853         }
 854
 855         DRM_DEBUG_DRIVER("MCR slice/subslice = %x\n", mcr);
 856
 857         wa_write_masked_or(wal, GEN8_MCR_SELECTOR, mcr_mask, mcr);
 858 }
 859
 860 static void
 861 cnl_gt_workarounds_init(struct drm_i915_private *i915, struct i915_wa_list *wal)
 862 {
 863         wa_init_mcr(i915, wal);
 864
 865         /* WaDisableI2mCycleOnWRPort:cnl (pre-prod) */
 866         if (IS_CNL_REVID(i915, CNL_REVID_B0, CNL_REVID_B0))
 867                 wa_write_or(wal,
 868                             GAMT_CHKN_BIT_REG,
 869                             GAMT_CHKN_DISABLE_I2M_CYCLE_ON_WR_PORT);
 870
 871         /* WaInPlaceDecompressionHang:cnl */
 872         wa_write_or(wal,
 873                     GEN9_GAMT_ECO_REG_RW_IA,
 874                     GAMT_ECO_ENABLE_IN_PLACE_DECOMPRESS);
 875 }
 876
 877 static void
 878 icl_gt_workarounds_init(struct drm_i915_private *i915, struct i915_wa_list *wal)
 879 {
 880         wa_init_mcr(i915, wal);
 881
 882         /* WaInPlaceDecompressionHang:icl */
 883         wa_write_or(wal,
 884                     GEN9_GAMT_ECO_REG_RW_IA,
 885                     GAMT_ECO_ENABLE_IN_PLACE_DECOMPRESS);
 886
 887         /* WaModifyGamTlbPartitioning:icl */
 888         wa_write_masked_or(wal,
 889                            GEN11_GACB_PERF_CTRL,
 890                            GEN11_HASH_CTRL_MASK,
 891                            GEN11_HASH_CTRL_BIT0 | GEN11_HASH_CTRL_BIT4);
 892
 893         /* Wa_1405766107:icl
 894          * Formerly known as WaCL2SFHalfMaxAlloc
 895          */
 896         wa_write_or(wal,
 897                     GEN11_LSN_UNSLCVC,
 898                     GEN11_LSN_UNSLCVC_GAFS_HALF_SF_MAXALLOC |
 899                     GEN11_LSN_UNSLCVC_GAFS_HALF_CL2_MAXALLOC);
 900
 901         /* Wa_220166154:icl
 902          * Formerly known as WaDisCtxReload
 903          */
 904         wa_write_or(wal,
 905                     GEN8_GAMW_ECO_DEV_RW_IA,
 906                     GAMW_ECO_DEV_CTX_RELOAD_DISABLE);
 907
 908         /* Wa_1405779004:icl (pre-prod) */
 909         if (IS_ICL_REVID(i915, ICL_REVID_A0, ICL_REVID_A0))
 910                 wa_write_or(wal,
 911                             SLICE_UNIT_LEVEL_CLKGATE,
 912                             MSCUNIT_CLKGATE_DIS);
 913
 914         /* Wa_1406680159:icl */
 915         wa_write_or(wal,
 916                     SUBSLICE_UNIT_LEVEL_CLKGATE,
 917                     GWUNIT_CLKGATE_DIS);
 918
 919         /* Wa_1406838659:icl (pre-prod) */
 920         if (IS_ICL_REVID(i915, ICL_REVID_A0, ICL_REVID_B0))
 921                 wa_write_or(wal,
 922                             INF_UNIT_LEVEL_CLKGATE,
 923                             CGPSF_CLKGATE_DIS);
 924
 925         /* Wa_1406463099:icl
 926          * Formerly known as WaGamTlbPendError
 927          */
 928         wa_write_or(wal,
 929                     GAMT_CHKN_BIT_REG,
 930                     GAMT_CHKN_DISABLE_L3_COH_PIPE);
 931
 932         /* Wa_1607087056:icl */
 933         wa_write_or(wal,
 934                     SLICE_UNIT_LEVEL_CLKGATE,
 935                     L3_CLKGATE_DIS | L3_CR2X_CLKGATE_DIS);
 936 }
 937
 938 static void
 939 tgl_gt_workarounds_init(struct drm_i915_private *i915, struct i915_wa_list *wal)
 940 {
 941         /* Wa_1409420604:tgl */
 942         if (IS_TGL_REVID(i915, TGL_REVID_A0, TGL_REVID_A0))
 943                 wa_write_or(wal,
 944                             SUBSLICE_UNIT_LEVEL_CLKGATE2,
 945                             CPSSUNIT_CLKGATE_DIS);
 946
 947         /* Wa_1607087056:tgl also know as BUG:1409180338 */
 948         if (IS_TGL_REVID(i915, TGL_REVID_A0, TGL_REVID_A0))
 949                 wa_write_or(wal,
 950                             SLICE_UNIT_LEVEL_CLKGATE,
 951                             L3_CLKGATE_DIS | L3_CR2X_CLKGATE_DIS);
 952 }
 953
 954 static void
 955 gt_init_workarounds(struct drm_i915_private *i915, struct i915_wa_list *wal)
 956 {
 957         if (IS_GEN(i915, 12))
 958                 tgl_gt_workarounds_init(i915, wal);
 959         else if (IS_GEN(i915, 11))
 960                 icl_gt_workarounds_init(i915, wal);
 961         else if (IS_CANNONLAKE(i915))
 962                 cnl_gt_workarounds_init(i915, wal);
 963         else if (IS_COFFEELAKE(i915))
 964                 cfl_gt_workarounds_init(i915, wal);
 965         else if (IS_GEMINILAKE(i915))
 966                 glk_gt_workarounds_init(i915, wal);
 967         else if (IS_KABYLAKE(i915))
 968                 kbl_gt_workarounds_init(i915, wal);
 969         else if (IS_BROXTON(i915))
 970                 bxt_gt_workarounds_init(i915, wal);
 971         else if (IS_SKYLAKE(i915))
 972                 skl_gt_workarounds_init(i915, wal);
 973         else if (INTEL_GEN(i915) <= 8)
 974                 return;
 975         else
 976                 MISSING_CASE(INTEL_GEN(i915));
 977 }
 978
 979 void intel_gt_init_workarounds(struct drm_i915_private *i915)
 980 {
 981         struct i915_wa_list *wal = &i915->gt_wa_list;
 982
 983         wa_init_start(wal, "GT", "global");
 984         gt_init_workarounds(i915, wal);
 985         wa_init_finish(wal);
 986 }
 987
 988 static enum forcewake_domains
 989 wal_get_fw_for_rmw(struct intel_uncore *uncore, const struct i915_wa_list *wal)
 990 {
 991         enum forcewake_domains fw = 0;
 992         struct i915_wa *wa;
 993         unsigned int i;
 994
 995         for (i = 0, wa = wal->list; i < wal->count; i++, wa++)
 996                 fw |= intel_uncore_forcewake_for_reg(uncore,
 997                                                      wa->reg,
 998                                                      FW_REG_READ |
 999                                                      FW_REG_WRITE);
1000
1001         return fw;
1002 }
1003
1004 static bool
1005 wa_verify(const struct i915_wa *wa, u32 cur, const char *name, const char *from)
1006 {
1007         if ((cur ^ wa->set) & wa->read) {
1008                 DRM_ERROR("%s workaround lost on %s! (%x=%x/%x, expected %x)\n",
1009                           name, from, i915_mmio_reg_offset(wa->reg),
1010                           cur, cur & wa->read, wa->set);
1011
1012                 return false;
1013         }
1014
1015         return true;
1016 }
1017
1018 static void
1019 wa_list_apply(struct intel_uncore *uncore, const struct i915_wa_list *wal)
1020 {
1021         enum forcewake_domains fw;
1022         unsigned long flags;
1023         struct i915_wa *wa;
1024         unsigned int i;
1025
1026         if (!wal->count)
1027                 return;
1028
1029         fw = wal_get_fw_for_rmw(uncore, wal);
1030
1031         spin_lock_irqsave(&uncore->lock, flags);
1032         intel_uncore_forcewake_get__locked(uncore, fw);
1033
1034         for (i = 0, wa = wal->list; i < wal->count; i++, wa++) {
1035                 if (wa->clr)
1036                         intel_uncore_rmw_fw(uncore, wa->reg, wa->clr, wa->set);
1037                 else
1038                         intel_uncore_write_fw(uncore, wa->reg, wa->set);
1039                 if (IS_ENABLED(CONFIG_DRM_I915_DEBUG_GEM))
1040                         wa_verify(wa,
1041                                   intel_uncore_read_fw(uncore, wa->reg),
1042                                   wal->name, "application");
1043         }
1044
1045         intel_uncore_forcewake_put__locked(uncore, fw);
1046         spin_unlock_irqrestore(&uncore->lock, flags);
1047 }
1048
1049 void intel_gt_apply_workarounds(struct intel_gt *gt)
1050 {
1051         wa_list_apply(gt->uncore, &gt->i915->gt_wa_list);
1052 }
1053
1054 static bool wa_list_verify(struct intel_uncore *uncore,
1055                            const struct i915_wa_list *wal,
1056                            const char *from)
1057 {
1058         struct i915_wa *wa;
1059         unsigned int i;
1060         bool ok = true;
1061
1062         for (i = 0, wa = wal->list; i < wal->count; i++, wa++)
1063                 ok &= wa_verify(wa,
1064                                 intel_uncore_read(uncore, wa->reg),
1065                                 wal->name, from);
1066
1067         return ok;
1068 }
1069
1070 bool intel_gt_verify_workarounds(struct intel_gt *gt, const char *from)
1071 {
1072         return wa_list_verify(gt->uncore, &gt->i915->gt_wa_list, from);
1073 }
1074
1075 static inline bool is_nonpriv_flags_valid(u32 flags)
1076 {
1077         /* Check only valid flag bits are set */
1078         if (flags & ~RING_FORCE_TO_NONPRIV_MASK_VALID)
1079                 return false;
1080
1081         /* NB: Only 3 out of 4 enum values are valid for access field */
1082         if ((flags & RING_FORCE_TO_NONPRIV_ACCESS_MASK) ==
1083             RING_FORCE_TO_NONPRIV_ACCESS_INVALID)
1084                 return false;
1085
1086         return true;
1087 }
1088
1089 static void
1090 whitelist_reg_ext(struct i915_wa_list *wal, i915_reg_t reg, u32 flags)
1091 {
1092         struct i915_wa wa = {
1093                 .reg = reg
1094         };
1095
1096         if (GEM_DEBUG_WARN_ON(wal->count >= RING_MAX_NONPRIV_SLOTS))
1097                 return;
1098
1099         if (GEM_DEBUG_WARN_ON(!is_nonpriv_flags_valid(flags)))
1100                 return;
1101
1102         wa.reg.reg |= flags;
1103         _wa_add(wal, &wa);
1104 }
1105
1106 static void
1107 whitelist_reg(struct i915_wa_list *wal, i915_reg_t reg)
1108 {
1109         whitelist_reg_ext(wal, reg, RING_FORCE_TO_NONPRIV_ACCESS_RW);
1110 }
1111
1112 static void gen9_whitelist_build(struct i915_wa_list *w)
1113 {
1114         /* WaVFEStateAfterPipeControlwithMediaStateClear:skl,bxt,glk,cfl */
1115         whitelist_reg(w, GEN9_CTX_PREEMPT_REG);
1116
1117         /* WaEnablePreemptionGranularityControlByUMD:skl,bxt,kbl,cfl,[cnl] */
1118         whitelist_reg(w, GEN8_CS_CHICKEN1);
1119
1120         /* WaAllowUMDToModifyHDCChicken1:skl,bxt,kbl,glk,cfl */
1121         whitelist_reg(w, GEN8_HDC_CHICKEN1);
1122
1123         /* WaSendPushConstantsFromMMIO:skl,bxt */
1124         whitelist_reg(w, COMMON_SLICE_CHICKEN2);
1125 }
1126
1127 static void skl_whitelist_build(struct intel_engine_cs *engine)
1128 {
1129         struct i915_wa_list *w = &engine->whitelist;
1130
1131         if (engine->class != RENDER_CLASS)
1132                 return;
1133
1134         gen9_whitelist_build(w);
1135
1136         /* WaDisableLSQCROPERFforOCL:skl */
1137         whitelist_reg(w, GEN8_L3SQCREG4);
1138 }
1139
1140 static void bxt_whitelist_build(struct intel_engine_cs *engine)
1141 {
1142         if (engine->class != RENDER_CLASS)
1143                 return;
1144
1145         gen9_whitelist_build(&engine->whitelist);
1146 }
1147
1148 static void kbl_whitelist_build(struct intel_engine_cs *engine)
1149 {
1150         struct i915_wa_list *w = &engine->whitelist;
1151
1152         if (engine->class != RENDER_CLASS)
1153                 return;
1154
1155         gen9_whitelist_build(w);
1156
1157         /* WaDisableLSQCROPERFforOCL:kbl */
1158         whitelist_reg(w, GEN8_L3SQCREG4);
1159 }
1160
1161 static void glk_whitelist_build(struct intel_engine_cs *engine)
1162 {
1163         struct i915_wa_list *w = &engine->whitelist;
1164
1165         if (engine->class != RENDER_CLASS)
1166                 return;
1167
1168         gen9_whitelist_build(w);
1169
1170         /* WA #0862: Userspace has to set "Barrier Mode" to avoid hangs. */
1171         whitelist_reg(w, GEN9_SLICE_COMMON_ECO_CHICKEN1);
1172 }
1173
1174 static void cfl_whitelist_build(struct intel_engine_cs *engine)
1175 {
1176         struct i915_wa_list *w = &engine->whitelist;
1177
1178         if (engine->class != RENDER_CLASS)
1179                 return;
1180
1181         gen9_whitelist_build(w);
1182
1183         /*
1184          * WaAllowPMDepthAndInvocationCountAccessFromUMD:cfl,whl,cml,aml
1185          *
1186          * This covers 4 register which are next to one another :
1187          *   - PS_INVOCATION_COUNT
1188          *   - PS_INVOCATION_COUNT_UDW
1189          *   - PS_DEPTH_COUNT
1190          *   - PS_DEPTH_COUNT_UDW
1191          */
1192         whitelist_reg_ext(w, PS_INVOCATION_COUNT,
1193                           RING_FORCE_TO_NONPRIV_ACCESS_RD |
1194                           RING_FORCE_TO_NONPRIV_RANGE_4);
1195 }
1196
1197 static void cnl_whitelist_build(struct intel_engine_cs *engine)
1198 {
1199         struct i915_wa_list *w = &engine->whitelist;
1200
1201         if (engine->class != RENDER_CLASS)
1202                 return;
1203
1204         /* WaEnablePreemptionGranularityControlByUMD:cnl */
1205         whitelist_reg(w, GEN8_CS_CHICKEN1);
1206 }
1207
1208 static void icl_whitelist_build(struct intel_engine_cs *engine)
1209 {
1210         struct i915_wa_list *w = &engine->whitelist;
1211
1212         switch (engine->class) {
1213         case RENDER_CLASS:
1214                 /* WaAllowUMDToModifyHalfSliceChicken7:icl */
1215                 whitelist_reg(w, GEN9_HALF_SLICE_CHICKEN7);
1216
1217                 /* WaAllowUMDToModifySamplerMode:icl */
1218                 whitelist_reg(w, GEN10_SAMPLER_MODE);
1219
1220                 /* WaEnableStateCacheRedirectToCS:icl */
1221                 whitelist_reg(w, GEN9_SLICE_COMMON_ECO_CHICKEN1);
1222
1223                 /*
1224                  * WaAllowPMDepthAndInvocationCountAccessFromUMD:icl
1225                  *
1226                  * This covers 4 register which are next to one another :
1227                  *   - PS_INVOCATION_COUNT
1228                  *   - PS_INVOCATION_COUNT_UDW
1229                  *   - PS_DEPTH_COUNT
1230                  *   - PS_DEPTH_COUNT_UDW
1231                  */
1232                 whitelist_reg_ext(w, PS_INVOCATION_COUNT,
1233                                   RING_FORCE_TO_NONPRIV_ACCESS_RD |
1234                                   RING_FORCE_TO_NONPRIV_RANGE_4);
1235                 break;
1236
1237         case VIDEO_DECODE_CLASS:
1238                 /* hucStatusRegOffset */
1239                 whitelist_reg_ext(w, _MMIO(0x2000 + engine->mmio_base),
1240                                   RING_FORCE_TO_NONPRIV_ACCESS_RD);
1241                 /* hucUKernelHdrInfoRegOffset */
1242                 whitelist_reg_ext(w, _MMIO(0x2014 + engine->mmio_base),
1243                                   RING_FORCE_TO_NONPRIV_ACCESS_RD);
1244                 /* hucStatus2RegOffset */
1245                 whitelist_reg_ext(w, _MMIO(0x23B0 + engine->mmio_base),
1246                                   RING_FORCE_TO_NONPRIV_ACCESS_RD);
1247                 break;
1248
1249         default:
1250                 break;
1251         }
1252 }
1253
1254 static void tgl_whitelist_build(struct intel_engine_cs *engine)
1255 {
1256         struct i915_wa_list *w = &engine->whitelist;
1257
1258         switch (engine->class) {
1259         case RENDER_CLASS:
1260                 /*
1261                  * WaAllowPMDepthAndInvocationCountAccessFromUMD:tgl
1262                  * Wa_1408556865:tgl
1263                  *
1264                  * This covers 4 registers which are next to one another :
1265                  *   - PS_INVOCATION_COUNT
1266                  *   - PS_INVOCATION_COUNT_UDW
1267                  *   - PS_DEPTH_COUNT
1268                  *   - PS_DEPTH_COUNT_UDW
1269                  */
1270                 whitelist_reg_ext(w, PS_INVOCATION_COUNT,
1271                                   RING_FORCE_TO_NONPRIV_ACCESS_RD |
1272                                   RING_FORCE_TO_NONPRIV_RANGE_4);
1273
1274                 /* Wa_1808121037:tgl */
1275                 whitelist_reg(w, GEN7_COMMON_SLICE_CHICKEN1);
1276
1277                 /* Wa_1806527549:tgl */
1278                 whitelist_reg(w, HIZ_CHICKEN);
1279                 break;
1280         default:
1281                 break;
1282         }
1283 }
1284
1285 void intel_engine_init_whitelist(struct intel_engine_cs *engine)
1286 {
1287         struct drm_i915_private *i915 = engine->i915;
1288         struct i915_wa_list *w = &engine->whitelist;
1289
1290         wa_init_start(w, "whitelist", engine->name);
1291
1292         if (IS_GEN(i915, 12))
1293                 tgl_whitelist_build(engine);
1294         else if (IS_GEN(i915, 11))
1295                 icl_whitelist_build(engine);
1296         else if (IS_CANNONLAKE(i915))
1297                 cnl_whitelist_build(engine);
1298         else if (IS_COFFEELAKE(i915))
1299                 cfl_whitelist_build(engine);
1300         else if (IS_GEMINILAKE(i915))
1301                 glk_whitelist_build(engine);
1302         else if (IS_KABYLAKE(i915))
1303                 kbl_whitelist_build(engine);
1304         else if (IS_BROXTON(i915))
1305                 bxt_whitelist_build(engine);
1306         else if (IS_SKYLAKE(i915))
1307                 skl_whitelist_build(engine);
1308         else if (INTEL_GEN(i915) <= 8)
1309                 return;
1310         else
1311                 MISSING_CASE(INTEL_GEN(i915));
1312
1313         wa_init_finish(w);
1314 }
1315
1316 void intel_engine_apply_whitelist(struct intel_engine_cs *engine)
1317 {
1318         const struct i915_wa_list *wal = &engine->whitelist;
1319         struct intel_uncore *uncore = engine->uncore;
1320         const u32 base = engine->mmio_base;
1321         struct i915_wa *wa;
1322         unsigned int i;
1323
1324         if (!wal->count)
1325                 return;
1326
1327         for (i = 0, wa = wal->list; i < wal->count; i++, wa++)
1328                 intel_uncore_write(uncore,
1329                                    RING_FORCE_TO_NONPRIV(base, i),
1330                                    i915_mmio_reg_offset(wa->reg));
1331
1332         /* And clear the rest just in case of garbage */
1333         for (; i < RING_MAX_NONPRIV_SLOTS; i++)
1334                 intel_uncore_write(uncore,
1335                                    RING_FORCE_TO_NONPRIV(base, i),
1336                                    i915_mmio_reg_offset(RING_NOPID(base)));
1337 }
1338
1339 static void
1340 rcs_engine_wa_init(struct intel_engine_cs *engine, struct i915_wa_list *wal)
1341 {
1342         struct drm_i915_private *i915 = engine->i915;
1343
1344         if (IS_TGL_REVID(i915, TGL_REVID_A0, TGL_REVID_A0)) {
1345                 /*
1346                  * Wa_1607138336:tgl
1347                  * Wa_1607063988:tgl
1348                  */
1349                 wa_write_or(wal,
1350                             GEN9_CTX_PREEMPT_REG,
1351                             GEN12_DISABLE_POSH_BUSY_FF_DOP_CG);
1352
1353                 /*
1354                  * Wa_1607030317:tgl
1355                  * Wa_1607186500:tgl
1356                  * Wa_1607297627:tgl there is 3 entries for this WA on BSpec, 2
1357                  * of then says it is fixed on B0 the other one says it is
1358                  * permanent
1359                  */
1360                 wa_masked_en(wal,
1361                              GEN6_RC_SLEEP_PSMI_CONTROL,
1362                              GEN12_WAIT_FOR_EVENT_POWER_DOWN_DISABLE |
1363                              GEN8_RC_SEMA_IDLE_MSG_DISABLE);
1364
1365                 /*
1366                  * Wa_1606679103:tgl
1367                  * (see also Wa_1606682166:icl)
1368                  */
1369                 wa_write_or(wal,
1370                             GEN7_SARCHKMD,
1371                             GEN7_DISABLE_SAMPLER_PREFETCH);
1372
1373                 /* Wa_1407928979:tgl */
1374                 wa_write_or(wal,
1375                             GEN7_FF_THREAD_MODE,
1376                             GEN12_FF_TESSELATION_DOP_GATE_DISABLE);
1377
1378                 /*
1379                  * Wa_1409085225:tgl
1380                  * Wa_14010229206:tgl
1381                  */
1382                 wa_masked_en(wal, GEN9_ROW_CHICKEN4, GEN12_DISABLE_TDL_PUSH);
1383
1384                 /* Wa_1408615072:tgl */
1385                 wa_write_or(wal, UNSLICE_UNIT_LEVEL_CLKGATE2,
1386                             VSUNIT_CLKGATE_DIS_TGL);
1387         }
1388
1389         if (IS_TIGERLAKE(i915)) {
1390                 /* Wa_1606931601:tgl */
1391                 wa_masked_en(wal, GEN7_ROW_CHICKEN2, GEN12_DISABLE_EARLY_READ);
1392
1393                 /* Wa_1409804808:tgl */
1394                 wa_masked_en(wal, GEN7_ROW_CHICKEN2,
1395                              GEN12_PUSH_CONST_DEREF_HOLD_DIS);
1396
1397                 /* Wa_1606700617:tgl */
1398                 wa_masked_en(wal,
1399                              GEN9_CS_DEBUG_MODE1,
1400                              FF_DOP_CLOCK_GATE_DISABLE);
1401         }
1402
1403         if (IS_GEN(i915, 11)) {
1404                 /* This is not an Wa. Enable for better image quality */
1405                 wa_masked_en(wal,
1406                              _3D_CHICKEN3,
1407                              _3D_CHICKEN3_AA_LINE_QUALITY_FIX_ENABLE);
1408
1409                 /* WaPipelineFlushCoherentLines:icl */
1410                 wa_write_or(wal,
1411                             GEN8_L3SQCREG4,
1412                             GEN8_LQSC_FLUSH_COHERENT_LINES);
1413
1414                 /*
1415                  * Wa_1405543622:icl
1416                  * Formerly known as WaGAPZPriorityScheme
1417                  */
1418                 wa_write_or(wal,
1419                             GEN8_GARBCNTL,
1420                             GEN11_ARBITRATION_PRIO_ORDER_MASK);
1421
1422                 /*
1423                  * Wa_1604223664:icl
1424                  * Formerly known as WaL3BankAddressHashing
1425                  */
1426                 wa_write_masked_or(wal,
1427                                    GEN8_GARBCNTL,
1428                                    GEN11_HASH_CTRL_EXCL_MASK,
1429                                    GEN11_HASH_CTRL_EXCL_BIT0);
1430                 wa_write_masked_or(wal,
1431                                    GEN11_GLBLINVL,
1432                                    GEN11_BANK_HASH_ADDR_EXCL_MASK,
1433                                    GEN11_BANK_HASH_ADDR_EXCL_BIT0);
1434
1435                 /*
1436                  * Wa_1405733216:icl
1437                  * Formerly known as WaDisableCleanEvicts
1438                  */
1439                 wa_write_or(wal,
1440                             GEN8_L3SQCREG4,
1441                             GEN11_LQSC_CLEAN_EVICT_DISABLE);
1442
1443                 /* WaForwardProgressSoftReset:icl */
1444                 wa_write_or(wal,
1445                             GEN10_SCRATCH_LNCF2,
1446                             PMFLUSHDONE_LNICRSDROP |
1447                             PMFLUSH_GAPL3UNBLOCK |
1448                             PMFLUSHDONE_LNEBLK);
1449
1450                 /* Wa_1406609255:icl (pre-prod) */
1451                 if (IS_ICL_REVID(i915, ICL_REVID_A0, ICL_REVID_B0))
1452                         wa_write_or(wal,
1453                                     GEN7_SARCHKMD,
1454                                     GEN7_DISABLE_DEMAND_PREFETCH);
1455
1456                 /* Wa_1606682166:icl */
1457                 wa_write_or(wal,
1458                             GEN7_SARCHKMD,
1459                             GEN7_DISABLE_SAMPLER_PREFETCH);
1460
1461                 /* Wa_1409178092:icl */
1462                 wa_write_masked_or(wal,
1463                                    GEN11_SCRATCH2,
1464                                    GEN11_COHERENT_PARTIAL_WRITE_MERGE_ENABLE,
1465                                    0);
1466
1467                 /* WaEnable32PlaneMode:icl */
1468                 wa_masked_en(wal, GEN9_CSFE_CHICKEN1_RCS,
1469                              GEN11_ENABLE_32_PLANE_MODE);
1470
1471                 /*
1472                  * Wa_1408615072:icl,ehl  (vsunit)
1473                  * Wa_1407596294:icl,ehl  (hsunit)
1474                  */
1475                 wa_write_or(wal, UNSLICE_UNIT_LEVEL_CLKGATE,
1476                             VSUNIT_CLKGATE_DIS | HSUNIT_CLKGATE_DIS);
1477
1478                 /* Wa_1407352427:icl,ehl */
1479                 wa_write_or(wal, UNSLICE_UNIT_LEVEL_CLKGATE2,
1480                             PSDUNIT_CLKGATE_DIS);
1481         }
1482
1483         if (IS_GEN_RANGE(i915, 9, 12)) {
1484                 /* FtrPerCtxtPreemptionGranularityControl:skl,bxt,kbl,cfl,cnl,icl,tgl */
1485                 wa_masked_en(wal,
1486                              GEN7_FF_SLICE_CS_CHICKEN1,
1487                              GEN9_FFSC_PERCTX_PREEMPT_CTRL);
1488         }
1489
1490         if (IS_SKYLAKE(i915) || IS_KABYLAKE(i915) || IS_COFFEELAKE(i915)) {
1491                 /* WaEnableGapsTsvCreditFix:skl,kbl,cfl */
1492                 wa_write_or(wal,
1493                             GEN8_GARBCNTL,
1494                             GEN9_GAPS_TSV_CREDIT_DISABLE);
1495         }
1496
1497         if (IS_BROXTON(i915)) {
1498                 /* WaDisablePooledEuLoadBalancingFix:bxt */
1499                 wa_masked_en(wal,
1500                              FF_SLICE_CS_CHICKEN2,
1501                              GEN9_POOLED_EU_LOAD_BALANCING_FIX_DISABLE);
1502         }
1503
1504         if (IS_GEN(i915, 9)) {
1505                 /* WaContextSwitchWithConcurrentTLBInvalidate:skl,bxt,kbl,glk,cfl */
1506                 wa_masked_en(wal,
1507                              GEN9_CSFE_CHICKEN1_RCS,
1508                              GEN9_PREEMPT_GPGPU_SYNC_SWITCH_DISABLE);
1509
1510                 /* WaEnableLbsSlaRetryTimerDecrement:skl,bxt,kbl,glk,cfl */
1511                 wa_write_or(wal,
1512                             BDW_SCRATCH1,
1513                             GEN9_LBS_SLA_RETRY_TIMER_DECREMENT_ENABLE);
1514
1515                 /* WaProgramL3SqcReg1DefaultForPerf:bxt,glk */
1516                 if (IS_GEN9_LP(i915))
1517                         wa_write_masked_or(wal,
1518                                            GEN8_L3SQCREG1,
1519                                            L3_PRIO_CREDITS_MASK,
1520                                            L3_GENERAL_PRIO_CREDITS(62) |
1521                                            L3_HIGH_PRIO_CREDITS(2));
1522
1523                 /* WaOCLCoherentLineFlush:skl,bxt,kbl,cfl */
1524                 wa_write_or(wal,
1525                             GEN8_L3SQCREG4,
1526                             GEN8_LQSC_FLUSH_COHERENT_LINES);
1527         }
1528
1529         if (IS_GEN(i915, 7))
1530                 /* WaBCSVCSTlbInvalidationMode:ivb,vlv,hsw */
1531                 wa_masked_en(wal,
1532                              GFX_MODE_GEN7,
1533                              GFX_TLB_INVALIDATE_EXPLICIT | GFX_REPLAY_MODE);
1534
1535         if (IS_GEN_RANGE(i915, 6, 7))
1536                 /*
1537                  * We need to disable the AsyncFlip performance optimisations in
1538                  * order to use MI_WAIT_FOR_EVENT within the CS. It should
1539                  * already be programmed to '1' on all products.
1540                  *
1541                  * WaDisableAsyncFlipPerfMode:snb,ivb,hsw,vlv
1542                  */
1543                 wa_masked_en(wal,
1544                              MI_MODE,
1545                              ASYNC_FLIP_PERF_DISABLE);
1546
1547         if (IS_GEN(i915, 6)) {
1548                 /*
1549                  * Required for the hardware to program scanline values for
1550                  * waiting
1551                  * WaEnableFlushTlbInvalidationMode:snb
1552                  */
1553                 wa_masked_en(wal,
1554                              GFX_MODE,
1555                              GFX_TLB_INVALIDATE_EXPLICIT);
1556
1557                 /*
1558                  * From the Sandybridge PRM, volume 1 part 3, page 24:
1559                  * "If this bit is set, STCunit will have LRA as replacement
1560                  *  policy. [...] This bit must be reset. LRA replacement
1561                  *  policy is not supported."
1562                  */
1563                 wa_masked_dis(wal,
1564                               CACHE_MODE_0,
1565                               CM0_STC_EVICT_DISABLE_LRA_SNB);
1566         }
1567
1568         if (IS_GEN_RANGE(i915, 4, 6))
1569                 /* WaTimedSingleVertexDispatch:cl,bw,ctg,elk,ilk,snb */
1570                 wa_add(wal, MI_MODE,
1571                        0, _MASKED_BIT_ENABLE(VS_TIMER_DISPATCH),
1572                        /* XXX bit doesn't stick on Broadwater */
1573                        IS_I965G(i915) ? 0 : VS_TIMER_DISPATCH);
1574 }
1575
1576 static void
1577 xcs_engine_wa_init(struct intel_engine_cs *engine, struct i915_wa_list *wal)
1578 {
1579         struct drm_i915_private *i915 = engine->i915;
1580
1581         /* WaKBLVECSSemaphoreWaitPoll:kbl */
1582         if (IS_KBL_REVID(i915, KBL_REVID_A0, KBL_REVID_E0)) {
1583                 wa_write(wal,
1584                          RING_SEMA_WAIT_POLL(engine->mmio_base),
1585                          1);
1586         }
1587 }
1588
1589 static void
1590 engine_init_workarounds(struct intel_engine_cs *engine, struct i915_wa_list *wal)
1591 {
1592         if (I915_SELFTEST_ONLY(INTEL_GEN(engine->i915) < 4))
1593                 return;
1594
1595         if (engine->class == RENDER_CLASS)
1596                 rcs_engine_wa_init(engine, wal);
1597         else
1598                 xcs_engine_wa_init(engine, wal);
1599 }
1600
1601 void intel_engine_init_workarounds(struct intel_engine_cs *engine)
1602 {
1603         struct i915_wa_list *wal = &engine->wa_list;
1604
1605         if (INTEL_GEN(engine->i915) < 4)
1606                 return;
1607
1608         wa_init_start(wal, "engine", engine->name);
1609         engine_init_workarounds(engine, wal);
1610         wa_init_finish(wal);
1611 }
1612
1613 void intel_engine_apply_workarounds(struct intel_engine_cs *engine)
1614 {
1615         wa_list_apply(engine->uncore, &engine->wa_list);
1616 }
1617
1618 static struct i915_vma *
1619 create_scratch(struct i915_address_space *vm, int count)
1620 {
1621         struct drm_i915_gem_object *obj;
1622         struct i915_vma *vma;
1623         unsigned int size;
1624         int err;
1625
1626         size = round_up(count * sizeof(u32), PAGE_SIZE);
1627         obj = i915_gem_object_create_internal(vm->i915, size);
1628         if (IS_ERR(obj))
1629                 return ERR_CAST(obj);
1630
1631         i915_gem_object_set_cache_coherency(obj, I915_CACHE_LLC);
1632
1633         vma = i915_vma_instance(obj, vm, NULL);
1634         if (IS_ERR(vma)) {
1635                 err = PTR_ERR(vma);
1636                 goto err_obj;
1637         }
1638
1639         err = i915_vma_pin(vma, 0, 0,
1640                            i915_vma_is_ggtt(vma) ? PIN_GLOBAL : PIN_USER);
1641         if (err)
1642                 goto err_obj;
1643
1644         return vma;
1645
1646 err_obj:
1647         i915_gem_object_put(obj);
1648         return ERR_PTR(err);
1649 }
1650
1651 static bool mcr_range(struct drm_i915_private *i915, u32 offset)
1652 {
1653         /*
1654          * Registers in this range are affected by the MCR selector
1655          * which only controls CPU initiated MMIO. Routing does not
1656          * work for CS access so we cannot verify them on this path.
1657          */
1658         if (INTEL_GEN(i915) >= 8 && (offset >= 0xb000 && offset <= 0xb4ff))
1659                 return true;
1660
1661         return false;
1662 }
1663
1664 static int
1665 wa_list_srm(struct i915_request *rq,
1666             const struct i915_wa_list *wal,
1667             struct i915_vma *vma)
1668 {
1669         struct drm_i915_private *i915 = rq->i915;
1670         unsigned int i, count = 0;
1671         const struct i915_wa *wa;
1672         u32 srm, *cs;
1673
1674         srm = MI_STORE_REGISTER_MEM | MI_SRM_LRM_GLOBAL_GTT;
1675         if (INTEL_GEN(i915) >= 8)
1676                 srm++;
1677
1678         for (i = 0, wa = wal->list; i < wal->count; i++, wa++) {
1679                 if (!mcr_range(i915, i915_mmio_reg_offset(wa->reg)))
1680                         count++;
1681         }
1682
1683         cs = intel_ring_begin(rq, 4 * count);
1684         if (IS_ERR(cs))
1685                 return PTR_ERR(cs);
1686
1687         for (i = 0, wa = wal->list; i < wal->count; i++, wa++) {
1688                 u32 offset = i915_mmio_reg_offset(wa->reg);
1689
1690                 if (mcr_range(i915, offset))
1691                         continue;
1692
1693                 *cs++ = srm;
1694                 *cs++ = offset;
1695                 *cs++ = i915_ggtt_offset(vma) + sizeof(u32) * i;
1696                 *cs++ = 0;
1697         }
1698         intel_ring_advance(rq, cs);
1699
1700         return 0;
1701 }
1702
1703 static int engine_wa_list_verify(struct intel_context *ce,
1704                                  const struct i915_wa_list * const wal,
1705                                  const char *from)
1706 {
1707         const struct i915_wa *wa;
1708         struct i915_request *rq;
1709         struct i915_vma *vma;
1710         unsigned int i;
1711         u32 *results;
1712         int err;
1713
1714         if (!wal->count)
1715                 return 0;
1716
1717         vma = create_scratch(&ce->engine->gt->ggtt->vm, wal->count);
1718         if (IS_ERR(vma))
1719                 return PTR_ERR(vma);
1720
1721         intel_engine_pm_get(ce->engine);
1722         rq = intel_context_create_request(ce);
1723         intel_engine_pm_put(ce->engine);
1724         if (IS_ERR(rq)) {
1725                 err = PTR_ERR(rq);
1726                 goto err_vma;
1727         }
1728
1729         i915_vma_lock(vma);
1730         err = i915_request_await_object(rq, vma->obj, true);
1731         if (err == 0)
1732                 err = i915_vma_move_to_active(vma, rq, EXEC_OBJECT_WRITE);
1733         i915_vma_unlock(vma);
1734         if (err) {
1735                 i915_request_add(rq);
1736                 goto err_vma;
1737         }
1738
1739         err = wa_list_srm(rq, wal, vma);
1740         if (err)
1741                 goto err_vma;
1742
1743         i915_request_get(rq);
1744         i915_request_add(rq);
1745         if (i915_request_wait(rq, 0, HZ / 5) < 0) {
1746                 err = -ETIME;
1747                 goto err_rq;
1748         }
1749
1750         results = i915_gem_object_pin_map(vma->obj, I915_MAP_WB);
1751         if (IS_ERR(results)) {
1752                 err = PTR_ERR(results);
1753                 goto err_rq;
1754         }
1755
1756         err = 0;
1757         for (i = 0, wa = wal->list; i < wal->count; i++, wa++) {
1758                 if (mcr_range(rq->i915, i915_mmio_reg_offset(wa->reg)))
1759                         continue;
1760
1761                 if (!wa_verify(wa, results[i], wal->name, from))
1762                         err = -ENXIO;
1763         }
1764
1765         i915_gem_object_unpin_map(vma->obj);
1766
1767 err_rq:
1768         i915_request_put(rq);
1769 err_vma:
1770         i915_vma_unpin(vma);
1771         i915_vma_put(vma);
1772         return err;
1773 }
1774
1775 int intel_engine_verify_workarounds(struct intel_engine_cs *engine,
1776                                     const char *from)
1777 {
1778         return engine_wa_list_verify(engine->kernel_context,
1779                                      &engine->wa_list,
1780                                      from);
1781 }
1782
1783 #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
1784 #include "selftest_workarounds.c"
1785 #endif