drivers/gpu/drm/i915/gt/intel_workarounds.c

   1 // SPDX-License-Identifier: MIT
   2 /*
   3  * Copyright © 2014-2018 Intel Corporation
   4  */
   5
   6 #include "i915_drv.h"
   7 #include "intel_context.h"
   8 #include "intel_engine_pm.h"
   9 #include "intel_gpu_commands.h"
  10 #include "intel_gt.h"
  11 #include "intel_ring.h"
  12 #include "intel_workarounds.h"
  13
  14 /**
  15  * DOC: Hardware workarounds
  16  *
  17  * This file is intended as a central place to implement most [1]_ of the
  18  * required workarounds for hardware to work as originally intended. They fall
  19  * in five basic categories depending on how/when they are applied:
  20  *
  21  * - Workarounds that touch registers that are saved/restored to/from the HW
  22  *   context image. The list is emitted (via Load Register Immediate commands)
  23  *   everytime a new context is created.
  24  * - GT workarounds. The list of these WAs is applied whenever these registers
  25  *   revert to default values (on GPU reset, suspend/resume [2]_, etc..).
  26  * - Display workarounds. The list is applied during display clock-gating
  27  *   initialization.
  28  * - Workarounds that whitelist a privileged register, so that UMDs can manage
  29  *   them directly. This is just a special case of a MMMIO workaround (as we
  30  *   write the list of these to/be-whitelisted registers to some special HW
  31  *   registers).
  32  * - Workaround batchbuffers, that get executed automatically by the hardware
  33  *   on every HW context restore.
  34  *
  35  * .. [1] Please notice that there are other WAs that, due to their nature,
  36  *    cannot be applied from a central place. Those are peppered around the rest
  37  *    of the code, as needed.
  38  *
  39  * .. [2] Technically, some registers are powercontext saved & restored, so they
  40  *    survive a suspend/resume. In practice, writing them again is not too
  41  *    costly and simplifies things. We can revisit this in the future.
  42  *
  43  * Layout
  44  * ~~~~~~
  45  *
  46  * Keep things in this file ordered by WA type, as per the above (context, GT,
  47  * display, register whitelist, batchbuffer). Then, inside each type, keep the
  48  * following order:
  49  *
  50  * - Infrastructure functions and macros
  51  * - WAs per platform in standard gen/chrono order
  52  * - Public functions to init or apply the given workaround type.
  53  */
  54
  55 static void wa_init_start(struct i915_wa_list *wal, const char *name, const char *engine_name)
  56 {
  57         wal->name = name;
  58         wal->engine_name = engine_name;
  59 }
  60
  61 #define WA_LIST_CHUNK (1 << 4)
  62
  63 static void wa_init_finish(struct i915_wa_list *wal)
  64 {
  65         /* Trim unused entries. */
  66         if (!IS_ALIGNED(wal->count, WA_LIST_CHUNK)) {
  67                 struct i915_wa *list = kmemdup(wal->list,
  68                                                wal->count * sizeof(*list),
  69                                                GFP_KERNEL);
  70
  71                 if (list) {
  72                         kfree(wal->list);
  73                         wal->list = list;
  74                 }
  75         }
  76
  77         if (!wal->count)
  78                 return;
  79
  80         DRM_DEBUG_DRIVER("Initialized %u %s workarounds on %s\n",
  81                          wal->wa_count, wal->name, wal->engine_name);
  82 }
  83
  84 static void _wa_add(struct i915_wa_list *wal, const struct i915_wa *wa)
  85 {
  86         unsigned int addr = i915_mmio_reg_offset(wa->reg);
  87         unsigned int start = 0, end = wal->count;
  88         const unsigned int grow = WA_LIST_CHUNK;
  89         struct i915_wa *wa_;
  90
  91         GEM_BUG_ON(!is_power_of_2(grow));
  92
  93         if (IS_ALIGNED(wal->count, grow)) { /* Either uninitialized or full. */
  94                 struct i915_wa *list;
  95
  96                 list = kmalloc_array(ALIGN(wal->count + 1, grow), sizeof(*wa),
  97                                      GFP_KERNEL);
  98                 if (!list) {
  99                         DRM_ERROR("No space for workaround init!\n");
 100                         return;
 101                 }
 102
 103                 if (wal->list) {
 104                         memcpy(list, wal->list, sizeof(*wa) * wal->count);
 105                         kfree(wal->list);
 106                 }
 107
 108                 wal->list = list;
 109         }
 110
 111         while (start < end) {
 112                 unsigned int mid = start + (end - start) / 2;
 113
 114                 if (i915_mmio_reg_offset(wal->list[mid].reg) < addr) {
 115                         start = mid + 1;
 116                 } else if (i915_mmio_reg_offset(wal->list[mid].reg) > addr) {
 117                         end = mid;
 118                 } else {
 119                         wa_ = &wal->list[mid];
 120
 121                         if ((wa->clr | wa_->clr) && !(wa->clr & ~wa_->clr)) {
 122                                 DRM_ERROR("Discarding overwritten w/a for reg %04x (clear: %08x, set: %08x)\n",
 123                                           i915_mmio_reg_offset(wa_->reg),
 124                                           wa_->clr, wa_->set);
 125
 126                                 wa_->set &= ~wa->clr;
 127                         }
 128
 129                         wal->wa_count++;
 130                         wa_->set |= wa->set;
 131                         wa_->clr |= wa->clr;
 132                         wa_->read |= wa->read;
 133                         return;
 134                 }
 135         }
 136
 137         wal->wa_count++;
 138         wa_ = &wal->list[wal->count++];
 139         *wa_ = *wa;
 140
 141         while (wa_-- > wal->list) {
 142                 GEM_BUG_ON(i915_mmio_reg_offset(wa_[0].reg) ==
 143                            i915_mmio_reg_offset(wa_[1].reg));
 144                 if (i915_mmio_reg_offset(wa_[1].reg) >
 145                     i915_mmio_reg_offset(wa_[0].reg))
 146                         break;
 147
 148                 swap(wa_[1], wa_[0]);
 149         }
 150 }
 151
 152 static void wa_add(struct i915_wa_list *wal, i915_reg_t reg,
 153                    u32 clear, u32 set, u32 read_mask)
 154 {
 155         struct i915_wa wa = {
 156                 .reg  = reg,
 157                 .clr  = clear,
 158                 .set  = set,
 159                 .read = read_mask,
 160         };
 161
 162         _wa_add(wal, &wa);
 163 }
 164
 165 static void
 166 wa_write_clr_set(struct i915_wa_list *wal, i915_reg_t reg, u32 clear, u32 set)
 167 {
 168         wa_add(wal, reg, clear, set, clear);
 169 }
 170
 171 static void
 172 wa_write(struct i915_wa_list *wal, i915_reg_t reg, u32 set)
 173 {
 174         wa_write_clr_set(wal, reg, ~0, set);
 175 }
 176
 177 static void
 178 wa_write_or(struct i915_wa_list *wal, i915_reg_t reg, u32 set)
 179 {
 180         wa_write_clr_set(wal, reg, set, set);
 181 }
 182
 183 static void
 184 wa_write_clr(struct i915_wa_list *wal, i915_reg_t reg, u32 clr)
 185 {
 186         wa_write_clr_set(wal, reg, clr, 0);
 187 }
 188
 189 /*
 190  * WA operations on "masked register". A masked register has the upper 16 bits
 191  * documented as "masked" in b-spec. Its purpose is to allow writing to just a
 192  * portion of the register without a rmw: you simply write in the upper 16 bits
 193  * the mask of bits you are going to modify.
 194  *
 195  * The wa_masked_* family of functions already does the necessary operations to
 196  * calculate the mask based on the parameters passed, so user only has to
 197  * provide the lower 16 bits of that register.
 198  */
 199
 200 static void
 201 wa_masked_en(struct i915_wa_list *wal, i915_reg_t reg, u32 val)
 202 {
 203         wa_add(wal, reg, 0, _MASKED_BIT_ENABLE(val), val);
 204 }
 205
 206 static void
 207 wa_masked_dis(struct i915_wa_list *wal, i915_reg_t reg, u32 val)
 208 {
 209         wa_add(wal, reg, 0, _MASKED_BIT_DISABLE(val), val);
 210 }
 211
 212 static void
 213 wa_masked_field_set(struct i915_wa_list *wal, i915_reg_t reg,
 214                     u32 mask, u32 val)
 215 {
 216         wa_add(wal, reg, 0, _MASKED_FIELD(mask, val), mask);
 217 }
 218
 219 static void gen6_ctx_workarounds_init(struct intel_engine_cs *engine,
 220                                       struct i915_wa_list *wal)
 221 {
 222         wa_masked_en(wal, INSTPM, INSTPM_FORCE_ORDERING);
 223 }
 224
 225 static void gen7_ctx_workarounds_init(struct intel_engine_cs *engine,
 226                                       struct i915_wa_list *wal)
 227 {
 228         wa_masked_en(wal, INSTPM, INSTPM_FORCE_ORDERING);
 229 }
 230
 231 static void gen8_ctx_workarounds_init(struct intel_engine_cs *engine,
 232                                       struct i915_wa_list *wal)
 233 {
 234         wa_masked_en(wal, INSTPM, INSTPM_FORCE_ORDERING);
 235
 236         /* WaDisableAsyncFlipPerfMode:bdw,chv */
 237         wa_masked_en(wal, MI_MODE, ASYNC_FLIP_PERF_DISABLE);
 238
 239         /* WaDisablePartialInstShootdown:bdw,chv */
 240         wa_masked_en(wal, GEN8_ROW_CHICKEN,
 241                      PARTIAL_INSTRUCTION_SHOOTDOWN_DISABLE);
 242
 243         /* Use Force Non-Coherent whenever executing a 3D context. This is a
 244          * workaround for a possible hang in the unlikely event a TLB
 245          * invalidation occurs during a PSD flush.
 246          */
 247         /* WaForceEnableNonCoherent:bdw,chv */
 248         /* WaHdcDisableFetchWhenMasked:bdw,chv */
 249         wa_masked_en(wal, HDC_CHICKEN0,
 250                      HDC_DONOT_FETCH_MEM_WHEN_MASKED |
 251                      HDC_FORCE_NON_COHERENT);
 252
 253         /* From the Haswell PRM, Command Reference: Registers, CACHE_MODE_0:
 254          * "The Hierarchical Z RAW Stall Optimization allows non-overlapping
 255          *  polygons in the same 8x4 pixel/sample area to be processed without
 256          *  stalling waiting for the earlier ones to write to Hierarchical Z
 257          *  buffer."
 258          *
 259          * This optimization is off by default for BDW and CHV; turn it on.
 260          */
 261         wa_masked_dis(wal, CACHE_MODE_0_GEN7, HIZ_RAW_STALL_OPT_DISABLE);
 262
 263         /* Wa4x4STCOptimizationDisable:bdw,chv */
 264         wa_masked_en(wal, CACHE_MODE_1, GEN8_4x4_STC_OPTIMIZATION_DISABLE);
 265
 266         /*
 267          * BSpec recommends 8x4 when MSAA is used,
 268          * however in practice 16x4 seems fastest.
 269          *
 270          * Note that PS/WM thread counts depend on the WIZ hashing
 271          * disable bit, which we don't touch here, but it's good
 272          * to keep in mind (see 3DSTATE_PS and 3DSTATE_WM).
 273          */
 274         wa_masked_field_set(wal, GEN7_GT_MODE,
 275                             GEN6_WIZ_HASHING_MASK,
 276                             GEN6_WIZ_HASHING_16x4);
 277 }
 278
 279 static void bdw_ctx_workarounds_init(struct intel_engine_cs *engine,
 280                                      struct i915_wa_list *wal)
 281 {
 282         struct drm_i915_private *i915 = engine->i915;
 283
 284         gen8_ctx_workarounds_init(engine, wal);
 285
 286         /* WaDisableThreadStallDopClockGating:bdw (pre-production) */
 287         wa_masked_en(wal, GEN8_ROW_CHICKEN, STALL_DOP_GATING_DISABLE);
 288
 289         /* WaDisableDopClockGating:bdw
 290          *
 291          * Also see the related UCGTCL1 write in bdw_init_clock_gating()
 292          * to disable EUTC clock gating.
 293          */
 294         wa_masked_en(wal, GEN7_ROW_CHICKEN2,
 295                      DOP_CLOCK_GATING_DISABLE);
 296
 297         wa_masked_en(wal, HALF_SLICE_CHICKEN3,
 298                      GEN8_SAMPLER_POWER_BYPASS_DIS);
 299
 300         wa_masked_en(wal, HDC_CHICKEN0,
 301                      /* WaForceContextSaveRestoreNonCoherent:bdw */
 302                      HDC_FORCE_CONTEXT_SAVE_RESTORE_NON_COHERENT |
 303                      /* WaDisableFenceDestinationToSLM:bdw (pre-prod) */
 304                      (IS_BDW_GT3(i915) ? HDC_FENCE_DEST_SLM_DISABLE : 0));
 305 }
 306
 307 static void chv_ctx_workarounds_init(struct intel_engine_cs *engine,
 308                                      struct i915_wa_list *wal)
 309 {
 310         gen8_ctx_workarounds_init(engine, wal);
 311
 312         /* WaDisableThreadStallDopClockGating:chv */
 313         wa_masked_en(wal, GEN8_ROW_CHICKEN, STALL_DOP_GATING_DISABLE);
 314
 315         /* Improve HiZ throughput on CHV. */
 316         wa_masked_en(wal, HIZ_CHICKEN, CHV_HZ_8X8_MODE_IN_1X);
 317 }
 318
 319 static void gen9_ctx_workarounds_init(struct intel_engine_cs *engine,
 320                                       struct i915_wa_list *wal)
 321 {
 322         struct drm_i915_private *i915 = engine->i915;
 323
 324         if (HAS_LLC(i915)) {
 325                 /* WaCompressedResourceSamplerPbeMediaNewHashMode:skl,kbl
 326                  *
 327                  * Must match Display Engine. See
 328                  * WaCompressedResourceDisplayNewHashMode.
 329                  */
 330                 wa_masked_en(wal, COMMON_SLICE_CHICKEN2,
 331                              GEN9_PBE_COMPRESSED_HASH_SELECTION);
 332                 wa_masked_en(wal, GEN9_HALF_SLICE_CHICKEN7,
 333                              GEN9_SAMPLER_HASH_COMPRESSED_READ_ADDR);
 334         }
 335
 336         /* WaClearFlowControlGpgpuContextSave:skl,bxt,kbl,glk,cfl */
 337         /* WaDisablePartialInstShootdown:skl,bxt,kbl,glk,cfl */
 338         wa_masked_en(wal, GEN8_ROW_CHICKEN,
 339                      FLOW_CONTROL_ENABLE |
 340                      PARTIAL_INSTRUCTION_SHOOTDOWN_DISABLE);
 341
 342         /* WaEnableYV12BugFixInHalfSliceChicken7:skl,bxt,kbl,glk,cfl */
 343         /* WaEnableSamplerGPGPUPreemptionSupport:skl,bxt,kbl,cfl */
 344         wa_masked_en(wal, GEN9_HALF_SLICE_CHICKEN7,
 345                      GEN9_ENABLE_YV12_BUGFIX |
 346                      GEN9_ENABLE_GPGPU_PREEMPTION);
 347
 348         /* Wa4x4STCOptimizationDisable:skl,bxt,kbl,glk,cfl */
 349         /* WaDisablePartialResolveInVc:skl,bxt,kbl,cfl */
 350         wa_masked_en(wal, CACHE_MODE_1,
 351                      GEN8_4x4_STC_OPTIMIZATION_DISABLE |
 352                      GEN9_PARTIAL_RESOLVE_IN_VC_DISABLE);
 353
 354         /* WaCcsTlbPrefetchDisable:skl,bxt,kbl,glk,cfl */
 355         wa_masked_dis(wal, GEN9_HALF_SLICE_CHICKEN5,
 356                       GEN9_CCS_TLB_PREFETCH_ENABLE);
 357
 358         /* WaForceContextSaveRestoreNonCoherent:skl,bxt,kbl,cfl */
 359         wa_masked_en(wal, HDC_CHICKEN0,
 360                      HDC_FORCE_CONTEXT_SAVE_RESTORE_NON_COHERENT |
 361                      HDC_FORCE_CSR_NON_COHERENT_OVR_DISABLE);
 362
 363         /* WaForceEnableNonCoherent and WaDisableHDCInvalidation are
 364          * both tied to WaForceContextSaveRestoreNonCoherent
 365          * in some hsds for skl. We keep the tie for all gen9. The
 366          * documentation is a bit hazy and so we want to get common behaviour,
 367          * even though there is no clear evidence we would need both on kbl/bxt.
 368          * This area has been source of system hangs so we play it safe
 369          * and mimic the skl regardless of what bspec says.
 370          *
 371          * Use Force Non-Coherent whenever executing a 3D context. This
 372          * is a workaround for a possible hang in the unlikely event
 373          * a TLB invalidation occurs during a PSD flush.
 374          */
 375
 376         /* WaForceEnableNonCoherent:skl,bxt,kbl,cfl */
 377         wa_masked_en(wal, HDC_CHICKEN0,
 378                      HDC_FORCE_NON_COHERENT);
 379
 380         /* WaDisableSamplerPowerBypassForSOPingPong:skl,bxt,kbl,cfl */
 381         if (IS_SKYLAKE(i915) ||
 382             IS_KABYLAKE(i915) ||
 383             IS_COFFEELAKE(i915) ||
 384             IS_COMETLAKE(i915))
 385                 wa_masked_en(wal, HALF_SLICE_CHICKEN3,
 386                              GEN8_SAMPLER_POWER_BYPASS_DIS);
 387
 388         /* WaDisableSTUnitPowerOptimization:skl,bxt,kbl,glk,cfl */
 389         wa_masked_en(wal, HALF_SLICE_CHICKEN2, GEN8_ST_PO_DISABLE);
 390
 391         /*
 392          * Supporting preemption with fine-granularity requires changes in the
 393          * batch buffer programming. Since we can't break old userspace, we
 394          * need to set our default preemption level to safe value. Userspace is
 395          * still able to use more fine-grained preemption levels, since in
 396          * WaEnablePreemptionGranularityControlByUMD we're whitelisting the
 397          * per-ctx register. As such, WaDisable{3D,GPGPU}MidCmdPreemption are
 398          * not real HW workarounds, but merely a way to start using preemption
 399          * while maintaining old contract with userspace.
 400          */
 401
 402         /* WaDisable3DMidCmdPreemption:skl,bxt,glk,cfl,[cnl] */
 403         wa_masked_dis(wal, GEN8_CS_CHICKEN1, GEN9_PREEMPT_3D_OBJECT_LEVEL);
 404
 405         /* WaDisableGPGPUMidCmdPreemption:skl,bxt,blk,cfl,[cnl] */
 406         wa_masked_field_set(wal, GEN8_CS_CHICKEN1,
 407                             GEN9_PREEMPT_GPGPU_LEVEL_MASK,
 408                             GEN9_PREEMPT_GPGPU_COMMAND_LEVEL);
 409
 410         /* WaClearHIZ_WM_CHICKEN3:bxt,glk */
 411         if (IS_GEN9_LP(i915))
 412                 wa_masked_en(wal, GEN9_WM_CHICKEN3, GEN9_FACTOR_IN_CLR_VAL_HIZ);
 413 }
 414
 415 static void skl_tune_iz_hashing(struct intel_engine_cs *engine,
 416                                 struct i915_wa_list *wal)
 417 {
 418         struct intel_gt *gt = engine->gt;
 419         u8 vals[3] = { 0, 0, 0 };
 420         unsigned int i;
 421
 422         for (i = 0; i < 3; i++) {
 423                 u8 ss;
 424
 425                 /*
 426                  * Only consider slices where one, and only one, subslice has 7
 427                  * EUs
 428                  */
 429                 if (!is_power_of_2(gt->info.sseu.subslice_7eu[i]))
 430                         continue;
 431
 432                 /*
 433                  * subslice_7eu[i] != 0 (because of the check above) and
 434                  * ss_max == 4 (maximum number of subslices possible per slice)
 435                  *
 436                  * ->    0 <= ss <= 3;
 437                  */
 438                 ss = ffs(gt->info.sseu.subslice_7eu[i]) - 1;
 439                 vals[i] = 3 - ss;
 440         }
 441
 442         if (vals[0] == 0 && vals[1] == 0 && vals[2] == 0)
 443                 return;
 444
 445         /* Tune IZ hashing. See intel_device_info_runtime_init() */
 446         wa_masked_field_set(wal, GEN7_GT_MODE,
 447                             GEN9_IZ_HASHING_MASK(2) |
 448                             GEN9_IZ_HASHING_MASK(1) |
 449                             GEN9_IZ_HASHING_MASK(0),
 450                             GEN9_IZ_HASHING(2, vals[2]) |
 451                             GEN9_IZ_HASHING(1, vals[1]) |
 452                             GEN9_IZ_HASHING(0, vals[0]));
 453 }
 454
 455 static void skl_ctx_workarounds_init(struct intel_engine_cs *engine,
 456                                      struct i915_wa_list *wal)
 457 {
 458         gen9_ctx_workarounds_init(engine, wal);
 459         skl_tune_iz_hashing(engine, wal);
 460 }
 461
 462 static void bxt_ctx_workarounds_init(struct intel_engine_cs *engine,
 463                                      struct i915_wa_list *wal)
 464 {
 465         gen9_ctx_workarounds_init(engine, wal);
 466
 467         /* WaDisableThreadStallDopClockGating:bxt */
 468         wa_masked_en(wal, GEN8_ROW_CHICKEN,
 469                      STALL_DOP_GATING_DISABLE);
 470
 471         /* WaToEnableHwFixForPushConstHWBug:bxt */
 472         wa_masked_en(wal, COMMON_SLICE_CHICKEN2,
 473                      GEN8_SBE_DISABLE_REPLAY_BUF_OPTIMIZATION);
 474 }
 475
 476 static void kbl_ctx_workarounds_init(struct intel_engine_cs *engine,
 477                                      struct i915_wa_list *wal)
 478 {
 479         struct drm_i915_private *i915 = engine->i915;
 480
 481         gen9_ctx_workarounds_init(engine, wal);
 482
 483         /* WaToEnableHwFixForPushConstHWBug:kbl */
 484         if (IS_KBL_GT_STEP(i915, STEP_C0, STEP_FOREVER))
 485                 wa_masked_en(wal, COMMON_SLICE_CHICKEN2,
 486                              GEN8_SBE_DISABLE_REPLAY_BUF_OPTIMIZATION);
 487
 488         /* WaDisableSbeCacheDispatchPortSharing:kbl */
 489         wa_masked_en(wal, GEN7_HALF_SLICE_CHICKEN1,
 490                      GEN7_SBE_SS_CACHE_DISPATCH_PORT_SHARING_DISABLE);
 491 }
 492
 493 static void glk_ctx_workarounds_init(struct intel_engine_cs *engine,
 494                                      struct i915_wa_list *wal)
 495 {
 496         gen9_ctx_workarounds_init(engine, wal);
 497
 498         /* WaToEnableHwFixForPushConstHWBug:glk */
 499         wa_masked_en(wal, COMMON_SLICE_CHICKEN2,
 500                      GEN8_SBE_DISABLE_REPLAY_BUF_OPTIMIZATION);
 501 }
 502
 503 static void cfl_ctx_workarounds_init(struct intel_engine_cs *engine,
 504                                      struct i915_wa_list *wal)
 505 {
 506         gen9_ctx_workarounds_init(engine, wal);
 507
 508         /* WaToEnableHwFixForPushConstHWBug:cfl */
 509         wa_masked_en(wal, COMMON_SLICE_CHICKEN2,
 510                      GEN8_SBE_DISABLE_REPLAY_BUF_OPTIMIZATION);
 511
 512         /* WaDisableSbeCacheDispatchPortSharing:cfl */
 513         wa_masked_en(wal, GEN7_HALF_SLICE_CHICKEN1,
 514                      GEN7_SBE_SS_CACHE_DISPATCH_PORT_SHARING_DISABLE);
 515 }
 516
 517 static void cnl_ctx_workarounds_init(struct intel_engine_cs *engine,
 518                                      struct i915_wa_list *wal)
 519 {
 520         /* WaForceContextSaveRestoreNonCoherent:cnl */
 521         wa_masked_en(wal, CNL_HDC_CHICKEN0,
 522                      HDC_FORCE_CONTEXT_SAVE_RESTORE_NON_COHERENT);
 523
 524         /* WaDisableReplayBufferBankArbitrationOptimization:cnl */
 525         wa_masked_en(wal, COMMON_SLICE_CHICKEN2,
 526                      GEN8_SBE_DISABLE_REPLAY_BUF_OPTIMIZATION);
 527
 528         /* WaPushConstantDereferenceHoldDisable:cnl */
 529         wa_masked_en(wal, GEN7_ROW_CHICKEN2, PUSH_CONSTANT_DEREF_DISABLE);
 530
 531         /* FtrEnableFastAnisoL1BankingFix:cnl */
 532         wa_masked_en(wal, HALF_SLICE_CHICKEN3, CNL_FAST_ANISO_L1_BANKING_FIX);
 533
 534         /* WaDisable3DMidCmdPreemption:cnl */
 535         wa_masked_dis(wal, GEN8_CS_CHICKEN1, GEN9_PREEMPT_3D_OBJECT_LEVEL);
 536
 537         /* WaDisableGPGPUMidCmdPreemption:cnl */
 538         wa_masked_field_set(wal, GEN8_CS_CHICKEN1,
 539                             GEN9_PREEMPT_GPGPU_LEVEL_MASK,
 540                             GEN9_PREEMPT_GPGPU_COMMAND_LEVEL);
 541
 542         /* WaDisableEarlyEOT:cnl */
 543         wa_masked_en(wal, GEN8_ROW_CHICKEN, DISABLE_EARLY_EOT);
 544 }
 545
 546 static void icl_ctx_workarounds_init(struct intel_engine_cs *engine,
 547                                      struct i915_wa_list *wal)
 548 {
 549         struct drm_i915_private *i915 = engine->i915;
 550
 551         /* WaDisableBankHangMode:icl */
 552         wa_write(wal,
 553                  GEN8_L3CNTLREG,
 554                  intel_uncore_read(engine->uncore, GEN8_L3CNTLREG) |
 555                  GEN8_ERRDETBCTRL);
 556
 557         /* Wa_1604370585:icl (pre-prod)
 558          * Formerly known as WaPushConstantDereferenceHoldDisable
 559          */
 560         if (IS_ICL_REVID(i915, ICL_REVID_A0, ICL_REVID_B0))
 561                 wa_masked_en(wal, GEN7_ROW_CHICKEN2,
 562                              PUSH_CONSTANT_DEREF_DISABLE);
 563
 564         /* WaForceEnableNonCoherent:icl
 565          * This is not the same workaround as in early Gen9 platforms, where
 566          * lacking this could cause system hangs, but coherency performance
 567          * overhead is high and only a few compute workloads really need it
 568          * (the register is whitelisted in hardware now, so UMDs can opt in
 569          * for coherency if they have a good reason).
 570          */
 571         wa_masked_en(wal, ICL_HDC_MODE, HDC_FORCE_NON_COHERENT);
 572
 573         /* Wa_2006611047:icl (pre-prod)
 574          * Formerly known as WaDisableImprovedTdlClkGating
 575          */
 576         if (IS_ICL_REVID(i915, ICL_REVID_A0, ICL_REVID_A0))
 577                 wa_masked_en(wal, GEN7_ROW_CHICKEN2,
 578                              GEN11_TDL_CLOCK_GATING_FIX_DISABLE);
 579
 580         /* Wa_2006665173:icl (pre-prod) */
 581         if (IS_ICL_REVID(i915, ICL_REVID_A0, ICL_REVID_A0))
 582                 wa_masked_en(wal, GEN11_COMMON_SLICE_CHICKEN3,
 583                              GEN11_BLEND_EMB_FIX_DISABLE_IN_RCC);
 584
 585         /* WaEnableFloatBlendOptimization:icl */
 586         wa_write_clr_set(wal,
 587                          GEN10_CACHE_MODE_SS,
 588                          0, /* write-only, so skip validation */
 589                          _MASKED_BIT_ENABLE(FLOAT_BLEND_OPTIMIZATION_ENABLE));
 590
 591         /* WaDisableGPGPUMidThreadPreemption:icl */
 592         wa_masked_field_set(wal, GEN8_CS_CHICKEN1,
 593                             GEN9_PREEMPT_GPGPU_LEVEL_MASK,
 594                             GEN9_PREEMPT_GPGPU_THREAD_GROUP_LEVEL);
 595
 596         /* allow headerless messages for preemptible GPGPU context */
 597         wa_masked_en(wal, GEN10_SAMPLER_MODE,
 598                      GEN11_SAMPLER_ENABLE_HEADLESS_MSG);
 599
 600         /* Wa_1604278689:icl,ehl */
 601         wa_write(wal, IVB_FBC_RT_BASE, 0xFFFFFFFF & ~ILK_FBC_RT_VALID);
 602         wa_write_clr_set(wal, IVB_FBC_RT_BASE_UPPER,
 603                          0, /* write-only register; skip validation */
 604                          0xFFFFFFFF);
 605
 606         /* Wa_1406306137:icl,ehl */
 607         wa_masked_en(wal, GEN9_ROW_CHICKEN4, GEN11_DIS_PICK_2ND_EU);
 608 }
 609
 610 /*
 611  * These settings aren't actually workarounds, but general tuning settings that
 612  * need to be programmed on several platforms.
 613  */
 614 static void gen12_ctx_gt_tuning_init(struct intel_engine_cs *engine,
 615                                      struct i915_wa_list *wal)
 616 {
 617         /*
 618          * Although some platforms refer to it as Wa_1604555607, we need to
 619          * program it even on those that don't explicitly list that
 620          * workaround.
 621          *
 622          * Note that the programming of this register is further modified
 623          * according to the FF_MODE2 guidance given by Wa_1608008084:gen12.
 624          * Wa_1608008084 tells us the FF_MODE2 register will return the wrong
 625          * value when read. The default value for this register is zero for all
 626          * fields and there are no bit masks. So instead of doing a RMW we
 627          * should just write TDS timer value. For the same reason read
 628          * verification is ignored.
 629          */
 630         wa_add(wal,
 631                FF_MODE2,
 632                FF_MODE2_TDS_TIMER_MASK,
 633                FF_MODE2_TDS_TIMER_128,
 634                0);
 635 }
 636
 637 static void gen12_ctx_workarounds_init(struct intel_engine_cs *engine,
 638                                        struct i915_wa_list *wal)
 639 {
 640         gen12_ctx_gt_tuning_init(engine, wal);
 641
 642         /*
 643          * Wa_1409142259:tgl
 644          * Wa_1409347922:tgl
 645          * Wa_1409252684:tgl
 646          * Wa_1409217633:tgl
 647          * Wa_1409207793:tgl
 648          * Wa_1409178076:tgl
 649          * Wa_1408979724:tgl
 650          * Wa_14010443199:rkl
 651          * Wa_14010698770:rkl
 652          */
 653         wa_masked_en(wal, GEN11_COMMON_SLICE_CHICKEN3,
 654                      GEN12_DISABLE_CPS_AWARE_COLOR_PIPE);
 655
 656         /* WaDisableGPGPUMidThreadPreemption:gen12 */
 657         wa_masked_field_set(wal, GEN8_CS_CHICKEN1,
 658                             GEN9_PREEMPT_GPGPU_LEVEL_MASK,
 659                             GEN9_PREEMPT_GPGPU_THREAD_GROUP_LEVEL);
 660
 661         /*
 662          * Wa_16011163337
 663          *
 664          * Like in gen12_ctx_gt_tuning_init(), read verification is ignored due
 665          * to Wa_1608008084.
 666          */
 667         wa_add(wal,
 668                FF_MODE2,
 669                FF_MODE2_GS_TIMER_MASK,
 670                FF_MODE2_GS_TIMER_224,
 671                0);
 672 }
 673
 674 static void dg1_ctx_workarounds_init(struct intel_engine_cs *engine,
 675                                      struct i915_wa_list *wal)
 676 {
 677         gen12_ctx_workarounds_init(engine, wal);
 678
 679         /* Wa_1409044764 */
 680         wa_masked_dis(wal, GEN11_COMMON_SLICE_CHICKEN3,
 681                       DG1_FLOAT_POINT_BLEND_OPT_STRICT_MODE_EN);
 682
 683         /* Wa_22010493298 */
 684         wa_masked_en(wal, HIZ_CHICKEN,
 685                      DG1_HZ_READ_SUPPRESSION_OPTIMIZATION_DISABLE);
 686 }
 687
 688 static void
 689 __intel_engine_init_ctx_wa(struct intel_engine_cs *engine,
 690                            struct i915_wa_list *wal,
 691                            const char *name)
 692 {
 693         struct drm_i915_private *i915 = engine->i915;
 694
 695         if (engine->class != RENDER_CLASS)
 696                 return;
 697
 698         wa_init_start(wal, name, engine->name);
 699
 700         if (IS_DG1(i915))
 701                 dg1_ctx_workarounds_init(engine, wal);
 702         else if (GRAPHICS_VER(i915) == 12)
 703                 gen12_ctx_workarounds_init(engine, wal);
 704         else if (GRAPHICS_VER(i915) == 11)
 705                 icl_ctx_workarounds_init(engine, wal);
 706         else if (IS_CANNONLAKE(i915))
 707                 cnl_ctx_workarounds_init(engine, wal);
 708         else if (IS_COFFEELAKE(i915) || IS_COMETLAKE(i915))
 709                 cfl_ctx_workarounds_init(engine, wal);
 710         else if (IS_GEMINILAKE(i915))
 711                 glk_ctx_workarounds_init(engine, wal);
 712         else if (IS_KABYLAKE(i915))
 713                 kbl_ctx_workarounds_init(engine, wal);
 714         else if (IS_BROXTON(i915))
 715                 bxt_ctx_workarounds_init(engine, wal);
 716         else if (IS_SKYLAKE(i915))
 717                 skl_ctx_workarounds_init(engine, wal);
 718         else if (IS_CHERRYVIEW(i915))
 719                 chv_ctx_workarounds_init(engine, wal);
 720         else if (IS_BROADWELL(i915))
 721                 bdw_ctx_workarounds_init(engine, wal);
 722         else if (GRAPHICS_VER(i915) == 7)
 723                 gen7_ctx_workarounds_init(engine, wal);
 724         else if (GRAPHICS_VER(i915) == 6)
 725                 gen6_ctx_workarounds_init(engine, wal);
 726         else if (GRAPHICS_VER(i915) < 8)
 727                 ;
 728         else
 729                 MISSING_CASE(GRAPHICS_VER(i915));
 730
 731         wa_init_finish(wal);
 732 }
 733
 734 void intel_engine_init_ctx_wa(struct intel_engine_cs *engine)
 735 {
 736         __intel_engine_init_ctx_wa(engine, &engine->ctx_wa_list, "context");
 737 }
 738
 739 int intel_engine_emit_ctx_wa(struct i915_request *rq)
 740 {
 741         struct i915_wa_list *wal = &rq->engine->ctx_wa_list;
 742         struct i915_wa *wa;
 743         unsigned int i;
 744         u32 *cs;
 745         int ret;
 746
 747         if (wal->count == 0)
 748                 return 0;
 749
 750         ret = rq->engine->emit_flush(rq, EMIT_BARRIER);
 751         if (ret)
 752                 return ret;
 753
 754         cs = intel_ring_begin(rq, (wal->count * 2 + 2));
 755         if (IS_ERR(cs))
 756                 return PTR_ERR(cs);
 757
 758         *cs++ = MI_LOAD_REGISTER_IMM(wal->count);
 759         for (i = 0, wa = wal->list; i < wal->count; i++, wa++) {
 760                 *cs++ = i915_mmio_reg_offset(wa->reg);
 761                 *cs++ = wa->set;
 762         }
 763         *cs++ = MI_NOOP;
 764
 765         intel_ring_advance(rq, cs);
 766
 767         ret = rq->engine->emit_flush(rq, EMIT_BARRIER);
 768         if (ret)
 769                 return ret;
 770
 771         return 0;
 772 }
 773
 774 static void
 775 gen4_gt_workarounds_init(struct drm_i915_private *i915,
 776                          struct i915_wa_list *wal)
 777 {
 778         /* WaDisable_RenderCache_OperationalFlush:gen4,ilk */
 779         wa_masked_dis(wal, CACHE_MODE_0, RC_OP_FLUSH_ENABLE);
 780 }
 781
 782 static void
 783 g4x_gt_workarounds_init(struct drm_i915_private *i915, struct i915_wa_list *wal)
 784 {
 785         gen4_gt_workarounds_init(i915, wal);
 786
 787         /* WaDisableRenderCachePipelinedFlush:g4x,ilk */
 788         wa_masked_en(wal, CACHE_MODE_0, CM0_PIPELINED_RENDER_FLUSH_DISABLE);
 789 }
 790
 791 static void
 792 ilk_gt_workarounds_init(struct drm_i915_private *i915, struct i915_wa_list *wal)
 793 {
 794         g4x_gt_workarounds_init(i915, wal);
 795
 796         wa_masked_en(wal, _3D_CHICKEN2, _3D_CHICKEN2_WM_READ_PIPELINED);
 797 }
 798
 799 static void
 800 snb_gt_workarounds_init(struct drm_i915_private *i915, struct i915_wa_list *wal)
 801 {
 802 }
 803
 804 static void
 805 ivb_gt_workarounds_init(struct drm_i915_private *i915, struct i915_wa_list *wal)
 806 {
 807         /* Apply the WaDisableRHWOOptimizationForRenderHang:ivb workaround. */
 808         wa_masked_dis(wal,
 809                       GEN7_COMMON_SLICE_CHICKEN1,
 810                       GEN7_CSC1_RHWO_OPT_DISABLE_IN_RCC);
 811
 812         /* WaApplyL3ControlAndL3ChickenMode:ivb */
 813         wa_write(wal, GEN7_L3CNTLREG1, GEN7_WA_FOR_GEN7_L3_CONTROL);
 814         wa_write(wal, GEN7_L3_CHICKEN_MODE_REGISTER, GEN7_WA_L3_CHICKEN_MODE);
 815
 816         /* WaForceL3Serialization:ivb */
 817         wa_write_clr(wal, GEN7_L3SQCREG4, L3SQ_URB_READ_CAM_MATCH_DISABLE);
 818 }
 819
 820 static void
 821 vlv_gt_workarounds_init(struct drm_i915_private *i915, struct i915_wa_list *wal)
 822 {
 823         /* WaForceL3Serialization:vlv */
 824         wa_write_clr(wal, GEN7_L3SQCREG4, L3SQ_URB_READ_CAM_MATCH_DISABLE);
 825
 826         /*
 827          * WaIncreaseL3CreditsForVLVB0:vlv
 828          * This is the hardware default actually.
 829          */
 830         wa_write(wal, GEN7_L3SQCREG1, VLV_B0_WA_L3SQCREG1_VALUE);
 831 }
 832
 833 static void
 834 hsw_gt_workarounds_init(struct drm_i915_private *i915, struct i915_wa_list *wal)
 835 {
 836         /* L3 caching of data atomics doesn't work -- disable it. */
 837         wa_write(wal, HSW_SCRATCH1, HSW_SCRATCH1_L3_DATA_ATOMICS_DISABLE);
 838
 839         wa_add(wal,
 840                HSW_ROW_CHICKEN3, 0,
 841                _MASKED_BIT_ENABLE(HSW_ROW_CHICKEN3_L3_GLOBAL_ATOMICS_DISABLE),
 842                 0 /* XXX does this reg exist? */);
 843
 844         /* WaVSRefCountFullforceMissDisable:hsw */
 845         wa_write_clr(wal, GEN7_FF_THREAD_MODE, GEN7_FF_VS_REF_CNT_FFME);
 846 }
 847
 848 static void
 849 gen9_gt_workarounds_init(struct drm_i915_private *i915, struct i915_wa_list *wal)
 850 {
 851         /* WaDisableKillLogic:bxt,skl,kbl */
 852         if (!IS_COFFEELAKE(i915) && !IS_COMETLAKE(i915))
 853                 wa_write_or(wal,
 854                             GAM_ECOCHK,
 855                             ECOCHK_DIS_TLB);
 856
 857         if (HAS_LLC(i915)) {
 858                 /* WaCompressedResourceSamplerPbeMediaNewHashMode:skl,kbl
 859                  *
 860                  * Must match Display Engine. See
 861                  * WaCompressedResourceDisplayNewHashMode.
 862                  */
 863                 wa_write_or(wal,
 864                             MMCD_MISC_CTRL,
 865                             MMCD_PCLA | MMCD_HOTSPOT_EN);
 866         }
 867
 868         /* WaDisableHDCInvalidation:skl,bxt,kbl,cfl */
 869         wa_write_or(wal,
 870                     GAM_ECOCHK,
 871                     BDW_DISABLE_HDC_INVALIDATION);
 872 }
 873
 874 static void
 875 skl_gt_workarounds_init(struct drm_i915_private *i915, struct i915_wa_list *wal)
 876 {
 877         gen9_gt_workarounds_init(i915, wal);
 878
 879         /* WaDisableGafsUnitClkGating:skl */
 880         wa_write_or(wal,
 881                     GEN7_UCGCTL4,
 882                     GEN8_EU_GAUNIT_CLOCK_GATE_DISABLE);
 883
 884         /* WaInPlaceDecompressionHang:skl */
 885         if (IS_SKL_REVID(i915, SKL_REVID_H0, REVID_FOREVER))
 886                 wa_write_or(wal,
 887                             GEN9_GAMT_ECO_REG_RW_IA,
 888                             GAMT_ECO_ENABLE_IN_PLACE_DECOMPRESS);
 889 }
 890
 891 static void
 892 bxt_gt_workarounds_init(struct drm_i915_private *i915, struct i915_wa_list *wal)
 893 {
 894         gen9_gt_workarounds_init(i915, wal);
 895
 896         /* WaInPlaceDecompressionHang:bxt */
 897         wa_write_or(wal,
 898                     GEN9_GAMT_ECO_REG_RW_IA,
 899                     GAMT_ECO_ENABLE_IN_PLACE_DECOMPRESS);
 900 }
 901
 902 static void
 903 kbl_gt_workarounds_init(struct drm_i915_private *i915, struct i915_wa_list *wal)
 904 {
 905         gen9_gt_workarounds_init(i915, wal);
 906
 907         /* WaDisableDynamicCreditSharing:kbl */
 908         if (IS_KBL_GT_STEP(i915, 0, STEP_B0))
 909                 wa_write_or(wal,
 910                             GAMT_CHKN_BIT_REG,
 911                             GAMT_CHKN_DISABLE_DYNAMIC_CREDIT_SHARING);
 912
 913         /* WaDisableGafsUnitClkGating:kbl */
 914         wa_write_or(wal,
 915                     GEN7_UCGCTL4,
 916                     GEN8_EU_GAUNIT_CLOCK_GATE_DISABLE);
 917
 918         /* WaInPlaceDecompressionHang:kbl */
 919         wa_write_or(wal,
 920                     GEN9_GAMT_ECO_REG_RW_IA,
 921                     GAMT_ECO_ENABLE_IN_PLACE_DECOMPRESS);
 922 }
 923
 924 static void
 925 glk_gt_workarounds_init(struct drm_i915_private *i915, struct i915_wa_list *wal)
 926 {
 927         gen9_gt_workarounds_init(i915, wal);
 928 }
 929
 930 static void
 931 cfl_gt_workarounds_init(struct drm_i915_private *i915, struct i915_wa_list *wal)
 932 {
 933         gen9_gt_workarounds_init(i915, wal);
 934
 935         /* WaDisableGafsUnitClkGating:cfl */
 936         wa_write_or(wal,
 937                     GEN7_UCGCTL4,
 938                     GEN8_EU_GAUNIT_CLOCK_GATE_DISABLE);
 939
 940         /* WaInPlaceDecompressionHang:cfl */
 941         wa_write_or(wal,
 942                     GEN9_GAMT_ECO_REG_RW_IA,
 943                     GAMT_ECO_ENABLE_IN_PLACE_DECOMPRESS);
 944 }
 945
 946 static void
 947 wa_init_mcr(struct drm_i915_private *i915, struct i915_wa_list *wal)
 948 {
 949         const struct sseu_dev_info *sseu = &i915->gt.info.sseu;
 950         unsigned int slice, subslice;
 951         u32 l3_en, mcr, mcr_mask;
 952
 953         GEM_BUG_ON(GRAPHICS_VER(i915) < 10);
 954
 955         /*
 956          * WaProgramMgsrForL3BankSpecificMmioReads: cnl,icl
 957          * L3Banks could be fused off in single slice scenario. If that is
 958          * the case, we might need to program MCR select to a valid L3Bank
 959          * by default, to make sure we correctly read certain registers
 960          * later on (in the range 0xB100 - 0xB3FF).
 961          *
 962          * WaProgramMgsrForCorrectSliceSpecificMmioReads:cnl,icl
 963          * Before any MMIO read into slice/subslice specific registers, MCR
 964          * packet control register needs to be programmed to point to any
 965          * enabled s/ss pair. Otherwise, incorrect values will be returned.
 966          * This means each subsequent MMIO read will be forwarded to an
 967          * specific s/ss combination, but this is OK since these registers
 968          * are consistent across s/ss in almost all cases. In the rare
 969          * occasions, such as INSTDONE, where this value is dependent
 970          * on s/ss combo, the read should be done with read_subslice_reg.
 971          *
 972          * Since GEN8_MCR_SELECTOR contains dual-purpose bits which select both
 973          * to which subslice, or to which L3 bank, the respective mmio reads
 974          * will go, we have to find a common index which works for both
 975          * accesses.
 976          *
 977          * Case where we cannot find a common index fortunately should not
 978          * happen in production hardware, so we only emit a warning instead of
 979          * implementing something more complex that requires checking the range
 980          * of every MMIO read.
 981          */
 982
 983         if (GRAPHICS_VER(i915) >= 10 && is_power_of_2(sseu->slice_mask)) {
 984                 u32 l3_fuse =
 985                         intel_uncore_read(&i915->uncore, GEN10_MIRROR_FUSE3) &
 986                         GEN10_L3BANK_MASK;
 987
 988                 drm_dbg(&i915->drm, "L3 fuse = %x\n", l3_fuse);
 989                 l3_en = ~(l3_fuse << GEN10_L3BANK_PAIR_COUNT | l3_fuse);
 990         } else {
 991                 l3_en = ~0;
 992         }
 993
 994         slice = fls(sseu->slice_mask) - 1;
 995         subslice = fls(l3_en & intel_sseu_get_subslices(sseu, slice));
 996         if (!subslice) {
 997                 drm_warn(&i915->drm,
 998                          "No common index found between subslice mask %x and L3 bank mask %x!\n",
 999                          intel_sseu_get_subslices(sseu, slice), l3_en);
1000                 subslice = fls(l3_en);
1001                 drm_WARN_ON(&i915->drm, !subslice);
1002         }
1003         subslice--;
1004
1005         if (GRAPHICS_VER(i915) >= 11) {
1006                 mcr = GEN11_MCR_SLICE(slice) | GEN11_MCR_SUBSLICE(subslice);
1007                 mcr_mask = GEN11_MCR_SLICE_MASK | GEN11_MCR_SUBSLICE_MASK;
1008         } else {
1009                 mcr = GEN8_MCR_SLICE(slice) | GEN8_MCR_SUBSLICE(subslice);
1010                 mcr_mask = GEN8_MCR_SLICE_MASK | GEN8_MCR_SUBSLICE_MASK;
1011         }
1012
1013         drm_dbg(&i915->drm, "MCR slice/subslice = %x\n", mcr);
1014
1015         wa_write_clr_set(wal, GEN8_MCR_SELECTOR, mcr_mask, mcr);
1016 }
1017
1018 static void
1019 cnl_gt_workarounds_init(struct drm_i915_private *i915, struct i915_wa_list *wal)
1020 {
1021         wa_init_mcr(i915, wal);
1022
1023         /* WaInPlaceDecompressionHang:cnl */
1024         wa_write_or(wal,
1025                     GEN9_GAMT_ECO_REG_RW_IA,
1026                     GAMT_ECO_ENABLE_IN_PLACE_DECOMPRESS);
1027 }
1028
1029 static void
1030 icl_gt_workarounds_init(struct drm_i915_private *i915, struct i915_wa_list *wal)
1031 {
1032         wa_init_mcr(i915, wal);
1033
1034         /* WaInPlaceDecompressionHang:icl */
1035         wa_write_or(wal,
1036                     GEN9_GAMT_ECO_REG_RW_IA,
1037                     GAMT_ECO_ENABLE_IN_PLACE_DECOMPRESS);
1038
1039         /* WaModifyGamTlbPartitioning:icl */
1040         wa_write_clr_set(wal,
1041                          GEN11_GACB_PERF_CTRL,
1042                          GEN11_HASH_CTRL_MASK,
1043                          GEN11_HASH_CTRL_BIT0 | GEN11_HASH_CTRL_BIT4);
1044
1045         /* Wa_1405766107:icl
1046          * Formerly known as WaCL2SFHalfMaxAlloc
1047          */
1048         wa_write_or(wal,
1049                     GEN11_LSN_UNSLCVC,
1050                     GEN11_LSN_UNSLCVC_GAFS_HALF_SF_MAXALLOC |
1051                     GEN11_LSN_UNSLCVC_GAFS_HALF_CL2_MAXALLOC);
1052
1053         /* Wa_220166154:icl
1054          * Formerly known as WaDisCtxReload
1055          */
1056         wa_write_or(wal,
1057                     GEN8_GAMW_ECO_DEV_RW_IA,
1058                     GAMW_ECO_DEV_CTX_RELOAD_DISABLE);
1059
1060         /* Wa_1405779004:icl (pre-prod) */
1061         if (IS_ICL_REVID(i915, ICL_REVID_A0, ICL_REVID_A0))
1062                 wa_write_or(wal,
1063                             SLICE_UNIT_LEVEL_CLKGATE,
1064                             MSCUNIT_CLKGATE_DIS);
1065
1066         /* Wa_1406838659:icl (pre-prod) */
1067         if (IS_ICL_REVID(i915, ICL_REVID_A0, ICL_REVID_B0))
1068                 wa_write_or(wal,
1069                             INF_UNIT_LEVEL_CLKGATE,
1070                             CGPSF_CLKGATE_DIS);
1071
1072         /* Wa_1406463099:icl
1073          * Formerly known as WaGamTlbPendError
1074          */
1075         wa_write_or(wal,
1076                     GAMT_CHKN_BIT_REG,
1077                     GAMT_CHKN_DISABLE_L3_COH_PIPE);
1078
1079         /* Wa_1607087056:icl,ehl,jsl */
1080         if (IS_ICELAKE(i915) ||
1081             IS_JSL_EHL_REVID(i915, EHL_REVID_A0, EHL_REVID_A0))
1082                 wa_write_or(wal,
1083                             SLICE_UNIT_LEVEL_CLKGATE,
1084                             L3_CLKGATE_DIS | L3_CR2X_CLKGATE_DIS);
1085 }
1086
1087 /*
1088  * Though there are per-engine instances of these registers,
1089  * they retain their value through engine resets and should
1090  * only be provided on the GT workaround list rather than
1091  * the engine-specific workaround list.
1092  */
1093 static void
1094 wa_14011060649(struct drm_i915_private *i915, struct i915_wa_list *wal)
1095 {
1096         struct intel_engine_cs *engine;
1097         struct intel_gt *gt = &i915->gt;
1098         int id;
1099
1100         for_each_engine(engine, gt, id) {
1101                 if (engine->class != VIDEO_DECODE_CLASS ||
1102                     (engine->instance % 2))
1103                         continue;
1104
1105                 wa_write_or(wal, VDBOX_CGCTL3F10(engine->mmio_base),
1106                             IECPUNIT_CLKGATE_DIS);
1107         }
1108 }
1109
1110 static void
1111 gen12_gt_workarounds_init(struct drm_i915_private *i915,
1112                           struct i915_wa_list *wal)
1113 {
1114         wa_init_mcr(i915, wal);
1115
1116         /* Wa_14011060649:tgl,rkl,dg1,adls */
1117         wa_14011060649(i915, wal);
1118 }
1119
1120 static void
1121 tgl_gt_workarounds_init(struct drm_i915_private *i915, struct i915_wa_list *wal)
1122 {
1123         gen12_gt_workarounds_init(i915, wal);
1124
1125         /* Wa_1409420604:tgl */
1126         if (IS_TGL_UY_GT_STEP(i915, STEP_A0, STEP_A0))
1127                 wa_write_or(wal,
1128                             SUBSLICE_UNIT_LEVEL_CLKGATE2,
1129                             CPSSUNIT_CLKGATE_DIS);
1130
1131         /* Wa_1607087056:tgl also know as BUG:1409180338 */
1132         if (IS_TGL_UY_GT_STEP(i915, STEP_A0, STEP_A0))
1133                 wa_write_or(wal,
1134                             SLICE_UNIT_LEVEL_CLKGATE,
1135                             L3_CLKGATE_DIS | L3_CR2X_CLKGATE_DIS);
1136
1137         /* Wa_1408615072:tgl[a0] */
1138         if (IS_TGL_UY_GT_STEP(i915, STEP_A0, STEP_A0))
1139                 wa_write_or(wal, UNSLICE_UNIT_LEVEL_CLKGATE2,
1140                             VSUNIT_CLKGATE_DIS_TGL);
1141 }
1142
1143 static void
1144 dg1_gt_workarounds_init(struct drm_i915_private *i915, struct i915_wa_list *wal)
1145 {
1146         gen12_gt_workarounds_init(i915, wal);
1147
1148         /* Wa_1607087056:dg1 */
1149         if (IS_DG1_REVID(i915, DG1_REVID_A0, DG1_REVID_A0))
1150                 wa_write_or(wal,
1151                             SLICE_UNIT_LEVEL_CLKGATE,
1152                             L3_CLKGATE_DIS | L3_CR2X_CLKGATE_DIS);
1153
1154         /* Wa_1409420604:dg1 */
1155         if (IS_DG1(i915))
1156                 wa_write_or(wal,
1157                             SUBSLICE_UNIT_LEVEL_CLKGATE2,
1158                             CPSSUNIT_CLKGATE_DIS);
1159
1160         /* Wa_1408615072:dg1 */
1161         /* Empirical testing shows this register is unaffected by engine reset. */
1162         if (IS_DG1(i915))
1163                 wa_write_or(wal, UNSLICE_UNIT_LEVEL_CLKGATE2,
1164                             VSUNIT_CLKGATE_DIS_TGL);
1165 }
1166
1167 static void
1168 gt_init_workarounds(struct drm_i915_private *i915, struct i915_wa_list *wal)
1169 {
1170         if (IS_DG1(i915))
1171                 dg1_gt_workarounds_init(i915, wal);
1172         else if (IS_TIGERLAKE(i915))
1173                 tgl_gt_workarounds_init(i915, wal);
1174         else if (GRAPHICS_VER(i915) == 12)
1175                 gen12_gt_workarounds_init(i915, wal);
1176         else if (GRAPHICS_VER(i915) == 11)
1177                 icl_gt_workarounds_init(i915, wal);
1178         else if (IS_CANNONLAKE(i915))
1179                 cnl_gt_workarounds_init(i915, wal);
1180         else if (IS_COFFEELAKE(i915) || IS_COMETLAKE(i915))
1181                 cfl_gt_workarounds_init(i915, wal);
1182         else if (IS_GEMINILAKE(i915))
1183                 glk_gt_workarounds_init(i915, wal);
1184         else if (IS_KABYLAKE(i915))
1185                 kbl_gt_workarounds_init(i915, wal);
1186         else if (IS_BROXTON(i915))
1187                 bxt_gt_workarounds_init(i915, wal);
1188         else if (IS_SKYLAKE(i915))
1189                 skl_gt_workarounds_init(i915, wal);
1190         else if (IS_HASWELL(i915))
1191                 hsw_gt_workarounds_init(i915, wal);
1192         else if (IS_VALLEYVIEW(i915))
1193                 vlv_gt_workarounds_init(i915, wal);
1194         else if (IS_IVYBRIDGE(i915))
1195                 ivb_gt_workarounds_init(i915, wal);
1196         else if (GRAPHICS_VER(i915) == 6)
1197                 snb_gt_workarounds_init(i915, wal);
1198         else if (GRAPHICS_VER(i915) == 5)
1199                 ilk_gt_workarounds_init(i915, wal);
1200         else if (IS_G4X(i915))
1201                 g4x_gt_workarounds_init(i915, wal);
1202         else if (GRAPHICS_VER(i915) == 4)
1203                 gen4_gt_workarounds_init(i915, wal);
1204         else if (GRAPHICS_VER(i915) <= 8)
1205                 ;
1206         else
1207                 MISSING_CASE(GRAPHICS_VER(i915));
1208 }
1209
1210 void intel_gt_init_workarounds(struct drm_i915_private *i915)
1211 {
1212         struct i915_wa_list *wal = &i915->gt_wa_list;
1213
1214         wa_init_start(wal, "GT", "global");
1215         gt_init_workarounds(i915, wal);
1216         wa_init_finish(wal);
1217 }
1218
1219 static enum forcewake_domains
1220 wal_get_fw_for_rmw(struct intel_uncore *uncore, const struct i915_wa_list *wal)
1221 {
1222         enum forcewake_domains fw = 0;
1223         struct i915_wa *wa;
1224         unsigned int i;
1225
1226         for (i = 0, wa = wal->list; i < wal->count; i++, wa++)
1227                 fw |= intel_uncore_forcewake_for_reg(uncore,
1228                                                      wa->reg,
1229                                                      FW_REG_READ |
1230                                                      FW_REG_WRITE);
1231
1232         return fw;
1233 }
1234
1235 static bool
1236 wa_verify(const struct i915_wa *wa, u32 cur, const char *name, const char *from)
1237 {
1238         if ((cur ^ wa->set) & wa->read) {
1239                 DRM_ERROR("%s workaround lost on %s! (reg[%x]=0x%x, relevant bits were 0x%x vs expected 0x%x)\n",
1240                           name, from, i915_mmio_reg_offset(wa->reg),
1241                           cur, cur & wa->read, wa->set & wa->read);
1242
1243                 return false;
1244         }
1245
1246         return true;
1247 }
1248
1249 static void
1250 wa_list_apply(struct intel_uncore *uncore, const struct i915_wa_list *wal)
1251 {
1252         enum forcewake_domains fw;
1253         unsigned long flags;
1254         struct i915_wa *wa;
1255         unsigned int i;
1256
1257         if (!wal->count)
1258                 return;
1259
1260         fw = wal_get_fw_for_rmw(uncore, wal);
1261
1262         spin_lock_irqsave(&uncore->lock, flags);
1263         intel_uncore_forcewake_get__locked(uncore, fw);
1264
1265         for (i = 0, wa = wal->list; i < wal->count; i++, wa++) {
1266                 if (wa->clr)
1267                         intel_uncore_rmw_fw(uncore, wa->reg, wa->clr, wa->set);
1268                 else
1269                         intel_uncore_write_fw(uncore, wa->reg, wa->set);
1270                 if (IS_ENABLED(CONFIG_DRM_I915_DEBUG_GEM))
1271                         wa_verify(wa,
1272                                   intel_uncore_read_fw(uncore, wa->reg),
1273                                   wal->name, "application");
1274         }
1275
1276         intel_uncore_forcewake_put__locked(uncore, fw);
1277         spin_unlock_irqrestore(&uncore->lock, flags);
1278 }
1279
1280 void intel_gt_apply_workarounds(struct intel_gt *gt)
1281 {
1282         wa_list_apply(gt->uncore, &gt->i915->gt_wa_list);
1283 }
1284
1285 static bool wa_list_verify(struct intel_uncore *uncore,
1286                            const struct i915_wa_list *wal,
1287                            const char *from)
1288 {
1289         struct i915_wa *wa;
1290         unsigned int i;
1291         bool ok = true;
1292
1293         for (i = 0, wa = wal->list; i < wal->count; i++, wa++)
1294                 ok &= wa_verify(wa,
1295                                 intel_uncore_read(uncore, wa->reg),
1296                                 wal->name, from);
1297
1298         return ok;
1299 }
1300
1301 bool intel_gt_verify_workarounds(struct intel_gt *gt, const char *from)
1302 {
1303         return wa_list_verify(gt->uncore, &gt->i915->gt_wa_list, from);
1304 }
1305
1306 __maybe_unused
1307 static bool is_nonpriv_flags_valid(u32 flags)
1308 {
1309         /* Check only valid flag bits are set */
1310         if (flags & ~RING_FORCE_TO_NONPRIV_MASK_VALID)
1311                 return false;
1312
1313         /* NB: Only 3 out of 4 enum values are valid for access field */
1314         if ((flags & RING_FORCE_TO_NONPRIV_ACCESS_MASK) ==
1315             RING_FORCE_TO_NONPRIV_ACCESS_INVALID)
1316                 return false;
1317
1318         return true;
1319 }
1320
1321 static void
1322 whitelist_reg_ext(struct i915_wa_list *wal, i915_reg_t reg, u32 flags)
1323 {
1324         struct i915_wa wa = {
1325                 .reg = reg
1326         };
1327
1328         if (GEM_DEBUG_WARN_ON(wal->count >= RING_MAX_NONPRIV_SLOTS))
1329                 return;
1330
1331         if (GEM_DEBUG_WARN_ON(!is_nonpriv_flags_valid(flags)))
1332                 return;
1333
1334         wa.reg.reg |= flags;
1335         _wa_add(wal, &wa);
1336 }
1337
1338 static void
1339 whitelist_reg(struct i915_wa_list *wal, i915_reg_t reg)
1340 {
1341         whitelist_reg_ext(wal, reg, RING_FORCE_TO_NONPRIV_ACCESS_RW);
1342 }
1343
1344 static void gen9_whitelist_build(struct i915_wa_list *w)
1345 {
1346         /* WaVFEStateAfterPipeControlwithMediaStateClear:skl,bxt,glk,cfl */
1347         whitelist_reg(w, GEN9_CTX_PREEMPT_REG);
1348
1349         /* WaEnablePreemptionGranularityControlByUMD:skl,bxt,kbl,cfl,[cnl] */
1350         whitelist_reg(w, GEN8_CS_CHICKEN1);
1351
1352         /* WaAllowUMDToModifyHDCChicken1:skl,bxt,kbl,glk,cfl */
1353         whitelist_reg(w, GEN8_HDC_CHICKEN1);
1354
1355         /* WaSendPushConstantsFromMMIO:skl,bxt */
1356         whitelist_reg(w, COMMON_SLICE_CHICKEN2);
1357 }
1358
1359 static void skl_whitelist_build(struct intel_engine_cs *engine)
1360 {
1361         struct i915_wa_list *w = &engine->whitelist;
1362
1363         if (engine->class != RENDER_CLASS)
1364                 return;
1365
1366         gen9_whitelist_build(w);
1367
1368         /* WaDisableLSQCROPERFforOCL:skl */
1369         whitelist_reg(w, GEN8_L3SQCREG4);
1370 }
1371
1372 static void bxt_whitelist_build(struct intel_engine_cs *engine)
1373 {
1374         if (engine->class != RENDER_CLASS)
1375                 return;
1376
1377         gen9_whitelist_build(&engine->whitelist);
1378 }
1379
1380 static void kbl_whitelist_build(struct intel_engine_cs *engine)
1381 {
1382         struct i915_wa_list *w = &engine->whitelist;
1383
1384         if (engine->class != RENDER_CLASS)
1385                 return;
1386
1387         gen9_whitelist_build(w);
1388
1389         /* WaDisableLSQCROPERFforOCL:kbl */
1390         whitelist_reg(w, GEN8_L3SQCREG4);
1391 }
1392
1393 static void glk_whitelist_build(struct intel_engine_cs *engine)
1394 {
1395         struct i915_wa_list *w = &engine->whitelist;
1396
1397         if (engine->class != RENDER_CLASS)
1398                 return;
1399
1400         gen9_whitelist_build(w);
1401
1402         /* WA #0862: Userspace has to set "Barrier Mode" to avoid hangs. */
1403         whitelist_reg(w, GEN9_SLICE_COMMON_ECO_CHICKEN1);
1404 }
1405
1406 static void cfl_whitelist_build(struct intel_engine_cs *engine)
1407 {
1408         struct i915_wa_list *w = &engine->whitelist;
1409
1410         if (engine->class != RENDER_CLASS)
1411                 return;
1412
1413         gen9_whitelist_build(w);
1414
1415         /*
1416          * WaAllowPMDepthAndInvocationCountAccessFromUMD:cfl,whl,cml,aml
1417          *
1418          * This covers 4 register which are next to one another :
1419          *   - PS_INVOCATION_COUNT
1420          *   - PS_INVOCATION_COUNT_UDW
1421          *   - PS_DEPTH_COUNT
1422          *   - PS_DEPTH_COUNT_UDW
1423          */
1424         whitelist_reg_ext(w, PS_INVOCATION_COUNT,
1425                           RING_FORCE_TO_NONPRIV_ACCESS_RD |
1426                           RING_FORCE_TO_NONPRIV_RANGE_4);
1427 }
1428
1429 static void cml_whitelist_build(struct intel_engine_cs *engine)
1430 {
1431         struct i915_wa_list *w = &engine->whitelist;
1432
1433         if (engine->class != RENDER_CLASS)
1434                 whitelist_reg_ext(w,
1435                                   RING_CTX_TIMESTAMP(engine->mmio_base),
1436                                   RING_FORCE_TO_NONPRIV_ACCESS_RD);
1437
1438         cfl_whitelist_build(engine);
1439 }
1440
1441 static void cnl_whitelist_build(struct intel_engine_cs *engine)
1442 {
1443         struct i915_wa_list *w = &engine->whitelist;
1444
1445         if (engine->class != RENDER_CLASS)
1446                 return;
1447
1448         /* WaEnablePreemptionGranularityControlByUMD:cnl */
1449         whitelist_reg(w, GEN8_CS_CHICKEN1);
1450 }
1451
1452 static void icl_whitelist_build(struct intel_engine_cs *engine)
1453 {
1454         struct i915_wa_list *w = &engine->whitelist;
1455
1456         switch (engine->class) {
1457         case RENDER_CLASS:
1458                 /* WaAllowUMDToModifyHalfSliceChicken7:icl */
1459                 whitelist_reg(w, GEN9_HALF_SLICE_CHICKEN7);
1460
1461                 /* WaAllowUMDToModifySamplerMode:icl */
1462                 whitelist_reg(w, GEN10_SAMPLER_MODE);
1463
1464                 /* WaEnableStateCacheRedirectToCS:icl */
1465                 whitelist_reg(w, GEN9_SLICE_COMMON_ECO_CHICKEN1);
1466
1467                 /*
1468                  * WaAllowPMDepthAndInvocationCountAccessFromUMD:icl
1469                  *
1470                  * This covers 4 register which are next to one another :
1471                  *   - PS_INVOCATION_COUNT
1472                  *   - PS_INVOCATION_COUNT_UDW
1473                  *   - PS_DEPTH_COUNT
1474                  *   - PS_DEPTH_COUNT_UDW
1475                  */
1476                 whitelist_reg_ext(w, PS_INVOCATION_COUNT,
1477                                   RING_FORCE_TO_NONPRIV_ACCESS_RD |
1478                                   RING_FORCE_TO_NONPRIV_RANGE_4);
1479                 break;
1480
1481         case VIDEO_DECODE_CLASS:
1482                 /* hucStatusRegOffset */
1483                 whitelist_reg_ext(w, _MMIO(0x2000 + engine->mmio_base),
1484                                   RING_FORCE_TO_NONPRIV_ACCESS_RD);
1485                 /* hucUKernelHdrInfoRegOffset */
1486                 whitelist_reg_ext(w, _MMIO(0x2014 + engine->mmio_base),
1487                                   RING_FORCE_TO_NONPRIV_ACCESS_RD);
1488                 /* hucStatus2RegOffset */
1489                 whitelist_reg_ext(w, _MMIO(0x23B0 + engine->mmio_base),
1490                                   RING_FORCE_TO_NONPRIV_ACCESS_RD);
1491                 whitelist_reg_ext(w,
1492                                   RING_CTX_TIMESTAMP(engine->mmio_base),
1493                                   RING_FORCE_TO_NONPRIV_ACCESS_RD);
1494                 break;
1495
1496         default:
1497                 whitelist_reg_ext(w,
1498                                   RING_CTX_TIMESTAMP(engine->mmio_base),
1499                                   RING_FORCE_TO_NONPRIV_ACCESS_RD);
1500                 break;
1501         }
1502 }
1503
1504 static void tgl_whitelist_build(struct intel_engine_cs *engine)
1505 {
1506         struct i915_wa_list *w = &engine->whitelist;
1507
1508         switch (engine->class) {
1509         case RENDER_CLASS:
1510                 /*
1511                  * WaAllowPMDepthAndInvocationCountAccessFromUMD:tgl
1512                  * Wa_1408556865:tgl
1513                  *
1514                  * This covers 4 registers which are next to one another :
1515                  *   - PS_INVOCATION_COUNT
1516                  *   - PS_INVOCATION_COUNT_UDW
1517                  *   - PS_DEPTH_COUNT
1518                  *   - PS_DEPTH_COUNT_UDW
1519                  */
1520                 whitelist_reg_ext(w, PS_INVOCATION_COUNT,
1521                                   RING_FORCE_TO_NONPRIV_ACCESS_RD |
1522                                   RING_FORCE_TO_NONPRIV_RANGE_4);
1523
1524                 /* Wa_1808121037:tgl */
1525                 whitelist_reg(w, GEN7_COMMON_SLICE_CHICKEN1);
1526
1527                 /* Wa_1806527549:tgl */
1528                 whitelist_reg(w, HIZ_CHICKEN);
1529                 break;
1530         default:
1531                 whitelist_reg_ext(w,
1532                                   RING_CTX_TIMESTAMP(engine->mmio_base),
1533                                   RING_FORCE_TO_NONPRIV_ACCESS_RD);
1534                 break;
1535         }
1536 }
1537
1538 static void dg1_whitelist_build(struct intel_engine_cs *engine)
1539 {
1540         struct i915_wa_list *w = &engine->whitelist;
1541
1542         tgl_whitelist_build(engine);
1543
1544         /* GEN:BUG:1409280441:dg1 */
1545         if (IS_DG1_REVID(engine->i915, DG1_REVID_A0, DG1_REVID_A0) &&
1546             (engine->class == RENDER_CLASS ||
1547              engine->class == COPY_ENGINE_CLASS))
1548                 whitelist_reg_ext(w, RING_ID(engine->mmio_base),
1549                                   RING_FORCE_TO_NONPRIV_ACCESS_RD);
1550 }
1551
1552 void intel_engine_init_whitelist(struct intel_engine_cs *engine)
1553 {
1554         struct drm_i915_private *i915 = engine->i915;
1555         struct i915_wa_list *w = &engine->whitelist;
1556
1557         wa_init_start(w, "whitelist", engine->name);
1558
1559         if (IS_DG1(i915))
1560                 dg1_whitelist_build(engine);
1561         else if (GRAPHICS_VER(i915) == 12)
1562                 tgl_whitelist_build(engine);
1563         else if (GRAPHICS_VER(i915) == 11)
1564                 icl_whitelist_build(engine);
1565         else if (IS_CANNONLAKE(i915))
1566                 cnl_whitelist_build(engine);
1567         else if (IS_COMETLAKE(i915))
1568                 cml_whitelist_build(engine);
1569         else if (IS_COFFEELAKE(i915))
1570                 cfl_whitelist_build(engine);
1571         else if (IS_GEMINILAKE(i915))
1572                 glk_whitelist_build(engine);
1573         else if (IS_KABYLAKE(i915))
1574                 kbl_whitelist_build(engine);
1575         else if (IS_BROXTON(i915))
1576                 bxt_whitelist_build(engine);
1577         else if (IS_SKYLAKE(i915))
1578                 skl_whitelist_build(engine);
1579         else if (GRAPHICS_VER(i915) <= 8)
1580                 ;
1581         else
1582                 MISSING_CASE(GRAPHICS_VER(i915));
1583
1584         wa_init_finish(w);
1585 }
1586
1587 void intel_engine_apply_whitelist(struct intel_engine_cs *engine)
1588 {
1589         const struct i915_wa_list *wal = &engine->whitelist;
1590         struct intel_uncore *uncore = engine->uncore;
1591         const u32 base = engine->mmio_base;
1592         struct i915_wa *wa;
1593         unsigned int i;
1594
1595         if (!wal->count)
1596                 return;
1597
1598         for (i = 0, wa = wal->list; i < wal->count; i++, wa++)
1599                 intel_uncore_write(uncore,
1600                                    RING_FORCE_TO_NONPRIV(base, i),
1601                                    i915_mmio_reg_offset(wa->reg));
1602
1603         /* And clear the rest just in case of garbage */
1604         for (; i < RING_MAX_NONPRIV_SLOTS; i++)
1605                 intel_uncore_write(uncore,
1606                                    RING_FORCE_TO_NONPRIV(base, i),
1607                                    i915_mmio_reg_offset(RING_NOPID(base)));
1608 }
1609
1610 static void
1611 rcs_engine_wa_init(struct intel_engine_cs *engine, struct i915_wa_list *wal)
1612 {
1613         struct drm_i915_private *i915 = engine->i915;
1614
1615         if (IS_DG1_REVID(i915, DG1_REVID_A0, DG1_REVID_A0) ||
1616             IS_TGL_UY_GT_STEP(i915, STEP_A0, STEP_A0)) {
1617                 /*
1618                  * Wa_1607138336:tgl[a0],dg1[a0]
1619                  * Wa_1607063988:tgl[a0],dg1[a0]
1620                  */
1621                 wa_write_or(wal,
1622                             GEN9_CTX_PREEMPT_REG,
1623                             GEN12_DISABLE_POSH_BUSY_FF_DOP_CG);
1624         }
1625
1626         if (IS_TGL_UY_GT_STEP(i915, STEP_A0, STEP_A0)) {
1627                 /*
1628                  * Wa_1606679103:tgl
1629                  * (see also Wa_1606682166:icl)
1630                  */
1631                 wa_write_or(wal,
1632                             GEN7_SARCHKMD,
1633                             GEN7_DISABLE_SAMPLER_PREFETCH);
1634         }
1635
1636         if (IS_ALDERLAKE_S(i915) || IS_DG1(i915) ||
1637             IS_ROCKETLAKE(i915) || IS_TIGERLAKE(i915)) {
1638                 /* Wa_1606931601:tgl,rkl,dg1,adl-s */
1639                 wa_masked_en(wal, GEN7_ROW_CHICKEN2, GEN12_DISABLE_EARLY_READ);
1640
1641                 /*
1642                  * Wa_1407928979:tgl A*
1643                  * Wa_18011464164:tgl[B0+],dg1[B0+]
1644                  * Wa_22010931296:tgl[B0+],dg1[B0+]
1645                  * Wa_14010919138:rkl,dg1,adl-s
1646                  */
1647                 wa_write_or(wal, GEN7_FF_THREAD_MODE,
1648                             GEN12_FF_TESSELATION_DOP_GATE_DISABLE);
1649
1650                 /*
1651                  * Wa_1606700617:tgl,dg1
1652                  * Wa_22010271021:tgl,rkl,dg1, adl-s
1653                  */
1654                 wa_masked_en(wal,
1655                              GEN9_CS_DEBUG_MODE1,
1656                              FF_DOP_CLOCK_GATE_DISABLE);
1657         }
1658
1659         if (IS_ALDERLAKE_S(i915) || IS_DG1_REVID(i915, DG1_REVID_A0, DG1_REVID_A0) ||
1660             IS_ROCKETLAKE(i915) || IS_TIGERLAKE(i915)) {
1661                 /* Wa_1409804808:tgl,rkl,dg1[a0],adl-s */
1662                 wa_masked_en(wal, GEN7_ROW_CHICKEN2,
1663                              GEN12_PUSH_CONST_DEREF_HOLD_DIS);
1664
1665                 /*
1666                  * Wa_1409085225:tgl
1667                  * Wa_14010229206:tgl,rkl,dg1[a0],adl-s
1668                  */
1669                 wa_masked_en(wal, GEN9_ROW_CHICKEN4, GEN12_DISABLE_TDL_PUSH);
1670         }
1671
1672
1673         if (IS_DG1_REVID(i915, DG1_REVID_A0, DG1_REVID_A0) ||
1674             IS_ROCKETLAKE(i915) || IS_TIGERLAKE(i915)) {
1675                 /*
1676                  * Wa_1607030317:tgl
1677                  * Wa_1607186500:tgl
1678                  * Wa_1607297627:tgl,rkl,dg1[a0]
1679                  *
1680                  * On TGL and RKL there are multiple entries for this WA in the
1681                  * BSpec; some indicate this is an A0-only WA, others indicate
1682                  * it applies to all steppings so we trust the "all steppings."
1683                  * For DG1 this only applies to A0.
1684                  */
1685                 wa_masked_en(wal,
1686                              GEN6_RC_SLEEP_PSMI_CONTROL,
1687                              GEN12_WAIT_FOR_EVENT_POWER_DOWN_DISABLE |
1688                              GEN8_RC_SEMA_IDLE_MSG_DISABLE);
1689         }
1690
1691         if (IS_DG1(i915) || IS_ROCKETLAKE(i915) || IS_TIGERLAKE(i915)) {
1692                 /* Wa_1406941453:tgl,rkl,dg1 */
1693                 wa_masked_en(wal,
1694                              GEN10_SAMPLER_MODE,
1695                              ENABLE_SMALLPL);
1696         }
1697
1698         if (GRAPHICS_VER(i915) == 11) {
1699                 /* This is not an Wa. Enable for better image quality */
1700                 wa_masked_en(wal,
1701                              _3D_CHICKEN3,
1702                              _3D_CHICKEN3_AA_LINE_QUALITY_FIX_ENABLE);
1703
1704                 /* WaPipelineFlushCoherentLines:icl */
1705                 wa_write_or(wal,
1706                             GEN8_L3SQCREG4,
1707                             GEN8_LQSC_FLUSH_COHERENT_LINES);
1708
1709                 /*
1710                  * Wa_1405543622:icl
1711                  * Formerly known as WaGAPZPriorityScheme
1712                  */
1713                 wa_write_or(wal,
1714                             GEN8_GARBCNTL,
1715                             GEN11_ARBITRATION_PRIO_ORDER_MASK);
1716
1717                 /*
1718                  * Wa_1604223664:icl
1719                  * Formerly known as WaL3BankAddressHashing
1720                  */
1721                 wa_write_clr_set(wal,
1722                                  GEN8_GARBCNTL,
1723                                  GEN11_HASH_CTRL_EXCL_MASK,
1724                                  GEN11_HASH_CTRL_EXCL_BIT0);
1725                 wa_write_clr_set(wal,
1726                                  GEN11_GLBLINVL,
1727                                  GEN11_BANK_HASH_ADDR_EXCL_MASK,
1728                                  GEN11_BANK_HASH_ADDR_EXCL_BIT0);
1729
1730                 /*
1731                  * Wa_1405733216:icl
1732                  * Formerly known as WaDisableCleanEvicts
1733                  */
1734                 wa_write_or(wal,
1735                             GEN8_L3SQCREG4,
1736                             GEN11_LQSC_CLEAN_EVICT_DISABLE);
1737
1738                 /* WaForwardProgressSoftReset:icl */
1739                 wa_write_or(wal,
1740                             GEN10_SCRATCH_LNCF2,
1741                             PMFLUSHDONE_LNICRSDROP |
1742                             PMFLUSH_GAPL3UNBLOCK |
1743                             PMFLUSHDONE_LNEBLK);
1744
1745                 /* Wa_1406609255:icl (pre-prod) */
1746                 if (IS_ICL_REVID(i915, ICL_REVID_A0, ICL_REVID_B0))
1747                         wa_write_or(wal,
1748                                     GEN7_SARCHKMD,
1749                                     GEN7_DISABLE_DEMAND_PREFETCH);
1750
1751                 /* Wa_1606682166:icl */
1752                 wa_write_or(wal,
1753                             GEN7_SARCHKMD,
1754                             GEN7_DISABLE_SAMPLER_PREFETCH);
1755
1756                 /* Wa_1409178092:icl */
1757                 wa_write_clr_set(wal,
1758                                  GEN11_SCRATCH2,
1759                                  GEN11_COHERENT_PARTIAL_WRITE_MERGE_ENABLE,
1760                                  0);
1761
1762                 /* WaEnable32PlaneMode:icl */
1763                 wa_masked_en(wal, GEN9_CSFE_CHICKEN1_RCS,
1764                              GEN11_ENABLE_32_PLANE_MODE);
1765
1766                 /*
1767                  * Wa_1408615072:icl,ehl  (vsunit)
1768                  * Wa_1407596294:icl,ehl  (hsunit)
1769                  */
1770                 wa_write_or(wal, UNSLICE_UNIT_LEVEL_CLKGATE,
1771                             VSUNIT_CLKGATE_DIS | HSUNIT_CLKGATE_DIS);
1772
1773                 /* Wa_1407352427:icl,ehl */
1774                 wa_write_or(wal, UNSLICE_UNIT_LEVEL_CLKGATE2,
1775                             PSDUNIT_CLKGATE_DIS);
1776
1777                 /* Wa_1406680159:icl,ehl */
1778                 wa_write_or(wal,
1779                             SUBSLICE_UNIT_LEVEL_CLKGATE,
1780                             GWUNIT_CLKGATE_DIS);
1781
1782                 /*
1783                  * Wa_1408767742:icl[a2..forever],ehl[all]
1784                  * Wa_1605460711:icl[a0..c0]
1785                  */
1786                 wa_write_or(wal,
1787                             GEN7_FF_THREAD_MODE,
1788                             GEN12_FF_TESSELATION_DOP_GATE_DISABLE);
1789
1790                 /* Wa_22010271021 */
1791                 wa_masked_en(wal,
1792                              GEN9_CS_DEBUG_MODE1,
1793                              FF_DOP_CLOCK_GATE_DISABLE);
1794         }
1795
1796         if (IS_GRAPHICS_VER(i915, 9, 12)) {
1797                 /* FtrPerCtxtPreemptionGranularityControl:skl,bxt,kbl,cfl,cnl,icl,tgl */
1798                 wa_masked_en(wal,
1799                              GEN7_FF_SLICE_CS_CHICKEN1,
1800                              GEN9_FFSC_PERCTX_PREEMPT_CTRL);
1801         }
1802
1803         if (IS_SKYLAKE(i915) ||
1804             IS_KABYLAKE(i915) ||
1805             IS_COFFEELAKE(i915) ||
1806             IS_COMETLAKE(i915)) {
1807                 /* WaEnableGapsTsvCreditFix:skl,kbl,cfl */
1808                 wa_write_or(wal,
1809                             GEN8_GARBCNTL,
1810                             GEN9_GAPS_TSV_CREDIT_DISABLE);
1811         }
1812
1813         if (IS_BROXTON(i915)) {
1814                 /* WaDisablePooledEuLoadBalancingFix:bxt */
1815                 wa_masked_en(wal,
1816                              FF_SLICE_CS_CHICKEN2,
1817                              GEN9_POOLED_EU_LOAD_BALANCING_FIX_DISABLE);
1818         }
1819
1820         if (GRAPHICS_VER(i915) == 9) {
1821                 /* WaContextSwitchWithConcurrentTLBInvalidate:skl,bxt,kbl,glk,cfl */
1822                 wa_masked_en(wal,
1823                              GEN9_CSFE_CHICKEN1_RCS,
1824                              GEN9_PREEMPT_GPGPU_SYNC_SWITCH_DISABLE);
1825
1826                 /* WaEnableLbsSlaRetryTimerDecrement:skl,bxt,kbl,glk,cfl */
1827                 wa_write_or(wal,
1828                             BDW_SCRATCH1,
1829                             GEN9_LBS_SLA_RETRY_TIMER_DECREMENT_ENABLE);
1830
1831                 /* WaProgramL3SqcReg1DefaultForPerf:bxt,glk */
1832                 if (IS_GEN9_LP(i915))
1833                         wa_write_clr_set(wal,
1834                                          GEN8_L3SQCREG1,
1835                                          L3_PRIO_CREDITS_MASK,
1836                                          L3_GENERAL_PRIO_CREDITS(62) |
1837                                          L3_HIGH_PRIO_CREDITS(2));
1838
1839                 /* WaOCLCoherentLineFlush:skl,bxt,kbl,cfl */
1840                 wa_write_or(wal,
1841                             GEN8_L3SQCREG4,
1842                             GEN8_LQSC_FLUSH_COHERENT_LINES);
1843
1844                 /* Disable atomics in L3 to prevent unrecoverable hangs */
1845                 wa_write_clr_set(wal, GEN9_SCRATCH_LNCF1,
1846                                  GEN9_LNCF_NONIA_COHERENT_ATOMICS_ENABLE, 0);
1847                 wa_write_clr_set(wal, GEN8_L3SQCREG4,
1848                                  GEN8_LQSQ_NONIA_COHERENT_ATOMICS_ENABLE, 0);
1849                 wa_write_clr_set(wal, GEN9_SCRATCH1,
1850                                  EVICTION_PERF_FIX_ENABLE, 0);
1851         }
1852
1853         if (IS_HASWELL(i915)) {
1854                 /* WaSampleCChickenBitEnable:hsw */
1855                 wa_masked_en(wal,
1856                              HALF_SLICE_CHICKEN3, HSW_SAMPLE_C_PERFORMANCE);
1857
1858                 wa_masked_dis(wal,
1859                               CACHE_MODE_0_GEN7,
1860                               /* enable HiZ Raw Stall Optimization */
1861                               HIZ_RAW_STALL_OPT_DISABLE);
1862         }
1863
1864         if (IS_VALLEYVIEW(i915)) {
1865                 /* WaDisableEarlyCull:vlv */
1866                 wa_masked_en(wal,
1867                              _3D_CHICKEN3,
1868                              _3D_CHICKEN_SF_DISABLE_OBJEND_CULL);
1869
1870                 /*
1871                  * WaVSThreadDispatchOverride:ivb,vlv
1872                  *
1873                  * This actually overrides the dispatch
1874                  * mode for all thread types.
1875                  */
1876                 wa_write_clr_set(wal,
1877                                  GEN7_FF_THREAD_MODE,
1878                                  GEN7_FF_SCHED_MASK,
1879                                  GEN7_FF_TS_SCHED_HW |
1880                                  GEN7_FF_VS_SCHED_HW |
1881                                  GEN7_FF_DS_SCHED_HW);
1882
1883                 /* WaPsdDispatchEnable:vlv */
1884                 /* WaDisablePSDDualDispatchEnable:vlv */
1885                 wa_masked_en(wal,
1886                              GEN7_HALF_SLICE_CHICKEN1,
1887                              GEN7_MAX_PS_THREAD_DEP |
1888                              GEN7_PSD_SINGLE_PORT_DISPATCH_ENABLE);
1889         }
1890
1891         if (IS_IVYBRIDGE(i915)) {
1892                 /* WaDisableEarlyCull:ivb */
1893                 wa_masked_en(wal,
1894                              _3D_CHICKEN3,
1895                              _3D_CHICKEN_SF_DISABLE_OBJEND_CULL);
1896
1897                 if (0) { /* causes HiZ corruption on ivb:gt1 */
1898                         /* enable HiZ Raw Stall Optimization */
1899                         wa_masked_dis(wal,
1900                                       CACHE_MODE_0_GEN7,
1901                                       HIZ_RAW_STALL_OPT_DISABLE);
1902                 }
1903
1904                 /*
1905                  * WaVSThreadDispatchOverride:ivb,vlv
1906                  *
1907                  * This actually overrides the dispatch
1908                  * mode for all thread types.
1909                  */
1910                 wa_write_clr_set(wal,
1911                                  GEN7_FF_THREAD_MODE,
1912                                  GEN7_FF_SCHED_MASK,
1913                                  GEN7_FF_TS_SCHED_HW |
1914                                  GEN7_FF_VS_SCHED_HW |
1915                                  GEN7_FF_DS_SCHED_HW);
1916
1917                 /* WaDisablePSDDualDispatchEnable:ivb */
1918                 if (IS_IVB_GT1(i915))
1919                         wa_masked_en(wal,
1920                                      GEN7_HALF_SLICE_CHICKEN1,
1921                                      GEN7_PSD_SINGLE_PORT_DISPATCH_ENABLE);
1922         }
1923
1924         if (GRAPHICS_VER(i915) == 7) {
1925                 /* WaBCSVCSTlbInvalidationMode:ivb,vlv,hsw */
1926                 wa_masked_en(wal,
1927                              GFX_MODE_GEN7,
1928                              GFX_TLB_INVALIDATE_EXPLICIT | GFX_REPLAY_MODE);
1929
1930                 /* WaDisable_RenderCache_OperationalFlush:ivb,vlv,hsw */
1931                 wa_masked_dis(wal, CACHE_MODE_0_GEN7, RC_OP_FLUSH_ENABLE);
1932
1933                 /*
1934                  * BSpec says this must be set, even though
1935                  * WaDisable4x2SubspanOptimization:ivb,hsw
1936                  * WaDisable4x2SubspanOptimization isn't listed for VLV.
1937                  */
1938                 wa_masked_en(wal,
1939                              CACHE_MODE_1,
1940                              PIXEL_SUBSPAN_COLLECT_OPT_DISABLE);
1941
1942                 /*
1943                  * BSpec recommends 8x4 when MSAA is used,
1944                  * however in practice 16x4 seems fastest.
1945                  *
1946                  * Note that PS/WM thread counts depend on the WIZ hashing
1947                  * disable bit, which we don't touch here, but it's good
1948                  * to keep in mind (see 3DSTATE_PS and 3DSTATE_WM).
1949                  */
1950                 wa_add(wal, GEN7_GT_MODE, 0,
1951                        _MASKED_FIELD(GEN6_WIZ_HASHING_MASK,
1952                                      GEN6_WIZ_HASHING_16x4),
1953                        GEN6_WIZ_HASHING_16x4);
1954         }
1955
1956         if (IS_GRAPHICS_VER(i915, 6, 7))
1957                 /*
1958                  * We need to disable the AsyncFlip performance optimisations in
1959                  * order to use MI_WAIT_FOR_EVENT within the CS. It should
1960                  * already be programmed to '1' on all products.
1961                  *
1962                  * WaDisableAsyncFlipPerfMode:snb,ivb,hsw,vlv
1963                  */
1964                 wa_masked_en(wal,
1965                              MI_MODE,
1966                              ASYNC_FLIP_PERF_DISABLE);
1967
1968         if (GRAPHICS_VER(i915) == 6) {
1969                 /*
1970                  * Required for the hardware to program scanline values for
1971                  * waiting
1972                  * WaEnableFlushTlbInvalidationMode:snb
1973                  */
1974                 wa_masked_en(wal,
1975                              GFX_MODE,
1976                              GFX_TLB_INVALIDATE_EXPLICIT);
1977
1978                 /* WaDisableHiZPlanesWhenMSAAEnabled:snb */
1979                 wa_masked_en(wal,
1980                              _3D_CHICKEN,
1981                              _3D_CHICKEN_HIZ_PLANE_DISABLE_MSAA_4X_SNB);
1982
1983                 wa_masked_en(wal,
1984                              _3D_CHICKEN3,
1985                              /* WaStripsFansDisableFastClipPerformanceFix:snb */
1986                              _3D_CHICKEN3_SF_DISABLE_FASTCLIP_CULL |
1987                              /*
1988                               * Bspec says:
1989                               * "This bit must be set if 3DSTATE_CLIP clip mode is set
1990                               * to normal and 3DSTATE_SF number of SF output attributes
1991                               * is more than 16."
1992                               */
1993                              _3D_CHICKEN3_SF_DISABLE_PIPELINED_ATTR_FETCH);
1994
1995                 /*
1996                  * BSpec recommends 8x4 when MSAA is used,
1997                  * however in practice 16x4 seems fastest.
1998                  *
1999                  * Note that PS/WM thread counts depend on the WIZ hashing
2000                  * disable bit, which we don't touch here, but it's good
2001                  * to keep in mind (see 3DSTATE_PS and 3DSTATE_WM).
2002                  */
2003                 wa_add(wal,
2004                        GEN6_GT_MODE, 0,
2005                        _MASKED_FIELD(GEN6_WIZ_HASHING_MASK, GEN6_WIZ_HASHING_16x4),
2006                        GEN6_WIZ_HASHING_16x4);
2007
2008                 /* WaDisable_RenderCache_OperationalFlush:snb */
2009                 wa_masked_dis(wal, CACHE_MODE_0, RC_OP_FLUSH_ENABLE);
2010
2011                 /*
2012                  * From the Sandybridge PRM, volume 1 part 3, page 24:
2013                  * "If this bit is set, STCunit will have LRA as replacement
2014                  *  policy. [...] This bit must be reset. LRA replacement
2015                  *  policy is not supported."
2016                  */
2017                 wa_masked_dis(wal,
2018                               CACHE_MODE_0,
2019                               CM0_STC_EVICT_DISABLE_LRA_SNB);
2020         }
2021
2022         if (IS_GRAPHICS_VER(i915, 4, 6))
2023                 /* WaTimedSingleVertexDispatch:cl,bw,ctg,elk,ilk,snb */
2024                 wa_add(wal, MI_MODE,
2025                        0, _MASKED_BIT_ENABLE(VS_TIMER_DISPATCH),
2026                        /* XXX bit doesn't stick on Broadwater */
2027                        IS_I965G(i915) ? 0 : VS_TIMER_DISPATCH);
2028
2029         if (GRAPHICS_VER(i915) == 4)
2030                 /*
2031                  * Disable CONSTANT_BUFFER before it is loaded from the context
2032                  * image. For as it is loaded, it is executed and the stored
2033                  * address may no longer be valid, leading to a GPU hang.
2034                  *
2035                  * This imposes the requirement that userspace reload their
2036                  * CONSTANT_BUFFER on every batch, fortunately a requirement
2037                  * they are already accustomed to from before contexts were
2038                  * enabled.
2039                  */
2040                 wa_add(wal, ECOSKPD,
2041                        0, _MASKED_BIT_ENABLE(ECO_CONSTANT_BUFFER_SR_DISABLE),
2042                        0 /* XXX bit doesn't stick on Broadwater */);
2043 }
2044
2045 static void
2046 xcs_engine_wa_init(struct intel_engine_cs *engine, struct i915_wa_list *wal)
2047 {
2048         struct drm_i915_private *i915 = engine->i915;
2049
2050         /* WaKBLVECSSemaphoreWaitPoll:kbl */
2051         if (IS_KBL_GT_STEP(i915, STEP_A0, STEP_E0)) {
2052                 wa_write(wal,
2053                          RING_SEMA_WAIT_POLL(engine->mmio_base),
2054                          1);
2055         }
2056 }
2057
2058 static void
2059 engine_init_workarounds(struct intel_engine_cs *engine, struct i915_wa_list *wal)
2060 {
2061         if (I915_SELFTEST_ONLY(GRAPHICS_VER(engine->i915) < 4))
2062                 return;
2063
2064         if (engine->class == RENDER_CLASS)
2065                 rcs_engine_wa_init(engine, wal);
2066         else
2067                 xcs_engine_wa_init(engine, wal);
2068 }
2069
2070 void intel_engine_init_workarounds(struct intel_engine_cs *engine)
2071 {
2072         struct i915_wa_list *wal = &engine->wa_list;
2073
2074         if (GRAPHICS_VER(engine->i915) < 4)
2075                 return;
2076
2077         wa_init_start(wal, "engine", engine->name);
2078         engine_init_workarounds(engine, wal);
2079         wa_init_finish(wal);
2080 }
2081
2082 void intel_engine_apply_workarounds(struct intel_engine_cs *engine)
2083 {
2084         wa_list_apply(engine->uncore, &engine->wa_list);
2085 }
2086
2087 struct mcr_range {
2088         u32 start;
2089         u32 end;
2090 };
2091
2092 static const struct mcr_range mcr_ranges_gen8[] = {
2093         { .start = 0x5500, .end = 0x55ff },
2094         { .start = 0x7000, .end = 0x7fff },
2095         { .start = 0x9400, .end = 0x97ff },
2096         { .start = 0xb000, .end = 0xb3ff },
2097         { .start = 0xe000, .end = 0xe7ff },
2098         {},
2099 };
2100
2101 static const struct mcr_range mcr_ranges_gen12[] = {
2102         { .start =  0x8150, .end =  0x815f },
2103         { .start =  0x9520, .end =  0x955f },
2104         { .start =  0xb100, .end =  0xb3ff },
2105         { .start =  0xde80, .end =  0xe8ff },
2106         { .start = 0x24a00, .end = 0x24a7f },
2107         {},
2108 };
2109
2110 static bool mcr_range(struct drm_i915_private *i915, u32 offset)
2111 {
2112         const struct mcr_range *mcr_ranges;
2113         int i;
2114
2115         if (GRAPHICS_VER(i915) >= 12)
2116                 mcr_ranges = mcr_ranges_gen12;
2117         else if (GRAPHICS_VER(i915) >= 8)
2118                 mcr_ranges = mcr_ranges_gen8;
2119         else
2120                 return false;
2121
2122         /*
2123          * Registers in these ranges are affected by the MCR selector
2124          * which only controls CPU initiated MMIO. Routing does not
2125          * work for CS access so we cannot verify them on this path.
2126          */
2127         for (i = 0; mcr_ranges[i].start; i++)
2128                 if (offset >= mcr_ranges[i].start &&
2129                     offset <= mcr_ranges[i].end)
2130                         return true;
2131
2132         return false;
2133 }
2134
2135 static int
2136 wa_list_srm(struct i915_request *rq,
2137             const struct i915_wa_list *wal,
2138             struct i915_vma *vma)
2139 {
2140         struct drm_i915_private *i915 = rq->engine->i915;
2141         unsigned int i, count = 0;
2142         const struct i915_wa *wa;
2143         u32 srm, *cs;
2144
2145         srm = MI_STORE_REGISTER_MEM | MI_SRM_LRM_GLOBAL_GTT;
2146         if (GRAPHICS_VER(i915) >= 8)
2147                 srm++;
2148
2149         for (i = 0, wa = wal->list; i < wal->count; i++, wa++) {
2150                 if (!mcr_range(i915, i915_mmio_reg_offset(wa->reg)))
2151                         count++;
2152         }
2153
2154         cs = intel_ring_begin(rq, 4 * count);
2155         if (IS_ERR(cs))
2156                 return PTR_ERR(cs);
2157
2158         for (i = 0, wa = wal->list; i < wal->count; i++, wa++) {
2159                 u32 offset = i915_mmio_reg_offset(wa->reg);
2160
2161                 if (mcr_range(i915, offset))
2162                         continue;
2163
2164                 *cs++ = srm;
2165                 *cs++ = offset;
2166                 *cs++ = i915_ggtt_offset(vma) + sizeof(u32) * i;
2167                 *cs++ = 0;
2168         }
2169         intel_ring_advance(rq, cs);
2170
2171         return 0;
2172 }
2173
2174 static int engine_wa_list_verify(struct intel_context *ce,
2175                                  const struct i915_wa_list * const wal,
2176                                  const char *from)
2177 {
2178         const struct i915_wa *wa;
2179         struct i915_request *rq;
2180         struct i915_vma *vma;
2181         struct i915_gem_ww_ctx ww;
2182         unsigned int i;
2183         u32 *results;
2184         int err;
2185
2186         if (!wal->count)
2187                 return 0;
2188
2189         vma = __vm_create_scratch_for_read(&ce->engine->gt->ggtt->vm,
2190                                            wal->count * sizeof(u32));
2191         if (IS_ERR(vma))
2192                 return PTR_ERR(vma);
2193
2194         intel_engine_pm_get(ce->engine);
2195         i915_gem_ww_ctx_init(&ww, false);
2196 retry:
2197         err = i915_gem_object_lock(vma->obj, &ww);
2198         if (err == 0)
2199                 err = intel_context_pin_ww(ce, &ww);
2200         if (err)
2201                 goto err_pm;
2202
2203         err = i915_vma_pin_ww(vma, &ww, 0, 0,
2204                            i915_vma_is_ggtt(vma) ? PIN_GLOBAL : PIN_USER);
2205         if (err)
2206                 goto err_unpin;
2207
2208         rq = i915_request_create(ce);
2209         if (IS_ERR(rq)) {
2210                 err = PTR_ERR(rq);
2211                 goto err_vma;
2212         }
2213
2214         err = i915_request_await_object(rq, vma->obj, true);
2215         if (err == 0)
2216                 err = i915_vma_move_to_active(vma, rq, EXEC_OBJECT_WRITE);
2217         if (err == 0)
2218                 err = wa_list_srm(rq, wal, vma);
2219
2220         i915_request_get(rq);
2221         if (err)
2222                 i915_request_set_error_once(rq, err);
2223         i915_request_add(rq);
2224
2225         if (err)
2226                 goto err_rq;
2227
2228         if (i915_request_wait(rq, 0, HZ / 5) < 0) {
2229                 err = -ETIME;
2230                 goto err_rq;
2231         }
2232
2233         results = i915_gem_object_pin_map(vma->obj, I915_MAP_WB);
2234         if (IS_ERR(results)) {
2235                 err = PTR_ERR(results);
2236                 goto err_rq;
2237         }
2238
2239         err = 0;
2240         for (i = 0, wa = wal->list; i < wal->count; i++, wa++) {
2241                 if (mcr_range(rq->engine->i915, i915_mmio_reg_offset(wa->reg)))
2242                         continue;
2243
2244                 if (!wa_verify(wa, results[i], wal->name, from))
2245                         err = -ENXIO;
2246         }
2247
2248         i915_gem_object_unpin_map(vma->obj);
2249
2250 err_rq:
2251         i915_request_put(rq);
2252 err_vma:
2253         i915_vma_unpin(vma);
2254 err_unpin:
2255         intel_context_unpin(ce);
2256 err_pm:
2257         if (err == -EDEADLK) {
2258                 err = i915_gem_ww_ctx_backoff(&ww);
2259                 if (!err)
2260                         goto retry;
2261         }
2262         i915_gem_ww_ctx_fini(&ww);
2263         intel_engine_pm_put(ce->engine);
2264         i915_vma_put(vma);
2265         return err;
2266 }
2267
2268 int intel_engine_verify_workarounds(struct intel_engine_cs *engine,
2269                                     const char *from)
2270 {
2271         return engine_wa_list_verify(engine->kernel_context,
2272                                      &engine->wa_list,
2273                                      from);
2274 }
2275
2276 #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
2277 #include "selftest_workarounds.c"
2278 #endif