drivers/gpu/drm/i915/gt/intel_workarounds.c

   1 // SPDX-License-Identifier: MIT
   2 /*
   3  * Copyright © 2014-2018 Intel Corporation
   4  */
   5
   6 #include "i915_drv.h"
   7 #include "i915_reg.h"
   8 #include "intel_context.h"
   9 #include "intel_engine_pm.h"
  10 #include "intel_engine_regs.h"
  11 #include "intel_gpu_commands.h"
  12 #include "intel_gt.h"
  13 #include "intel_gt_mcr.h"
  14 #include "intel_gt_regs.h"
  15 #include "intel_ring.h"
  16 #include "intel_workarounds.h"
  17
  18 /**
  19  * DOC: Hardware workarounds
  20  *
  21  * Hardware workarounds are register programming documented to be executed in
  22  * the driver that fall outside of the normal programming sequences for a
  23  * platform. There are some basic categories of workarounds, depending on
  24  * how/when they are applied:
  25  *
  26  * - Context workarounds: workarounds that touch registers that are
  27  *   saved/restored to/from the HW context image. The list is emitted (via Load
  28  *   Register Immediate commands) once when initializing the device and saved in
  29  *   the default context. That default context is then used on every context
  30  *   creation to have a "primed golden context", i.e. a context image that
  31  *   already contains the changes needed to all the registers.
  32  *
  33  * - Engine workarounds: the list of these WAs is applied whenever the specific
  34  *   engine is reset. It's also possible that a set of engine classes share a
  35  *   common power domain and they are reset together. This happens on some
  36  *   platforms with render and compute engines. In this case (at least) one of
  37  *   them need to keeep the workaround programming: the approach taken in the
  38  *   driver is to tie those workarounds to the first compute/render engine that
  39  *   is registered.  When executing with GuC submission, engine resets are
  40  *   outside of kernel driver control, hence the list of registers involved in
  41  *   written once, on engine initialization, and then passed to GuC, that
  42  *   saves/restores their values before/after the reset takes place. See
  43  *   ``drivers/gpu/drm/i915/gt/uc/intel_guc_ads.c`` for reference.
  44  *
  45  * - GT workarounds: the list of these WAs is applied whenever these registers
  46  *   revert to their default values: on GPU reset, suspend/resume [1]_, etc.
  47  *
  48  * - Register whitelist: some workarounds need to be implemented in userspace,
  49  *   but need to touch privileged registers. The whitelist in the kernel
  50  *   instructs the hardware to allow the access to happen. From the kernel side,
  51  *   this is just a special case of a MMIO workaround (as we write the list of
  52  *   these to/be-whitelisted registers to some special HW registers).
  53  *
  54  * - Workaround batchbuffers: buffers that get executed automatically by the
  55  *   hardware on every HW context restore. These buffers are created and
  56  *   programmed in the default context so the hardware always go through those
  57  *   programming sequences when switching contexts. The support for workaround
  58  *   batchbuffers is enabled these hardware mechanisms:
  59  *
  60  *   #. INDIRECT_CTX: A batchbuffer and an offset are provided in the default
  61  *      context, pointing the hardware to jump to that location when that offset
  62  *      is reached in the context restore. Workaround batchbuffer in the driver
  63  *      currently uses this mechanism for all platforms.
  64  *
  65  *   #. BB_PER_CTX_PTR: A batchbuffer is provided in the default context,
  66  *      pointing the hardware to a buffer to continue executing after the
  67  *      engine registers are restored in a context restore sequence. This is
  68  *      currently not used in the driver.
  69  *
  70  * - Other:  There are WAs that, due to their nature, cannot be applied from a
  71  *   central place. Those are peppered around the rest of the code, as needed.
  72  *   Workarounds related to the display IP are the main example.
  73  *
  74  * .. [1] Technically, some registers are powercontext saved & restored, so they
  75  *    survive a suspend/resume. In practice, writing them again is not too
  76  *    costly and simplifies things, so it's the approach taken in the driver.
  77  */
  78
  79 static void wa_init_start(struct i915_wa_list *wal, struct intel_gt *gt,
  80                           const char *name, const char *engine_name)
  81 {
  82         wal->gt = gt;
  83         wal->name = name;
  84         wal->engine_name = engine_name;
  85 }
  86
  87 #define WA_LIST_CHUNK (1 << 4)
  88
  89 static void wa_init_finish(struct i915_wa_list *wal)
  90 {
  91         /* Trim unused entries. */
  92         if (!IS_ALIGNED(wal->count, WA_LIST_CHUNK)) {
  93                 struct i915_wa *list = kmemdup(wal->list,
  94                                                wal->count * sizeof(*list),
  95                                                GFP_KERNEL);
  96
  97                 if (list) {
  98                         kfree(wal->list);
  99                         wal->list = list;
 100                 }
 101         }
 102
 103         if (!wal->count)
 104                 return;
 105
 106         drm_dbg(&wal->gt->i915->drm, "Initialized %u %s workarounds on %s\n",
 107                 wal->wa_count, wal->name, wal->engine_name);
 108 }
 109
 110 static void _wa_add(struct i915_wa_list *wal, const struct i915_wa *wa)
 111 {
 112         unsigned int addr = i915_mmio_reg_offset(wa->reg);
 113         struct drm_i915_private *i915 = wal->gt->i915;
 114         unsigned int start = 0, end = wal->count;
 115         const unsigned int grow = WA_LIST_CHUNK;
 116         struct i915_wa *wa_;
 117
 118         GEM_BUG_ON(!is_power_of_2(grow));
 119
 120         if (IS_ALIGNED(wal->count, grow)) { /* Either uninitialized or full. */
 121                 struct i915_wa *list;
 122
 123                 list = kmalloc_array(ALIGN(wal->count + 1, grow), sizeof(*wa),
 124                                      GFP_KERNEL);
 125                 if (!list) {
 126                         drm_err(&i915->drm, "No space for workaround init!\n");
 127                         return;
 128                 }
 129
 130                 if (wal->list) {
 131                         memcpy(list, wal->list, sizeof(*wa) * wal->count);
 132                         kfree(wal->list);
 133                 }
 134
 135                 wal->list = list;
 136         }
 137
 138         while (start < end) {
 139                 unsigned int mid = start + (end - start) / 2;
 140
 141                 if (i915_mmio_reg_offset(wal->list[mid].reg) < addr) {
 142                         start = mid + 1;
 143                 } else if (i915_mmio_reg_offset(wal->list[mid].reg) > addr) {
 144                         end = mid;
 145                 } else {
 146                         wa_ = &wal->list[mid];
 147
 148                         if ((wa->clr | wa_->clr) && !(wa->clr & ~wa_->clr)) {
 149                                 drm_err(&i915->drm,
 150                                         "Discarding overwritten w/a for reg %04x (clear: %08x, set: %08x)\n",
 151                                         i915_mmio_reg_offset(wa_->reg),
 152                                         wa_->clr, wa_->set);
 153
 154                                 wa_->set &= ~wa->clr;
 155                         }
 156
 157                         wal->wa_count++;
 158                         wa_->set |= wa->set;
 159                         wa_->clr |= wa->clr;
 160                         wa_->read |= wa->read;
 161                         return;
 162                 }
 163         }
 164
 165         wal->wa_count++;
 166         wa_ = &wal->list[wal->count++];
 167         *wa_ = *wa;
 168
 169         while (wa_-- > wal->list) {
 170                 GEM_BUG_ON(i915_mmio_reg_offset(wa_[0].reg) ==
 171                            i915_mmio_reg_offset(wa_[1].reg));
 172                 if (i915_mmio_reg_offset(wa_[1].reg) >
 173                     i915_mmio_reg_offset(wa_[0].reg))
 174                         break;
 175
 176                 swap(wa_[1], wa_[0]);
 177         }
 178 }
 179
 180 static void wa_add(struct i915_wa_list *wal, i915_reg_t reg,
 181                    u32 clear, u32 set, u32 read_mask, bool masked_reg)
 182 {
 183         struct i915_wa wa = {
 184                 .reg  = reg,
 185                 .clr  = clear,
 186                 .set  = set,
 187                 .read = read_mask,
 188                 .masked_reg = masked_reg,
 189         };
 190
 191         _wa_add(wal, &wa);
 192 }
 193
 194 static void wa_mcr_add(struct i915_wa_list *wal, i915_mcr_reg_t reg,
 195                        u32 clear, u32 set, u32 read_mask, bool masked_reg)
 196 {
 197         struct i915_wa wa = {
 198                 .mcr_reg = reg,
 199                 .clr  = clear,
 200                 .set  = set,
 201                 .read = read_mask,
 202                 .masked_reg = masked_reg,
 203                 .is_mcr = 1,
 204         };
 205
 206         _wa_add(wal, &wa);
 207 }
 208
 209 static void
 210 wa_write_clr_set(struct i915_wa_list *wal, i915_reg_t reg, u32 clear, u32 set)
 211 {
 212         wa_add(wal, reg, clear, set, clear, false);
 213 }
 214
 215 static void
 216 wa_mcr_write_clr_set(struct i915_wa_list *wal, i915_mcr_reg_t reg, u32 clear, u32 set)
 217 {
 218         wa_mcr_add(wal, reg, clear, set, clear, false);
 219 }
 220
 221 static void
 222 wa_write(struct i915_wa_list *wal, i915_reg_t reg, u32 set)
 223 {
 224         wa_write_clr_set(wal, reg, ~0, set);
 225 }
 226
 227 static void
 228 wa_write_or(struct i915_wa_list *wal, i915_reg_t reg, u32 set)
 229 {
 230         wa_write_clr_set(wal, reg, set, set);
 231 }
 232
 233 static void
 234 wa_mcr_write_or(struct i915_wa_list *wal, i915_mcr_reg_t reg, u32 set)
 235 {
 236         wa_mcr_write_clr_set(wal, reg, set, set);
 237 }
 238
 239 static void
 240 wa_write_clr(struct i915_wa_list *wal, i915_reg_t reg, u32 clr)
 241 {
 242         wa_write_clr_set(wal, reg, clr, 0);
 243 }
 244
 245 static void
 246 wa_mcr_write_clr(struct i915_wa_list *wal, i915_mcr_reg_t reg, u32 clr)
 247 {
 248         wa_mcr_write_clr_set(wal, reg, clr, 0);
 249 }
 250
 251 /*
 252  * WA operations on "masked register". A masked register has the upper 16 bits
 253  * documented as "masked" in b-spec. Its purpose is to allow writing to just a
 254  * portion of the register without a rmw: you simply write in the upper 16 bits
 255  * the mask of bits you are going to modify.
 256  *
 257  * The wa_masked_* family of functions already does the necessary operations to
 258  * calculate the mask based on the parameters passed, so user only has to
 259  * provide the lower 16 bits of that register.
 260  */
 261
 262 static void
 263 wa_masked_en(struct i915_wa_list *wal, i915_reg_t reg, u32 val)
 264 {
 265         wa_add(wal, reg, 0, _MASKED_BIT_ENABLE(val), val, true);
 266 }
 267
 268 static void
 269 wa_mcr_masked_en(struct i915_wa_list *wal, i915_mcr_reg_t reg, u32 val)
 270 {
 271         wa_mcr_add(wal, reg, 0, _MASKED_BIT_ENABLE(val), val, true);
 272 }
 273
 274 static void
 275 wa_masked_dis(struct i915_wa_list *wal, i915_reg_t reg, u32 val)
 276 {
 277         wa_add(wal, reg, 0, _MASKED_BIT_DISABLE(val), val, true);
 278 }
 279
 280 static void
 281 wa_mcr_masked_dis(struct i915_wa_list *wal, i915_mcr_reg_t reg, u32 val)
 282 {
 283         wa_mcr_add(wal, reg, 0, _MASKED_BIT_DISABLE(val), val, true);
 284 }
 285
 286 static void
 287 wa_masked_field_set(struct i915_wa_list *wal, i915_reg_t reg,
 288                     u32 mask, u32 val)
 289 {
 290         wa_add(wal, reg, 0, _MASKED_FIELD(mask, val), mask, true);
 291 }
 292
 293 static void
 294 wa_mcr_masked_field_set(struct i915_wa_list *wal, i915_mcr_reg_t reg,
 295                         u32 mask, u32 val)
 296 {
 297         wa_mcr_add(wal, reg, 0, _MASKED_FIELD(mask, val), mask, true);
 298 }
 299
 300 static void gen6_ctx_workarounds_init(struct intel_engine_cs *engine,
 301                                       struct i915_wa_list *wal)
 302 {
 303         wa_masked_en(wal, INSTPM, INSTPM_FORCE_ORDERING);
 304 }
 305
 306 static void gen7_ctx_workarounds_init(struct intel_engine_cs *engine,
 307                                       struct i915_wa_list *wal)
 308 {
 309         wa_masked_en(wal, INSTPM, INSTPM_FORCE_ORDERING);
 310 }
 311
 312 static void gen8_ctx_workarounds_init(struct intel_engine_cs *engine,
 313                                       struct i915_wa_list *wal)
 314 {
 315         wa_masked_en(wal, INSTPM, INSTPM_FORCE_ORDERING);
 316
 317         /* WaDisableAsyncFlipPerfMode:bdw,chv */
 318         wa_masked_en(wal, RING_MI_MODE(RENDER_RING_BASE), ASYNC_FLIP_PERF_DISABLE);
 319
 320         /* WaDisablePartialInstShootdown:bdw,chv */
 321         wa_mcr_masked_en(wal, GEN8_ROW_CHICKEN,
 322                          PARTIAL_INSTRUCTION_SHOOTDOWN_DISABLE);
 323
 324         /* Use Force Non-Coherent whenever executing a 3D context. This is a
 325          * workaround for a possible hang in the unlikely event a TLB
 326          * invalidation occurs during a PSD flush.
 327          */
 328         /* WaForceEnableNonCoherent:bdw,chv */
 329         /* WaHdcDisableFetchWhenMasked:bdw,chv */
 330         wa_masked_en(wal, HDC_CHICKEN0,
 331                      HDC_DONOT_FETCH_MEM_WHEN_MASKED |
 332                      HDC_FORCE_NON_COHERENT);
 333
 334         /* From the Haswell PRM, Command Reference: Registers, CACHE_MODE_0:
 335          * "The Hierarchical Z RAW Stall Optimization allows non-overlapping
 336          *  polygons in the same 8x4 pixel/sample area to be processed without
 337          *  stalling waiting for the earlier ones to write to Hierarchical Z
 338          *  buffer."
 339          *
 340          * This optimization is off by default for BDW and CHV; turn it on.
 341          */
 342         wa_masked_dis(wal, CACHE_MODE_0_GEN7, HIZ_RAW_STALL_OPT_DISABLE);
 343
 344         /* Wa4x4STCOptimizationDisable:bdw,chv */
 345         wa_masked_en(wal, CACHE_MODE_1, GEN8_4x4_STC_OPTIMIZATION_DISABLE);
 346
 347         /*
 348          * BSpec recommends 8x4 when MSAA is used,
 349          * however in practice 16x4 seems fastest.
 350          *
 351          * Note that PS/WM thread counts depend on the WIZ hashing
 352          * disable bit, which we don't touch here, but it's good
 353          * to keep in mind (see 3DSTATE_PS and 3DSTATE_WM).
 354          */
 355         wa_masked_field_set(wal, GEN7_GT_MODE,
 356                             GEN6_WIZ_HASHING_MASK,
 357                             GEN6_WIZ_HASHING_16x4);
 358 }
 359
 360 static void bdw_ctx_workarounds_init(struct intel_engine_cs *engine,
 361                                      struct i915_wa_list *wal)
 362 {
 363         struct drm_i915_private *i915 = engine->i915;
 364
 365         gen8_ctx_workarounds_init(engine, wal);
 366
 367         /* WaDisableThreadStallDopClockGating:bdw (pre-production) */
 368         wa_mcr_masked_en(wal, GEN8_ROW_CHICKEN, STALL_DOP_GATING_DISABLE);
 369
 370         /* WaDisableDopClockGating:bdw
 371          *
 372          * Also see the related UCGTCL1 write in bdw_init_clock_gating()
 373          * to disable EUTC clock gating.
 374          */
 375         wa_mcr_masked_en(wal, GEN8_ROW_CHICKEN2,
 376                          DOP_CLOCK_GATING_DISABLE);
 377
 378         wa_mcr_masked_en(wal, GEN8_HALF_SLICE_CHICKEN3,
 379                          GEN8_SAMPLER_POWER_BYPASS_DIS);
 380
 381         wa_masked_en(wal, HDC_CHICKEN0,
 382                      /* WaForceContextSaveRestoreNonCoherent:bdw */
 383                      HDC_FORCE_CONTEXT_SAVE_RESTORE_NON_COHERENT |
 384                      /* WaDisableFenceDestinationToSLM:bdw (pre-prod) */
 385                      (IS_BDW_GT3(i915) ? HDC_FENCE_DEST_SLM_DISABLE : 0));
 386 }
 387
 388 static void chv_ctx_workarounds_init(struct intel_engine_cs *engine,
 389                                      struct i915_wa_list *wal)
 390 {
 391         gen8_ctx_workarounds_init(engine, wal);
 392
 393         /* WaDisableThreadStallDopClockGating:chv */
 394         wa_mcr_masked_en(wal, GEN8_ROW_CHICKEN, STALL_DOP_GATING_DISABLE);
 395
 396         /* Improve HiZ throughput on CHV. */
 397         wa_masked_en(wal, HIZ_CHICKEN, CHV_HZ_8X8_MODE_IN_1X);
 398 }
 399
 400 static void gen9_ctx_workarounds_init(struct intel_engine_cs *engine,
 401                                       struct i915_wa_list *wal)
 402 {
 403         struct drm_i915_private *i915 = engine->i915;
 404
 405         if (HAS_LLC(i915)) {
 406                 /* WaCompressedResourceSamplerPbeMediaNewHashMode:skl,kbl
 407                  *
 408                  * Must match Display Engine. See
 409                  * WaCompressedResourceDisplayNewHashMode.
 410                  */
 411                 wa_masked_en(wal, COMMON_SLICE_CHICKEN2,
 412                              GEN9_PBE_COMPRESSED_HASH_SELECTION);
 413                 wa_mcr_masked_en(wal, GEN9_HALF_SLICE_CHICKEN7,
 414                                  GEN9_SAMPLER_HASH_COMPRESSED_READ_ADDR);
 415         }
 416
 417         /* WaClearFlowControlGpgpuContextSave:skl,bxt,kbl,glk,cfl */
 418         /* WaDisablePartialInstShootdown:skl,bxt,kbl,glk,cfl */
 419         wa_mcr_masked_en(wal, GEN8_ROW_CHICKEN,
 420                          FLOW_CONTROL_ENABLE |
 421                          PARTIAL_INSTRUCTION_SHOOTDOWN_DISABLE);
 422
 423         /* WaEnableYV12BugFixInHalfSliceChicken7:skl,bxt,kbl,glk,cfl */
 424         /* WaEnableSamplerGPGPUPreemptionSupport:skl,bxt,kbl,cfl */
 425         wa_mcr_masked_en(wal, GEN9_HALF_SLICE_CHICKEN7,
 426                          GEN9_ENABLE_YV12_BUGFIX |
 427                          GEN9_ENABLE_GPGPU_PREEMPTION);
 428
 429         /* Wa4x4STCOptimizationDisable:skl,bxt,kbl,glk,cfl */
 430         /* WaDisablePartialResolveInVc:skl,bxt,kbl,cfl */
 431         wa_masked_en(wal, CACHE_MODE_1,
 432                      GEN8_4x4_STC_OPTIMIZATION_DISABLE |
 433                      GEN9_PARTIAL_RESOLVE_IN_VC_DISABLE);
 434
 435         /* WaCcsTlbPrefetchDisable:skl,bxt,kbl,glk,cfl */
 436         wa_mcr_masked_dis(wal, GEN9_HALF_SLICE_CHICKEN5,
 437                           GEN9_CCS_TLB_PREFETCH_ENABLE);
 438
 439         /* WaForceContextSaveRestoreNonCoherent:skl,bxt,kbl,cfl */
 440         wa_masked_en(wal, HDC_CHICKEN0,
 441                      HDC_FORCE_CONTEXT_SAVE_RESTORE_NON_COHERENT |
 442                      HDC_FORCE_CSR_NON_COHERENT_OVR_DISABLE);
 443
 444         /* WaForceEnableNonCoherent and WaDisableHDCInvalidation are
 445          * both tied to WaForceContextSaveRestoreNonCoherent
 446          * in some hsds for skl. We keep the tie for all gen9. The
 447          * documentation is a bit hazy and so we want to get common behaviour,
 448          * even though there is no clear evidence we would need both on kbl/bxt.
 449          * This area has been source of system hangs so we play it safe
 450          * and mimic the skl regardless of what bspec says.
 451          *
 452          * Use Force Non-Coherent whenever executing a 3D context. This
 453          * is a workaround for a possible hang in the unlikely event
 454          * a TLB invalidation occurs during a PSD flush.
 455          */
 456
 457         /* WaForceEnableNonCoherent:skl,bxt,kbl,cfl */
 458         wa_masked_en(wal, HDC_CHICKEN0,
 459                      HDC_FORCE_NON_COHERENT);
 460
 461         /* WaDisableSamplerPowerBypassForSOPingPong:skl,bxt,kbl,cfl */
 462         if (IS_SKYLAKE(i915) ||
 463             IS_KABYLAKE(i915) ||
 464             IS_COFFEELAKE(i915) ||
 465             IS_COMETLAKE(i915))
 466                 wa_mcr_masked_en(wal, GEN8_HALF_SLICE_CHICKEN3,
 467                                  GEN8_SAMPLER_POWER_BYPASS_DIS);
 468
 469         /* WaDisableSTUnitPowerOptimization:skl,bxt,kbl,glk,cfl */
 470         wa_mcr_masked_en(wal, HALF_SLICE_CHICKEN2, GEN8_ST_PO_DISABLE);
 471
 472         /*
 473          * Supporting preemption with fine-granularity requires changes in the
 474          * batch buffer programming. Since we can't break old userspace, we
 475          * need to set our default preemption level to safe value. Userspace is
 476          * still able to use more fine-grained preemption levels, since in
 477          * WaEnablePreemptionGranularityControlByUMD we're whitelisting the
 478          * per-ctx register. As such, WaDisable{3D,GPGPU}MidCmdPreemption are
 479          * not real HW workarounds, but merely a way to start using preemption
 480          * while maintaining old contract with userspace.
 481          */
 482
 483         /* WaDisable3DMidCmdPreemption:skl,bxt,glk,cfl,[cnl] */
 484         wa_masked_dis(wal, GEN8_CS_CHICKEN1, GEN9_PREEMPT_3D_OBJECT_LEVEL);
 485
 486         /* WaDisableGPGPUMidCmdPreemption:skl,bxt,blk,cfl,[cnl] */
 487         wa_masked_field_set(wal, GEN8_CS_CHICKEN1,
 488                             GEN9_PREEMPT_GPGPU_LEVEL_MASK,
 489                             GEN9_PREEMPT_GPGPU_COMMAND_LEVEL);
 490
 491         /* WaClearHIZ_WM_CHICKEN3:bxt,glk */
 492         if (IS_GEN9_LP(i915))
 493                 wa_masked_en(wal, GEN9_WM_CHICKEN3, GEN9_FACTOR_IN_CLR_VAL_HIZ);
 494 }
 495
 496 static void skl_tune_iz_hashing(struct intel_engine_cs *engine,
 497                                 struct i915_wa_list *wal)
 498 {
 499         struct intel_gt *gt = engine->gt;
 500         u8 vals[3] = { 0, 0, 0 };
 501         unsigned int i;
 502
 503         for (i = 0; i < 3; i++) {
 504                 u8 ss;
 505
 506                 /*
 507                  * Only consider slices where one, and only one, subslice has 7
 508                  * EUs
 509                  */
 510                 if (!is_power_of_2(gt->info.sseu.subslice_7eu[i]))
 511                         continue;
 512
 513                 /*
 514                  * subslice_7eu[i] != 0 (because of the check above) and
 515                  * ss_max == 4 (maximum number of subslices possible per slice)
 516                  *
 517                  * ->    0 <= ss <= 3;
 518                  */
 519                 ss = ffs(gt->info.sseu.subslice_7eu[i]) - 1;
 520                 vals[i] = 3 - ss;
 521         }
 522
 523         if (vals[0] == 0 && vals[1] == 0 && vals[2] == 0)
 524                 return;
 525
 526         /* Tune IZ hashing. See intel_device_info_runtime_init() */
 527         wa_masked_field_set(wal, GEN7_GT_MODE,
 528                             GEN9_IZ_HASHING_MASK(2) |
 529                             GEN9_IZ_HASHING_MASK(1) |
 530                             GEN9_IZ_HASHING_MASK(0),
 531                             GEN9_IZ_HASHING(2, vals[2]) |
 532                             GEN9_IZ_HASHING(1, vals[1]) |
 533                             GEN9_IZ_HASHING(0, vals[0]));
 534 }
 535
 536 static void skl_ctx_workarounds_init(struct intel_engine_cs *engine,
 537                                      struct i915_wa_list *wal)
 538 {
 539         gen9_ctx_workarounds_init(engine, wal);
 540         skl_tune_iz_hashing(engine, wal);
 541 }
 542
 543 static void bxt_ctx_workarounds_init(struct intel_engine_cs *engine,
 544                                      struct i915_wa_list *wal)
 545 {
 546         gen9_ctx_workarounds_init(engine, wal);
 547
 548         /* WaDisableThreadStallDopClockGating:bxt */
 549         wa_mcr_masked_en(wal, GEN8_ROW_CHICKEN,
 550                          STALL_DOP_GATING_DISABLE);
 551
 552         /* WaToEnableHwFixForPushConstHWBug:bxt */
 553         wa_masked_en(wal, COMMON_SLICE_CHICKEN2,
 554                      GEN8_SBE_DISABLE_REPLAY_BUF_OPTIMIZATION);
 555 }
 556
 557 static void kbl_ctx_workarounds_init(struct intel_engine_cs *engine,
 558                                      struct i915_wa_list *wal)
 559 {
 560         struct drm_i915_private *i915 = engine->i915;
 561
 562         gen9_ctx_workarounds_init(engine, wal);
 563
 564         /* WaToEnableHwFixForPushConstHWBug:kbl */
 565         if (IS_KBL_GRAPHICS_STEP(i915, STEP_C0, STEP_FOREVER))
 566                 wa_masked_en(wal, COMMON_SLICE_CHICKEN2,
 567                              GEN8_SBE_DISABLE_REPLAY_BUF_OPTIMIZATION);
 568
 569         /* WaDisableSbeCacheDispatchPortSharing:kbl */
 570         wa_mcr_masked_en(wal, GEN8_HALF_SLICE_CHICKEN1,
 571                          GEN7_SBE_SS_CACHE_DISPATCH_PORT_SHARING_DISABLE);
 572 }
 573
 574 static void glk_ctx_workarounds_init(struct intel_engine_cs *engine,
 575                                      struct i915_wa_list *wal)
 576 {
 577         gen9_ctx_workarounds_init(engine, wal);
 578
 579         /* WaToEnableHwFixForPushConstHWBug:glk */
 580         wa_masked_en(wal, COMMON_SLICE_CHICKEN2,
 581                      GEN8_SBE_DISABLE_REPLAY_BUF_OPTIMIZATION);
 582 }
 583
 584 static void cfl_ctx_workarounds_init(struct intel_engine_cs *engine,
 585                                      struct i915_wa_list *wal)
 586 {
 587         gen9_ctx_workarounds_init(engine, wal);
 588
 589         /* WaToEnableHwFixForPushConstHWBug:cfl */
 590         wa_masked_en(wal, COMMON_SLICE_CHICKEN2,
 591                      GEN8_SBE_DISABLE_REPLAY_BUF_OPTIMIZATION);
 592
 593         /* WaDisableSbeCacheDispatchPortSharing:cfl */
 594         wa_mcr_masked_en(wal, GEN8_HALF_SLICE_CHICKEN1,
 595                          GEN7_SBE_SS_CACHE_DISPATCH_PORT_SHARING_DISABLE);
 596 }
 597
 598 static void icl_ctx_workarounds_init(struct intel_engine_cs *engine,
 599                                      struct i915_wa_list *wal)
 600 {
 601         /* Wa_1406697149 (WaDisableBankHangMode:icl) */
 602         wa_write(wal,
 603                  GEN8_L3CNTLREG,
 604                  intel_uncore_read(engine->uncore, GEN8_L3CNTLREG) |
 605                  GEN8_ERRDETBCTRL);
 606
 607         /* WaForceEnableNonCoherent:icl
 608          * This is not the same workaround as in early Gen9 platforms, where
 609          * lacking this could cause system hangs, but coherency performance
 610          * overhead is high and only a few compute workloads really need it
 611          * (the register is whitelisted in hardware now, so UMDs can opt in
 612          * for coherency if they have a good reason).
 613          */
 614         wa_mcr_masked_en(wal, ICL_HDC_MODE, HDC_FORCE_NON_COHERENT);
 615
 616         /* WaEnableFloatBlendOptimization:icl */
 617         wa_mcr_add(wal, GEN10_CACHE_MODE_SS, 0,
 618                    _MASKED_BIT_ENABLE(FLOAT_BLEND_OPTIMIZATION_ENABLE),
 619                    0 /* write-only, so skip validation */,
 620                    true);
 621
 622         /* WaDisableGPGPUMidThreadPreemption:icl */
 623         wa_masked_field_set(wal, GEN8_CS_CHICKEN1,
 624                             GEN9_PREEMPT_GPGPU_LEVEL_MASK,
 625                             GEN9_PREEMPT_GPGPU_THREAD_GROUP_LEVEL);
 626
 627         /* allow headerless messages for preemptible GPGPU context */
 628         wa_mcr_masked_en(wal, GEN10_SAMPLER_MODE,
 629                          GEN11_SAMPLER_ENABLE_HEADLESS_MSG);
 630
 631         /* Wa_1604278689:icl,ehl */
 632         wa_write(wal, IVB_FBC_RT_BASE, 0xFFFFFFFF & ~ILK_FBC_RT_VALID);
 633         wa_write_clr_set(wal, IVB_FBC_RT_BASE_UPPER,
 634                          0, /* write-only register; skip validation */
 635                          0xFFFFFFFF);
 636
 637         /* Wa_1406306137:icl,ehl */
 638         wa_mcr_masked_en(wal, GEN9_ROW_CHICKEN4, GEN11_DIS_PICK_2ND_EU);
 639 }
 640
 641 /*
 642  * These settings aren't actually workarounds, but general tuning settings that
 643  * need to be programmed on dg2 platform.
 644  */
 645 static void dg2_ctx_gt_tuning_init(struct intel_engine_cs *engine,
 646                                    struct i915_wa_list *wal)
 647 {
 648         wa_mcr_masked_en(wal, CHICKEN_RASTER_2, TBIMR_FAST_CLIP);
 649         wa_mcr_write_clr_set(wal, XEHP_L3SQCREG5, L3_PWM_TIMER_INIT_VAL_MASK,
 650                              REG_FIELD_PREP(L3_PWM_TIMER_INIT_VAL_MASK, 0x7f));
 651         wa_mcr_add(wal,
 652                    XEHP_FF_MODE2,
 653                    FF_MODE2_TDS_TIMER_MASK,
 654                    FF_MODE2_TDS_TIMER_128,
 655                    0, false);
 656 }
 657
 658 /*
 659  * These settings aren't actually workarounds, but general tuning settings that
 660  * need to be programmed on several platforms.
 661  */
 662 static void gen12_ctx_gt_tuning_init(struct intel_engine_cs *engine,
 663                                      struct i915_wa_list *wal)
 664 {
 665         /*
 666          * Although some platforms refer to it as Wa_1604555607, we need to
 667          * program it even on those that don't explicitly list that
 668          * workaround.
 669          *
 670          * Note that the programming of this register is further modified
 671          * according to the FF_MODE2 guidance given by Wa_1608008084:gen12.
 672          * Wa_1608008084 tells us the FF_MODE2 register will return the wrong
 673          * value when read. The default value for this register is zero for all
 674          * fields and there are no bit masks. So instead of doing a RMW we
 675          * should just write TDS timer value. For the same reason read
 676          * verification is ignored.
 677          */
 678         wa_add(wal,
 679                GEN12_FF_MODE2,
 680                FF_MODE2_TDS_TIMER_MASK,
 681                FF_MODE2_TDS_TIMER_128,
 682                0, false);
 683 }
 684
 685 static void gen12_ctx_workarounds_init(struct intel_engine_cs *engine,
 686                                        struct i915_wa_list *wal)
 687 {
 688         struct drm_i915_private *i915 = engine->i915;
 689
 690         gen12_ctx_gt_tuning_init(engine, wal);
 691
 692         /*
 693          * Wa_1409142259:tgl,dg1,adl-p
 694          * Wa_1409347922:tgl,dg1,adl-p
 695          * Wa_1409252684:tgl,dg1,adl-p
 696          * Wa_1409217633:tgl,dg1,adl-p
 697          * Wa_1409207793:tgl,dg1,adl-p
 698          * Wa_1409178076:tgl,dg1,adl-p
 699          * Wa_1408979724:tgl,dg1,adl-p
 700          * Wa_14010443199:tgl,rkl,dg1,adl-p
 701          * Wa_14010698770:tgl,rkl,dg1,adl-s,adl-p
 702          * Wa_1409342910:tgl,rkl,dg1,adl-s,adl-p
 703          */
 704         wa_masked_en(wal, GEN11_COMMON_SLICE_CHICKEN3,
 705                      GEN12_DISABLE_CPS_AWARE_COLOR_PIPE);
 706
 707         /* WaDisableGPGPUMidThreadPreemption:gen12 */
 708         wa_masked_field_set(wal, GEN8_CS_CHICKEN1,
 709                             GEN9_PREEMPT_GPGPU_LEVEL_MASK,
 710                             GEN9_PREEMPT_GPGPU_THREAD_GROUP_LEVEL);
 711
 712         /*
 713          * Wa_16011163337
 714          *
 715          * Like in gen12_ctx_gt_tuning_init(), read verification is ignored due
 716          * to Wa_1608008084.
 717          */
 718         wa_add(wal,
 719                GEN12_FF_MODE2,
 720                FF_MODE2_GS_TIMER_MASK,
 721                FF_MODE2_GS_TIMER_224,
 722                0, false);
 723
 724         if (!IS_DG1(i915))
 725                 /* Wa_1806527549 */
 726                 wa_masked_en(wal, HIZ_CHICKEN, HZ_DEPTH_TEST_LE_GE_OPT_DISABLE);
 727 }
 728
 729 static void dg1_ctx_workarounds_init(struct intel_engine_cs *engine,
 730                                      struct i915_wa_list *wal)
 731 {
 732         gen12_ctx_workarounds_init(engine, wal);
 733
 734         /* Wa_1409044764 */
 735         wa_masked_dis(wal, GEN11_COMMON_SLICE_CHICKEN3,
 736                       DG1_FLOAT_POINT_BLEND_OPT_STRICT_MODE_EN);
 737
 738         /* Wa_22010493298 */
 739         wa_masked_en(wal, HIZ_CHICKEN,
 740                      DG1_HZ_READ_SUPPRESSION_OPTIMIZATION_DISABLE);
 741 }
 742
 743 static void dg2_ctx_workarounds_init(struct intel_engine_cs *engine,
 744                                      struct i915_wa_list *wal)
 745 {
 746         dg2_ctx_gt_tuning_init(engine, wal);
 747
 748         /* Wa_16011186671:dg2_g11 */
 749         if (IS_DG2_GRAPHICS_STEP(engine->i915, G11, STEP_A0, STEP_B0)) {
 750                 wa_mcr_masked_dis(wal, VFLSKPD, DIS_MULT_MISS_RD_SQUASH);
 751                 wa_mcr_masked_en(wal, VFLSKPD, DIS_OVER_FETCH_CACHE);
 752         }
 753
 754         if (IS_DG2_GRAPHICS_STEP(engine->i915, G10, STEP_A0, STEP_B0)) {
 755                 /* Wa_14010469329:dg2_g10 */
 756                 wa_mcr_masked_en(wal, XEHP_COMMON_SLICE_CHICKEN3,
 757                                  XEHP_DUAL_SIMD8_SEQ_MERGE_DISABLE);
 758
 759                 /*
 760                  * Wa_22010465075:dg2_g10
 761                  * Wa_22010613112:dg2_g10
 762                  * Wa_14010698770:dg2_g10
 763                  */
 764                 wa_mcr_masked_en(wal, XEHP_COMMON_SLICE_CHICKEN3,
 765                                  GEN12_DISABLE_CPS_AWARE_COLOR_PIPE);
 766         }
 767
 768         /* Wa_16013271637:dg2 */
 769         wa_mcr_masked_en(wal, XEHP_SLICE_COMMON_ECO_CHICKEN1,
 770                          MSC_MSAA_REODER_BUF_BYPASS_DISABLE);
 771
 772         /* Wa_14014947963:dg2 */
 773         if (IS_DG2_GRAPHICS_STEP(engine->i915, G10, STEP_B0, STEP_FOREVER) ||
 774             IS_DG2_G11(engine->i915) || IS_DG2_G12(engine->i915))
 775                 wa_masked_field_set(wal, VF_PREEMPTION, PREEMPTION_VERTEX_COUNT, 0x4000);
 776
 777         /* Wa_18018764978:dg2 */
 778         if (IS_DG2_GRAPHICS_STEP(engine->i915, G10, STEP_C0, STEP_FOREVER) ||
 779             IS_DG2_G11(engine->i915) || IS_DG2_G12(engine->i915))
 780                 wa_masked_en(wal, PSS_MODE2, SCOREBOARD_STALL_FLUSH_CONTROL);
 781
 782         /* Wa_15010599737:dg2 */
 783         wa_mcr_masked_en(wal, CHICKEN_RASTER_1, DIS_SF_ROUND_NEAREST_EVEN);
 784
 785         /* Wa_18019271663:dg2 */
 786         wa_masked_en(wal, CACHE_MODE_1, MSAA_OPTIMIZATION_REDUC_DISABLE);
 787 }
 788
 789 static void fakewa_disable_nestedbb_mode(struct intel_engine_cs *engine,
 790                                          struct i915_wa_list *wal)
 791 {
 792         /*
 793          * This is a "fake" workaround defined by software to ensure we
 794          * maintain reliable, backward-compatible behavior for userspace with
 795          * regards to how nested MI_BATCH_BUFFER_START commands are handled.
 796          *
 797          * The per-context setting of MI_MODE[12] determines whether the bits
 798          * of a nested MI_BATCH_BUFFER_START instruction should be interpreted
 799          * in the traditional manner or whether they should instead use a new
 800          * tgl+ meaning that breaks backward compatibility, but allows nesting
 801          * into 3rd-level batchbuffers.  When this new capability was first
 802          * added in TGL, it remained off by default unless a context
 803          * intentionally opted in to the new behavior.  However Xe_HPG now
 804          * flips this on by default and requires that we explicitly opt out if
 805          * we don't want the new behavior.
 806          *
 807          * From a SW perspective, we want to maintain the backward-compatible
 808          * behavior for userspace, so we'll apply a fake workaround to set it
 809          * back to the legacy behavior on platforms where the hardware default
 810          * is to break compatibility.  At the moment there is no Linux
 811          * userspace that utilizes third-level batchbuffers, so this will avoid
 812          * userspace from needing to make any changes.  using the legacy
 813          * meaning is the correct thing to do.  If/when we have userspace
 814          * consumers that want to utilize third-level batch nesting, we can
 815          * provide a context parameter to allow them to opt-in.
 816          */
 817         wa_masked_dis(wal, RING_MI_MODE(engine->mmio_base), TGL_NESTED_BB_EN);
 818 }
 819
 820 static void gen12_ctx_gt_mocs_init(struct intel_engine_cs *engine,
 821                                    struct i915_wa_list *wal)
 822 {
 823         u8 mocs;
 824
 825         /*
 826          * Some blitter commands do not have a field for MOCS, those
 827          * commands will use MOCS index pointed by BLIT_CCTL.
 828          * BLIT_CCTL registers are needed to be programmed to un-cached.
 829          */
 830         if (engine->class == COPY_ENGINE_CLASS) {
 831                 mocs = engine->gt->mocs.uc_index;
 832                 wa_write_clr_set(wal,
 833                                  BLIT_CCTL(engine->mmio_base),
 834                                  BLIT_CCTL_MASK,
 835                                  BLIT_CCTL_MOCS(mocs, mocs));
 836         }
 837 }
 838
 839 /*
 840  * gen12_ctx_gt_fake_wa_init() aren't programmingan official workaround
 841  * defined by the hardware team, but it programming general context registers.
 842  * Adding those context register programming in context workaround
 843  * allow us to use the wa framework for proper application and validation.
 844  */
 845 static void
 846 gen12_ctx_gt_fake_wa_init(struct intel_engine_cs *engine,
 847                           struct i915_wa_list *wal)
 848 {
 849         if (GRAPHICS_VER_FULL(engine->i915) >= IP_VER(12, 55))
 850                 fakewa_disable_nestedbb_mode(engine, wal);
 851
 852         gen12_ctx_gt_mocs_init(engine, wal);
 853 }
 854
 855 static void
 856 __intel_engine_init_ctx_wa(struct intel_engine_cs *engine,
 857                            struct i915_wa_list *wal,
 858                            const char *name)
 859 {
 860         struct drm_i915_private *i915 = engine->i915;
 861
 862         wa_init_start(wal, engine->gt, name, engine->name);
 863
 864         /* Applies to all engines */
 865         /*
 866          * Fake workarounds are not the actual workaround but
 867          * programming of context registers using workaround framework.
 868          */
 869         if (GRAPHICS_VER(i915) >= 12)
 870                 gen12_ctx_gt_fake_wa_init(engine, wal);
 871
 872         if (engine->class != RENDER_CLASS)
 873                 goto done;
 874
 875         if (IS_PONTEVECCHIO(i915))
 876                 ; /* noop; none at this time */
 877         else if (IS_DG2(i915))
 878                 dg2_ctx_workarounds_init(engine, wal);
 879         else if (IS_XEHPSDV(i915))
 880                 ; /* noop; none at this time */
 881         else if (IS_DG1(i915))
 882                 dg1_ctx_workarounds_init(engine, wal);
 883         else if (GRAPHICS_VER(i915) == 12)
 884                 gen12_ctx_workarounds_init(engine, wal);
 885         else if (GRAPHICS_VER(i915) == 11)
 886                 icl_ctx_workarounds_init(engine, wal);
 887         else if (IS_COFFEELAKE(i915) || IS_COMETLAKE(i915))
 888                 cfl_ctx_workarounds_init(engine, wal);
 889         else if (IS_GEMINILAKE(i915))
 890                 glk_ctx_workarounds_init(engine, wal);
 891         else if (IS_KABYLAKE(i915))
 892                 kbl_ctx_workarounds_init(engine, wal);
 893         else if (IS_BROXTON(i915))
 894                 bxt_ctx_workarounds_init(engine, wal);
 895         else if (IS_SKYLAKE(i915))
 896                 skl_ctx_workarounds_init(engine, wal);
 897         else if (IS_CHERRYVIEW(i915))
 898                 chv_ctx_workarounds_init(engine, wal);
 899         else if (IS_BROADWELL(i915))
 900                 bdw_ctx_workarounds_init(engine, wal);
 901         else if (GRAPHICS_VER(i915) == 7)
 902                 gen7_ctx_workarounds_init(engine, wal);
 903         else if (GRAPHICS_VER(i915) == 6)
 904                 gen6_ctx_workarounds_init(engine, wal);
 905         else if (GRAPHICS_VER(i915) < 8)
 906                 ;
 907         else
 908                 MISSING_CASE(GRAPHICS_VER(i915));
 909
 910 done:
 911         wa_init_finish(wal);
 912 }
 913
 914 void intel_engine_init_ctx_wa(struct intel_engine_cs *engine)
 915 {
 916         __intel_engine_init_ctx_wa(engine, &engine->ctx_wa_list, "context");
 917 }
 918
 919 int intel_engine_emit_ctx_wa(struct i915_request *rq)
 920 {
 921         struct i915_wa_list *wal = &rq->engine->ctx_wa_list;
 922         struct i915_wa *wa;
 923         unsigned int i;
 924         u32 *cs;
 925         int ret;
 926
 927         if (wal->count == 0)
 928                 return 0;
 929
 930         ret = rq->engine->emit_flush(rq, EMIT_BARRIER);
 931         if (ret)
 932                 return ret;
 933
 934         cs = intel_ring_begin(rq, (wal->count * 2 + 2));
 935         if (IS_ERR(cs))
 936                 return PTR_ERR(cs);
 937
 938         *cs++ = MI_LOAD_REGISTER_IMM(wal->count);
 939         for (i = 0, wa = wal->list; i < wal->count; i++, wa++) {
 940                 *cs++ = i915_mmio_reg_offset(wa->reg);
 941                 *cs++ = wa->set;
 942         }
 943         *cs++ = MI_NOOP;
 944
 945         intel_ring_advance(rq, cs);
 946
 947         ret = rq->engine->emit_flush(rq, EMIT_BARRIER);
 948         if (ret)
 949                 return ret;
 950
 951         return 0;
 952 }
 953
 954 static void
 955 gen4_gt_workarounds_init(struct intel_gt *gt,
 956                          struct i915_wa_list *wal)
 957 {
 958         /* WaDisable_RenderCache_OperationalFlush:gen4,ilk */
 959         wa_masked_dis(wal, CACHE_MODE_0, RC_OP_FLUSH_ENABLE);
 960 }
 961
 962 static void
 963 g4x_gt_workarounds_init(struct intel_gt *gt, struct i915_wa_list *wal)
 964 {
 965         gen4_gt_workarounds_init(gt, wal);
 966
 967         /* WaDisableRenderCachePipelinedFlush:g4x,ilk */
 968         wa_masked_en(wal, CACHE_MODE_0, CM0_PIPELINED_RENDER_FLUSH_DISABLE);
 969 }
 970
 971 static void
 972 ilk_gt_workarounds_init(struct intel_gt *gt, struct i915_wa_list *wal)
 973 {
 974         g4x_gt_workarounds_init(gt, wal);
 975
 976         wa_masked_en(wal, _3D_CHICKEN2, _3D_CHICKEN2_WM_READ_PIPELINED);
 977 }
 978
 979 static void
 980 snb_gt_workarounds_init(struct intel_gt *gt, struct i915_wa_list *wal)
 981 {
 982 }
 983
 984 static void
 985 ivb_gt_workarounds_init(struct intel_gt *gt, struct i915_wa_list *wal)
 986 {
 987         /* Apply the WaDisableRHWOOptimizationForRenderHang:ivb workaround. */
 988         wa_masked_dis(wal,
 989                       GEN7_COMMON_SLICE_CHICKEN1,
 990                       GEN7_CSC1_RHWO_OPT_DISABLE_IN_RCC);
 991
 992         /* WaApplyL3ControlAndL3ChickenMode:ivb */
 993         wa_write(wal, GEN7_L3CNTLREG1, GEN7_WA_FOR_GEN7_L3_CONTROL);
 994         wa_write(wal, GEN7_L3_CHICKEN_MODE_REGISTER, GEN7_WA_L3_CHICKEN_MODE);
 995
 996         /* WaForceL3Serialization:ivb */
 997         wa_write_clr(wal, GEN7_L3SQCREG4, L3SQ_URB_READ_CAM_MATCH_DISABLE);
 998 }
 999
1000 static void
1001 vlv_gt_workarounds_init(struct intel_gt *gt, struct i915_wa_list *wal)
1002 {
1003         /* WaForceL3Serialization:vlv */
1004         wa_write_clr(wal, GEN7_L3SQCREG4, L3SQ_URB_READ_CAM_MATCH_DISABLE);
1005
1006         /*
1007          * WaIncreaseL3CreditsForVLVB0:vlv
1008          * This is the hardware default actually.
1009          */
1010         wa_write(wal, GEN7_L3SQCREG1, VLV_B0_WA_L3SQCREG1_VALUE);
1011 }
1012
1013 static void
1014 hsw_gt_workarounds_init(struct intel_gt *gt, struct i915_wa_list *wal)
1015 {
1016         /* L3 caching of data atomics doesn't work -- disable it. */
1017         wa_write(wal, HSW_SCRATCH1, HSW_SCRATCH1_L3_DATA_ATOMICS_DISABLE);
1018
1019         wa_add(wal,
1020                HSW_ROW_CHICKEN3, 0,
1021                _MASKED_BIT_ENABLE(HSW_ROW_CHICKEN3_L3_GLOBAL_ATOMICS_DISABLE),
1022                0 /* XXX does this reg exist? */, true);
1023
1024         /* WaVSRefCountFullforceMissDisable:hsw */
1025         wa_write_clr(wal, GEN7_FF_THREAD_MODE, GEN7_FF_VS_REF_CNT_FFME);
1026 }
1027
1028 static void
1029 gen9_wa_init_mcr(struct drm_i915_private *i915, struct i915_wa_list *wal)
1030 {
1031         const struct sseu_dev_info *sseu = &to_gt(i915)->info.sseu;
1032         unsigned int slice, subslice;
1033         u32 mcr, mcr_mask;
1034
1035         GEM_BUG_ON(GRAPHICS_VER(i915) != 9);
1036
1037         /*
1038          * WaProgramMgsrForCorrectSliceSpecificMmioReads:gen9,glk,kbl,cml
1039          * Before any MMIO read into slice/subslice specific registers, MCR
1040          * packet control register needs to be programmed to point to any
1041          * enabled s/ss pair. Otherwise, incorrect values will be returned.
1042          * This means each subsequent MMIO read will be forwarded to an
1043          * specific s/ss combination, but this is OK since these registers
1044          * are consistent across s/ss in almost all cases. In the rare
1045          * occasions, such as INSTDONE, where this value is dependent
1046          * on s/ss combo, the read should be done with read_subslice_reg.
1047          */
1048         slice = ffs(sseu->slice_mask) - 1;
1049         GEM_BUG_ON(slice >= ARRAY_SIZE(sseu->subslice_mask.hsw));
1050         subslice = ffs(intel_sseu_get_hsw_subslices(sseu, slice));
1051         GEM_BUG_ON(!subslice);
1052         subslice--;
1053
1054         /*
1055          * We use GEN8_MCR..() macros to calculate the |mcr| value for
1056          * Gen9 to address WaProgramMgsrForCorrectSliceSpecificMmioReads
1057          */
1058         mcr = GEN8_MCR_SLICE(slice) | GEN8_MCR_SUBSLICE(subslice);
1059         mcr_mask = GEN8_MCR_SLICE_MASK | GEN8_MCR_SUBSLICE_MASK;
1060
1061         drm_dbg(&i915->drm, "MCR slice:%d/subslice:%d = %x\n", slice, subslice, mcr);
1062
1063         wa_write_clr_set(wal, GEN8_MCR_SELECTOR, mcr_mask, mcr);
1064 }
1065
1066 static void
1067 gen9_gt_workarounds_init(struct intel_gt *gt, struct i915_wa_list *wal)
1068 {
1069         struct drm_i915_private *i915 = gt->i915;
1070
1071         /* WaProgramMgsrForCorrectSliceSpecificMmioReads:glk,kbl,cml,gen9 */
1072         gen9_wa_init_mcr(i915, wal);
1073
1074         /* WaDisableKillLogic:bxt,skl,kbl */
1075         if (!IS_COFFEELAKE(i915) && !IS_COMETLAKE(i915))
1076                 wa_write_or(wal,
1077                             GAM_ECOCHK,
1078                             ECOCHK_DIS_TLB);
1079
1080         if (HAS_LLC(i915)) {
1081                 /* WaCompressedResourceSamplerPbeMediaNewHashMode:skl,kbl
1082                  *
1083                  * Must match Display Engine. See
1084                  * WaCompressedResourceDisplayNewHashMode.
1085                  */
1086                 wa_write_or(wal,
1087                             MMCD_MISC_CTRL,
1088                             MMCD_PCLA | MMCD_HOTSPOT_EN);
1089         }
1090
1091         /* WaDisableHDCInvalidation:skl,bxt,kbl,cfl */
1092         wa_write_or(wal,
1093                     GAM_ECOCHK,
1094                     BDW_DISABLE_HDC_INVALIDATION);
1095 }
1096
1097 static void
1098 skl_gt_workarounds_init(struct intel_gt *gt, struct i915_wa_list *wal)
1099 {
1100         gen9_gt_workarounds_init(gt, wal);
1101
1102         /* WaDisableGafsUnitClkGating:skl */
1103         wa_write_or(wal,
1104                     GEN7_UCGCTL4,
1105                     GEN8_EU_GAUNIT_CLOCK_GATE_DISABLE);
1106
1107         /* WaInPlaceDecompressionHang:skl */
1108         if (IS_SKL_GRAPHICS_STEP(gt->i915, STEP_A0, STEP_H0))
1109                 wa_write_or(wal,
1110                             GEN9_GAMT_ECO_REG_RW_IA,
1111                             GAMT_ECO_ENABLE_IN_PLACE_DECOMPRESS);
1112 }
1113
1114 static void
1115 kbl_gt_workarounds_init(struct intel_gt *gt, struct i915_wa_list *wal)
1116 {
1117         gen9_gt_workarounds_init(gt, wal);
1118
1119         /* WaDisableDynamicCreditSharing:kbl */
1120         if (IS_KBL_GRAPHICS_STEP(gt->i915, 0, STEP_C0))
1121                 wa_write_or(wal,
1122                             GAMT_CHKN_BIT_REG,
1123                             GAMT_CHKN_DISABLE_DYNAMIC_CREDIT_SHARING);
1124
1125         /* WaDisableGafsUnitClkGating:kbl */
1126         wa_write_or(wal,
1127                     GEN7_UCGCTL4,
1128                     GEN8_EU_GAUNIT_CLOCK_GATE_DISABLE);
1129
1130         /* WaInPlaceDecompressionHang:kbl */
1131         wa_write_or(wal,
1132                     GEN9_GAMT_ECO_REG_RW_IA,
1133                     GAMT_ECO_ENABLE_IN_PLACE_DECOMPRESS);
1134 }
1135
1136 static void
1137 glk_gt_workarounds_init(struct intel_gt *gt, struct i915_wa_list *wal)
1138 {
1139         gen9_gt_workarounds_init(gt, wal);
1140 }
1141
1142 static void
1143 cfl_gt_workarounds_init(struct intel_gt *gt, struct i915_wa_list *wal)
1144 {
1145         gen9_gt_workarounds_init(gt, wal);
1146
1147         /* WaDisableGafsUnitClkGating:cfl */
1148         wa_write_or(wal,
1149                     GEN7_UCGCTL4,
1150                     GEN8_EU_GAUNIT_CLOCK_GATE_DISABLE);
1151
1152         /* WaInPlaceDecompressionHang:cfl */
1153         wa_write_or(wal,
1154                     GEN9_GAMT_ECO_REG_RW_IA,
1155                     GAMT_ECO_ENABLE_IN_PLACE_DECOMPRESS);
1156 }
1157
1158 static void __set_mcr_steering(struct i915_wa_list *wal,
1159                                i915_reg_t steering_reg,
1160                                unsigned int slice, unsigned int subslice)
1161 {
1162         u32 mcr, mcr_mask;
1163
1164         mcr = GEN11_MCR_SLICE(slice) | GEN11_MCR_SUBSLICE(subslice);
1165         mcr_mask = GEN11_MCR_SLICE_MASK | GEN11_MCR_SUBSLICE_MASK;
1166
1167         wa_write_clr_set(wal, steering_reg, mcr_mask, mcr);
1168 }
1169
1170 static void debug_dump_steering(struct intel_gt *gt)
1171 {
1172         struct drm_printer p = drm_debug_printer("MCR Steering:");
1173
1174         if (drm_debug_enabled(DRM_UT_DRIVER))
1175                 intel_gt_mcr_report_steering(&p, gt, false);
1176 }
1177
1178 static void __add_mcr_wa(struct intel_gt *gt, struct i915_wa_list *wal,
1179                          unsigned int slice, unsigned int subslice)
1180 {
1181         __set_mcr_steering(wal, GEN8_MCR_SELECTOR, slice, subslice);
1182
1183         gt->default_steering.groupid = slice;
1184         gt->default_steering.instanceid = subslice;
1185
1186         debug_dump_steering(gt);
1187 }
1188
1189 static void
1190 icl_wa_init_mcr(struct intel_gt *gt, struct i915_wa_list *wal)
1191 {
1192         const struct sseu_dev_info *sseu = &gt->info.sseu;
1193         unsigned int subslice;
1194
1195         GEM_BUG_ON(GRAPHICS_VER(gt->i915) < 11);
1196         GEM_BUG_ON(hweight8(sseu->slice_mask) > 1);
1197
1198         /*
1199          * Although a platform may have subslices, we need to always steer
1200          * reads to the lowest instance that isn't fused off.  When Render
1201          * Power Gating is enabled, grabbing forcewake will only power up a
1202          * single subslice (the "minconfig") if there isn't a real workload
1203          * that needs to be run; this means that if we steer register reads to
1204          * one of the higher subslices, we run the risk of reading back 0's or
1205          * random garbage.
1206          */
1207         subslice = __ffs(intel_sseu_get_hsw_subslices(sseu, 0));
1208
1209         /*
1210          * If the subslice we picked above also steers us to a valid L3 bank,
1211          * then we can just rely on the default steering and won't need to
1212          * worry about explicitly re-steering L3BANK reads later.
1213          */
1214         if (gt->info.l3bank_mask & BIT(subslice))
1215                 gt->steering_table[L3BANK] = NULL;
1216
1217         __add_mcr_wa(gt, wal, 0, subslice);
1218 }
1219
1220 static void
1221 xehp_init_mcr(struct intel_gt *gt, struct i915_wa_list *wal)
1222 {
1223         const struct sseu_dev_info *sseu = &gt->info.sseu;
1224         unsigned long slice, subslice = 0, slice_mask = 0;
1225         u32 lncf_mask = 0;
1226         int i;
1227
1228         /*
1229          * On Xe_HP the steering increases in complexity. There are now several
1230          * more units that require steering and we're not guaranteed to be able
1231          * to find a common setting for all of them. These are:
1232          * - GSLICE (fusable)
1233          * - DSS (sub-unit within gslice; fusable)
1234          * - L3 Bank (fusable)
1235          * - MSLICE (fusable)
1236          * - LNCF (sub-unit within mslice; always present if mslice is present)
1237          *
1238          * We'll do our default/implicit steering based on GSLICE (in the
1239          * sliceid field) and DSS (in the subsliceid field).  If we can
1240          * find overlap between the valid MSLICE and/or LNCF values with
1241          * a suitable GSLICE, then we can just re-use the default value and
1242          * skip and explicit steering at runtime.
1243          *
1244          * We only need to look for overlap between GSLICE/MSLICE/LNCF to find
1245          * a valid sliceid value.  DSS steering is the only type of steering
1246          * that utilizes the 'subsliceid' bits.
1247          *
1248          * Also note that, even though the steering domain is called "GSlice"
1249          * and it is encoded in the register using the gslice format, the spec
1250          * says that the combined (geometry | compute) fuse should be used to
1251          * select the steering.
1252          */
1253
1254         /* Find the potential gslice candidates */
1255         slice_mask = intel_slicemask_from_xehp_dssmask(sseu->subslice_mask,
1256                                                        GEN_DSS_PER_GSLICE);
1257
1258         /*
1259          * Find the potential LNCF candidates.  Either LNCF within a valid
1260          * mslice is fine.
1261          */
1262         for_each_set_bit(i, &gt->info.mslice_mask, GEN12_MAX_MSLICES)
1263                 lncf_mask |= (0x3 << (i * 2));
1264
1265         /*
1266          * Are there any sliceid values that work for both GSLICE and LNCF
1267          * steering?
1268          */
1269         if (slice_mask & lncf_mask) {
1270                 slice_mask &= lncf_mask;
1271                 gt->steering_table[LNCF] = NULL;
1272         }
1273
1274         /* How about sliceid values that also work for MSLICE steering? */
1275         if (slice_mask & gt->info.mslice_mask) {
1276                 slice_mask &= gt->info.mslice_mask;
1277                 gt->steering_table[MSLICE] = NULL;
1278         }
1279
1280         if (IS_XEHPSDV(gt->i915) && slice_mask & BIT(0))
1281                 gt->steering_table[GAM] = NULL;
1282
1283         slice = __ffs(slice_mask);
1284         subslice = intel_sseu_find_first_xehp_dss(sseu, GEN_DSS_PER_GSLICE, slice) %
1285                 GEN_DSS_PER_GSLICE;
1286
1287         __add_mcr_wa(gt, wal, slice, subslice);
1288
1289         /*
1290          * SQIDI ranges are special because they use different steering
1291          * registers than everything else we work with.  On XeHP SDV and
1292          * DG2-G10, any value in the steering registers will work fine since
1293          * all instances are present, but DG2-G11 only has SQIDI instances at
1294          * ID's 2 and 3, so we need to steer to one of those.  For simplicity
1295          * we'll just steer to a hardcoded "2" since that value will work
1296          * everywhere.
1297          */
1298         __set_mcr_steering(wal, MCFG_MCR_SELECTOR, 0, 2);
1299         __set_mcr_steering(wal, SF_MCR_SELECTOR, 0, 2);
1300
1301         /*
1302          * On DG2, GAM registers have a dedicated steering control register
1303          * and must always be programmed to a hardcoded groupid of "1."
1304          */
1305         if (IS_DG2(gt->i915))
1306                 __set_mcr_steering(wal, GAM_MCR_SELECTOR, 1, 0);
1307 }
1308
1309 static void
1310 pvc_init_mcr(struct intel_gt *gt, struct i915_wa_list *wal)
1311 {
1312         unsigned int dss;
1313
1314         /*
1315          * Setup implicit steering for COMPUTE and DSS ranges to the first
1316          * non-fused-off DSS.  All other types of MCR registers will be
1317          * explicitly steered.
1318          */
1319         dss = intel_sseu_find_first_xehp_dss(&gt->info.sseu, 0, 0);
1320         __add_mcr_wa(gt, wal, dss / GEN_DSS_PER_CSLICE, dss % GEN_DSS_PER_CSLICE);
1321 }
1322
1323 static void
1324 icl_gt_workarounds_init(struct intel_gt *gt, struct i915_wa_list *wal)
1325 {
1326         struct drm_i915_private *i915 = gt->i915;
1327
1328         icl_wa_init_mcr(gt, wal);
1329
1330         /* WaModifyGamTlbPartitioning:icl */
1331         wa_write_clr_set(wal,
1332                          GEN11_GACB_PERF_CTRL,
1333                          GEN11_HASH_CTRL_MASK,
1334                          GEN11_HASH_CTRL_BIT0 | GEN11_HASH_CTRL_BIT4);
1335
1336         /* Wa_1405766107:icl
1337          * Formerly known as WaCL2SFHalfMaxAlloc
1338          */
1339         wa_write_or(wal,
1340                     GEN11_LSN_UNSLCVC,
1341                     GEN11_LSN_UNSLCVC_GAFS_HALF_SF_MAXALLOC |
1342                     GEN11_LSN_UNSLCVC_GAFS_HALF_CL2_MAXALLOC);
1343
1344         /* Wa_220166154:icl
1345          * Formerly known as WaDisCtxReload
1346          */
1347         wa_write_or(wal,
1348                     GEN8_GAMW_ECO_DEV_RW_IA,
1349                     GAMW_ECO_DEV_CTX_RELOAD_DISABLE);
1350
1351         /* Wa_1406463099:icl
1352          * Formerly known as WaGamTlbPendError
1353          */
1354         wa_write_or(wal,
1355                     GAMT_CHKN_BIT_REG,
1356                     GAMT_CHKN_DISABLE_L3_COH_PIPE);
1357
1358         /*
1359          * Wa_1408615072:icl,ehl  (vsunit)
1360          * Wa_1407596294:icl,ehl  (hsunit)
1361          */
1362         wa_write_or(wal, UNSLICE_UNIT_LEVEL_CLKGATE,
1363                     VSUNIT_CLKGATE_DIS | HSUNIT_CLKGATE_DIS);
1364
1365         /* Wa_1407352427:icl,ehl */
1366         wa_write_or(wal, UNSLICE_UNIT_LEVEL_CLKGATE2,
1367                     PSDUNIT_CLKGATE_DIS);
1368
1369         /* Wa_1406680159:icl,ehl */
1370         wa_mcr_write_or(wal,
1371                         GEN11_SUBSLICE_UNIT_LEVEL_CLKGATE,
1372                         GWUNIT_CLKGATE_DIS);
1373
1374         /* Wa_1607087056:icl,ehl,jsl */
1375         if (IS_ICELAKE(i915) ||
1376             IS_JSL_EHL_GRAPHICS_STEP(i915, STEP_A0, STEP_B0))
1377                 wa_write_or(wal,
1378                             GEN11_SLICE_UNIT_LEVEL_CLKGATE,
1379                             L3_CLKGATE_DIS | L3_CR2X_CLKGATE_DIS);
1380
1381         /*
1382          * This is not a documented workaround, but rather an optimization
1383          * to reduce sampler power.
1384          */
1385         wa_mcr_write_clr(wal, GEN10_DFR_RATIO_EN_AND_CHICKEN, DFR_DISABLE);
1386 }
1387
1388 /*
1389  * Though there are per-engine instances of these registers,
1390  * they retain their value through engine resets and should
1391  * only be provided on the GT workaround list rather than
1392  * the engine-specific workaround list.
1393  */
1394 static void
1395 wa_14011060649(struct intel_gt *gt, struct i915_wa_list *wal)
1396 {
1397         struct intel_engine_cs *engine;
1398         int id;
1399
1400         for_each_engine(engine, gt, id) {
1401                 if (engine->class != VIDEO_DECODE_CLASS ||
1402                     (engine->instance % 2))
1403                         continue;
1404
1405                 wa_write_or(wal, VDBOX_CGCTL3F10(engine->mmio_base),
1406                             IECPUNIT_CLKGATE_DIS);
1407         }
1408 }
1409
1410 static void
1411 gen12_gt_workarounds_init(struct intel_gt *gt, struct i915_wa_list *wal)
1412 {
1413         icl_wa_init_mcr(gt, wal);
1414
1415         /* Wa_14011060649:tgl,rkl,dg1,adl-s,adl-p */
1416         wa_14011060649(gt, wal);
1417
1418         /* Wa_14011059788:tgl,rkl,adl-s,dg1,adl-p */
1419         wa_mcr_write_or(wal, GEN10_DFR_RATIO_EN_AND_CHICKEN, DFR_DISABLE);
1420 }
1421
1422 static void
1423 tgl_gt_workarounds_init(struct intel_gt *gt, struct i915_wa_list *wal)
1424 {
1425         struct drm_i915_private *i915 = gt->i915;
1426
1427         gen12_gt_workarounds_init(gt, wal);
1428
1429         /* Wa_1409420604:tgl */
1430         if (IS_TGL_UY_GRAPHICS_STEP(i915, STEP_A0, STEP_B0))
1431                 wa_mcr_write_or(wal,
1432                                 SUBSLICE_UNIT_LEVEL_CLKGATE2,
1433                                 CPSSUNIT_CLKGATE_DIS);
1434
1435         /* Wa_1607087056:tgl also know as BUG:1409180338 */
1436         if (IS_TGL_UY_GRAPHICS_STEP(i915, STEP_A0, STEP_B0))
1437                 wa_write_or(wal,
1438                             GEN11_SLICE_UNIT_LEVEL_CLKGATE,
1439                             L3_CLKGATE_DIS | L3_CR2X_CLKGATE_DIS);
1440
1441         /* Wa_1408615072:tgl[a0] */
1442         if (IS_TGL_UY_GRAPHICS_STEP(i915, STEP_A0, STEP_B0))
1443                 wa_write_or(wal, UNSLICE_UNIT_LEVEL_CLKGATE2,
1444                             VSUNIT_CLKGATE_DIS_TGL);
1445 }
1446
1447 static void
1448 dg1_gt_workarounds_init(struct intel_gt *gt, struct i915_wa_list *wal)
1449 {
1450         struct drm_i915_private *i915 = gt->i915;
1451
1452         gen12_gt_workarounds_init(gt, wal);
1453
1454         /* Wa_1607087056:dg1 */
1455         if (IS_DG1_GRAPHICS_STEP(i915, STEP_A0, STEP_B0))
1456                 wa_write_or(wal,
1457                             GEN11_SLICE_UNIT_LEVEL_CLKGATE,
1458                             L3_CLKGATE_DIS | L3_CR2X_CLKGATE_DIS);
1459
1460         /* Wa_1409420604:dg1 */
1461         if (IS_DG1(i915))
1462                 wa_mcr_write_or(wal,
1463                                 SUBSLICE_UNIT_LEVEL_CLKGATE2,
1464                                 CPSSUNIT_CLKGATE_DIS);
1465
1466         /* Wa_1408615072:dg1 */
1467         /* Empirical testing shows this register is unaffected by engine reset. */
1468         if (IS_DG1(i915))
1469                 wa_write_or(wal, UNSLICE_UNIT_LEVEL_CLKGATE2,
1470                             VSUNIT_CLKGATE_DIS_TGL);
1471 }
1472
1473 static void
1474 xehpsdv_gt_workarounds_init(struct intel_gt *gt, struct i915_wa_list *wal)
1475 {
1476         struct drm_i915_private *i915 = gt->i915;
1477
1478         xehp_init_mcr(gt, wal);
1479
1480         /* Wa_1409757795:xehpsdv */
1481         wa_mcr_write_or(wal, SCCGCTL94DC, CG3DDISURB);
1482
1483         /* Wa_16011155590:xehpsdv */
1484         if (IS_XEHPSDV_GRAPHICS_STEP(i915, STEP_A0, STEP_B0))
1485                 wa_write_or(wal, UNSLICE_UNIT_LEVEL_CLKGATE,
1486                             TSGUNIT_CLKGATE_DIS);
1487
1488         /* Wa_14011780169:xehpsdv */
1489         if (IS_XEHPSDV_GRAPHICS_STEP(i915, STEP_B0, STEP_FOREVER)) {
1490                 wa_write_or(wal, UNSLCGCTL9440, GAMTLBOACS_CLKGATE_DIS |
1491                             GAMTLBVDBOX7_CLKGATE_DIS |
1492                             GAMTLBVDBOX6_CLKGATE_DIS |
1493                             GAMTLBVDBOX5_CLKGATE_DIS |
1494                             GAMTLBVDBOX4_CLKGATE_DIS |
1495                             GAMTLBVDBOX3_CLKGATE_DIS |
1496                             GAMTLBVDBOX2_CLKGATE_DIS |
1497                             GAMTLBVDBOX1_CLKGATE_DIS |
1498                             GAMTLBVDBOX0_CLKGATE_DIS |
1499                             GAMTLBKCR_CLKGATE_DIS |
1500                             GAMTLBGUC_CLKGATE_DIS |
1501                             GAMTLBBLT_CLKGATE_DIS);
1502                 wa_write_or(wal, UNSLCGCTL9444, GAMTLBGFXA0_CLKGATE_DIS |
1503                             GAMTLBGFXA1_CLKGATE_DIS |
1504                             GAMTLBCOMPA0_CLKGATE_DIS |
1505                             GAMTLBCOMPA1_CLKGATE_DIS |
1506                             GAMTLBCOMPB0_CLKGATE_DIS |
1507                             GAMTLBCOMPB1_CLKGATE_DIS |
1508                             GAMTLBCOMPC0_CLKGATE_DIS |
1509                             GAMTLBCOMPC1_CLKGATE_DIS |
1510                             GAMTLBCOMPD0_CLKGATE_DIS |
1511                             GAMTLBCOMPD1_CLKGATE_DIS |
1512                             GAMTLBMERT_CLKGATE_DIS   |
1513                             GAMTLBVEBOX3_CLKGATE_DIS |
1514                             GAMTLBVEBOX2_CLKGATE_DIS |
1515                             GAMTLBVEBOX1_CLKGATE_DIS |
1516                             GAMTLBVEBOX0_CLKGATE_DIS);
1517         }
1518
1519         /* Wa_16012725990:xehpsdv */
1520         if (IS_XEHPSDV_GRAPHICS_STEP(i915, STEP_A1, STEP_FOREVER))
1521                 wa_write_or(wal, UNSLICE_UNIT_LEVEL_CLKGATE, VFUNIT_CLKGATE_DIS);
1522
1523         /* Wa_14011060649:xehpsdv */
1524         wa_14011060649(gt, wal);
1525 }
1526
1527 static void
1528 dg2_gt_workarounds_init(struct intel_gt *gt, struct i915_wa_list *wal)
1529 {
1530         struct intel_engine_cs *engine;
1531         int id;
1532
1533         xehp_init_mcr(gt, wal);
1534
1535         /* Wa_14011060649:dg2 */
1536         wa_14011060649(gt, wal);
1537
1538         /*
1539          * Although there are per-engine instances of these registers,
1540          * they technically exist outside the engine itself and are not
1541          * impacted by engine resets.  Furthermore, they're part of the
1542          * GuC blacklist so trying to treat them as engine workarounds
1543          * will result in GuC initialization failure and a wedged GPU.
1544          */
1545         for_each_engine(engine, gt, id) {
1546                 if (engine->class != VIDEO_DECODE_CLASS)
1547                         continue;
1548
1549                 /* Wa_16010515920:dg2_g10 */
1550                 if (IS_DG2_GRAPHICS_STEP(gt->i915, G10, STEP_A0, STEP_B0))
1551                         wa_write_or(wal, VDBOX_CGCTL3F18(engine->mmio_base),
1552                                     ALNUNIT_CLKGATE_DIS);
1553         }
1554
1555         if (IS_DG2_G10(gt->i915)) {
1556                 /* Wa_22010523718:dg2 */
1557                 wa_write_or(wal, UNSLICE_UNIT_LEVEL_CLKGATE,
1558                             CG3DDISCFEG_CLKGATE_DIS);
1559
1560                 /* Wa_14011006942:dg2 */
1561                 wa_mcr_write_or(wal, GEN11_SUBSLICE_UNIT_LEVEL_CLKGATE,
1562                                 DSS_ROUTER_CLKGATE_DIS);
1563         }
1564
1565         if (IS_DG2_GRAPHICS_STEP(gt->i915, G10, STEP_A0, STEP_B0)) {
1566                 /* Wa_14010948348:dg2_g10 */
1567                 wa_write_or(wal, UNSLCGCTL9430, MSQDUNIT_CLKGATE_DIS);
1568
1569                 /* Wa_14011037102:dg2_g10 */
1570                 wa_write_or(wal, UNSLCGCTL9444, LTCDD_CLKGATE_DIS);
1571
1572                 /* Wa_14011371254:dg2_g10 */
1573                 wa_mcr_write_or(wal, XEHP_SLICE_UNIT_LEVEL_CLKGATE, NODEDSS_CLKGATE_DIS);
1574
1575                 /* Wa_14011431319:dg2_g10 */
1576                 wa_write_or(wal, UNSLCGCTL9440, GAMTLBOACS_CLKGATE_DIS |
1577                             GAMTLBVDBOX7_CLKGATE_DIS |
1578                             GAMTLBVDBOX6_CLKGATE_DIS |
1579                             GAMTLBVDBOX5_CLKGATE_DIS |
1580                             GAMTLBVDBOX4_CLKGATE_DIS |
1581                             GAMTLBVDBOX3_CLKGATE_DIS |
1582                             GAMTLBVDBOX2_CLKGATE_DIS |
1583                             GAMTLBVDBOX1_CLKGATE_DIS |
1584                             GAMTLBVDBOX0_CLKGATE_DIS |
1585                             GAMTLBKCR_CLKGATE_DIS |
1586                             GAMTLBGUC_CLKGATE_DIS |
1587                             GAMTLBBLT_CLKGATE_DIS);
1588                 wa_write_or(wal, UNSLCGCTL9444, GAMTLBGFXA0_CLKGATE_DIS |
1589                             GAMTLBGFXA1_CLKGATE_DIS |
1590                             GAMTLBCOMPA0_CLKGATE_DIS |
1591                             GAMTLBCOMPA1_CLKGATE_DIS |
1592                             GAMTLBCOMPB0_CLKGATE_DIS |
1593                             GAMTLBCOMPB1_CLKGATE_DIS |
1594                             GAMTLBCOMPC0_CLKGATE_DIS |
1595                             GAMTLBCOMPC1_CLKGATE_DIS |
1596                             GAMTLBCOMPD0_CLKGATE_DIS |
1597                             GAMTLBCOMPD1_CLKGATE_DIS |
1598                             GAMTLBMERT_CLKGATE_DIS   |
1599                             GAMTLBVEBOX3_CLKGATE_DIS |
1600                             GAMTLBVEBOX2_CLKGATE_DIS |
1601                             GAMTLBVEBOX1_CLKGATE_DIS |
1602                             GAMTLBVEBOX0_CLKGATE_DIS);
1603
1604                 /* Wa_14010569222:dg2_g10 */
1605                 wa_write_or(wal, UNSLICE_UNIT_LEVEL_CLKGATE,
1606                             GAMEDIA_CLKGATE_DIS);
1607
1608                 /* Wa_14011028019:dg2_g10 */
1609                 wa_mcr_write_or(wal, SSMCGCTL9530, RTFUNIT_CLKGATE_DIS);
1610         }
1611
1612         /* Wa_14014830051:dg2 */
1613         wa_mcr_write_clr(wal, SARB_CHICKEN1, COMP_CKN_IN);
1614
1615         /*
1616          * The following are not actually "workarounds" but rather
1617          * recommended tuning settings documented in the bspec's
1618          * performance guide section.
1619          */
1620         wa_mcr_write_or(wal, XEHP_SQCM, EN_32B_ACCESS);
1621
1622         /* Wa_14015795083 */
1623         wa_mcr_write_clr(wal, GEN8_MISCCPCTL, GEN12_DOP_CLOCK_GATE_RENDER_ENABLE);
1624 }
1625
1626 static void
1627 pvc_gt_workarounds_init(struct intel_gt *gt, struct i915_wa_list *wal)
1628 {
1629         pvc_init_mcr(gt, wal);
1630
1631         /* Wa_14015795083 */
1632         wa_mcr_write_clr(wal, GEN8_MISCCPCTL, GEN12_DOP_CLOCK_GATE_RENDER_ENABLE);
1633 }
1634
1635 static void
1636 xelpg_gt_workarounds_init(struct intel_gt *gt, struct i915_wa_list *wal)
1637 {
1638         /* FIXME: Actual workarounds will be added in future patch(es) */
1639
1640         /*
1641          * Unlike older platforms, we no longer setup implicit steering here;
1642          * all MCR accesses are explicitly steered.
1643          */
1644         debug_dump_steering(gt);
1645 }
1646
1647 static void
1648 xelpmp_gt_workarounds_init(struct intel_gt *gt, struct i915_wa_list *wal)
1649 {
1650         /* FIXME: Actual workarounds will be added in future patch(es) */
1651
1652         debug_dump_steering(gt);
1653 }
1654
1655 static void
1656 gt_init_workarounds(struct intel_gt *gt, struct i915_wa_list *wal)
1657 {
1658         struct drm_i915_private *i915 = gt->i915;
1659
1660         if (gt->type == GT_MEDIA) {
1661                 if (MEDIA_VER(i915) >= 13)
1662                         xelpmp_gt_workarounds_init(gt, wal);
1663                 else
1664                         MISSING_CASE(MEDIA_VER(i915));
1665
1666                 return;
1667         }
1668
1669         if (GRAPHICS_VER_FULL(i915) >= IP_VER(12, 70))
1670                 xelpg_gt_workarounds_init(gt, wal);
1671         else if (IS_PONTEVECCHIO(i915))
1672                 pvc_gt_workarounds_init(gt, wal);
1673         else if (IS_DG2(i915))
1674                 dg2_gt_workarounds_init(gt, wal);
1675         else if (IS_XEHPSDV(i915))
1676                 xehpsdv_gt_workarounds_init(gt, wal);
1677         else if (IS_DG1(i915))
1678                 dg1_gt_workarounds_init(gt, wal);
1679         else if (IS_TIGERLAKE(i915))
1680                 tgl_gt_workarounds_init(gt, wal);
1681         else if (GRAPHICS_VER(i915) == 12)
1682                 gen12_gt_workarounds_init(gt, wal);
1683         else if (GRAPHICS_VER(i915) == 11)
1684                 icl_gt_workarounds_init(gt, wal);
1685         else if (IS_COFFEELAKE(i915) || IS_COMETLAKE(i915))
1686                 cfl_gt_workarounds_init(gt, wal);
1687         else if (IS_GEMINILAKE(i915))
1688                 glk_gt_workarounds_init(gt, wal);
1689         else if (IS_KABYLAKE(i915))
1690                 kbl_gt_workarounds_init(gt, wal);
1691         else if (IS_BROXTON(i915))
1692                 gen9_gt_workarounds_init(gt, wal);
1693         else if (IS_SKYLAKE(i915))
1694                 skl_gt_workarounds_init(gt, wal);
1695         else if (IS_HASWELL(i915))
1696                 hsw_gt_workarounds_init(gt, wal);
1697         else if (IS_VALLEYVIEW(i915))
1698                 vlv_gt_workarounds_init(gt, wal);
1699         else if (IS_IVYBRIDGE(i915))
1700                 ivb_gt_workarounds_init(gt, wal);
1701         else if (GRAPHICS_VER(i915) == 6)
1702                 snb_gt_workarounds_init(gt, wal);
1703         else if (GRAPHICS_VER(i915) == 5)
1704                 ilk_gt_workarounds_init(gt, wal);
1705         else if (IS_G4X(i915))
1706                 g4x_gt_workarounds_init(gt, wal);
1707         else if (GRAPHICS_VER(i915) == 4)
1708                 gen4_gt_workarounds_init(gt, wal);
1709         else if (GRAPHICS_VER(i915) <= 8)
1710                 ;
1711         else
1712                 MISSING_CASE(GRAPHICS_VER(i915));
1713 }
1714
1715 void intel_gt_init_workarounds(struct intel_gt *gt)
1716 {
1717         struct i915_wa_list *wal = &gt->wa_list;
1718
1719         wa_init_start(wal, gt, "GT", "global");
1720         gt_init_workarounds(gt, wal);
1721         wa_init_finish(wal);
1722 }
1723
1724 static enum forcewake_domains
1725 wal_get_fw_for_rmw(struct intel_uncore *uncore, const struct i915_wa_list *wal)
1726 {
1727         enum forcewake_domains fw = 0;
1728         struct i915_wa *wa;
1729         unsigned int i;
1730
1731         for (i = 0, wa = wal->list; i < wal->count; i++, wa++)
1732                 fw |= intel_uncore_forcewake_for_reg(uncore,
1733                                                      wa->reg,
1734                                                      FW_REG_READ |
1735                                                      FW_REG_WRITE);
1736
1737         return fw;
1738 }
1739
1740 static bool
1741 wa_verify(struct intel_gt *gt, const struct i915_wa *wa, u32 cur,
1742           const char *name, const char *from)
1743 {
1744         if ((cur ^ wa->set) & wa->read) {
1745                 drm_err(&gt->i915->drm,
1746                         "%s workaround lost on %s! (reg[%x]=0x%x, relevant bits were 0x%x vs expected 0x%x)\n",
1747                         name, from, i915_mmio_reg_offset(wa->reg),
1748                         cur, cur & wa->read, wa->set & wa->read);
1749
1750                 return false;
1751         }
1752
1753         return true;
1754 }
1755
1756 static void wa_list_apply(const struct i915_wa_list *wal)
1757 {
1758         struct intel_gt *gt = wal->gt;
1759         struct intel_uncore *uncore = gt->uncore;
1760         enum forcewake_domains fw;
1761         unsigned long flags;
1762         struct i915_wa *wa;
1763         unsigned int i;
1764
1765         if (!wal->count)
1766                 return;
1767
1768         fw = wal_get_fw_for_rmw(uncore, wal);
1769
1770         spin_lock_irqsave(&uncore->lock, flags);
1771         intel_uncore_forcewake_get__locked(uncore, fw);
1772
1773         for (i = 0, wa = wal->list; i < wal->count; i++, wa++) {
1774                 u32 val, old = 0;
1775
1776                 /* open-coded rmw due to steering */
1777                 if (wa->clr)
1778                         old = wa->is_mcr ?
1779                                 intel_gt_mcr_read_any_fw(gt, wa->mcr_reg) :
1780                                 intel_uncore_read_fw(uncore, wa->reg);
1781                 val = (old & ~wa->clr) | wa->set;
1782                 if (val != old || !wa->clr) {
1783                         if (wa->is_mcr)
1784                                 intel_gt_mcr_multicast_write_fw(gt, wa->mcr_reg, val);
1785                         else
1786                                 intel_uncore_write_fw(uncore, wa->reg, val);
1787                 }
1788
1789                 if (IS_ENABLED(CONFIG_DRM_I915_DEBUG_GEM)) {
1790                         u32 val = wa->is_mcr ?
1791                                 intel_gt_mcr_read_any_fw(gt, wa->mcr_reg) :
1792                                 intel_uncore_read_fw(uncore, wa->reg);
1793
1794                         wa_verify(gt, wa, val, wal->name, "application");
1795                 }
1796         }
1797
1798         intel_uncore_forcewake_put__locked(uncore, fw);
1799         spin_unlock_irqrestore(&uncore->lock, flags);
1800 }
1801
1802 void intel_gt_apply_workarounds(struct intel_gt *gt)
1803 {
1804         wa_list_apply(&gt->wa_list);
1805 }
1806
1807 static bool wa_list_verify(struct intel_gt *gt,
1808                            const struct i915_wa_list *wal,
1809                            const char *from)
1810 {
1811         struct intel_uncore *uncore = gt->uncore;
1812         struct i915_wa *wa;
1813         enum forcewake_domains fw;
1814         unsigned long flags;
1815         unsigned int i;
1816         bool ok = true;
1817
1818         fw = wal_get_fw_for_rmw(uncore, wal);
1819
1820         spin_lock_irqsave(&uncore->lock, flags);
1821         intel_uncore_forcewake_get__locked(uncore, fw);
1822
1823         for (i = 0, wa = wal->list; i < wal->count; i++, wa++)
1824                 ok &= wa_verify(wal->gt, wa, wa->is_mcr ?
1825                                 intel_gt_mcr_read_any_fw(gt, wa->mcr_reg) :
1826                                 intel_uncore_read_fw(uncore, wa->reg),
1827                                 wal->name, from);
1828
1829         intel_uncore_forcewake_put__locked(uncore, fw);
1830         spin_unlock_irqrestore(&uncore->lock, flags);
1831
1832         return ok;
1833 }
1834
1835 bool intel_gt_verify_workarounds(struct intel_gt *gt, const char *from)
1836 {
1837         return wa_list_verify(gt, &gt->wa_list, from);
1838 }
1839
1840 __maybe_unused
1841 static bool is_nonpriv_flags_valid(u32 flags)
1842 {
1843         /* Check only valid flag bits are set */
1844         if (flags & ~RING_FORCE_TO_NONPRIV_MASK_VALID)
1845                 return false;
1846
1847         /* NB: Only 3 out of 4 enum values are valid for access field */
1848         if ((flags & RING_FORCE_TO_NONPRIV_ACCESS_MASK) ==
1849             RING_FORCE_TO_NONPRIV_ACCESS_INVALID)
1850                 return false;
1851
1852         return true;
1853 }
1854
1855 static void
1856 whitelist_reg_ext(struct i915_wa_list *wal, i915_reg_t reg, u32 flags)
1857 {
1858         struct i915_wa wa = {
1859                 .reg = reg
1860         };
1861
1862         if (GEM_DEBUG_WARN_ON(wal->count >= RING_MAX_NONPRIV_SLOTS))
1863                 return;
1864
1865         if (GEM_DEBUG_WARN_ON(!is_nonpriv_flags_valid(flags)))
1866                 return;
1867
1868         wa.reg.reg |= flags;
1869         _wa_add(wal, &wa);
1870 }
1871
1872 static void
1873 whitelist_mcr_reg_ext(struct i915_wa_list *wal, i915_mcr_reg_t reg, u32 flags)
1874 {
1875         struct i915_wa wa = {
1876                 .mcr_reg = reg,
1877                 .is_mcr = 1,
1878         };
1879
1880         if (GEM_DEBUG_WARN_ON(wal->count >= RING_MAX_NONPRIV_SLOTS))
1881                 return;
1882
1883         if (GEM_DEBUG_WARN_ON(!is_nonpriv_flags_valid(flags)))
1884                 return;
1885
1886         wa.mcr_reg.reg |= flags;
1887         _wa_add(wal, &wa);
1888 }
1889
1890 static void
1891 whitelist_reg(struct i915_wa_list *wal, i915_reg_t reg)
1892 {
1893         whitelist_reg_ext(wal, reg, RING_FORCE_TO_NONPRIV_ACCESS_RW);
1894 }
1895
1896 static void
1897 whitelist_mcr_reg(struct i915_wa_list *wal, i915_mcr_reg_t reg)
1898 {
1899         whitelist_mcr_reg_ext(wal, reg, RING_FORCE_TO_NONPRIV_ACCESS_RW);
1900 }
1901
1902 static void gen9_whitelist_build(struct i915_wa_list *w)
1903 {
1904         /* WaVFEStateAfterPipeControlwithMediaStateClear:skl,bxt,glk,cfl */
1905         whitelist_reg(w, GEN9_CTX_PREEMPT_REG);
1906
1907         /* WaEnablePreemptionGranularityControlByUMD:skl,bxt,kbl,cfl,[cnl] */
1908         whitelist_reg(w, GEN8_CS_CHICKEN1);
1909
1910         /* WaAllowUMDToModifyHDCChicken1:skl,bxt,kbl,glk,cfl */
1911         whitelist_reg(w, GEN8_HDC_CHICKEN1);
1912
1913         /* WaSendPushConstantsFromMMIO:skl,bxt */
1914         whitelist_reg(w, COMMON_SLICE_CHICKEN2);
1915 }
1916
1917 static void skl_whitelist_build(struct intel_engine_cs *engine)
1918 {
1919         struct i915_wa_list *w = &engine->whitelist;
1920
1921         if (engine->class != RENDER_CLASS)
1922                 return;
1923
1924         gen9_whitelist_build(w);
1925
1926         /* WaDisableLSQCROPERFforOCL:skl */
1927         whitelist_mcr_reg(w, GEN8_L3SQCREG4);
1928 }
1929
1930 static void bxt_whitelist_build(struct intel_engine_cs *engine)
1931 {
1932         if (engine->class != RENDER_CLASS)
1933                 return;
1934
1935         gen9_whitelist_build(&engine->whitelist);
1936 }
1937
1938 static void kbl_whitelist_build(struct intel_engine_cs *engine)
1939 {
1940         struct i915_wa_list *w = &engine->whitelist;
1941
1942         if (engine->class != RENDER_CLASS)
1943                 return;
1944
1945         gen9_whitelist_build(w);
1946
1947         /* WaDisableLSQCROPERFforOCL:kbl */
1948         whitelist_mcr_reg(w, GEN8_L3SQCREG4);
1949 }
1950
1951 static void glk_whitelist_build(struct intel_engine_cs *engine)
1952 {
1953         struct i915_wa_list *w = &engine->whitelist;
1954
1955         if (engine->class != RENDER_CLASS)
1956                 return;
1957
1958         gen9_whitelist_build(w);
1959
1960         /* WA #0862: Userspace has to set "Barrier Mode" to avoid hangs. */
1961         whitelist_reg(w, GEN9_SLICE_COMMON_ECO_CHICKEN1);
1962 }
1963
1964 static void cfl_whitelist_build(struct intel_engine_cs *engine)
1965 {
1966         struct i915_wa_list *w = &engine->whitelist;
1967
1968         if (engine->class != RENDER_CLASS)
1969                 return;
1970
1971         gen9_whitelist_build(w);
1972
1973         /*
1974          * WaAllowPMDepthAndInvocationCountAccessFromUMD:cfl,whl,cml,aml
1975          *
1976          * This covers 4 register which are next to one another :
1977          *   - PS_INVOCATION_COUNT
1978          *   - PS_INVOCATION_COUNT_UDW
1979          *   - PS_DEPTH_COUNT
1980          *   - PS_DEPTH_COUNT_UDW
1981          */
1982         whitelist_reg_ext(w, PS_INVOCATION_COUNT,
1983                           RING_FORCE_TO_NONPRIV_ACCESS_RD |
1984                           RING_FORCE_TO_NONPRIV_RANGE_4);
1985 }
1986
1987 static void allow_read_ctx_timestamp(struct intel_engine_cs *engine)
1988 {
1989         struct i915_wa_list *w = &engine->whitelist;
1990
1991         if (engine->class != RENDER_CLASS)
1992                 whitelist_reg_ext(w,
1993                                   RING_CTX_TIMESTAMP(engine->mmio_base),
1994                                   RING_FORCE_TO_NONPRIV_ACCESS_RD);
1995 }
1996
1997 static void cml_whitelist_build(struct intel_engine_cs *engine)
1998 {
1999         allow_read_ctx_timestamp(engine);
2000
2001         cfl_whitelist_build(engine);
2002 }
2003
2004 static void icl_whitelist_build(struct intel_engine_cs *engine)
2005 {
2006         struct i915_wa_list *w = &engine->whitelist;
2007
2008         allow_read_ctx_timestamp(engine);
2009
2010         switch (engine->class) {
2011         case RENDER_CLASS:
2012                 /* WaAllowUMDToModifyHalfSliceChicken7:icl */
2013                 whitelist_mcr_reg(w, GEN9_HALF_SLICE_CHICKEN7);
2014
2015                 /* WaAllowUMDToModifySamplerMode:icl */
2016                 whitelist_mcr_reg(w, GEN10_SAMPLER_MODE);
2017
2018                 /* WaEnableStateCacheRedirectToCS:icl */
2019                 whitelist_reg(w, GEN9_SLICE_COMMON_ECO_CHICKEN1);
2020
2021                 /*
2022                  * WaAllowPMDepthAndInvocationCountAccessFromUMD:icl
2023                  *
2024                  * This covers 4 register which are next to one another :
2025                  *   - PS_INVOCATION_COUNT
2026                  *   - PS_INVOCATION_COUNT_UDW
2027                  *   - PS_DEPTH_COUNT
2028                  *   - PS_DEPTH_COUNT_UDW
2029                  */
2030                 whitelist_reg_ext(w, PS_INVOCATION_COUNT,
2031                                   RING_FORCE_TO_NONPRIV_ACCESS_RD |
2032                                   RING_FORCE_TO_NONPRIV_RANGE_4);
2033                 break;
2034
2035         case VIDEO_DECODE_CLASS:
2036                 /* hucStatusRegOffset */
2037                 whitelist_reg_ext(w, _MMIO(0x2000 + engine->mmio_base),
2038                                   RING_FORCE_TO_NONPRIV_ACCESS_RD);
2039                 /* hucUKernelHdrInfoRegOffset */
2040                 whitelist_reg_ext(w, _MMIO(0x2014 + engine->mmio_base),
2041                                   RING_FORCE_TO_NONPRIV_ACCESS_RD);
2042                 /* hucStatus2RegOffset */
2043                 whitelist_reg_ext(w, _MMIO(0x23B0 + engine->mmio_base),
2044                                   RING_FORCE_TO_NONPRIV_ACCESS_RD);
2045                 break;
2046
2047         default:
2048                 break;
2049         }
2050 }
2051
2052 static void tgl_whitelist_build(struct intel_engine_cs *engine)
2053 {
2054         struct i915_wa_list *w = &engine->whitelist;
2055
2056         allow_read_ctx_timestamp(engine);
2057
2058         switch (engine->class) {
2059         case RENDER_CLASS:
2060                 /*
2061                  * WaAllowPMDepthAndInvocationCountAccessFromUMD:tgl
2062                  * Wa_1408556865:tgl
2063                  *
2064                  * This covers 4 registers which are next to one another :
2065                  *   - PS_INVOCATION_COUNT
2066                  *   - PS_INVOCATION_COUNT_UDW
2067                  *   - PS_DEPTH_COUNT
2068                  *   - PS_DEPTH_COUNT_UDW
2069                  */
2070                 whitelist_reg_ext(w, PS_INVOCATION_COUNT,
2071                                   RING_FORCE_TO_NONPRIV_ACCESS_RD |
2072                                   RING_FORCE_TO_NONPRIV_RANGE_4);
2073
2074                 /*
2075                  * Wa_1808121037:tgl
2076                  * Wa_14012131227:dg1
2077                  * Wa_1508744258:tgl,rkl,dg1,adl-s,adl-p
2078                  */
2079                 whitelist_reg(w, GEN7_COMMON_SLICE_CHICKEN1);
2080
2081                 /* Wa_1806527549:tgl */
2082                 whitelist_reg(w, HIZ_CHICKEN);
2083                 break;
2084         default:
2085                 break;
2086         }
2087 }
2088
2089 static void dg1_whitelist_build(struct intel_engine_cs *engine)
2090 {
2091         struct i915_wa_list *w = &engine->whitelist;
2092
2093         tgl_whitelist_build(engine);
2094
2095         /* GEN:BUG:1409280441:dg1 */
2096         if (IS_DG1_GRAPHICS_STEP(engine->i915, STEP_A0, STEP_B0) &&
2097             (engine->class == RENDER_CLASS ||
2098              engine->class == COPY_ENGINE_CLASS))
2099                 whitelist_reg_ext(w, RING_ID(engine->mmio_base),
2100                                   RING_FORCE_TO_NONPRIV_ACCESS_RD);
2101 }
2102
2103 static void xehpsdv_whitelist_build(struct intel_engine_cs *engine)
2104 {
2105         allow_read_ctx_timestamp(engine);
2106 }
2107
2108 static void dg2_whitelist_build(struct intel_engine_cs *engine)
2109 {
2110         struct i915_wa_list *w = &engine->whitelist;
2111
2112         allow_read_ctx_timestamp(engine);
2113
2114         switch (engine->class) {
2115         case RENDER_CLASS:
2116                 /*
2117                  * Wa_1507100340:dg2_g10
2118                  *
2119                  * This covers 4 registers which are next to one another :
2120                  *   - PS_INVOCATION_COUNT
2121                  *   - PS_INVOCATION_COUNT_UDW
2122                  *   - PS_DEPTH_COUNT
2123                  *   - PS_DEPTH_COUNT_UDW
2124                  */
2125                 if (IS_DG2_GRAPHICS_STEP(engine->i915, G10, STEP_A0, STEP_B0))
2126                         whitelist_reg_ext(w, PS_INVOCATION_COUNT,
2127                                           RING_FORCE_TO_NONPRIV_ACCESS_RD |
2128                                           RING_FORCE_TO_NONPRIV_RANGE_4);
2129
2130                 break;
2131         case COMPUTE_CLASS:
2132                 /* Wa_16011157294:dg2_g10 */
2133                 if (IS_DG2_GRAPHICS_STEP(engine->i915, G10, STEP_A0, STEP_B0))
2134                         whitelist_reg(w, GEN9_CTX_PREEMPT_REG);
2135                 break;
2136         default:
2137                 break;
2138         }
2139 }
2140
2141 static void blacklist_trtt(struct intel_engine_cs *engine)
2142 {
2143         struct i915_wa_list *w = &engine->whitelist;
2144
2145         /*
2146          * Prevent read/write access to [0x4400, 0x4600) which covers
2147          * the TRTT range across all engines. Note that normally userspace
2148          * cannot access the other engines' trtt control, but for simplicity
2149          * we cover the entire range on each engine.
2150          */
2151         whitelist_reg_ext(w, _MMIO(0x4400),
2152                           RING_FORCE_TO_NONPRIV_DENY |
2153                           RING_FORCE_TO_NONPRIV_RANGE_64);
2154         whitelist_reg_ext(w, _MMIO(0x4500),
2155                           RING_FORCE_TO_NONPRIV_DENY |
2156                           RING_FORCE_TO_NONPRIV_RANGE_64);
2157 }
2158
2159 static void pvc_whitelist_build(struct intel_engine_cs *engine)
2160 {
2161         allow_read_ctx_timestamp(engine);
2162
2163         /* Wa_16014440446:pvc */
2164         blacklist_trtt(engine);
2165 }
2166
2167 void intel_engine_init_whitelist(struct intel_engine_cs *engine)
2168 {
2169         struct drm_i915_private *i915 = engine->i915;
2170         struct i915_wa_list *w = &engine->whitelist;
2171
2172         wa_init_start(w, engine->gt, "whitelist", engine->name);
2173
2174         if (IS_PONTEVECCHIO(i915))
2175                 pvc_whitelist_build(engine);
2176         else if (IS_DG2(i915))
2177                 dg2_whitelist_build(engine);
2178         else if (IS_XEHPSDV(i915))
2179                 xehpsdv_whitelist_build(engine);
2180         else if (IS_DG1(i915))
2181                 dg1_whitelist_build(engine);
2182         else if (GRAPHICS_VER(i915) == 12)
2183                 tgl_whitelist_build(engine);
2184         else if (GRAPHICS_VER(i915) == 11)
2185                 icl_whitelist_build(engine);
2186         else if (IS_COMETLAKE(i915))
2187                 cml_whitelist_build(engine);
2188         else if (IS_COFFEELAKE(i915))
2189                 cfl_whitelist_build(engine);
2190         else if (IS_GEMINILAKE(i915))
2191                 glk_whitelist_build(engine);
2192         else if (IS_KABYLAKE(i915))
2193                 kbl_whitelist_build(engine);
2194         else if (IS_BROXTON(i915))
2195                 bxt_whitelist_build(engine);
2196         else if (IS_SKYLAKE(i915))
2197                 skl_whitelist_build(engine);
2198         else if (GRAPHICS_VER(i915) <= 8)
2199                 ;
2200         else
2201                 MISSING_CASE(GRAPHICS_VER(i915));
2202
2203         wa_init_finish(w);
2204 }
2205
2206 void intel_engine_apply_whitelist(struct intel_engine_cs *engine)
2207 {
2208         const struct i915_wa_list *wal = &engine->whitelist;
2209         struct intel_uncore *uncore = engine->uncore;
2210         const u32 base = engine->mmio_base;
2211         struct i915_wa *wa;
2212         unsigned int i;
2213
2214         if (!wal->count)
2215                 return;
2216
2217         for (i = 0, wa = wal->list; i < wal->count; i++, wa++)
2218                 intel_uncore_write(uncore,
2219                                    RING_FORCE_TO_NONPRIV(base, i),
2220                                    i915_mmio_reg_offset(wa->reg));
2221
2222         /* And clear the rest just in case of garbage */
2223         for (; i < RING_MAX_NONPRIV_SLOTS; i++)
2224                 intel_uncore_write(uncore,
2225                                    RING_FORCE_TO_NONPRIV(base, i),
2226                                    i915_mmio_reg_offset(RING_NOPID(base)));
2227 }
2228
2229 /*
2230  * engine_fake_wa_init(), a place holder to program the registers
2231  * which are not part of an official workaround defined by the
2232  * hardware team.
2233  * Adding programming of those register inside workaround will
2234  * allow utilizing wa framework to proper application and verification.
2235  */
2236 static void
2237 engine_fake_wa_init(struct intel_engine_cs *engine, struct i915_wa_list *wal)
2238 {
2239         u8 mocs_w, mocs_r;
2240
2241         /*
2242          * RING_CMD_CCTL specifies the default MOCS entry that will be used
2243          * by the command streamer when executing commands that don't have
2244          * a way to explicitly specify a MOCS setting.  The default should
2245          * usually reference whichever MOCS entry corresponds to uncached
2246          * behavior, although use of a WB cached entry is recommended by the
2247          * spec in certain circumstances on specific platforms.
2248          */
2249         if (GRAPHICS_VER(engine->i915) >= 12) {
2250                 mocs_r = engine->gt->mocs.uc_index;
2251                 mocs_w = engine->gt->mocs.uc_index;
2252
2253                 if (HAS_L3_CCS_READ(engine->i915) &&
2254                     engine->class == COMPUTE_CLASS) {
2255                         mocs_r = engine->gt->mocs.wb_index;
2256
2257                         /*
2258                          * Even on the few platforms where MOCS 0 is a
2259                          * legitimate table entry, it's never the correct
2260                          * setting to use here; we can assume the MOCS init
2261                          * just forgot to initialize wb_index.
2262                          */
2263                         drm_WARN_ON(&engine->i915->drm, mocs_r == 0);
2264                 }
2265
2266                 wa_masked_field_set(wal,
2267                                     RING_CMD_CCTL(engine->mmio_base),
2268                                     CMD_CCTL_MOCS_MASK,
2269                                     CMD_CCTL_MOCS_OVERRIDE(mocs_w, mocs_r));
2270         }
2271 }
2272
2273 static bool needs_wa_1308578152(struct intel_engine_cs *engine)
2274 {
2275         return intel_sseu_find_first_xehp_dss(&engine->gt->info.sseu, 0, 0) >=
2276                 GEN_DSS_PER_GSLICE;
2277 }
2278
2279 static void
2280 rcs_engine_wa_init(struct intel_engine_cs *engine, struct i915_wa_list *wal)
2281 {
2282         struct drm_i915_private *i915 = engine->i915;
2283
2284         if (IS_DG2(i915)) {
2285                 /* Wa_1509235366:dg2 */
2286                 wa_write_or(wal, GEN12_GAMCNTRL_CTRL, INVALIDATION_BROADCAST_MODE_DIS |
2287                             GLOBAL_INVALIDATION_MODE);
2288         }
2289
2290         if (IS_DG2_GRAPHICS_STEP(i915, G11, STEP_A0, STEP_B0)) {
2291                 /* Wa_14013392000:dg2_g11 */
2292                 wa_mcr_masked_en(wal, GEN8_ROW_CHICKEN2, GEN12_ENABLE_LARGE_GRF_MODE);
2293         }
2294
2295         if (IS_DG2_GRAPHICS_STEP(i915, G10, STEP_B0, STEP_FOREVER) ||
2296             IS_DG2_G11(i915) || IS_DG2_G12(i915)) {
2297                 /* Wa_1509727124:dg2 */
2298                 wa_mcr_masked_en(wal, GEN10_SAMPLER_MODE,
2299                                  SC_DISABLE_POWER_OPTIMIZATION_EBB);
2300         }
2301
2302         if (IS_DG2_GRAPHICS_STEP(i915, G10, STEP_A0, STEP_B0) ||
2303             IS_DG2_GRAPHICS_STEP(i915, G11, STEP_A0, STEP_B0)) {
2304                 /* Wa_14012419201:dg2 */
2305                 wa_mcr_masked_en(wal, GEN9_ROW_CHICKEN4,
2306                                  GEN12_DISABLE_HDR_PAST_PAYLOAD_HOLD_FIX);
2307         }
2308
2309         if (IS_DG2_GRAPHICS_STEP(i915, G10, STEP_B0, STEP_C0) ||
2310             IS_DG2_G11(i915)) {
2311                 /*
2312                  * Wa_22012826095:dg2
2313                  * Wa_22013059131:dg2
2314                  */
2315                 wa_mcr_write_clr_set(wal, LSC_CHICKEN_BIT_0_UDW,
2316                                      MAXREQS_PER_BANK,
2317                                      REG_FIELD_PREP(MAXREQS_PER_BANK, 2));
2318
2319                 /* Wa_22013059131:dg2 */
2320                 wa_mcr_write_or(wal, LSC_CHICKEN_BIT_0,
2321                                 FORCE_1_SUB_MESSAGE_PER_FRAGMENT);
2322         }
2323
2324         /* Wa_1308578152:dg2_g10 when first gslice is fused off */
2325         if (IS_DG2_GRAPHICS_STEP(i915, G10, STEP_B0, STEP_C0) &&
2326             needs_wa_1308578152(engine)) {
2327                 wa_masked_dis(wal, GEN12_CS_DEBUG_MODE1_CCCSUNIT_BE_COMMON,
2328                               GEN12_REPLAY_MODE_GRANULARITY);
2329         }
2330
2331         if (IS_DG2_GRAPHICS_STEP(i915, G10, STEP_B0, STEP_FOREVER) ||
2332             IS_DG2_G11(i915) || IS_DG2_G12(i915)) {
2333                 /* Wa_22013037850:dg2 */
2334                 wa_mcr_write_or(wal, LSC_CHICKEN_BIT_0_UDW,
2335                                 DISABLE_128B_EVICTION_COMMAND_UDW);
2336
2337                 /* Wa_22012856258:dg2 */
2338                 wa_mcr_masked_en(wal, GEN8_ROW_CHICKEN2,
2339                                  GEN12_DISABLE_READ_SUPPRESSION);
2340
2341                 /*
2342                  * Wa_22010960976:dg2
2343                  * Wa_14013347512:dg2
2344                  */
2345                 wa_mcr_masked_dis(wal, XEHP_HDC_CHICKEN0,
2346                                   LSC_L1_FLUSH_CTL_3D_DATAPORT_FLUSH_EVENTS_MASK);
2347         }
2348
2349         if (IS_DG2_GRAPHICS_STEP(i915, G10, STEP_A0, STEP_B0)) {
2350                 /*
2351                  * Wa_1608949956:dg2_g10
2352                  * Wa_14010198302:dg2_g10
2353                  */
2354                 wa_mcr_masked_en(wal, GEN8_ROW_CHICKEN,
2355                                  MDQ_ARBITRATION_MODE | UGM_BACKUP_MODE);
2356
2357                 /*
2358                  * Wa_14010918519:dg2_g10
2359                  *
2360                  * LSC_CHICKEN_BIT_0 always reads back as 0 is this stepping,
2361                  * so ignoring verification.
2362                  */
2363                 wa_mcr_add(wal, LSC_CHICKEN_BIT_0_UDW, 0,
2364                            FORCE_SLM_FENCE_SCOPE_TO_TILE | FORCE_UGM_FENCE_SCOPE_TO_TILE,
2365                            0, false);
2366         }
2367
2368         if (IS_DG2_GRAPHICS_STEP(i915, G10, STEP_A0, STEP_B0)) {
2369                 /* Wa_22010430635:dg2 */
2370                 wa_mcr_masked_en(wal,
2371                                  GEN9_ROW_CHICKEN4,
2372                                  GEN12_DISABLE_GRF_CLEAR);
2373
2374                 /* Wa_14010648519:dg2 */
2375                 wa_mcr_write_or(wal, XEHP_L3NODEARBCFG, XEHP_LNESPARE);
2376         }
2377
2378         /* Wa_14013202645:dg2 */
2379         if (IS_DG2_GRAPHICS_STEP(i915, G10, STEP_B0, STEP_C0) ||
2380             IS_DG2_GRAPHICS_STEP(i915, G11, STEP_A0, STEP_B0))
2381                 wa_mcr_write_or(wal, RT_CTRL, DIS_NULL_QUERY);
2382
2383         /* Wa_22012532006:dg2 */
2384         if (IS_DG2_GRAPHICS_STEP(engine->i915, G10, STEP_A0, STEP_C0) ||
2385             IS_DG2_GRAPHICS_STEP(engine->i915, G11, STEP_A0, STEP_B0))
2386                 wa_mcr_masked_en(wal, GEN9_HALF_SLICE_CHICKEN7,
2387                                  DG2_DISABLE_ROUND_ENABLE_ALLOW_FOR_SSLA);
2388
2389         if (IS_DG2_GRAPHICS_STEP(engine->i915, G10, STEP_A0, STEP_B0)) {
2390                 /* Wa_14010680813:dg2_g10 */
2391                 wa_write_or(wal, GEN12_GAMSTLB_CTRL, CONTROL_BLOCK_CLKGATE_DIS |
2392                             EGRESS_BLOCK_CLKGATE_DIS | TAG_BLOCK_CLKGATE_DIS);
2393         }
2394
2395         if (IS_DG2_GRAPHICS_STEP(engine->i915, G10, STEP_A0, STEP_B0) ||
2396             IS_DG2_GRAPHICS_STEP(engine->i915, G11, STEP_A0, STEP_B0)) {
2397                 /* Wa_14012362059:dg2 */
2398                 wa_mcr_write_or(wal, XEHP_MERT_MOD_CTRL, FORCE_MISS_FTLB);
2399         }
2400
2401         if (IS_DG2_GRAPHICS_STEP(i915, G11, STEP_B0, STEP_FOREVER) ||
2402             IS_DG2_G10(i915)) {
2403                 /* Wa_22014600077:dg2 */
2404                 wa_mcr_add(wal, GEN10_CACHE_MODE_SS, 0,
2405                            _MASKED_BIT_ENABLE(ENABLE_EU_COUNT_FOR_TDL_FLUSH),
2406                            0 /* Wa_14012342262 write-only reg, so skip verification */,
2407                            true);
2408         }
2409
2410         if (IS_DG1_GRAPHICS_STEP(i915, STEP_A0, STEP_B0) ||
2411             IS_TGL_UY_GRAPHICS_STEP(i915, STEP_A0, STEP_B0)) {
2412                 /*
2413                  * Wa_1607138336:tgl[a0],dg1[a0]
2414                  * Wa_1607063988:tgl[a0],dg1[a0]
2415                  */
2416                 wa_write_or(wal,
2417                             GEN9_CTX_PREEMPT_REG,
2418                             GEN12_DISABLE_POSH_BUSY_FF_DOP_CG);
2419         }
2420
2421         if (IS_TGL_UY_GRAPHICS_STEP(i915, STEP_A0, STEP_B0)) {
2422                 /*
2423                  * Wa_1606679103:tgl
2424                  * (see also Wa_1606682166:icl)
2425                  */
2426                 wa_write_or(wal,
2427                             GEN7_SARCHKMD,
2428                             GEN7_DISABLE_SAMPLER_PREFETCH);
2429         }
2430
2431         if (IS_ALDERLAKE_P(i915) || IS_ALDERLAKE_S(i915) || IS_DG1(i915) ||
2432             IS_ROCKETLAKE(i915) || IS_TIGERLAKE(i915)) {
2433                 /* Wa_1606931601:tgl,rkl,dg1,adl-s,adl-p */
2434                 wa_mcr_masked_en(wal, GEN8_ROW_CHICKEN2, GEN12_DISABLE_EARLY_READ);
2435
2436                 /*
2437                  * Wa_1407928979:tgl A*
2438                  * Wa_18011464164:tgl[B0+],dg1[B0+]
2439                  * Wa_22010931296:tgl[B0+],dg1[B0+]
2440                  * Wa_14010919138:rkl,dg1,adl-s,adl-p
2441                  */
2442                 wa_write_or(wal, GEN7_FF_THREAD_MODE,
2443                             GEN12_FF_TESSELATION_DOP_GATE_DISABLE);
2444         }
2445
2446         if (IS_ALDERLAKE_P(i915) || IS_DG2(i915) || IS_ALDERLAKE_S(i915) ||
2447             IS_DG1(i915) || IS_ROCKETLAKE(i915) || IS_TIGERLAKE(i915)) {
2448                 /*
2449                  * Wa_1606700617:tgl,dg1,adl-p
2450                  * Wa_22010271021:tgl,rkl,dg1,adl-s,adl-p
2451                  * Wa_14010826681:tgl,dg1,rkl,adl-p
2452                  * Wa_18019627453:dg2
2453                  */
2454                 wa_masked_en(wal,
2455                              GEN9_CS_DEBUG_MODE1,
2456                              FF_DOP_CLOCK_GATE_DISABLE);
2457         }
2458
2459         if (IS_ALDERLAKE_P(i915) || IS_ALDERLAKE_S(i915) ||
2460             IS_DG1_GRAPHICS_STEP(i915, STEP_A0, STEP_B0) ||
2461             IS_ROCKETLAKE(i915) || IS_TIGERLAKE(i915)) {
2462                 /* Wa_1409804808:tgl,rkl,dg1[a0],adl-s,adl-p */
2463                 wa_mcr_masked_en(wal, GEN8_ROW_CHICKEN2,
2464                                  GEN12_PUSH_CONST_DEREF_HOLD_DIS);
2465
2466                 /*
2467                  * Wa_1409085225:tgl
2468                  * Wa_14010229206:tgl,rkl,dg1[a0],adl-s,adl-p
2469                  */
2470                 wa_mcr_masked_en(wal, GEN9_ROW_CHICKEN4, GEN12_DISABLE_TDL_PUSH);
2471         }
2472
2473         if (IS_DG1_GRAPHICS_STEP(i915, STEP_A0, STEP_B0) ||
2474             IS_ROCKETLAKE(i915) || IS_TIGERLAKE(i915) || IS_ALDERLAKE_P(i915)) {
2475                 /*
2476                  * Wa_1607030317:tgl
2477                  * Wa_1607186500:tgl
2478                  * Wa_1607297627:tgl,rkl,dg1[a0],adlp
2479                  *
2480                  * On TGL and RKL there are multiple entries for this WA in the
2481                  * BSpec; some indicate this is an A0-only WA, others indicate
2482                  * it applies to all steppings so we trust the "all steppings."
2483                  * For DG1 this only applies to A0.
2484                  */
2485                 wa_masked_en(wal,
2486                              RING_PSMI_CTL(RENDER_RING_BASE),
2487                              GEN12_WAIT_FOR_EVENT_POWER_DOWN_DISABLE |
2488                              GEN8_RC_SEMA_IDLE_MSG_DISABLE);
2489         }
2490
2491         if (IS_DG1(i915) || IS_ROCKETLAKE(i915) || IS_TIGERLAKE(i915) ||
2492             IS_ALDERLAKE_S(i915) || IS_ALDERLAKE_P(i915)) {
2493                 /* Wa_1406941453:tgl,rkl,dg1,adl-s,adl-p */
2494                 wa_mcr_masked_en(wal,
2495                                  GEN10_SAMPLER_MODE,
2496                                  ENABLE_SMALLPL);
2497         }
2498
2499         if (GRAPHICS_VER(i915) == 11) {
2500                 /* This is not an Wa. Enable for better image quality */
2501                 wa_masked_en(wal,
2502                              _3D_CHICKEN3,
2503                              _3D_CHICKEN3_AA_LINE_QUALITY_FIX_ENABLE);
2504
2505                 /*
2506                  * Wa_1405543622:icl
2507                  * Formerly known as WaGAPZPriorityScheme
2508                  */
2509                 wa_write_or(wal,
2510                             GEN8_GARBCNTL,
2511                             GEN11_ARBITRATION_PRIO_ORDER_MASK);
2512
2513                 /*
2514                  * Wa_1604223664:icl
2515                  * Formerly known as WaL3BankAddressHashing
2516                  */
2517                 wa_write_clr_set(wal,
2518                                  GEN8_GARBCNTL,
2519                                  GEN11_HASH_CTRL_EXCL_MASK,
2520                                  GEN11_HASH_CTRL_EXCL_BIT0);
2521                 wa_write_clr_set(wal,
2522                                  GEN11_GLBLINVL,
2523                                  GEN11_BANK_HASH_ADDR_EXCL_MASK,
2524                                  GEN11_BANK_HASH_ADDR_EXCL_BIT0);
2525
2526                 /*
2527                  * Wa_1405733216:icl
2528                  * Formerly known as WaDisableCleanEvicts
2529                  */
2530                 wa_mcr_write_or(wal,
2531                                 GEN8_L3SQCREG4,
2532                                 GEN11_LQSC_CLEAN_EVICT_DISABLE);
2533
2534                 /* Wa_1606682166:icl */
2535                 wa_write_or(wal,
2536                             GEN7_SARCHKMD,
2537                             GEN7_DISABLE_SAMPLER_PREFETCH);
2538
2539                 /* Wa_1409178092:icl */
2540                 wa_mcr_write_clr_set(wal,
2541                                      GEN11_SCRATCH2,
2542                                      GEN11_COHERENT_PARTIAL_WRITE_MERGE_ENABLE,
2543                                      0);
2544
2545                 /* WaEnable32PlaneMode:icl */
2546                 wa_masked_en(wal, GEN9_CSFE_CHICKEN1_RCS,
2547                              GEN11_ENABLE_32_PLANE_MODE);
2548
2549                 /*
2550                  * Wa_1408767742:icl[a2..forever],ehl[all]
2551                  * Wa_1605460711:icl[a0..c0]
2552                  */
2553                 wa_write_or(wal,
2554                             GEN7_FF_THREAD_MODE,
2555                             GEN12_FF_TESSELATION_DOP_GATE_DISABLE);
2556
2557                 /* Wa_22010271021 */
2558                 wa_masked_en(wal,
2559                              GEN9_CS_DEBUG_MODE1,
2560                              FF_DOP_CLOCK_GATE_DISABLE);
2561         }
2562
2563         /*
2564          * Intel platforms that support fine-grained preemption (i.e., gen9 and
2565          * beyond) allow the kernel-mode driver to choose between two different
2566          * options for controlling preemption granularity and behavior.
2567          *
2568          * Option 1 (hardware default):
2569          *   Preemption settings are controlled in a global manner via
2570          *   kernel-only register CS_DEBUG_MODE1 (0x20EC).  Any granularity
2571          *   and settings chosen by the kernel-mode driver will apply to all
2572          *   userspace clients.
2573          *
2574          * Option 2:
2575          *   Preemption settings are controlled on a per-context basis via
2576          *   register CS_CHICKEN1 (0x2580).  CS_CHICKEN1 is saved/restored on
2577          *   context switch and is writable by userspace (e.g., via
2578          *   MI_LOAD_REGISTER_IMMEDIATE instructions placed in a batch buffer)
2579          *   which allows different userspace drivers/clients to select
2580          *   different settings, or to change those settings on the fly in
2581          *   response to runtime needs.  This option was known by name
2582          *   "FtrPerCtxtPreemptionGranularityControl" at one time, although
2583          *   that name is somewhat misleading as other non-granularity
2584          *   preemption settings are also impacted by this decision.
2585          *
2586          * On Linux, our policy has always been to let userspace drivers
2587          * control preemption granularity/settings (Option 2).  This was
2588          * originally mandatory on gen9 to prevent ABI breakage (old gen9
2589          * userspace developed before object-level preemption was enabled would
2590          * not behave well if i915 were to go with Option 1 and enable that
2591          * preemption in a global manner).  On gen9 each context would have
2592          * object-level preemption disabled by default (see
2593          * WaDisable3DMidCmdPreemption in gen9_ctx_workarounds_init), but
2594          * userspace drivers could opt-in to object-level preemption as they
2595          * saw fit.  For post-gen9 platforms, we continue to utilize Option 2;
2596          * even though it is no longer necessary for ABI compatibility when
2597          * enabling a new platform, it does ensure that userspace will be able
2598          * to implement any workarounds that show up requiring temporary
2599          * adjustments to preemption behavior at runtime.
2600          *
2601          * Notes/Workarounds:
2602          *  - Wa_14015141709:  On DG2 and early steppings of MTL,
2603          *      CS_CHICKEN1[0] does not disable object-level preemption as
2604          *      it is supposed to (nor does CS_DEBUG_MODE1[0] if we had been
2605          *      using Option 1).  Effectively this means userspace is unable
2606          *      to disable object-level preemption on these platforms/steppings
2607          *      despite the setting here.
2608          *
2609          *  - Wa_16013994831:  May require that userspace program
2610          *      CS_CHICKEN1[10] when certain runtime conditions are true.
2611          *      Userspace requires Option 2 to be in effect for their update of
2612          *      CS_CHICKEN1[10] to be effective.
2613          *
2614          * Other workarounds may appear in the future that will also require
2615          * Option 2 behavior to allow proper userspace implementation.
2616          */
2617         if (GRAPHICS_VER(i915) >= 9)
2618                 wa_masked_en(wal,
2619                              GEN7_FF_SLICE_CS_CHICKEN1,
2620                              GEN9_FFSC_PERCTX_PREEMPT_CTRL);
2621
2622         if (IS_SKYLAKE(i915) ||
2623             IS_KABYLAKE(i915) ||
2624             IS_COFFEELAKE(i915) ||
2625             IS_COMETLAKE(i915)) {
2626                 /* WaEnableGapsTsvCreditFix:skl,kbl,cfl */
2627                 wa_write_or(wal,
2628                             GEN8_GARBCNTL,
2629                             GEN9_GAPS_TSV_CREDIT_DISABLE);
2630         }
2631
2632         if (IS_BROXTON(i915)) {
2633                 /* WaDisablePooledEuLoadBalancingFix:bxt */
2634                 wa_masked_en(wal,
2635                              FF_SLICE_CS_CHICKEN2,
2636                              GEN9_POOLED_EU_LOAD_BALANCING_FIX_DISABLE);
2637         }
2638
2639         if (GRAPHICS_VER(i915) == 9) {
2640                 /* WaContextSwitchWithConcurrentTLBInvalidate:skl,bxt,kbl,glk,cfl */
2641                 wa_masked_en(wal,
2642                              GEN9_CSFE_CHICKEN1_RCS,
2643                              GEN9_PREEMPT_GPGPU_SYNC_SWITCH_DISABLE);
2644
2645                 /* WaEnableLbsSlaRetryTimerDecrement:skl,bxt,kbl,glk,cfl */
2646                 wa_mcr_write_or(wal,
2647                                 BDW_SCRATCH1,
2648                                 GEN9_LBS_SLA_RETRY_TIMER_DECREMENT_ENABLE);
2649
2650                 /* WaProgramL3SqcReg1DefaultForPerf:bxt,glk */
2651                 if (IS_GEN9_LP(i915))
2652                         wa_mcr_write_clr_set(wal,
2653                                              GEN8_L3SQCREG1,
2654                                              L3_PRIO_CREDITS_MASK,
2655                                              L3_GENERAL_PRIO_CREDITS(62) |
2656                                              L3_HIGH_PRIO_CREDITS(2));
2657
2658                 /* WaOCLCoherentLineFlush:skl,bxt,kbl,cfl */
2659                 wa_mcr_write_or(wal,
2660                                 GEN8_L3SQCREG4,
2661                                 GEN8_LQSC_FLUSH_COHERENT_LINES);
2662
2663                 /* Disable atomics in L3 to prevent unrecoverable hangs */
2664                 wa_write_clr_set(wal, GEN9_SCRATCH_LNCF1,
2665                                  GEN9_LNCF_NONIA_COHERENT_ATOMICS_ENABLE, 0);
2666                 wa_mcr_write_clr_set(wal, GEN8_L3SQCREG4,
2667                                      GEN8_LQSQ_NONIA_COHERENT_ATOMICS_ENABLE, 0);
2668                 wa_mcr_write_clr_set(wal, GEN9_SCRATCH1,
2669                                      EVICTION_PERF_FIX_ENABLE, 0);
2670         }
2671
2672         if (IS_HASWELL(i915)) {
2673                 /* WaSampleCChickenBitEnable:hsw */
2674                 wa_masked_en(wal,
2675                              HSW_HALF_SLICE_CHICKEN3, HSW_SAMPLE_C_PERFORMANCE);
2676
2677                 wa_masked_dis(wal,
2678                               CACHE_MODE_0_GEN7,
2679                               /* enable HiZ Raw Stall Optimization */
2680                               HIZ_RAW_STALL_OPT_DISABLE);
2681         }
2682
2683         if (IS_VALLEYVIEW(i915)) {
2684                 /* WaDisableEarlyCull:vlv */
2685                 wa_masked_en(wal,
2686                              _3D_CHICKEN3,
2687                              _3D_CHICKEN_SF_DISABLE_OBJEND_CULL);
2688
2689                 /*
2690                  * WaVSThreadDispatchOverride:ivb,vlv
2691                  *
2692                  * This actually overrides the dispatch
2693                  * mode for all thread types.
2694                  */
2695                 wa_write_clr_set(wal,
2696                                  GEN7_FF_THREAD_MODE,
2697                                  GEN7_FF_SCHED_MASK,
2698                                  GEN7_FF_TS_SCHED_HW |
2699                                  GEN7_FF_VS_SCHED_HW |
2700                                  GEN7_FF_DS_SCHED_HW);
2701
2702                 /* WaPsdDispatchEnable:vlv */
2703                 /* WaDisablePSDDualDispatchEnable:vlv */
2704                 wa_masked_en(wal,
2705                              GEN7_HALF_SLICE_CHICKEN1,
2706                              GEN7_MAX_PS_THREAD_DEP |
2707                              GEN7_PSD_SINGLE_PORT_DISPATCH_ENABLE);
2708         }
2709
2710         if (IS_IVYBRIDGE(i915)) {
2711                 /* WaDisableEarlyCull:ivb */
2712                 wa_masked_en(wal,
2713                              _3D_CHICKEN3,
2714                              _3D_CHICKEN_SF_DISABLE_OBJEND_CULL);
2715
2716                 if (0) { /* causes HiZ corruption on ivb:gt1 */
2717                         /* enable HiZ Raw Stall Optimization */
2718                         wa_masked_dis(wal,
2719                                       CACHE_MODE_0_GEN7,
2720                                       HIZ_RAW_STALL_OPT_DISABLE);
2721                 }
2722
2723                 /*
2724                  * WaVSThreadDispatchOverride:ivb,vlv
2725                  *
2726                  * This actually overrides the dispatch
2727                  * mode for all thread types.
2728                  */
2729                 wa_write_clr_set(wal,
2730                                  GEN7_FF_THREAD_MODE,
2731                                  GEN7_FF_SCHED_MASK,
2732                                  GEN7_FF_TS_SCHED_HW |
2733                                  GEN7_FF_VS_SCHED_HW |
2734                                  GEN7_FF_DS_SCHED_HW);
2735
2736                 /* WaDisablePSDDualDispatchEnable:ivb */
2737                 if (IS_IVB_GT1(i915))
2738                         wa_masked_en(wal,
2739                                      GEN7_HALF_SLICE_CHICKEN1,
2740                                      GEN7_PSD_SINGLE_PORT_DISPATCH_ENABLE);
2741         }
2742
2743         if (GRAPHICS_VER(i915) == 7) {
2744                 /* WaBCSVCSTlbInvalidationMode:ivb,vlv,hsw */
2745                 wa_masked_en(wal,
2746                              RING_MODE_GEN7(RENDER_RING_BASE),
2747                              GFX_TLB_INVALIDATE_EXPLICIT | GFX_REPLAY_MODE);
2748
2749                 /* WaDisable_RenderCache_OperationalFlush:ivb,vlv,hsw */
2750                 wa_masked_dis(wal, CACHE_MODE_0_GEN7, RC_OP_FLUSH_ENABLE);
2751
2752                 /*
2753                  * BSpec says this must be set, even though
2754                  * WaDisable4x2SubspanOptimization:ivb,hsw
2755                  * WaDisable4x2SubspanOptimization isn't listed for VLV.
2756                  */
2757                 wa_masked_en(wal,
2758                              CACHE_MODE_1,
2759                              PIXEL_SUBSPAN_COLLECT_OPT_DISABLE);
2760
2761                 /*
2762                  * BSpec recommends 8x4 when MSAA is used,
2763                  * however in practice 16x4 seems fastest.
2764                  *
2765                  * Note that PS/WM thread counts depend on the WIZ hashing
2766                  * disable bit, which we don't touch here, but it's good
2767                  * to keep in mind (see 3DSTATE_PS and 3DSTATE_WM).
2768                  */
2769                 wa_masked_field_set(wal,
2770                                     GEN7_GT_MODE,
2771                                     GEN6_WIZ_HASHING_MASK,
2772                                     GEN6_WIZ_HASHING_16x4);
2773         }
2774
2775         if (IS_GRAPHICS_VER(i915, 6, 7))
2776                 /*
2777                  * We need to disable the AsyncFlip performance optimisations in
2778                  * order to use MI_WAIT_FOR_EVENT within the CS. It should
2779                  * already be programmed to '1' on all products.
2780                  *
2781                  * WaDisableAsyncFlipPerfMode:snb,ivb,hsw,vlv
2782                  */
2783                 wa_masked_en(wal,
2784                              RING_MI_MODE(RENDER_RING_BASE),
2785                              ASYNC_FLIP_PERF_DISABLE);
2786
2787         if (GRAPHICS_VER(i915) == 6) {
2788                 /*
2789                  * Required for the hardware to program scanline values for
2790                  * waiting
2791                  * WaEnableFlushTlbInvalidationMode:snb
2792                  */
2793                 wa_masked_en(wal,
2794                              GFX_MODE,
2795                              GFX_TLB_INVALIDATE_EXPLICIT);
2796
2797                 /* WaDisableHiZPlanesWhenMSAAEnabled:snb */
2798                 wa_masked_en(wal,
2799                              _3D_CHICKEN,
2800                              _3D_CHICKEN_HIZ_PLANE_DISABLE_MSAA_4X_SNB);
2801
2802                 wa_masked_en(wal,
2803                              _3D_CHICKEN3,
2804                              /* WaStripsFansDisableFastClipPerformanceFix:snb */
2805                              _3D_CHICKEN3_SF_DISABLE_FASTCLIP_CULL |
2806                              /*
2807                               * Bspec says:
2808                               * "This bit must be set if 3DSTATE_CLIP clip mode is set
2809                               * to normal and 3DSTATE_SF number of SF output attributes
2810                               * is more than 16."
2811                               */
2812                              _3D_CHICKEN3_SF_DISABLE_PIPELINED_ATTR_FETCH);
2813
2814                 /*
2815                  * BSpec recommends 8x4 when MSAA is used,
2816                  * however in practice 16x4 seems fastest.
2817                  *
2818                  * Note that PS/WM thread counts depend on the WIZ hashing
2819                  * disable bit, which we don't touch here, but it's good
2820                  * to keep in mind (see 3DSTATE_PS and 3DSTATE_WM).
2821                  */
2822                 wa_masked_field_set(wal,
2823                                     GEN6_GT_MODE,
2824                                     GEN6_WIZ_HASHING_MASK,
2825                                     GEN6_WIZ_HASHING_16x4);
2826
2827                 /* WaDisable_RenderCache_OperationalFlush:snb */
2828                 wa_masked_dis(wal, CACHE_MODE_0, RC_OP_FLUSH_ENABLE);
2829
2830                 /*
2831                  * From the Sandybridge PRM, volume 1 part 3, page 24:
2832                  * "If this bit is set, STCunit will have LRA as replacement
2833                  *  policy. [...] This bit must be reset. LRA replacement
2834                  *  policy is not supported."
2835                  */
2836                 wa_masked_dis(wal,
2837                               CACHE_MODE_0,
2838                               CM0_STC_EVICT_DISABLE_LRA_SNB);
2839         }
2840
2841         if (IS_GRAPHICS_VER(i915, 4, 6))
2842                 /* WaTimedSingleVertexDispatch:cl,bw,ctg,elk,ilk,snb */
2843                 wa_add(wal, RING_MI_MODE(RENDER_RING_BASE),
2844                        0, _MASKED_BIT_ENABLE(VS_TIMER_DISPATCH),
2845                        /* XXX bit doesn't stick on Broadwater */
2846                        IS_I965G(i915) ? 0 : VS_TIMER_DISPATCH, true);
2847
2848         if (GRAPHICS_VER(i915) == 4)
2849                 /*
2850                  * Disable CONSTANT_BUFFER before it is loaded from the context
2851                  * image. For as it is loaded, it is executed and the stored
2852                  * address may no longer be valid, leading to a GPU hang.
2853                  *
2854                  * This imposes the requirement that userspace reload their
2855                  * CONSTANT_BUFFER on every batch, fortunately a requirement
2856                  * they are already accustomed to from before contexts were
2857                  * enabled.
2858                  */
2859                 wa_add(wal, ECOSKPD(RENDER_RING_BASE),
2860                        0, _MASKED_BIT_ENABLE(ECO_CONSTANT_BUFFER_SR_DISABLE),
2861                        0 /* XXX bit doesn't stick on Broadwater */,
2862                        true);
2863 }
2864
2865 static void
2866 xcs_engine_wa_init(struct intel_engine_cs *engine, struct i915_wa_list *wal)
2867 {
2868         struct drm_i915_private *i915 = engine->i915;
2869
2870         /* WaKBLVECSSemaphoreWaitPoll:kbl */
2871         if (IS_KBL_GRAPHICS_STEP(i915, STEP_A0, STEP_F0)) {
2872                 wa_write(wal,
2873                          RING_SEMA_WAIT_POLL(engine->mmio_base),
2874                          1);
2875         }
2876 }
2877
2878 static void
2879 ccs_engine_wa_init(struct intel_engine_cs *engine, struct i915_wa_list *wal)
2880 {
2881         if (IS_PVC_CT_STEP(engine->i915, STEP_A0, STEP_C0)) {
2882                 /* Wa_14014999345:pvc */
2883                 wa_mcr_masked_en(wal, GEN10_CACHE_MODE_SS, DISABLE_ECC);
2884         }
2885 }
2886
2887 /*
2888  * The bspec performance guide has recommended MMIO tuning settings.  These
2889  * aren't truly "workarounds" but we want to program them with the same
2890  * workaround infrastructure to ensure that they're automatically added to
2891  * the GuC save/restore lists, re-applied at the right times, and checked for
2892  * any conflicting programming requested by real workarounds.
2893  *
2894  * Programming settings should be added here only if their registers are not
2895  * part of an engine's register state context.  If a register is part of a
2896  * context, then any tuning settings should be programmed in an appropriate
2897  * function invoked by __intel_engine_init_ctx_wa().
2898  */
2899 static void
2900 add_render_compute_tuning_settings(struct drm_i915_private *i915,
2901                                    struct i915_wa_list *wal)
2902 {
2903         if (IS_PONTEVECCHIO(i915)) {
2904                 wa_write(wal, XEHPC_L3SCRUB,
2905                          SCRUB_CL_DWNGRADE_SHARED | SCRUB_RATE_4B_PER_CLK);
2906         }
2907
2908         if (IS_DG2(i915)) {
2909                 wa_mcr_write_or(wal, XEHP_L3SCQREG7, BLEND_FILL_CACHING_OPT_DIS);
2910                 wa_mcr_write_clr_set(wal, RT_CTRL, STACKID_CTRL, STACKID_CTRL_512);
2911
2912                 /*
2913                  * This is also listed as Wa_22012654132 for certain DG2
2914                  * steppings, but the tuning setting programming is a superset
2915                  * since it applies to all DG2 variants and steppings.
2916                  *
2917                  * Note that register 0xE420 is write-only and cannot be read
2918                  * back for verification on DG2 (due to Wa_14012342262), so
2919                  * we need to explicitly skip the readback.
2920                  */
2921                 wa_mcr_add(wal, GEN10_CACHE_MODE_SS, 0,
2922                            _MASKED_BIT_ENABLE(ENABLE_PREFETCH_INTO_IC),
2923                            0 /* write-only, so skip validation */,
2924                            true);
2925         }
2926
2927         /*
2928          * This tuning setting proves beneficial only on ATS-M designs; the
2929          * default "age based" setting is optimal on regular DG2 and other
2930          * platforms.
2931          */
2932         if (INTEL_INFO(i915)->tuning_thread_rr_after_dep)
2933                 wa_mcr_masked_field_set(wal, GEN9_ROW_CHICKEN4, THREAD_EX_ARB_MODE,
2934                                         THREAD_EX_ARB_MODE_RR_AFTER_DEP);
2935 }
2936
2937 /*
2938  * The workarounds in this function apply to shared registers in
2939  * the general render reset domain that aren't tied to a
2940  * specific engine.  Since all render+compute engines get reset
2941  * together, and the contents of these registers are lost during
2942  * the shared render domain reset, we'll define such workarounds
2943  * here and then add them to just a single RCS or CCS engine's
2944  * workaround list (whichever engine has the XXXX flag).
2945  */
2946 static void
2947 general_render_compute_wa_init(struct intel_engine_cs *engine, struct i915_wa_list *wal)
2948 {
2949         struct drm_i915_private *i915 = engine->i915;
2950
2951         add_render_compute_tuning_settings(i915, wal);
2952
2953         if (IS_PONTEVECCHIO(i915)) {
2954                 /* Wa_16016694945 */
2955                 wa_masked_en(wal, XEHPC_LNCFMISCCFGREG0, XEHPC_OVRLSCCC);
2956         }
2957
2958         if (IS_XEHPSDV(i915)) {
2959                 /* Wa_1409954639 */
2960                 wa_mcr_masked_en(wal,
2961                                  GEN8_ROW_CHICKEN,
2962                                  SYSTOLIC_DOP_CLOCK_GATING_DIS);
2963
2964                 /* Wa_1607196519 */
2965                 wa_mcr_masked_en(wal,
2966                                  GEN9_ROW_CHICKEN4,
2967                                  GEN12_DISABLE_GRF_CLEAR);
2968
2969                 /* Wa_14010670810:xehpsdv */
2970                 wa_mcr_write_or(wal, XEHP_L3NODEARBCFG, XEHP_LNESPARE);
2971
2972                 /* Wa_14010449647:xehpsdv */
2973                 wa_mcr_masked_en(wal, GEN8_HALF_SLICE_CHICKEN1,
2974                                  GEN7_PSD_SINGLE_PORT_DISPATCH_ENABLE);
2975
2976                 /* Wa_18011725039:xehpsdv */
2977                 if (IS_XEHPSDV_GRAPHICS_STEP(i915, STEP_A1, STEP_B0)) {
2978                         wa_mcr_masked_dis(wal, MLTICTXCTL, TDONRENDER);
2979                         wa_mcr_write_or(wal, L3SQCREG1_CCS0, FLUSHALLNONCOH);
2980                 }
2981
2982                 /* Wa_14012362059:xehpsdv */
2983                 wa_mcr_write_or(wal, XEHP_MERT_MOD_CTRL, FORCE_MISS_FTLB);
2984
2985                 /* Wa_14014368820:xehpsdv */
2986                 wa_write_or(wal, GEN12_GAMCNTRL_CTRL, INVALIDATION_BROADCAST_MODE_DIS |
2987                                 GLOBAL_INVALIDATION_MODE);
2988         }
2989
2990         if (IS_DG2(i915) || IS_PONTEVECCHIO(i915)) {
2991                 /* Wa_14015227452:dg2,pvc */
2992                 wa_mcr_masked_en(wal, GEN9_ROW_CHICKEN4, XEHP_DIS_BBL_SYSPIPE);
2993
2994                 /* Wa_22014226127:dg2,pvc */
2995                 wa_mcr_write_or(wal, LSC_CHICKEN_BIT_0, DISABLE_D8_D16_COASLESCE);
2996
2997                 /* Wa_16015675438:dg2,pvc */
2998                 wa_masked_en(wal, FF_SLICE_CS_CHICKEN2, GEN12_PERF_FIX_BALANCING_CFE_DISABLE);
2999
3000                 /* Wa_18018781329:dg2,pvc */
3001                 wa_mcr_write_or(wal, RENDER_MOD_CTRL, FORCE_MISS_FTLB);
3002                 wa_mcr_write_or(wal, COMP_MOD_CTRL, FORCE_MISS_FTLB);
3003                 wa_mcr_write_or(wal, VDBX_MOD_CTRL, FORCE_MISS_FTLB);
3004                 wa_mcr_write_or(wal, VEBX_MOD_CTRL, FORCE_MISS_FTLB);
3005         }
3006
3007         if (IS_DG2(i915)) {
3008                 /*
3009                  * Wa_16011620976:dg2_g11
3010                  * Wa_22015475538:dg2
3011                  */
3012                 wa_mcr_write_or(wal, LSC_CHICKEN_BIT_0_UDW, DIS_CHAIN_2XSIMD8);
3013
3014                 /* Wa_18017747507:dg2 */
3015                 wa_masked_en(wal, VFG_PREEMPTION_CHICKEN, POLYGON_TRIFAN_LINELOOP_DISABLE);
3016         }
3017 }
3018
3019 static void
3020 engine_init_workarounds(struct intel_engine_cs *engine, struct i915_wa_list *wal)
3021 {
3022         if (GRAPHICS_VER(engine->i915) < 4)
3023                 return;
3024
3025         engine_fake_wa_init(engine, wal);
3026
3027         /*
3028          * These are common workarounds that just need to applied
3029          * to a single RCS/CCS engine's workaround list since
3030          * they're reset as part of the general render domain reset.
3031          */
3032         if (engine->flags & I915_ENGINE_FIRST_RENDER_COMPUTE)
3033                 general_render_compute_wa_init(engine, wal);
3034
3035         if (engine->class == COMPUTE_CLASS)
3036                 ccs_engine_wa_init(engine, wal);
3037         else if (engine->class == RENDER_CLASS)
3038                 rcs_engine_wa_init(engine, wal);
3039         else
3040                 xcs_engine_wa_init(engine, wal);
3041 }
3042
3043 void intel_engine_init_workarounds(struct intel_engine_cs *engine)
3044 {
3045         struct i915_wa_list *wal = &engine->wa_list;
3046
3047         wa_init_start(wal, engine->gt, "engine", engine->name);
3048         engine_init_workarounds(engine, wal);
3049         wa_init_finish(wal);
3050 }
3051
3052 void intel_engine_apply_workarounds(struct intel_engine_cs *engine)
3053 {
3054         wa_list_apply(&engine->wa_list);
3055 }
3056
3057 static const struct i915_range mcr_ranges_gen8[] = {
3058         { .start = 0x5500, .end = 0x55ff },
3059         { .start = 0x7000, .end = 0x7fff },
3060         { .start = 0x9400, .end = 0x97ff },
3061         { .start = 0xb000, .end = 0xb3ff },
3062         { .start = 0xe000, .end = 0xe7ff },
3063         {},
3064 };
3065
3066 static const struct i915_range mcr_ranges_gen12[] = {
3067         { .start =  0x8150, .end =  0x815f },
3068         { .start =  0x9520, .end =  0x955f },
3069         { .start =  0xb100, .end =  0xb3ff },
3070         { .start =  0xde80, .end =  0xe8ff },
3071         { .start = 0x24a00, .end = 0x24a7f },
3072         {},
3073 };
3074
3075 static const struct i915_range mcr_ranges_xehp[] = {
3076         { .start =  0x4000, .end =  0x4aff },
3077         { .start =  0x5200, .end =  0x52ff },
3078         { .start =  0x5400, .end =  0x7fff },
3079         { .start =  0x8140, .end =  0x815f },
3080         { .start =  0x8c80, .end =  0x8dff },
3081         { .start =  0x94d0, .end =  0x955f },
3082         { .start =  0x9680, .end =  0x96ff },
3083         { .start =  0xb000, .end =  0xb3ff },
3084         { .start =  0xc800, .end =  0xcfff },
3085         { .start =  0xd800, .end =  0xd8ff },
3086         { .start =  0xdc00, .end =  0xffff },
3087         { .start = 0x17000, .end = 0x17fff },
3088         { .start = 0x24a00, .end = 0x24a7f },
3089         {},
3090 };
3091
3092 static bool mcr_range(struct drm_i915_private *i915, u32 offset)
3093 {
3094         const struct i915_range *mcr_ranges;
3095         int i;
3096
3097         if (GRAPHICS_VER_FULL(i915) >= IP_VER(12, 50))
3098                 mcr_ranges = mcr_ranges_xehp;
3099         else if (GRAPHICS_VER(i915) >= 12)
3100                 mcr_ranges = mcr_ranges_gen12;
3101         else if (GRAPHICS_VER(i915) >= 8)
3102                 mcr_ranges = mcr_ranges_gen8;
3103         else
3104                 return false;
3105
3106         /*
3107          * Registers in these ranges are affected by the MCR selector
3108          * which only controls CPU initiated MMIO. Routing does not
3109          * work for CS access so we cannot verify them on this path.
3110          */
3111         for (i = 0; mcr_ranges[i].start; i++)
3112                 if (offset >= mcr_ranges[i].start &&
3113                     offset <= mcr_ranges[i].end)
3114                         return true;
3115
3116         return false;
3117 }
3118
3119 static int
3120 wa_list_srm(struct i915_request *rq,
3121             const struct i915_wa_list *wal,
3122             struct i915_vma *vma)
3123 {
3124         struct drm_i915_private *i915 = rq->engine->i915;
3125         unsigned int i, count = 0;
3126         const struct i915_wa *wa;
3127         u32 srm, *cs;
3128
3129         srm = MI_STORE_REGISTER_MEM | MI_SRM_LRM_GLOBAL_GTT;
3130         if (GRAPHICS_VER(i915) >= 8)
3131                 srm++;
3132
3133         for (i = 0, wa = wal->list; i < wal->count; i++, wa++) {
3134                 if (!mcr_range(i915, i915_mmio_reg_offset(wa->reg)))
3135                         count++;
3136         }
3137
3138         cs = intel_ring_begin(rq, 4 * count);
3139         if (IS_ERR(cs))
3140                 return PTR_ERR(cs);
3141
3142         for (i = 0, wa = wal->list; i < wal->count; i++, wa++) {
3143                 u32 offset = i915_mmio_reg_offset(wa->reg);
3144
3145                 if (mcr_range(i915, offset))
3146                         continue;
3147
3148                 *cs++ = srm;
3149                 *cs++ = offset;
3150                 *cs++ = i915_ggtt_offset(vma) + sizeof(u32) * i;
3151                 *cs++ = 0;
3152         }
3153         intel_ring_advance(rq, cs);
3154
3155         return 0;
3156 }
3157
3158 static int engine_wa_list_verify(struct intel_context *ce,
3159                                  const struct i915_wa_list * const wal,
3160                                  const char *from)
3161 {
3162         const struct i915_wa *wa;
3163         struct i915_request *rq;
3164         struct i915_vma *vma;
3165         struct i915_gem_ww_ctx ww;
3166         unsigned int i;
3167         u32 *results;
3168         int err;
3169
3170         if (!wal->count)
3171                 return 0;
3172
3173         vma = __vm_create_scratch_for_read(&ce->engine->gt->ggtt->vm,
3174                                            wal->count * sizeof(u32));
3175         if (IS_ERR(vma))
3176                 return PTR_ERR(vma);
3177
3178         intel_engine_pm_get(ce->engine);
3179         i915_gem_ww_ctx_init(&ww, false);
3180 retry:
3181         err = i915_gem_object_lock(vma->obj, &ww);
3182         if (err == 0)
3183                 err = intel_context_pin_ww(ce, &ww);
3184         if (err)
3185                 goto err_pm;
3186
3187         err = i915_vma_pin_ww(vma, &ww, 0, 0,
3188                            i915_vma_is_ggtt(vma) ? PIN_GLOBAL : PIN_USER);
3189         if (err)
3190                 goto err_unpin;
3191
3192         rq = i915_request_create(ce);
3193         if (IS_ERR(rq)) {
3194                 err = PTR_ERR(rq);
3195                 goto err_vma;
3196         }
3197
3198         err = i915_vma_move_to_active(vma, rq, EXEC_OBJECT_WRITE);
3199         if (err == 0)
3200                 err = wa_list_srm(rq, wal, vma);
3201
3202         i915_request_get(rq);
3203         if (err)
3204                 i915_request_set_error_once(rq, err);
3205         i915_request_add(rq);
3206
3207         if (err)
3208                 goto err_rq;
3209
3210         if (i915_request_wait(rq, 0, HZ / 5) < 0) {
3211                 err = -ETIME;
3212                 goto err_rq;
3213         }
3214
3215         results = i915_gem_object_pin_map(vma->obj, I915_MAP_WB);
3216         if (IS_ERR(results)) {
3217                 err = PTR_ERR(results);
3218                 goto err_rq;
3219         }
3220
3221         err = 0;
3222         for (i = 0, wa = wal->list; i < wal->count; i++, wa++) {
3223                 if (mcr_range(rq->engine->i915, i915_mmio_reg_offset(wa->reg)))
3224                         continue;
3225
3226                 if (!wa_verify(wal->gt, wa, results[i], wal->name, from))
3227                         err = -ENXIO;
3228         }
3229
3230         i915_gem_object_unpin_map(vma->obj);
3231
3232 err_rq:
3233         i915_request_put(rq);
3234 err_vma:
3235         i915_vma_unpin(vma);
3236 err_unpin:
3237         intel_context_unpin(ce);
3238 err_pm:
3239         if (err == -EDEADLK) {
3240                 err = i915_gem_ww_ctx_backoff(&ww);
3241                 if (!err)
3242                         goto retry;
3243         }
3244         i915_gem_ww_ctx_fini(&ww);
3245         intel_engine_pm_put(ce->engine);
3246         i915_vma_put(vma);
3247         return err;
3248 }
3249
3250 int intel_engine_verify_workarounds(struct intel_engine_cs *engine,
3251                                     const char *from)
3252 {
3253         return engine_wa_list_verify(engine->kernel_context,
3254                                      &engine->wa_list,
3255                                      from);
3256 }
3257
3258 #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
3259 #include "selftest_workarounds.c"
3260 #endif