drivers/gpu/drm/i915/gt/intel_workarounds.c

   1 // SPDX-License-Identifier: MIT
   2 /*
   3  * Copyright © 2014-2018 Intel Corporation
   4  */
   5
   6 #include "i915_drv.h"
   7 #include "intel_context.h"
   8 #include "intel_engine_pm.h"
   9 #include "intel_engine_regs.h"
  10 #include "intel_gpu_commands.h"
  11 #include "intel_gt.h"
  12 #include "intel_gt_mcr.h"
  13 #include "intel_gt_regs.h"
  14 #include "intel_ring.h"
  15 #include "intel_workarounds.h"
  16
  17 /**
  18  * DOC: Hardware workarounds
  19  *
  20  * This file is intended as a central place to implement most [1]_ of the
  21  * required workarounds for hardware to work as originally intended. They fall
  22  * in five basic categories depending on how/when they are applied:
  23  *
  24  * - Workarounds that touch registers that are saved/restored to/from the HW
  25  *   context image. The list is emitted (via Load Register Immediate commands)
  26  *   everytime a new context is created.
  27  * - GT workarounds. The list of these WAs is applied whenever these registers
  28  *   revert to default values (on GPU reset, suspend/resume [2]_, etc..).
  29  * - Display workarounds. The list is applied during display clock-gating
  30  *   initialization.
  31  * - Workarounds that whitelist a privileged register, so that UMDs can manage
  32  *   them directly. This is just a special case of a MMMIO workaround (as we
  33  *   write the list of these to/be-whitelisted registers to some special HW
  34  *   registers).
  35  * - Workaround batchbuffers, that get executed automatically by the hardware
  36  *   on every HW context restore.
  37  *
  38  * .. [1] Please notice that there are other WAs that, due to their nature,
  39  *    cannot be applied from a central place. Those are peppered around the rest
  40  *    of the code, as needed.
  41  *
  42  * .. [2] Technically, some registers are powercontext saved & restored, so they
  43  *    survive a suspend/resume. In practice, writing them again is not too
  44  *    costly and simplifies things. We can revisit this in the future.
  45  *
  46  * Layout
  47  * ~~~~~~
  48  *
  49  * Keep things in this file ordered by WA type, as per the above (context, GT,
  50  * display, register whitelist, batchbuffer). Then, inside each type, keep the
  51  * following order:
  52  *
  53  * - Infrastructure functions and macros
  54  * - WAs per platform in standard gen/chrono order
  55  * - Public functions to init or apply the given workaround type.
  56  */
  57
  58 static void wa_init_start(struct i915_wa_list *wal, const char *name, const char *engine_name)
  59 {
  60         wal->name = name;
  61         wal->engine_name = engine_name;
  62 }
  63
  64 #define WA_LIST_CHUNK (1 << 4)
  65
  66 static void wa_init_finish(struct i915_wa_list *wal)
  67 {
  68         /* Trim unused entries. */
  69         if (!IS_ALIGNED(wal->count, WA_LIST_CHUNK)) {
  70                 struct i915_wa *list = kmemdup(wal->list,
  71                                                wal->count * sizeof(*list),
  72                                                GFP_KERNEL);
  73
  74                 if (list) {
  75                         kfree(wal->list);
  76                         wal->list = list;
  77                 }
  78         }
  79
  80         if (!wal->count)
  81                 return;
  82
  83         DRM_DEBUG_DRIVER("Initialized %u %s workarounds on %s\n",
  84                          wal->wa_count, wal->name, wal->engine_name);
  85 }
  86
  87 static void _wa_add(struct i915_wa_list *wal, const struct i915_wa *wa)
  88 {
  89         unsigned int addr = i915_mmio_reg_offset(wa->reg);
  90         unsigned int start = 0, end = wal->count;
  91         const unsigned int grow = WA_LIST_CHUNK;
  92         struct i915_wa *wa_;
  93
  94         GEM_BUG_ON(!is_power_of_2(grow));
  95
  96         if (IS_ALIGNED(wal->count, grow)) { /* Either uninitialized or full. */
  97                 struct i915_wa *list;
  98
  99                 list = kmalloc_array(ALIGN(wal->count + 1, grow), sizeof(*wa),
 100                                      GFP_KERNEL);
 101                 if (!list) {
 102                         DRM_ERROR("No space for workaround init!\n");
 103                         return;
 104                 }
 105
 106                 if (wal->list) {
 107                         memcpy(list, wal->list, sizeof(*wa) * wal->count);
 108                         kfree(wal->list);
 109                 }
 110
 111                 wal->list = list;
 112         }
 113
 114         while (start < end) {
 115                 unsigned int mid = start + (end - start) / 2;
 116
 117                 if (i915_mmio_reg_offset(wal->list[mid].reg) < addr) {
 118                         start = mid + 1;
 119                 } else if (i915_mmio_reg_offset(wal->list[mid].reg) > addr) {
 120                         end = mid;
 121                 } else {
 122                         wa_ = &wal->list[mid];
 123
 124                         if ((wa->clr | wa_->clr) && !(wa->clr & ~wa_->clr)) {
 125                                 DRM_ERROR("Discarding overwritten w/a for reg %04x (clear: %08x, set: %08x)\n",
 126                                           i915_mmio_reg_offset(wa_->reg),
 127                                           wa_->clr, wa_->set);
 128
 129                                 wa_->set &= ~wa->clr;
 130                         }
 131
 132                         wal->wa_count++;
 133                         wa_->set |= wa->set;
 134                         wa_->clr |= wa->clr;
 135                         wa_->read |= wa->read;
 136                         return;
 137                 }
 138         }
 139
 140         wal->wa_count++;
 141         wa_ = &wal->list[wal->count++];
 142         *wa_ = *wa;
 143
 144         while (wa_-- > wal->list) {
 145                 GEM_BUG_ON(i915_mmio_reg_offset(wa_[0].reg) ==
 146                            i915_mmio_reg_offset(wa_[1].reg));
 147                 if (i915_mmio_reg_offset(wa_[1].reg) >
 148                     i915_mmio_reg_offset(wa_[0].reg))
 149                         break;
 150
 151                 swap(wa_[1], wa_[0]);
 152         }
 153 }
 154
 155 static void wa_add(struct i915_wa_list *wal, i915_reg_t reg,
 156                    u32 clear, u32 set, u32 read_mask, bool masked_reg)
 157 {
 158         struct i915_wa wa = {
 159                 .reg  = reg,
 160                 .clr  = clear,
 161                 .set  = set,
 162                 .read = read_mask,
 163                 .masked_reg = masked_reg,
 164         };
 165
 166         _wa_add(wal, &wa);
 167 }
 168
 169 static void
 170 wa_write_clr_set(struct i915_wa_list *wal, i915_reg_t reg, u32 clear, u32 set)
 171 {
 172         wa_add(wal, reg, clear, set, clear, false);
 173 }
 174
 175 static void
 176 wa_write(struct i915_wa_list *wal, i915_reg_t reg, u32 set)
 177 {
 178         wa_write_clr_set(wal, reg, ~0, set);
 179 }
 180
 181 static void
 182 wa_write_or(struct i915_wa_list *wal, i915_reg_t reg, u32 set)
 183 {
 184         wa_write_clr_set(wal, reg, set, set);
 185 }
 186
 187 static void
 188 wa_write_clr(struct i915_wa_list *wal, i915_reg_t reg, u32 clr)
 189 {
 190         wa_write_clr_set(wal, reg, clr, 0);
 191 }
 192
 193 /*
 194  * WA operations on "masked register". A masked register has the upper 16 bits
 195  * documented as "masked" in b-spec. Its purpose is to allow writing to just a
 196  * portion of the register without a rmw: you simply write in the upper 16 bits
 197  * the mask of bits you are going to modify.
 198  *
 199  * The wa_masked_* family of functions already does the necessary operations to
 200  * calculate the mask based on the parameters passed, so user only has to
 201  * provide the lower 16 bits of that register.
 202  */
 203
 204 static void
 205 wa_masked_en(struct i915_wa_list *wal, i915_reg_t reg, u32 val)
 206 {
 207         wa_add(wal, reg, 0, _MASKED_BIT_ENABLE(val), val, true);
 208 }
 209
 210 static void
 211 wa_masked_dis(struct i915_wa_list *wal, i915_reg_t reg, u32 val)
 212 {
 213         wa_add(wal, reg, 0, _MASKED_BIT_DISABLE(val), val, true);
 214 }
 215
 216 static void
 217 wa_masked_field_set(struct i915_wa_list *wal, i915_reg_t reg,
 218                     u32 mask, u32 val)
 219 {
 220         wa_add(wal, reg, 0, _MASKED_FIELD(mask, val), mask, true);
 221 }
 222
 223 static void gen6_ctx_workarounds_init(struct intel_engine_cs *engine,
 224                                       struct i915_wa_list *wal)
 225 {
 226         wa_masked_en(wal, INSTPM, INSTPM_FORCE_ORDERING);
 227 }
 228
 229 static void gen7_ctx_workarounds_init(struct intel_engine_cs *engine,
 230                                       struct i915_wa_list *wal)
 231 {
 232         wa_masked_en(wal, INSTPM, INSTPM_FORCE_ORDERING);
 233 }
 234
 235 static void gen8_ctx_workarounds_init(struct intel_engine_cs *engine,
 236                                       struct i915_wa_list *wal)
 237 {
 238         wa_masked_en(wal, INSTPM, INSTPM_FORCE_ORDERING);
 239
 240         /* WaDisableAsyncFlipPerfMode:bdw,chv */
 241         wa_masked_en(wal, RING_MI_MODE(RENDER_RING_BASE), ASYNC_FLIP_PERF_DISABLE);
 242
 243         /* WaDisablePartialInstShootdown:bdw,chv */
 244         wa_masked_en(wal, GEN8_ROW_CHICKEN,
 245                      PARTIAL_INSTRUCTION_SHOOTDOWN_DISABLE);
 246
 247         /* Use Force Non-Coherent whenever executing a 3D context. This is a
 248          * workaround for a possible hang in the unlikely event a TLB
 249          * invalidation occurs during a PSD flush.
 250          */
 251         /* WaForceEnableNonCoherent:bdw,chv */
 252         /* WaHdcDisableFetchWhenMasked:bdw,chv */
 253         wa_masked_en(wal, HDC_CHICKEN0,
 254                      HDC_DONOT_FETCH_MEM_WHEN_MASKED |
 255                      HDC_FORCE_NON_COHERENT);
 256
 257         /* From the Haswell PRM, Command Reference: Registers, CACHE_MODE_0:
 258          * "The Hierarchical Z RAW Stall Optimization allows non-overlapping
 259          *  polygons in the same 8x4 pixel/sample area to be processed without
 260          *  stalling waiting for the earlier ones to write to Hierarchical Z
 261          *  buffer."
 262          *
 263          * This optimization is off by default for BDW and CHV; turn it on.
 264          */
 265         wa_masked_dis(wal, CACHE_MODE_0_GEN7, HIZ_RAW_STALL_OPT_DISABLE);
 266
 267         /* Wa4x4STCOptimizationDisable:bdw,chv */
 268         wa_masked_en(wal, CACHE_MODE_1, GEN8_4x4_STC_OPTIMIZATION_DISABLE);
 269
 270         /*
 271          * BSpec recommends 8x4 when MSAA is used,
 272          * however in practice 16x4 seems fastest.
 273          *
 274          * Note that PS/WM thread counts depend on the WIZ hashing
 275          * disable bit, which we don't touch here, but it's good
 276          * to keep in mind (see 3DSTATE_PS and 3DSTATE_WM).
 277          */
 278         wa_masked_field_set(wal, GEN7_GT_MODE,
 279                             GEN6_WIZ_HASHING_MASK,
 280                             GEN6_WIZ_HASHING_16x4);
 281 }
 282
 283 static void bdw_ctx_workarounds_init(struct intel_engine_cs *engine,
 284                                      struct i915_wa_list *wal)
 285 {
 286         struct drm_i915_private *i915 = engine->i915;
 287
 288         gen8_ctx_workarounds_init(engine, wal);
 289
 290         /* WaDisableThreadStallDopClockGating:bdw (pre-production) */
 291         wa_masked_en(wal, GEN8_ROW_CHICKEN, STALL_DOP_GATING_DISABLE);
 292
 293         /* WaDisableDopClockGating:bdw
 294          *
 295          * Also see the related UCGTCL1 write in bdw_init_clock_gating()
 296          * to disable EUTC clock gating.
 297          */
 298         wa_masked_en(wal, GEN7_ROW_CHICKEN2,
 299                      DOP_CLOCK_GATING_DISABLE);
 300
 301         wa_masked_en(wal, HALF_SLICE_CHICKEN3,
 302                      GEN8_SAMPLER_POWER_BYPASS_DIS);
 303
 304         wa_masked_en(wal, HDC_CHICKEN0,
 305                      /* WaForceContextSaveRestoreNonCoherent:bdw */
 306                      HDC_FORCE_CONTEXT_SAVE_RESTORE_NON_COHERENT |
 307                      /* WaDisableFenceDestinationToSLM:bdw (pre-prod) */
 308                      (IS_BDW_GT3(i915) ? HDC_FENCE_DEST_SLM_DISABLE : 0));
 309 }
 310
 311 static void chv_ctx_workarounds_init(struct intel_engine_cs *engine,
 312                                      struct i915_wa_list *wal)
 313 {
 314         gen8_ctx_workarounds_init(engine, wal);
 315
 316         /* WaDisableThreadStallDopClockGating:chv */
 317         wa_masked_en(wal, GEN8_ROW_CHICKEN, STALL_DOP_GATING_DISABLE);
 318
 319         /* Improve HiZ throughput on CHV. */
 320         wa_masked_en(wal, HIZ_CHICKEN, CHV_HZ_8X8_MODE_IN_1X);
 321 }
 322
 323 static void gen9_ctx_workarounds_init(struct intel_engine_cs *engine,
 324                                       struct i915_wa_list *wal)
 325 {
 326         struct drm_i915_private *i915 = engine->i915;
 327
 328         if (HAS_LLC(i915)) {
 329                 /* WaCompressedResourceSamplerPbeMediaNewHashMode:skl,kbl
 330                  *
 331                  * Must match Display Engine. See
 332                  * WaCompressedResourceDisplayNewHashMode.
 333                  */
 334                 wa_masked_en(wal, COMMON_SLICE_CHICKEN2,
 335                              GEN9_PBE_COMPRESSED_HASH_SELECTION);
 336                 wa_masked_en(wal, GEN9_HALF_SLICE_CHICKEN7,
 337                              GEN9_SAMPLER_HASH_COMPRESSED_READ_ADDR);
 338         }
 339
 340         /* WaClearFlowControlGpgpuContextSave:skl,bxt,kbl,glk,cfl */
 341         /* WaDisablePartialInstShootdown:skl,bxt,kbl,glk,cfl */
 342         wa_masked_en(wal, GEN8_ROW_CHICKEN,
 343                      FLOW_CONTROL_ENABLE |
 344                      PARTIAL_INSTRUCTION_SHOOTDOWN_DISABLE);
 345
 346         /* WaEnableYV12BugFixInHalfSliceChicken7:skl,bxt,kbl,glk,cfl */
 347         /* WaEnableSamplerGPGPUPreemptionSupport:skl,bxt,kbl,cfl */
 348         wa_masked_en(wal, GEN9_HALF_SLICE_CHICKEN7,
 349                      GEN9_ENABLE_YV12_BUGFIX |
 350                      GEN9_ENABLE_GPGPU_PREEMPTION);
 351
 352         /* Wa4x4STCOptimizationDisable:skl,bxt,kbl,glk,cfl */
 353         /* WaDisablePartialResolveInVc:skl,bxt,kbl,cfl */
 354         wa_masked_en(wal, CACHE_MODE_1,
 355                      GEN8_4x4_STC_OPTIMIZATION_DISABLE |
 356                      GEN9_PARTIAL_RESOLVE_IN_VC_DISABLE);
 357
 358         /* WaCcsTlbPrefetchDisable:skl,bxt,kbl,glk,cfl */
 359         wa_masked_dis(wal, GEN9_HALF_SLICE_CHICKEN5,
 360                       GEN9_CCS_TLB_PREFETCH_ENABLE);
 361
 362         /* WaForceContextSaveRestoreNonCoherent:skl,bxt,kbl,cfl */
 363         wa_masked_en(wal, HDC_CHICKEN0,
 364                      HDC_FORCE_CONTEXT_SAVE_RESTORE_NON_COHERENT |
 365                      HDC_FORCE_CSR_NON_COHERENT_OVR_DISABLE);
 366
 367         /* WaForceEnableNonCoherent and WaDisableHDCInvalidation are
 368          * both tied to WaForceContextSaveRestoreNonCoherent
 369          * in some hsds for skl. We keep the tie for all gen9. The
 370          * documentation is a bit hazy and so we want to get common behaviour,
 371          * even though there is no clear evidence we would need both on kbl/bxt.
 372          * This area has been source of system hangs so we play it safe
 373          * and mimic the skl regardless of what bspec says.
 374          *
 375          * Use Force Non-Coherent whenever executing a 3D context. This
 376          * is a workaround for a possible hang in the unlikely event
 377          * a TLB invalidation occurs during a PSD flush.
 378          */
 379
 380         /* WaForceEnableNonCoherent:skl,bxt,kbl,cfl */
 381         wa_masked_en(wal, HDC_CHICKEN0,
 382                      HDC_FORCE_NON_COHERENT);
 383
 384         /* WaDisableSamplerPowerBypassForSOPingPong:skl,bxt,kbl,cfl */
 385         if (IS_SKYLAKE(i915) ||
 386             IS_KABYLAKE(i915) ||
 387             IS_COFFEELAKE(i915) ||
 388             IS_COMETLAKE(i915))
 389                 wa_masked_en(wal, HALF_SLICE_CHICKEN3,
 390                              GEN8_SAMPLER_POWER_BYPASS_DIS);
 391
 392         /* WaDisableSTUnitPowerOptimization:skl,bxt,kbl,glk,cfl */
 393         wa_masked_en(wal, HALF_SLICE_CHICKEN2, GEN8_ST_PO_DISABLE);
 394
 395         /*
 396          * Supporting preemption with fine-granularity requires changes in the
 397          * batch buffer programming. Since we can't break old userspace, we
 398          * need to set our default preemption level to safe value. Userspace is
 399          * still able to use more fine-grained preemption levels, since in
 400          * WaEnablePreemptionGranularityControlByUMD we're whitelisting the
 401          * per-ctx register. As such, WaDisable{3D,GPGPU}MidCmdPreemption are
 402          * not real HW workarounds, but merely a way to start using preemption
 403          * while maintaining old contract with userspace.
 404          */
 405
 406         /* WaDisable3DMidCmdPreemption:skl,bxt,glk,cfl,[cnl] */
 407         wa_masked_dis(wal, GEN8_CS_CHICKEN1, GEN9_PREEMPT_3D_OBJECT_LEVEL);
 408
 409         /* WaDisableGPGPUMidCmdPreemption:skl,bxt,blk,cfl,[cnl] */
 410         wa_masked_field_set(wal, GEN8_CS_CHICKEN1,
 411                             GEN9_PREEMPT_GPGPU_LEVEL_MASK,
 412                             GEN9_PREEMPT_GPGPU_COMMAND_LEVEL);
 413
 414         /* WaClearHIZ_WM_CHICKEN3:bxt,glk */
 415         if (IS_GEN9_LP(i915))
 416                 wa_masked_en(wal, GEN9_WM_CHICKEN3, GEN9_FACTOR_IN_CLR_VAL_HIZ);
 417 }
 418
 419 static void skl_tune_iz_hashing(struct intel_engine_cs *engine,
 420                                 struct i915_wa_list *wal)
 421 {
 422         struct intel_gt *gt = engine->gt;
 423         u8 vals[3] = { 0, 0, 0 };
 424         unsigned int i;
 425
 426         for (i = 0; i < 3; i++) {
 427                 u8 ss;
 428
 429                 /*
 430                  * Only consider slices where one, and only one, subslice has 7
 431                  * EUs
 432                  */
 433                 if (!is_power_of_2(gt->info.sseu.subslice_7eu[i]))
 434                         continue;
 435
 436                 /*
 437                  * subslice_7eu[i] != 0 (because of the check above) and
 438                  * ss_max == 4 (maximum number of subslices possible per slice)
 439                  *
 440                  * ->    0 <= ss <= 3;
 441                  */
 442                 ss = ffs(gt->info.sseu.subslice_7eu[i]) - 1;
 443                 vals[i] = 3 - ss;
 444         }
 445
 446         if (vals[0] == 0 && vals[1] == 0 && vals[2] == 0)
 447                 return;
 448
 449         /* Tune IZ hashing. See intel_device_info_runtime_init() */
 450         wa_masked_field_set(wal, GEN7_GT_MODE,
 451                             GEN9_IZ_HASHING_MASK(2) |
 452                             GEN9_IZ_HASHING_MASK(1) |
 453                             GEN9_IZ_HASHING_MASK(0),
 454                             GEN9_IZ_HASHING(2, vals[2]) |
 455                             GEN9_IZ_HASHING(1, vals[1]) |
 456                             GEN9_IZ_HASHING(0, vals[0]));
 457 }
 458
 459 static void skl_ctx_workarounds_init(struct intel_engine_cs *engine,
 460                                      struct i915_wa_list *wal)
 461 {
 462         gen9_ctx_workarounds_init(engine, wal);
 463         skl_tune_iz_hashing(engine, wal);
 464 }
 465
 466 static void bxt_ctx_workarounds_init(struct intel_engine_cs *engine,
 467                                      struct i915_wa_list *wal)
 468 {
 469         gen9_ctx_workarounds_init(engine, wal);
 470
 471         /* WaDisableThreadStallDopClockGating:bxt */
 472         wa_masked_en(wal, GEN8_ROW_CHICKEN,
 473                      STALL_DOP_GATING_DISABLE);
 474
 475         /* WaToEnableHwFixForPushConstHWBug:bxt */
 476         wa_masked_en(wal, COMMON_SLICE_CHICKEN2,
 477                      GEN8_SBE_DISABLE_REPLAY_BUF_OPTIMIZATION);
 478 }
 479
 480 static void kbl_ctx_workarounds_init(struct intel_engine_cs *engine,
 481                                      struct i915_wa_list *wal)
 482 {
 483         struct drm_i915_private *i915 = engine->i915;
 484
 485         gen9_ctx_workarounds_init(engine, wal);
 486
 487         /* WaToEnableHwFixForPushConstHWBug:kbl */
 488         if (IS_KBL_GRAPHICS_STEP(i915, STEP_C0, STEP_FOREVER))
 489                 wa_masked_en(wal, COMMON_SLICE_CHICKEN2,
 490                              GEN8_SBE_DISABLE_REPLAY_BUF_OPTIMIZATION);
 491
 492         /* WaDisableSbeCacheDispatchPortSharing:kbl */
 493         wa_masked_en(wal, GEN7_HALF_SLICE_CHICKEN1,
 494                      GEN7_SBE_SS_CACHE_DISPATCH_PORT_SHARING_DISABLE);
 495 }
 496
 497 static void glk_ctx_workarounds_init(struct intel_engine_cs *engine,
 498                                      struct i915_wa_list *wal)
 499 {
 500         gen9_ctx_workarounds_init(engine, wal);
 501
 502         /* WaToEnableHwFixForPushConstHWBug:glk */
 503         wa_masked_en(wal, COMMON_SLICE_CHICKEN2,
 504                      GEN8_SBE_DISABLE_REPLAY_BUF_OPTIMIZATION);
 505 }
 506
 507 static void cfl_ctx_workarounds_init(struct intel_engine_cs *engine,
 508                                      struct i915_wa_list *wal)
 509 {
 510         gen9_ctx_workarounds_init(engine, wal);
 511
 512         /* WaToEnableHwFixForPushConstHWBug:cfl */
 513         wa_masked_en(wal, COMMON_SLICE_CHICKEN2,
 514                      GEN8_SBE_DISABLE_REPLAY_BUF_OPTIMIZATION);
 515
 516         /* WaDisableSbeCacheDispatchPortSharing:cfl */
 517         wa_masked_en(wal, GEN7_HALF_SLICE_CHICKEN1,
 518                      GEN7_SBE_SS_CACHE_DISPATCH_PORT_SHARING_DISABLE);
 519 }
 520
 521 static void icl_ctx_workarounds_init(struct intel_engine_cs *engine,
 522                                      struct i915_wa_list *wal)
 523 {
 524         /* Wa_1406697149 (WaDisableBankHangMode:icl) */
 525         wa_write(wal,
 526                  GEN8_L3CNTLREG,
 527                  intel_uncore_read(engine->uncore, GEN8_L3CNTLREG) |
 528                  GEN8_ERRDETBCTRL);
 529
 530         /* WaForceEnableNonCoherent:icl
 531          * This is not the same workaround as in early Gen9 platforms, where
 532          * lacking this could cause system hangs, but coherency performance
 533          * overhead is high and only a few compute workloads really need it
 534          * (the register is whitelisted in hardware now, so UMDs can opt in
 535          * for coherency if they have a good reason).
 536          */
 537         wa_masked_en(wal, ICL_HDC_MODE, HDC_FORCE_NON_COHERENT);
 538
 539         /* WaEnableFloatBlendOptimization:icl */
 540         wa_add(wal, GEN10_CACHE_MODE_SS, 0,
 541                _MASKED_BIT_ENABLE(FLOAT_BLEND_OPTIMIZATION_ENABLE),
 542                0 /* write-only, so skip validation */,
 543                true);
 544
 545         /* WaDisableGPGPUMidThreadPreemption:icl */
 546         wa_masked_field_set(wal, GEN8_CS_CHICKEN1,
 547                             GEN9_PREEMPT_GPGPU_LEVEL_MASK,
 548                             GEN9_PREEMPT_GPGPU_THREAD_GROUP_LEVEL);
 549
 550         /* allow headerless messages for preemptible GPGPU context */
 551         wa_masked_en(wal, GEN10_SAMPLER_MODE,
 552                      GEN11_SAMPLER_ENABLE_HEADLESS_MSG);
 553
 554         /* Wa_1604278689:icl,ehl */
 555         wa_write(wal, IVB_FBC_RT_BASE, 0xFFFFFFFF & ~ILK_FBC_RT_VALID);
 556         wa_write_clr_set(wal, IVB_FBC_RT_BASE_UPPER,
 557                          0, /* write-only register; skip validation */
 558                          0xFFFFFFFF);
 559
 560         /* Wa_1406306137:icl,ehl */
 561         wa_masked_en(wal, GEN9_ROW_CHICKEN4, GEN11_DIS_PICK_2ND_EU);
 562 }
 563
 564 /*
 565  * These settings aren't actually workarounds, but general tuning settings that
 566  * need to be programmed on dg2 platform.
 567  */
 568 static void dg2_ctx_gt_tuning_init(struct intel_engine_cs *engine,
 569                                    struct i915_wa_list *wal)
 570 {
 571         wa_write_clr_set(wal, GEN11_L3SQCREG5, L3_PWM_TIMER_INIT_VAL_MASK,
 572                          REG_FIELD_PREP(L3_PWM_TIMER_INIT_VAL_MASK, 0x7f));
 573         wa_add(wal,
 574                FF_MODE2,
 575                FF_MODE2_TDS_TIMER_MASK,
 576                FF_MODE2_TDS_TIMER_128,
 577                0, false);
 578 }
 579
 580 /*
 581  * These settings aren't actually workarounds, but general tuning settings that
 582  * need to be programmed on several platforms.
 583  */
 584 static void gen12_ctx_gt_tuning_init(struct intel_engine_cs *engine,
 585                                      struct i915_wa_list *wal)
 586 {
 587         /*
 588          * Although some platforms refer to it as Wa_1604555607, we need to
 589          * program it even on those that don't explicitly list that
 590          * workaround.
 591          *
 592          * Note that the programming of this register is further modified
 593          * according to the FF_MODE2 guidance given by Wa_1608008084:gen12.
 594          * Wa_1608008084 tells us the FF_MODE2 register will return the wrong
 595          * value when read. The default value for this register is zero for all
 596          * fields and there are no bit masks. So instead of doing a RMW we
 597          * should just write TDS timer value. For the same reason read
 598          * verification is ignored.
 599          */
 600         wa_add(wal,
 601                FF_MODE2,
 602                FF_MODE2_TDS_TIMER_MASK,
 603                FF_MODE2_TDS_TIMER_128,
 604                0, false);
 605 }
 606
 607 static void gen12_ctx_workarounds_init(struct intel_engine_cs *engine,
 608                                        struct i915_wa_list *wal)
 609 {
 610         gen12_ctx_gt_tuning_init(engine, wal);
 611
 612         /*
 613          * Wa_1409142259:tgl,dg1,adl-p
 614          * Wa_1409347922:tgl,dg1,adl-p
 615          * Wa_1409252684:tgl,dg1,adl-p
 616          * Wa_1409217633:tgl,dg1,adl-p
 617          * Wa_1409207793:tgl,dg1,adl-p
 618          * Wa_1409178076:tgl,dg1,adl-p
 619          * Wa_1408979724:tgl,dg1,adl-p
 620          * Wa_14010443199:tgl,rkl,dg1,adl-p
 621          * Wa_14010698770:tgl,rkl,dg1,adl-s,adl-p
 622          * Wa_1409342910:tgl,rkl,dg1,adl-s,adl-p
 623          */
 624         wa_masked_en(wal, GEN11_COMMON_SLICE_CHICKEN3,
 625                      GEN12_DISABLE_CPS_AWARE_COLOR_PIPE);
 626
 627         /* WaDisableGPGPUMidThreadPreemption:gen12 */
 628         wa_masked_field_set(wal, GEN8_CS_CHICKEN1,
 629                             GEN9_PREEMPT_GPGPU_LEVEL_MASK,
 630                             GEN9_PREEMPT_GPGPU_THREAD_GROUP_LEVEL);
 631
 632         /*
 633          * Wa_16011163337
 634          *
 635          * Like in gen12_ctx_gt_tuning_init(), read verification is ignored due
 636          * to Wa_1608008084.
 637          */
 638         wa_add(wal,
 639                FF_MODE2,
 640                FF_MODE2_GS_TIMER_MASK,
 641                FF_MODE2_GS_TIMER_224,
 642                0, false);
 643 }
 644
 645 static void dg1_ctx_workarounds_init(struct intel_engine_cs *engine,
 646                                      struct i915_wa_list *wal)
 647 {
 648         gen12_ctx_workarounds_init(engine, wal);
 649
 650         /* Wa_1409044764 */
 651         wa_masked_dis(wal, GEN11_COMMON_SLICE_CHICKEN3,
 652                       DG1_FLOAT_POINT_BLEND_OPT_STRICT_MODE_EN);
 653
 654         /* Wa_22010493298 */
 655         wa_masked_en(wal, HIZ_CHICKEN,
 656                      DG1_HZ_READ_SUPPRESSION_OPTIMIZATION_DISABLE);
 657 }
 658
 659 static void dg2_ctx_workarounds_init(struct intel_engine_cs *engine,
 660                                      struct i915_wa_list *wal)
 661 {
 662         dg2_ctx_gt_tuning_init(engine, wal);
 663
 664         /* Wa_16011186671:dg2_g11 */
 665         if (IS_DG2_GRAPHICS_STEP(engine->i915, G11, STEP_A0, STEP_B0)) {
 666                 wa_masked_dis(wal, VFLSKPD, DIS_MULT_MISS_RD_SQUASH);
 667                 wa_masked_en(wal, VFLSKPD, DIS_OVER_FETCH_CACHE);
 668         }
 669
 670         if (IS_DG2_GRAPHICS_STEP(engine->i915, G10, STEP_A0, STEP_B0)) {
 671                 /* Wa_14010469329:dg2_g10 */
 672                 wa_masked_en(wal, GEN11_COMMON_SLICE_CHICKEN3,
 673                              XEHP_DUAL_SIMD8_SEQ_MERGE_DISABLE);
 674
 675                 /*
 676                  * Wa_22010465075:dg2_g10
 677                  * Wa_22010613112:dg2_g10
 678                  * Wa_14010698770:dg2_g10
 679                  */
 680                 wa_masked_en(wal, GEN11_COMMON_SLICE_CHICKEN3,
 681                              GEN12_DISABLE_CPS_AWARE_COLOR_PIPE);
 682         }
 683
 684         /* Wa_16013271637:dg2 */
 685         wa_masked_en(wal, SLICE_COMMON_ECO_CHICKEN1,
 686                      MSC_MSAA_REODER_BUF_BYPASS_DISABLE);
 687
 688         /* Wa_14014947963:dg2 */
 689         if (IS_DG2_GRAPHICS_STEP(engine->i915, G10, STEP_B0, STEP_FOREVER) ||
 690                 IS_DG2_G11(engine->i915) || IS_DG2_G12(engine->i915))
 691                 wa_masked_field_set(wal, VF_PREEMPTION, PREEMPTION_VERTEX_COUNT, 0x4000);
 692
 693         /* Wa_15010599737:dg2 */
 694         wa_masked_en(wal, CHICKEN_RASTER_1, DIS_SF_ROUND_NEAREST_EVEN);
 695 }
 696
 697 static void fakewa_disable_nestedbb_mode(struct intel_engine_cs *engine,
 698                                          struct i915_wa_list *wal)
 699 {
 700         /*
 701          * This is a "fake" workaround defined by software to ensure we
 702          * maintain reliable, backward-compatible behavior for userspace with
 703          * regards to how nested MI_BATCH_BUFFER_START commands are handled.
 704          *
 705          * The per-context setting of MI_MODE[12] determines whether the bits
 706          * of a nested MI_BATCH_BUFFER_START instruction should be interpreted
 707          * in the traditional manner or whether they should instead use a new
 708          * tgl+ meaning that breaks backward compatibility, but allows nesting
 709          * into 3rd-level batchbuffers.  When this new capability was first
 710          * added in TGL, it remained off by default unless a context
 711          * intentionally opted in to the new behavior.  However Xe_HPG now
 712          * flips this on by default and requires that we explicitly opt out if
 713          * we don't want the new behavior.
 714          *
 715          * From a SW perspective, we want to maintain the backward-compatible
 716          * behavior for userspace, so we'll apply a fake workaround to set it
 717          * back to the legacy behavior on platforms where the hardware default
 718          * is to break compatibility.  At the moment there is no Linux
 719          * userspace that utilizes third-level batchbuffers, so this will avoid
 720          * userspace from needing to make any changes.  using the legacy
 721          * meaning is the correct thing to do.  If/when we have userspace
 722          * consumers that want to utilize third-level batch nesting, we can
 723          * provide a context parameter to allow them to opt-in.
 724          */
 725         wa_masked_dis(wal, RING_MI_MODE(engine->mmio_base), TGL_NESTED_BB_EN);
 726 }
 727
 728 static void gen12_ctx_gt_mocs_init(struct intel_engine_cs *engine,
 729                                    struct i915_wa_list *wal)
 730 {
 731         u8 mocs;
 732
 733         /*
 734          * Some blitter commands do not have a field for MOCS, those
 735          * commands will use MOCS index pointed by BLIT_CCTL.
 736          * BLIT_CCTL registers are needed to be programmed to un-cached.
 737          */
 738         if (engine->class == COPY_ENGINE_CLASS) {
 739                 mocs = engine->gt->mocs.uc_index;
 740                 wa_write_clr_set(wal,
 741                                  BLIT_CCTL(engine->mmio_base),
 742                                  BLIT_CCTL_MASK,
 743                                  BLIT_CCTL_MOCS(mocs, mocs));
 744         }
 745 }
 746
 747 /*
 748  * gen12_ctx_gt_fake_wa_init() aren't programmingan official workaround
 749  * defined by the hardware team, but it programming general context registers.
 750  * Adding those context register programming in context workaround
 751  * allow us to use the wa framework for proper application and validation.
 752  */
 753 static void
 754 gen12_ctx_gt_fake_wa_init(struct intel_engine_cs *engine,
 755                           struct i915_wa_list *wal)
 756 {
 757         if (GRAPHICS_VER_FULL(engine->i915) >= IP_VER(12, 55))
 758                 fakewa_disable_nestedbb_mode(engine, wal);
 759
 760         gen12_ctx_gt_mocs_init(engine, wal);
 761 }
 762
 763 static void
 764 __intel_engine_init_ctx_wa(struct intel_engine_cs *engine,
 765                            struct i915_wa_list *wal,
 766                            const char *name)
 767 {
 768         struct drm_i915_private *i915 = engine->i915;
 769
 770         wa_init_start(wal, name, engine->name);
 771
 772         /* Applies to all engines */
 773         /*
 774          * Fake workarounds are not the actual workaround but
 775          * programming of context registers using workaround framework.
 776          */
 777         if (GRAPHICS_VER(i915) >= 12)
 778                 gen12_ctx_gt_fake_wa_init(engine, wal);
 779
 780         if (engine->class != RENDER_CLASS)
 781                 goto done;
 782
 783         if (IS_PONTEVECCHIO(i915))
 784                 ; /* noop; none at this time */
 785         else if (IS_DG2(i915))
 786                 dg2_ctx_workarounds_init(engine, wal);
 787         else if (IS_XEHPSDV(i915))
 788                 ; /* noop; none at this time */
 789         else if (IS_DG1(i915))
 790                 dg1_ctx_workarounds_init(engine, wal);
 791         else if (GRAPHICS_VER(i915) == 12)
 792                 gen12_ctx_workarounds_init(engine, wal);
 793         else if (GRAPHICS_VER(i915) == 11)
 794                 icl_ctx_workarounds_init(engine, wal);
 795         else if (IS_COFFEELAKE(i915) || IS_COMETLAKE(i915))
 796                 cfl_ctx_workarounds_init(engine, wal);
 797         else if (IS_GEMINILAKE(i915))
 798                 glk_ctx_workarounds_init(engine, wal);
 799         else if (IS_KABYLAKE(i915))
 800                 kbl_ctx_workarounds_init(engine, wal);
 801         else if (IS_BROXTON(i915))
 802                 bxt_ctx_workarounds_init(engine, wal);
 803         else if (IS_SKYLAKE(i915))
 804                 skl_ctx_workarounds_init(engine, wal);
 805         else if (IS_CHERRYVIEW(i915))
 806                 chv_ctx_workarounds_init(engine, wal);
 807         else if (IS_BROADWELL(i915))
 808                 bdw_ctx_workarounds_init(engine, wal);
 809         else if (GRAPHICS_VER(i915) == 7)
 810                 gen7_ctx_workarounds_init(engine, wal);
 811         else if (GRAPHICS_VER(i915) == 6)
 812                 gen6_ctx_workarounds_init(engine, wal);
 813         else if (GRAPHICS_VER(i915) < 8)
 814                 ;
 815         else
 816                 MISSING_CASE(GRAPHICS_VER(i915));
 817
 818 done:
 819         wa_init_finish(wal);
 820 }
 821
 822 void intel_engine_init_ctx_wa(struct intel_engine_cs *engine)
 823 {
 824         __intel_engine_init_ctx_wa(engine, &engine->ctx_wa_list, "context");
 825 }
 826
 827 int intel_engine_emit_ctx_wa(struct i915_request *rq)
 828 {
 829         struct i915_wa_list *wal = &rq->engine->ctx_wa_list;
 830         struct i915_wa *wa;
 831         unsigned int i;
 832         u32 *cs;
 833         int ret;
 834
 835         if (wal->count == 0)
 836                 return 0;
 837
 838         ret = rq->engine->emit_flush(rq, EMIT_BARRIER);
 839         if (ret)
 840                 return ret;
 841
 842         cs = intel_ring_begin(rq, (wal->count * 2 + 2));
 843         if (IS_ERR(cs))
 844                 return PTR_ERR(cs);
 845
 846         *cs++ = MI_LOAD_REGISTER_IMM(wal->count);
 847         for (i = 0, wa = wal->list; i < wal->count; i++, wa++) {
 848                 *cs++ = i915_mmio_reg_offset(wa->reg);
 849                 *cs++ = wa->set;
 850         }
 851         *cs++ = MI_NOOP;
 852
 853         intel_ring_advance(rq, cs);
 854
 855         ret = rq->engine->emit_flush(rq, EMIT_BARRIER);
 856         if (ret)
 857                 return ret;
 858
 859         return 0;
 860 }
 861
 862 static void
 863 gen4_gt_workarounds_init(struct intel_gt *gt,
 864                          struct i915_wa_list *wal)
 865 {
 866         /* WaDisable_RenderCache_OperationalFlush:gen4,ilk */
 867         wa_masked_dis(wal, CACHE_MODE_0, RC_OP_FLUSH_ENABLE);
 868 }
 869
 870 static void
 871 g4x_gt_workarounds_init(struct intel_gt *gt, struct i915_wa_list *wal)
 872 {
 873         gen4_gt_workarounds_init(gt, wal);
 874
 875         /* WaDisableRenderCachePipelinedFlush:g4x,ilk */
 876         wa_masked_en(wal, CACHE_MODE_0, CM0_PIPELINED_RENDER_FLUSH_DISABLE);
 877 }
 878
 879 static void
 880 ilk_gt_workarounds_init(struct intel_gt *gt, struct i915_wa_list *wal)
 881 {
 882         g4x_gt_workarounds_init(gt, wal);
 883
 884         wa_masked_en(wal, _3D_CHICKEN2, _3D_CHICKEN2_WM_READ_PIPELINED);
 885 }
 886
 887 static void
 888 snb_gt_workarounds_init(struct intel_gt *gt, struct i915_wa_list *wal)
 889 {
 890 }
 891
 892 static void
 893 ivb_gt_workarounds_init(struct intel_gt *gt, struct i915_wa_list *wal)
 894 {
 895         /* Apply the WaDisableRHWOOptimizationForRenderHang:ivb workaround. */
 896         wa_masked_dis(wal,
 897                       GEN7_COMMON_SLICE_CHICKEN1,
 898                       GEN7_CSC1_RHWO_OPT_DISABLE_IN_RCC);
 899
 900         /* WaApplyL3ControlAndL3ChickenMode:ivb */
 901         wa_write(wal, GEN7_L3CNTLREG1, GEN7_WA_FOR_GEN7_L3_CONTROL);
 902         wa_write(wal, GEN7_L3_CHICKEN_MODE_REGISTER, GEN7_WA_L3_CHICKEN_MODE);
 903
 904         /* WaForceL3Serialization:ivb */
 905         wa_write_clr(wal, GEN7_L3SQCREG4, L3SQ_URB_READ_CAM_MATCH_DISABLE);
 906 }
 907
 908 static void
 909 vlv_gt_workarounds_init(struct intel_gt *gt, struct i915_wa_list *wal)
 910 {
 911         /* WaForceL3Serialization:vlv */
 912         wa_write_clr(wal, GEN7_L3SQCREG4, L3SQ_URB_READ_CAM_MATCH_DISABLE);
 913
 914         /*
 915          * WaIncreaseL3CreditsForVLVB0:vlv
 916          * This is the hardware default actually.
 917          */
 918         wa_write(wal, GEN7_L3SQCREG1, VLV_B0_WA_L3SQCREG1_VALUE);
 919 }
 920
 921 static void
 922 hsw_gt_workarounds_init(struct intel_gt *gt, struct i915_wa_list *wal)
 923 {
 924         /* L3 caching of data atomics doesn't work -- disable it. */
 925         wa_write(wal, HSW_SCRATCH1, HSW_SCRATCH1_L3_DATA_ATOMICS_DISABLE);
 926
 927         wa_add(wal,
 928                HSW_ROW_CHICKEN3, 0,
 929                _MASKED_BIT_ENABLE(HSW_ROW_CHICKEN3_L3_GLOBAL_ATOMICS_DISABLE),
 930                0 /* XXX does this reg exist? */, true);
 931
 932         /* WaVSRefCountFullforceMissDisable:hsw */
 933         wa_write_clr(wal, GEN7_FF_THREAD_MODE, GEN7_FF_VS_REF_CNT_FFME);
 934 }
 935
 936 static void
 937 gen9_wa_init_mcr(struct drm_i915_private *i915, struct i915_wa_list *wal)
 938 {
 939         const struct sseu_dev_info *sseu = &to_gt(i915)->info.sseu;
 940         unsigned int slice, subslice;
 941         u32 mcr, mcr_mask;
 942
 943         GEM_BUG_ON(GRAPHICS_VER(i915) != 9);
 944
 945         /*
 946          * WaProgramMgsrForCorrectSliceSpecificMmioReads:gen9,glk,kbl,cml
 947          * Before any MMIO read into slice/subslice specific registers, MCR
 948          * packet control register needs to be programmed to point to any
 949          * enabled s/ss pair. Otherwise, incorrect values will be returned.
 950          * This means each subsequent MMIO read will be forwarded to an
 951          * specific s/ss combination, but this is OK since these registers
 952          * are consistent across s/ss in almost all cases. In the rare
 953          * occasions, such as INSTDONE, where this value is dependent
 954          * on s/ss combo, the read should be done with read_subslice_reg.
 955          */
 956         slice = ffs(sseu->slice_mask) - 1;
 957         GEM_BUG_ON(slice >= ARRAY_SIZE(sseu->subslice_mask.hsw));
 958         subslice = ffs(intel_sseu_get_hsw_subslices(sseu, slice));
 959         GEM_BUG_ON(!subslice);
 960         subslice--;
 961
 962         /*
 963          * We use GEN8_MCR..() macros to calculate the |mcr| value for
 964          * Gen9 to address WaProgramMgsrForCorrectSliceSpecificMmioReads
 965          */
 966         mcr = GEN8_MCR_SLICE(slice) | GEN8_MCR_SUBSLICE(subslice);
 967         mcr_mask = GEN8_MCR_SLICE_MASK | GEN8_MCR_SUBSLICE_MASK;
 968
 969         drm_dbg(&i915->drm, "MCR slice:%d/subslice:%d = %x\n", slice, subslice, mcr);
 970
 971         wa_write_clr_set(wal, GEN8_MCR_SELECTOR, mcr_mask, mcr);
 972 }
 973
 974 static void
 975 gen9_gt_workarounds_init(struct intel_gt *gt, struct i915_wa_list *wal)
 976 {
 977         struct drm_i915_private *i915 = gt->i915;
 978
 979         /* WaProgramMgsrForCorrectSliceSpecificMmioReads:glk,kbl,cml,gen9 */
 980         gen9_wa_init_mcr(i915, wal);
 981
 982         /* WaDisableKillLogic:bxt,skl,kbl */
 983         if (!IS_COFFEELAKE(i915) && !IS_COMETLAKE(i915))
 984                 wa_write_or(wal,
 985                             GAM_ECOCHK,
 986                             ECOCHK_DIS_TLB);
 987
 988         if (HAS_LLC(i915)) {
 989                 /* WaCompressedResourceSamplerPbeMediaNewHashMode:skl,kbl
 990                  *
 991                  * Must match Display Engine. See
 992                  * WaCompressedResourceDisplayNewHashMode.
 993                  */
 994                 wa_write_or(wal,
 995                             MMCD_MISC_CTRL,
 996                             MMCD_PCLA | MMCD_HOTSPOT_EN);
 997         }
 998
 999         /* WaDisableHDCInvalidation:skl,bxt,kbl,cfl */
1000         wa_write_or(wal,
1001                     GAM_ECOCHK,
1002                     BDW_DISABLE_HDC_INVALIDATION);
1003 }
1004
1005 static void
1006 skl_gt_workarounds_init(struct intel_gt *gt, struct i915_wa_list *wal)
1007 {
1008         gen9_gt_workarounds_init(gt, wal);
1009
1010         /* WaDisableGafsUnitClkGating:skl */
1011         wa_write_or(wal,
1012                     GEN7_UCGCTL4,
1013                     GEN8_EU_GAUNIT_CLOCK_GATE_DISABLE);
1014
1015         /* WaInPlaceDecompressionHang:skl */
1016         if (IS_SKL_GRAPHICS_STEP(gt->i915, STEP_A0, STEP_H0))
1017                 wa_write_or(wal,
1018                             GEN9_GAMT_ECO_REG_RW_IA,
1019                             GAMT_ECO_ENABLE_IN_PLACE_DECOMPRESS);
1020 }
1021
1022 static void
1023 kbl_gt_workarounds_init(struct intel_gt *gt, struct i915_wa_list *wal)
1024 {
1025         gen9_gt_workarounds_init(gt, wal);
1026
1027         /* WaDisableDynamicCreditSharing:kbl */
1028         if (IS_KBL_GRAPHICS_STEP(gt->i915, 0, STEP_C0))
1029                 wa_write_or(wal,
1030                             GAMT_CHKN_BIT_REG,
1031                             GAMT_CHKN_DISABLE_DYNAMIC_CREDIT_SHARING);
1032
1033         /* WaDisableGafsUnitClkGating:kbl */
1034         wa_write_or(wal,
1035                     GEN7_UCGCTL4,
1036                     GEN8_EU_GAUNIT_CLOCK_GATE_DISABLE);
1037
1038         /* WaInPlaceDecompressionHang:kbl */
1039         wa_write_or(wal,
1040                     GEN9_GAMT_ECO_REG_RW_IA,
1041                     GAMT_ECO_ENABLE_IN_PLACE_DECOMPRESS);
1042 }
1043
1044 static void
1045 glk_gt_workarounds_init(struct intel_gt *gt, struct i915_wa_list *wal)
1046 {
1047         gen9_gt_workarounds_init(gt, wal);
1048 }
1049
1050 static void
1051 cfl_gt_workarounds_init(struct intel_gt *gt, struct i915_wa_list *wal)
1052 {
1053         gen9_gt_workarounds_init(gt, wal);
1054
1055         /* WaDisableGafsUnitClkGating:cfl */
1056         wa_write_or(wal,
1057                     GEN7_UCGCTL4,
1058                     GEN8_EU_GAUNIT_CLOCK_GATE_DISABLE);
1059
1060         /* WaInPlaceDecompressionHang:cfl */
1061         wa_write_or(wal,
1062                     GEN9_GAMT_ECO_REG_RW_IA,
1063                     GAMT_ECO_ENABLE_IN_PLACE_DECOMPRESS);
1064 }
1065
1066 static void __set_mcr_steering(struct i915_wa_list *wal,
1067                                i915_reg_t steering_reg,
1068                                unsigned int slice, unsigned int subslice)
1069 {
1070         u32 mcr, mcr_mask;
1071
1072         mcr = GEN11_MCR_SLICE(slice) | GEN11_MCR_SUBSLICE(subslice);
1073         mcr_mask = GEN11_MCR_SLICE_MASK | GEN11_MCR_SUBSLICE_MASK;
1074
1075         wa_write_clr_set(wal, steering_reg, mcr_mask, mcr);
1076 }
1077
1078 static void __add_mcr_wa(struct intel_gt *gt, struct i915_wa_list *wal,
1079                          unsigned int slice, unsigned int subslice)
1080 {
1081         struct drm_printer p = drm_debug_printer("MCR Steering:");
1082
1083         __set_mcr_steering(wal, GEN8_MCR_SELECTOR, slice, subslice);
1084
1085         gt->default_steering.groupid = slice;
1086         gt->default_steering.instanceid = subslice;
1087
1088         if (drm_debug_enabled(DRM_UT_DRIVER))
1089                 intel_gt_mcr_report_steering(&p, gt, false);
1090 }
1091
1092 static void
1093 icl_wa_init_mcr(struct intel_gt *gt, struct i915_wa_list *wal)
1094 {
1095         const struct sseu_dev_info *sseu = &gt->info.sseu;
1096         unsigned int subslice;
1097
1098         GEM_BUG_ON(GRAPHICS_VER(gt->i915) < 11);
1099         GEM_BUG_ON(hweight8(sseu->slice_mask) > 1);
1100
1101         /*
1102          * Although a platform may have subslices, we need to always steer
1103          * reads to the lowest instance that isn't fused off.  When Render
1104          * Power Gating is enabled, grabbing forcewake will only power up a
1105          * single subslice (the "minconfig") if there isn't a real workload
1106          * that needs to be run; this means that if we steer register reads to
1107          * one of the higher subslices, we run the risk of reading back 0's or
1108          * random garbage.
1109          */
1110         subslice = __ffs(intel_sseu_get_hsw_subslices(sseu, 0));
1111
1112         /*
1113          * If the subslice we picked above also steers us to a valid L3 bank,
1114          * then we can just rely on the default steering and won't need to
1115          * worry about explicitly re-steering L3BANK reads later.
1116          */
1117         if (gt->info.l3bank_mask & BIT(subslice))
1118                 gt->steering_table[L3BANK] = NULL;
1119
1120         __add_mcr_wa(gt, wal, 0, subslice);
1121 }
1122
1123 static void
1124 xehp_init_mcr(struct intel_gt *gt, struct i915_wa_list *wal)
1125 {
1126         const struct sseu_dev_info *sseu = &gt->info.sseu;
1127         unsigned long slice, subslice = 0, slice_mask = 0;
1128         u32 lncf_mask = 0;
1129         int i;
1130
1131         /*
1132          * On Xe_HP the steering increases in complexity. There are now several
1133          * more units that require steering and we're not guaranteed to be able
1134          * to find a common setting for all of them. These are:
1135          * - GSLICE (fusable)
1136          * - DSS (sub-unit within gslice; fusable)
1137          * - L3 Bank (fusable)
1138          * - MSLICE (fusable)
1139          * - LNCF (sub-unit within mslice; always present if mslice is present)
1140          *
1141          * We'll do our default/implicit steering based on GSLICE (in the
1142          * sliceid field) and DSS (in the subsliceid field).  If we can
1143          * find overlap between the valid MSLICE and/or LNCF values with
1144          * a suitable GSLICE, then we can just re-use the default value and
1145          * skip and explicit steering at runtime.
1146          *
1147          * We only need to look for overlap between GSLICE/MSLICE/LNCF to find
1148          * a valid sliceid value.  DSS steering is the only type of steering
1149          * that utilizes the 'subsliceid' bits.
1150          *
1151          * Also note that, even though the steering domain is called "GSlice"
1152          * and it is encoded in the register using the gslice format, the spec
1153          * says that the combined (geometry | compute) fuse should be used to
1154          * select the steering.
1155          */
1156
1157         /* Find the potential gslice candidates */
1158         slice_mask = intel_slicemask_from_xehp_dssmask(sseu->subslice_mask,
1159                                                        GEN_DSS_PER_GSLICE);
1160
1161         /*
1162          * Find the potential LNCF candidates.  Either LNCF within a valid
1163          * mslice is fine.
1164          */
1165         for_each_set_bit(i, &gt->info.mslice_mask, GEN12_MAX_MSLICES)
1166                 lncf_mask |= (0x3 << (i * 2));
1167
1168         /*
1169          * Are there any sliceid values that work for both GSLICE and LNCF
1170          * steering?
1171          */
1172         if (slice_mask & lncf_mask) {
1173                 slice_mask &= lncf_mask;
1174                 gt->steering_table[LNCF] = NULL;
1175         }
1176
1177         /* How about sliceid values that also work for MSLICE steering? */
1178         if (slice_mask & gt->info.mslice_mask) {
1179                 slice_mask &= gt->info.mslice_mask;
1180                 gt->steering_table[MSLICE] = NULL;
1181         }
1182
1183         slice = __ffs(slice_mask);
1184         subslice = intel_sseu_find_first_xehp_dss(sseu, GEN_DSS_PER_GSLICE, slice) %
1185                 GEN_DSS_PER_GSLICE;
1186
1187         __add_mcr_wa(gt, wal, slice, subslice);
1188
1189         /*
1190          * SQIDI ranges are special because they use different steering
1191          * registers than everything else we work with.  On XeHP SDV and
1192          * DG2-G10, any value in the steering registers will work fine since
1193          * all instances are present, but DG2-G11 only has SQIDI instances at
1194          * ID's 2 and 3, so we need to steer to one of those.  For simplicity
1195          * we'll just steer to a hardcoded "2" since that value will work
1196          * everywhere.
1197          */
1198         __set_mcr_steering(wal, MCFG_MCR_SELECTOR, 0, 2);
1199         __set_mcr_steering(wal, SF_MCR_SELECTOR, 0, 2);
1200 }
1201
1202 static void
1203 pvc_init_mcr(struct intel_gt *gt, struct i915_wa_list *wal)
1204 {
1205         unsigned int dss;
1206
1207         /*
1208          * Setup implicit steering for COMPUTE and DSS ranges to the first
1209          * non-fused-off DSS.  All other types of MCR registers will be
1210          * explicitly steered.
1211          */
1212         dss = intel_sseu_find_first_xehp_dss(&gt->info.sseu, 0, 0);
1213         __add_mcr_wa(gt, wal, dss / GEN_DSS_PER_CSLICE, dss % GEN_DSS_PER_CSLICE);
1214 }
1215
1216 static void
1217 icl_gt_workarounds_init(struct intel_gt *gt, struct i915_wa_list *wal)
1218 {
1219         struct drm_i915_private *i915 = gt->i915;
1220
1221         icl_wa_init_mcr(gt, wal);
1222
1223         /* WaModifyGamTlbPartitioning:icl */
1224         wa_write_clr_set(wal,
1225                          GEN11_GACB_PERF_CTRL,
1226                          GEN11_HASH_CTRL_MASK,
1227                          GEN11_HASH_CTRL_BIT0 | GEN11_HASH_CTRL_BIT4);
1228
1229         /* Wa_1405766107:icl
1230          * Formerly known as WaCL2SFHalfMaxAlloc
1231          */
1232         wa_write_or(wal,
1233                     GEN11_LSN_UNSLCVC,
1234                     GEN11_LSN_UNSLCVC_GAFS_HALF_SF_MAXALLOC |
1235                     GEN11_LSN_UNSLCVC_GAFS_HALF_CL2_MAXALLOC);
1236
1237         /* Wa_220166154:icl
1238          * Formerly known as WaDisCtxReload
1239          */
1240         wa_write_or(wal,
1241                     GEN8_GAMW_ECO_DEV_RW_IA,
1242                     GAMW_ECO_DEV_CTX_RELOAD_DISABLE);
1243
1244         /* Wa_1406463099:icl
1245          * Formerly known as WaGamTlbPendError
1246          */
1247         wa_write_or(wal,
1248                     GAMT_CHKN_BIT_REG,
1249                     GAMT_CHKN_DISABLE_L3_COH_PIPE);
1250
1251         /* Wa_1407352427:icl,ehl */
1252         wa_write_or(wal, UNSLICE_UNIT_LEVEL_CLKGATE2,
1253                     PSDUNIT_CLKGATE_DIS);
1254
1255         /* Wa_1406680159:icl,ehl */
1256         wa_write_or(wal,
1257                     SUBSLICE_UNIT_LEVEL_CLKGATE,
1258                     GWUNIT_CLKGATE_DIS);
1259
1260         /* Wa_1607087056:icl,ehl,jsl */
1261         if (IS_ICELAKE(i915) ||
1262             IS_JSL_EHL_GRAPHICS_STEP(i915, STEP_A0, STEP_B0))
1263                 wa_write_or(wal,
1264                             SLICE_UNIT_LEVEL_CLKGATE,
1265                             L3_CLKGATE_DIS | L3_CR2X_CLKGATE_DIS);
1266
1267         /*
1268          * This is not a documented workaround, but rather an optimization
1269          * to reduce sampler power.
1270          */
1271         wa_write_clr(wal, GEN10_DFR_RATIO_EN_AND_CHICKEN, DFR_DISABLE);
1272 }
1273
1274 /*
1275  * Though there are per-engine instances of these registers,
1276  * they retain their value through engine resets and should
1277  * only be provided on the GT workaround list rather than
1278  * the engine-specific workaround list.
1279  */
1280 static void
1281 wa_14011060649(struct intel_gt *gt, struct i915_wa_list *wal)
1282 {
1283         struct intel_engine_cs *engine;
1284         int id;
1285
1286         for_each_engine(engine, gt, id) {
1287                 if (engine->class != VIDEO_DECODE_CLASS ||
1288                     (engine->instance % 2))
1289                         continue;
1290
1291                 wa_write_or(wal, VDBOX_CGCTL3F10(engine->mmio_base),
1292                             IECPUNIT_CLKGATE_DIS);
1293         }
1294 }
1295
1296 static void
1297 gen12_gt_workarounds_init(struct intel_gt *gt, struct i915_wa_list *wal)
1298 {
1299         icl_wa_init_mcr(gt, wal);
1300
1301         /* Wa_14011060649:tgl,rkl,dg1,adl-s,adl-p */
1302         wa_14011060649(gt, wal);
1303
1304         /* Wa_14011059788:tgl,rkl,adl-s,dg1,adl-p */
1305         wa_write_or(wal, GEN10_DFR_RATIO_EN_AND_CHICKEN, DFR_DISABLE);
1306 }
1307
1308 static void
1309 tgl_gt_workarounds_init(struct intel_gt *gt, struct i915_wa_list *wal)
1310 {
1311         struct drm_i915_private *i915 = gt->i915;
1312
1313         gen12_gt_workarounds_init(gt, wal);
1314
1315         /* Wa_1409420604:tgl */
1316         if (IS_TGL_UY_GRAPHICS_STEP(i915, STEP_A0, STEP_B0))
1317                 wa_write_or(wal,
1318                             SUBSLICE_UNIT_LEVEL_CLKGATE2,
1319                             CPSSUNIT_CLKGATE_DIS);
1320
1321         /* Wa_1607087056:tgl also know as BUG:1409180338 */
1322         if (IS_TGL_UY_GRAPHICS_STEP(i915, STEP_A0, STEP_B0))
1323                 wa_write_or(wal,
1324                             SLICE_UNIT_LEVEL_CLKGATE,
1325                             L3_CLKGATE_DIS | L3_CR2X_CLKGATE_DIS);
1326
1327         /* Wa_1408615072:tgl[a0] */
1328         if (IS_TGL_UY_GRAPHICS_STEP(i915, STEP_A0, STEP_B0))
1329                 wa_write_or(wal, UNSLICE_UNIT_LEVEL_CLKGATE2,
1330                             VSUNIT_CLKGATE_DIS_TGL);
1331 }
1332
1333 static void
1334 dg1_gt_workarounds_init(struct intel_gt *gt, struct i915_wa_list *wal)
1335 {
1336         struct drm_i915_private *i915 = gt->i915;
1337
1338         gen12_gt_workarounds_init(gt, wal);
1339
1340         /* Wa_1607087056:dg1 */
1341         if (IS_DG1_GRAPHICS_STEP(i915, STEP_A0, STEP_B0))
1342                 wa_write_or(wal,
1343                             SLICE_UNIT_LEVEL_CLKGATE,
1344                             L3_CLKGATE_DIS | L3_CR2X_CLKGATE_DIS);
1345
1346         /* Wa_1409420604:dg1 */
1347         if (IS_DG1(i915))
1348                 wa_write_or(wal,
1349                             SUBSLICE_UNIT_LEVEL_CLKGATE2,
1350                             CPSSUNIT_CLKGATE_DIS);
1351
1352         /* Wa_1408615072:dg1 */
1353         /* Empirical testing shows this register is unaffected by engine reset. */
1354         if (IS_DG1(i915))
1355                 wa_write_or(wal, UNSLICE_UNIT_LEVEL_CLKGATE2,
1356                             VSUNIT_CLKGATE_DIS_TGL);
1357 }
1358
1359 static void
1360 xehpsdv_gt_workarounds_init(struct intel_gt *gt, struct i915_wa_list *wal)
1361 {
1362         struct drm_i915_private *i915 = gt->i915;
1363
1364         xehp_init_mcr(gt, wal);
1365
1366         /* Wa_1409757795:xehpsdv */
1367         wa_write_or(wal, SCCGCTL94DC, CG3DDISURB);
1368
1369         /* Wa_16011155590:xehpsdv */
1370         if (IS_XEHPSDV_GRAPHICS_STEP(i915, STEP_A0, STEP_B0))
1371                 wa_write_or(wal, UNSLICE_UNIT_LEVEL_CLKGATE,
1372                             TSGUNIT_CLKGATE_DIS);
1373
1374         /* Wa_14011780169:xehpsdv */
1375         if (IS_XEHPSDV_GRAPHICS_STEP(i915, STEP_B0, STEP_FOREVER)) {
1376                 wa_write_or(wal, UNSLCGCTL9440, GAMTLBOACS_CLKGATE_DIS |
1377                             GAMTLBVDBOX7_CLKGATE_DIS |
1378                             GAMTLBVDBOX6_CLKGATE_DIS |
1379                             GAMTLBVDBOX5_CLKGATE_DIS |
1380                             GAMTLBVDBOX4_CLKGATE_DIS |
1381                             GAMTLBVDBOX3_CLKGATE_DIS |
1382                             GAMTLBVDBOX2_CLKGATE_DIS |
1383                             GAMTLBVDBOX1_CLKGATE_DIS |
1384                             GAMTLBVDBOX0_CLKGATE_DIS |
1385                             GAMTLBKCR_CLKGATE_DIS |
1386                             GAMTLBGUC_CLKGATE_DIS |
1387                             GAMTLBBLT_CLKGATE_DIS);
1388                 wa_write_or(wal, UNSLCGCTL9444, GAMTLBGFXA0_CLKGATE_DIS |
1389                             GAMTLBGFXA1_CLKGATE_DIS |
1390                             GAMTLBCOMPA0_CLKGATE_DIS |
1391                             GAMTLBCOMPA1_CLKGATE_DIS |
1392                             GAMTLBCOMPB0_CLKGATE_DIS |
1393                             GAMTLBCOMPB1_CLKGATE_DIS |
1394                             GAMTLBCOMPC0_CLKGATE_DIS |
1395                             GAMTLBCOMPC1_CLKGATE_DIS |
1396                             GAMTLBCOMPD0_CLKGATE_DIS |
1397                             GAMTLBCOMPD1_CLKGATE_DIS |
1398                             GAMTLBMERT_CLKGATE_DIS   |
1399                             GAMTLBVEBOX3_CLKGATE_DIS |
1400                             GAMTLBVEBOX2_CLKGATE_DIS |
1401                             GAMTLBVEBOX1_CLKGATE_DIS |
1402                             GAMTLBVEBOX0_CLKGATE_DIS);
1403         }
1404
1405         /* Wa_16012725990:xehpsdv */
1406         if (IS_XEHPSDV_GRAPHICS_STEP(i915, STEP_A1, STEP_FOREVER))
1407                 wa_write_or(wal, UNSLICE_UNIT_LEVEL_CLKGATE, VFUNIT_CLKGATE_DIS);
1408
1409         /* Wa_14011060649:xehpsdv */
1410         wa_14011060649(gt, wal);
1411 }
1412
1413 static void
1414 dg2_gt_workarounds_init(struct intel_gt *gt, struct i915_wa_list *wal)
1415 {
1416         struct intel_engine_cs *engine;
1417         int id;
1418
1419         xehp_init_mcr(gt, wal);
1420
1421         /* Wa_14011060649:dg2 */
1422         wa_14011060649(gt, wal);
1423
1424         /*
1425          * Although there are per-engine instances of these registers,
1426          * they technically exist outside the engine itself and are not
1427          * impacted by engine resets.  Furthermore, they're part of the
1428          * GuC blacklist so trying to treat them as engine workarounds
1429          * will result in GuC initialization failure and a wedged GPU.
1430          */
1431         for_each_engine(engine, gt, id) {
1432                 if (engine->class != VIDEO_DECODE_CLASS)
1433                         continue;
1434
1435                 /* Wa_16010515920:dg2_g10 */
1436                 if (IS_DG2_GRAPHICS_STEP(gt->i915, G10, STEP_A0, STEP_B0))
1437                         wa_write_or(wal, VDBOX_CGCTL3F18(engine->mmio_base),
1438                                     ALNUNIT_CLKGATE_DIS);
1439         }
1440
1441         if (IS_DG2_G10(gt->i915)) {
1442                 /* Wa_22010523718:dg2 */
1443                 wa_write_or(wal, UNSLICE_UNIT_LEVEL_CLKGATE,
1444                             CG3DDISCFEG_CLKGATE_DIS);
1445
1446                 /* Wa_14011006942:dg2 */
1447                 wa_write_or(wal, SUBSLICE_UNIT_LEVEL_CLKGATE,
1448                             DSS_ROUTER_CLKGATE_DIS);
1449         }
1450
1451         if (IS_DG2_GRAPHICS_STEP(gt->i915, G10, STEP_A0, STEP_B0)) {
1452                 /* Wa_14010948348:dg2_g10 */
1453                 wa_write_or(wal, UNSLCGCTL9430, MSQDUNIT_CLKGATE_DIS);
1454
1455                 /* Wa_14011037102:dg2_g10 */
1456                 wa_write_or(wal, UNSLCGCTL9444, LTCDD_CLKGATE_DIS);
1457
1458                 /* Wa_14011371254:dg2_g10 */
1459                 wa_write_or(wal, SLICE_UNIT_LEVEL_CLKGATE, NODEDSS_CLKGATE_DIS);
1460
1461                 /* Wa_14011431319:dg2_g10 */
1462                 wa_write_or(wal, UNSLCGCTL9440, GAMTLBOACS_CLKGATE_DIS |
1463                             GAMTLBVDBOX7_CLKGATE_DIS |
1464                             GAMTLBVDBOX6_CLKGATE_DIS |
1465                             GAMTLBVDBOX5_CLKGATE_DIS |
1466                             GAMTLBVDBOX4_CLKGATE_DIS |
1467                             GAMTLBVDBOX3_CLKGATE_DIS |
1468                             GAMTLBVDBOX2_CLKGATE_DIS |
1469                             GAMTLBVDBOX1_CLKGATE_DIS |
1470                             GAMTLBVDBOX0_CLKGATE_DIS |
1471                             GAMTLBKCR_CLKGATE_DIS |
1472                             GAMTLBGUC_CLKGATE_DIS |
1473                             GAMTLBBLT_CLKGATE_DIS);
1474                 wa_write_or(wal, UNSLCGCTL9444, GAMTLBGFXA0_CLKGATE_DIS |
1475                             GAMTLBGFXA1_CLKGATE_DIS |
1476                             GAMTLBCOMPA0_CLKGATE_DIS |
1477                             GAMTLBCOMPA1_CLKGATE_DIS |
1478                             GAMTLBCOMPB0_CLKGATE_DIS |
1479                             GAMTLBCOMPB1_CLKGATE_DIS |
1480                             GAMTLBCOMPC0_CLKGATE_DIS |
1481                             GAMTLBCOMPC1_CLKGATE_DIS |
1482                             GAMTLBCOMPD0_CLKGATE_DIS |
1483                             GAMTLBCOMPD1_CLKGATE_DIS |
1484                             GAMTLBMERT_CLKGATE_DIS   |
1485                             GAMTLBVEBOX3_CLKGATE_DIS |
1486                             GAMTLBVEBOX2_CLKGATE_DIS |
1487                             GAMTLBVEBOX1_CLKGATE_DIS |
1488                             GAMTLBVEBOX0_CLKGATE_DIS);
1489
1490                 /* Wa_14010569222:dg2_g10 */
1491                 wa_write_or(wal, UNSLICE_UNIT_LEVEL_CLKGATE,
1492                             GAMEDIA_CLKGATE_DIS);
1493
1494                 /* Wa_14011028019:dg2_g10 */
1495                 wa_write_or(wal, SSMCGCTL9530, RTFUNIT_CLKGATE_DIS);
1496         }
1497
1498         /* Wa_14014830051:dg2 */
1499         wa_write_clr(wal, SARB_CHICKEN1, COMP_CKN_IN);
1500
1501         /*
1502          * The following are not actually "workarounds" but rather
1503          * recommended tuning settings documented in the bspec's
1504          * performance guide section.
1505          */
1506         wa_write_or(wal, GEN12_SQCM, EN_32B_ACCESS);
1507
1508         /* Wa_14015795083 */
1509         wa_write_clr(wal, GEN7_MISCCPCTL, GEN12_DOP_CLOCK_GATE_RENDER_ENABLE);
1510 }
1511
1512 static void
1513 pvc_gt_workarounds_init(struct intel_gt *gt, struct i915_wa_list *wal)
1514 {
1515         pvc_init_mcr(gt, wal);
1516
1517         /* Wa_14015795083 */
1518         wa_write_clr(wal, GEN7_MISCCPCTL, GEN12_DOP_CLOCK_GATE_RENDER_ENABLE);
1519 }
1520
1521 static void
1522 gt_init_workarounds(struct intel_gt *gt, struct i915_wa_list *wal)
1523 {
1524         struct drm_i915_private *i915 = gt->i915;
1525
1526         if (IS_PONTEVECCHIO(i915))
1527                 pvc_gt_workarounds_init(gt, wal);
1528         else if (IS_DG2(i915))
1529                 dg2_gt_workarounds_init(gt, wal);
1530         else if (IS_XEHPSDV(i915))
1531                 xehpsdv_gt_workarounds_init(gt, wal);
1532         else if (IS_DG1(i915))
1533                 dg1_gt_workarounds_init(gt, wal);
1534         else if (IS_TIGERLAKE(i915))
1535                 tgl_gt_workarounds_init(gt, wal);
1536         else if (GRAPHICS_VER(i915) == 12)
1537                 gen12_gt_workarounds_init(gt, wal);
1538         else if (GRAPHICS_VER(i915) == 11)
1539                 icl_gt_workarounds_init(gt, wal);
1540         else if (IS_COFFEELAKE(i915) || IS_COMETLAKE(i915))
1541                 cfl_gt_workarounds_init(gt, wal);
1542         else if (IS_GEMINILAKE(i915))
1543                 glk_gt_workarounds_init(gt, wal);
1544         else if (IS_KABYLAKE(i915))
1545                 kbl_gt_workarounds_init(gt, wal);
1546         else if (IS_BROXTON(i915))
1547                 gen9_gt_workarounds_init(gt, wal);
1548         else if (IS_SKYLAKE(i915))
1549                 skl_gt_workarounds_init(gt, wal);
1550         else if (IS_HASWELL(i915))
1551                 hsw_gt_workarounds_init(gt, wal);
1552         else if (IS_VALLEYVIEW(i915))
1553                 vlv_gt_workarounds_init(gt, wal);
1554         else if (IS_IVYBRIDGE(i915))
1555                 ivb_gt_workarounds_init(gt, wal);
1556         else if (GRAPHICS_VER(i915) == 6)
1557                 snb_gt_workarounds_init(gt, wal);
1558         else if (GRAPHICS_VER(i915) == 5)
1559                 ilk_gt_workarounds_init(gt, wal);
1560         else if (IS_G4X(i915))
1561                 g4x_gt_workarounds_init(gt, wal);
1562         else if (GRAPHICS_VER(i915) == 4)
1563                 gen4_gt_workarounds_init(gt, wal);
1564         else if (GRAPHICS_VER(i915) <= 8)
1565                 ;
1566         else
1567                 MISSING_CASE(GRAPHICS_VER(i915));
1568 }
1569
1570 void intel_gt_init_workarounds(struct intel_gt *gt)
1571 {
1572         struct i915_wa_list *wal = &gt->wa_list;
1573
1574         wa_init_start(wal, "GT", "global");
1575         gt_init_workarounds(gt, wal);
1576         wa_init_finish(wal);
1577 }
1578
1579 static enum forcewake_domains
1580 wal_get_fw_for_rmw(struct intel_uncore *uncore, const struct i915_wa_list *wal)
1581 {
1582         enum forcewake_domains fw = 0;
1583         struct i915_wa *wa;
1584         unsigned int i;
1585
1586         for (i = 0, wa = wal->list; i < wal->count; i++, wa++)
1587                 fw |= intel_uncore_forcewake_for_reg(uncore,
1588                                                      wa->reg,
1589                                                      FW_REG_READ |
1590                                                      FW_REG_WRITE);
1591
1592         return fw;
1593 }
1594
1595 static bool
1596 wa_verify(const struct i915_wa *wa, u32 cur, const char *name, const char *from)
1597 {
1598         if ((cur ^ wa->set) & wa->read) {
1599                 DRM_ERROR("%s workaround lost on %s! (reg[%x]=0x%x, relevant bits were 0x%x vs expected 0x%x)\n",
1600                           name, from, i915_mmio_reg_offset(wa->reg),
1601                           cur, cur & wa->read, wa->set & wa->read);
1602
1603                 return false;
1604         }
1605
1606         return true;
1607 }
1608
1609 static void
1610 wa_list_apply(struct intel_gt *gt, const struct i915_wa_list *wal)
1611 {
1612         struct intel_uncore *uncore = gt->uncore;
1613         enum forcewake_domains fw;
1614         unsigned long flags;
1615         struct i915_wa *wa;
1616         unsigned int i;
1617
1618         if (!wal->count)
1619                 return;
1620
1621         fw = wal_get_fw_for_rmw(uncore, wal);
1622
1623         spin_lock_irqsave(&uncore->lock, flags);
1624         intel_uncore_forcewake_get__locked(uncore, fw);
1625
1626         for (i = 0, wa = wal->list; i < wal->count; i++, wa++) {
1627                 u32 val, old = 0;
1628
1629                 /* open-coded rmw due to steering */
1630                 old = wa->clr ? intel_gt_mcr_read_any_fw(gt, wa->reg) : 0;
1631                 val = (old & ~wa->clr) | wa->set;
1632                 if (val != old || !wa->clr)
1633                         intel_uncore_write_fw(uncore, wa->reg, val);
1634
1635                 if (IS_ENABLED(CONFIG_DRM_I915_DEBUG_GEM))
1636                         wa_verify(wa, intel_gt_mcr_read_any_fw(gt, wa->reg),
1637                                   wal->name, "application");
1638         }
1639
1640         intel_uncore_forcewake_put__locked(uncore, fw);
1641         spin_unlock_irqrestore(&uncore->lock, flags);
1642 }
1643
1644 void intel_gt_apply_workarounds(struct intel_gt *gt)
1645 {
1646         wa_list_apply(gt, &gt->wa_list);
1647 }
1648
1649 static bool wa_list_verify(struct intel_gt *gt,
1650                            const struct i915_wa_list *wal,
1651                            const char *from)
1652 {
1653         struct intel_uncore *uncore = gt->uncore;
1654         struct i915_wa *wa;
1655         enum forcewake_domains fw;
1656         unsigned long flags;
1657         unsigned int i;
1658         bool ok = true;
1659
1660         fw = wal_get_fw_for_rmw(uncore, wal);
1661
1662         spin_lock_irqsave(&uncore->lock, flags);
1663         intel_uncore_forcewake_get__locked(uncore, fw);
1664
1665         for (i = 0, wa = wal->list; i < wal->count; i++, wa++)
1666                 ok &= wa_verify(wa,
1667                                 intel_gt_mcr_read_any_fw(gt, wa->reg),
1668                                 wal->name, from);
1669
1670         intel_uncore_forcewake_put__locked(uncore, fw);
1671         spin_unlock_irqrestore(&uncore->lock, flags);
1672
1673         return ok;
1674 }
1675
1676 bool intel_gt_verify_workarounds(struct intel_gt *gt, const char *from)
1677 {
1678         return wa_list_verify(gt, &gt->wa_list, from);
1679 }
1680
1681 __maybe_unused
1682 static bool is_nonpriv_flags_valid(u32 flags)
1683 {
1684         /* Check only valid flag bits are set */
1685         if (flags & ~RING_FORCE_TO_NONPRIV_MASK_VALID)
1686                 return false;
1687
1688         /* NB: Only 3 out of 4 enum values are valid for access field */
1689         if ((flags & RING_FORCE_TO_NONPRIV_ACCESS_MASK) ==
1690             RING_FORCE_TO_NONPRIV_ACCESS_INVALID)
1691                 return false;
1692
1693         return true;
1694 }
1695
1696 static void
1697 whitelist_reg_ext(struct i915_wa_list *wal, i915_reg_t reg, u32 flags)
1698 {
1699         struct i915_wa wa = {
1700                 .reg = reg
1701         };
1702
1703         if (GEM_DEBUG_WARN_ON(wal->count >= RING_MAX_NONPRIV_SLOTS))
1704                 return;
1705
1706         if (GEM_DEBUG_WARN_ON(!is_nonpriv_flags_valid(flags)))
1707                 return;
1708
1709         wa.reg.reg |= flags;
1710         _wa_add(wal, &wa);
1711 }
1712
1713 static void
1714 whitelist_reg(struct i915_wa_list *wal, i915_reg_t reg)
1715 {
1716         whitelist_reg_ext(wal, reg, RING_FORCE_TO_NONPRIV_ACCESS_RW);
1717 }
1718
1719 static void gen9_whitelist_build(struct i915_wa_list *w)
1720 {
1721         /* WaVFEStateAfterPipeControlwithMediaStateClear:skl,bxt,glk,cfl */
1722         whitelist_reg(w, GEN9_CTX_PREEMPT_REG);
1723
1724         /* WaEnablePreemptionGranularityControlByUMD:skl,bxt,kbl,cfl,[cnl] */
1725         whitelist_reg(w, GEN8_CS_CHICKEN1);
1726
1727         /* WaAllowUMDToModifyHDCChicken1:skl,bxt,kbl,glk,cfl */
1728         whitelist_reg(w, GEN8_HDC_CHICKEN1);
1729
1730         /* WaSendPushConstantsFromMMIO:skl,bxt */
1731         whitelist_reg(w, COMMON_SLICE_CHICKEN2);
1732 }
1733
1734 static void skl_whitelist_build(struct intel_engine_cs *engine)
1735 {
1736         struct i915_wa_list *w = &engine->whitelist;
1737
1738         if (engine->class != RENDER_CLASS)
1739                 return;
1740
1741         gen9_whitelist_build(w);
1742
1743         /* WaDisableLSQCROPERFforOCL:skl */
1744         whitelist_reg(w, GEN8_L3SQCREG4);
1745 }
1746
1747 static void bxt_whitelist_build(struct intel_engine_cs *engine)
1748 {
1749         if (engine->class != RENDER_CLASS)
1750                 return;
1751
1752         gen9_whitelist_build(&engine->whitelist);
1753 }
1754
1755 static void kbl_whitelist_build(struct intel_engine_cs *engine)
1756 {
1757         struct i915_wa_list *w = &engine->whitelist;
1758
1759         if (engine->class != RENDER_CLASS)
1760                 return;
1761
1762         gen9_whitelist_build(w);
1763
1764         /* WaDisableLSQCROPERFforOCL:kbl */
1765         whitelist_reg(w, GEN8_L3SQCREG4);
1766 }
1767
1768 static void glk_whitelist_build(struct intel_engine_cs *engine)
1769 {
1770         struct i915_wa_list *w = &engine->whitelist;
1771
1772         if (engine->class != RENDER_CLASS)
1773                 return;
1774
1775         gen9_whitelist_build(w);
1776
1777         /* WA #0862: Userspace has to set "Barrier Mode" to avoid hangs. */
1778         whitelist_reg(w, GEN9_SLICE_COMMON_ECO_CHICKEN1);
1779 }
1780
1781 static void cfl_whitelist_build(struct intel_engine_cs *engine)
1782 {
1783         struct i915_wa_list *w = &engine->whitelist;
1784
1785         if (engine->class != RENDER_CLASS)
1786                 return;
1787
1788         gen9_whitelist_build(w);
1789
1790         /*
1791          * WaAllowPMDepthAndInvocationCountAccessFromUMD:cfl,whl,cml,aml
1792          *
1793          * This covers 4 register which are next to one another :
1794          *   - PS_INVOCATION_COUNT
1795          *   - PS_INVOCATION_COUNT_UDW
1796          *   - PS_DEPTH_COUNT
1797          *   - PS_DEPTH_COUNT_UDW
1798          */
1799         whitelist_reg_ext(w, PS_INVOCATION_COUNT,
1800                           RING_FORCE_TO_NONPRIV_ACCESS_RD |
1801                           RING_FORCE_TO_NONPRIV_RANGE_4);
1802 }
1803
1804 static void allow_read_ctx_timestamp(struct intel_engine_cs *engine)
1805 {
1806         struct i915_wa_list *w = &engine->whitelist;
1807
1808         if (engine->class != RENDER_CLASS)
1809                 whitelist_reg_ext(w,
1810                                   RING_CTX_TIMESTAMP(engine->mmio_base),
1811                                   RING_FORCE_TO_NONPRIV_ACCESS_RD);
1812 }
1813
1814 static void cml_whitelist_build(struct intel_engine_cs *engine)
1815 {
1816         allow_read_ctx_timestamp(engine);
1817
1818         cfl_whitelist_build(engine);
1819 }
1820
1821 static void icl_whitelist_build(struct intel_engine_cs *engine)
1822 {
1823         struct i915_wa_list *w = &engine->whitelist;
1824
1825         allow_read_ctx_timestamp(engine);
1826
1827         switch (engine->class) {
1828         case RENDER_CLASS:
1829                 /* WaAllowUMDToModifyHalfSliceChicken7:icl */
1830                 whitelist_reg(w, GEN9_HALF_SLICE_CHICKEN7);
1831
1832                 /* WaAllowUMDToModifySamplerMode:icl */
1833                 whitelist_reg(w, GEN10_SAMPLER_MODE);
1834
1835                 /* WaEnableStateCacheRedirectToCS:icl */
1836                 whitelist_reg(w, GEN9_SLICE_COMMON_ECO_CHICKEN1);
1837
1838                 /*
1839                  * WaAllowPMDepthAndInvocationCountAccessFromUMD:icl
1840                  *
1841                  * This covers 4 register which are next to one another :
1842                  *   - PS_INVOCATION_COUNT
1843                  *   - PS_INVOCATION_COUNT_UDW
1844                  *   - PS_DEPTH_COUNT
1845                  *   - PS_DEPTH_COUNT_UDW
1846                  */
1847                 whitelist_reg_ext(w, PS_INVOCATION_COUNT,
1848                                   RING_FORCE_TO_NONPRIV_ACCESS_RD |
1849                                   RING_FORCE_TO_NONPRIV_RANGE_4);
1850                 break;
1851
1852         case VIDEO_DECODE_CLASS:
1853                 /* hucStatusRegOffset */
1854                 whitelist_reg_ext(w, _MMIO(0x2000 + engine->mmio_base),
1855                                   RING_FORCE_TO_NONPRIV_ACCESS_RD);
1856                 /* hucUKernelHdrInfoRegOffset */
1857                 whitelist_reg_ext(w, _MMIO(0x2014 + engine->mmio_base),
1858                                   RING_FORCE_TO_NONPRIV_ACCESS_RD);
1859                 /* hucStatus2RegOffset */
1860                 whitelist_reg_ext(w, _MMIO(0x23B0 + engine->mmio_base),
1861                                   RING_FORCE_TO_NONPRIV_ACCESS_RD);
1862                 break;
1863
1864         default:
1865                 break;
1866         }
1867 }
1868
1869 static void tgl_whitelist_build(struct intel_engine_cs *engine)
1870 {
1871         struct i915_wa_list *w = &engine->whitelist;
1872
1873         allow_read_ctx_timestamp(engine);
1874
1875         switch (engine->class) {
1876         case RENDER_CLASS:
1877                 /*
1878                  * WaAllowPMDepthAndInvocationCountAccessFromUMD:tgl
1879                  * Wa_1408556865:tgl
1880                  *
1881                  * This covers 4 registers which are next to one another :
1882                  *   - PS_INVOCATION_COUNT
1883                  *   - PS_INVOCATION_COUNT_UDW
1884                  *   - PS_DEPTH_COUNT
1885                  *   - PS_DEPTH_COUNT_UDW
1886                  */
1887                 whitelist_reg_ext(w, PS_INVOCATION_COUNT,
1888                                   RING_FORCE_TO_NONPRIV_ACCESS_RD |
1889                                   RING_FORCE_TO_NONPRIV_RANGE_4);
1890
1891                 /*
1892                  * Wa_1808121037:tgl
1893                  * Wa_14012131227:dg1
1894                  * Wa_1508744258:tgl,rkl,dg1,adl-s,adl-p
1895                  */
1896                 whitelist_reg(w, GEN7_COMMON_SLICE_CHICKEN1);
1897
1898                 /* Wa_1806527549:tgl */
1899                 whitelist_reg(w, HIZ_CHICKEN);
1900                 break;
1901         default:
1902                 break;
1903         }
1904 }
1905
1906 static void dg1_whitelist_build(struct intel_engine_cs *engine)
1907 {
1908         struct i915_wa_list *w = &engine->whitelist;
1909
1910         tgl_whitelist_build(engine);
1911
1912         /* GEN:BUG:1409280441:dg1 */
1913         if (IS_DG1_GRAPHICS_STEP(engine->i915, STEP_A0, STEP_B0) &&
1914             (engine->class == RENDER_CLASS ||
1915              engine->class == COPY_ENGINE_CLASS))
1916                 whitelist_reg_ext(w, RING_ID(engine->mmio_base),
1917                                   RING_FORCE_TO_NONPRIV_ACCESS_RD);
1918 }
1919
1920 static void xehpsdv_whitelist_build(struct intel_engine_cs *engine)
1921 {
1922         allow_read_ctx_timestamp(engine);
1923 }
1924
1925 static void dg2_whitelist_build(struct intel_engine_cs *engine)
1926 {
1927         struct i915_wa_list *w = &engine->whitelist;
1928
1929         allow_read_ctx_timestamp(engine);
1930
1931         switch (engine->class) {
1932         case RENDER_CLASS:
1933                 /*
1934                  * Wa_1507100340:dg2_g10
1935                  *
1936                  * This covers 4 registers which are next to one another :
1937                  *   - PS_INVOCATION_COUNT
1938                  *   - PS_INVOCATION_COUNT_UDW
1939                  *   - PS_DEPTH_COUNT
1940                  *   - PS_DEPTH_COUNT_UDW
1941                  */
1942                 if (IS_DG2_GRAPHICS_STEP(engine->i915, G10, STEP_A0, STEP_B0))
1943                         whitelist_reg_ext(w, PS_INVOCATION_COUNT,
1944                                           RING_FORCE_TO_NONPRIV_ACCESS_RD |
1945                                           RING_FORCE_TO_NONPRIV_RANGE_4);
1946
1947                 break;
1948         case COMPUTE_CLASS:
1949                 /* Wa_16011157294:dg2_g10 */
1950                 if (IS_DG2_GRAPHICS_STEP(engine->i915, G10, STEP_A0, STEP_B0))
1951                         whitelist_reg(w, GEN9_CTX_PREEMPT_REG);
1952                 break;
1953         default:
1954                 break;
1955         }
1956 }
1957
1958 static void blacklist_trtt(struct intel_engine_cs *engine)
1959 {
1960         struct i915_wa_list *w = &engine->whitelist;
1961
1962         /*
1963          * Prevent read/write access to [0x4400, 0x4600) which covers
1964          * the TRTT range across all engines. Note that normally userspace
1965          * cannot access the other engines' trtt control, but for simplicity
1966          * we cover the entire range on each engine.
1967          */
1968         whitelist_reg_ext(w, _MMIO(0x4400),
1969                           RING_FORCE_TO_NONPRIV_DENY |
1970                           RING_FORCE_TO_NONPRIV_RANGE_64);
1971         whitelist_reg_ext(w, _MMIO(0x4500),
1972                           RING_FORCE_TO_NONPRIV_DENY |
1973                           RING_FORCE_TO_NONPRIV_RANGE_64);
1974 }
1975
1976 static void pvc_whitelist_build(struct intel_engine_cs *engine)
1977 {
1978         allow_read_ctx_timestamp(engine);
1979
1980         /* Wa_16014440446:pvc */
1981         blacklist_trtt(engine);
1982 }
1983
1984 void intel_engine_init_whitelist(struct intel_engine_cs *engine)
1985 {
1986         struct drm_i915_private *i915 = engine->i915;
1987         struct i915_wa_list *w = &engine->whitelist;
1988
1989         wa_init_start(w, "whitelist", engine->name);
1990
1991         if (IS_PONTEVECCHIO(i915))
1992                 pvc_whitelist_build(engine);
1993         else if (IS_DG2(i915))
1994                 dg2_whitelist_build(engine);
1995         else if (IS_XEHPSDV(i915))
1996                 xehpsdv_whitelist_build(engine);
1997         else if (IS_DG1(i915))
1998                 dg1_whitelist_build(engine);
1999         else if (GRAPHICS_VER(i915) == 12)
2000                 tgl_whitelist_build(engine);
2001         else if (GRAPHICS_VER(i915) == 11)
2002                 icl_whitelist_build(engine);
2003         else if (IS_COMETLAKE(i915))
2004                 cml_whitelist_build(engine);
2005         else if (IS_COFFEELAKE(i915))
2006                 cfl_whitelist_build(engine);
2007         else if (IS_GEMINILAKE(i915))
2008                 glk_whitelist_build(engine);
2009         else if (IS_KABYLAKE(i915))
2010                 kbl_whitelist_build(engine);
2011         else if (IS_BROXTON(i915))
2012                 bxt_whitelist_build(engine);
2013         else if (IS_SKYLAKE(i915))
2014                 skl_whitelist_build(engine);
2015         else if (GRAPHICS_VER(i915) <= 8)
2016                 ;
2017         else
2018                 MISSING_CASE(GRAPHICS_VER(i915));
2019
2020         wa_init_finish(w);
2021 }
2022
2023 void intel_engine_apply_whitelist(struct intel_engine_cs *engine)
2024 {
2025         const struct i915_wa_list *wal = &engine->whitelist;
2026         struct intel_uncore *uncore = engine->uncore;
2027         const u32 base = engine->mmio_base;
2028         struct i915_wa *wa;
2029         unsigned int i;
2030
2031         if (!wal->count)
2032                 return;
2033
2034         for (i = 0, wa = wal->list; i < wal->count; i++, wa++)
2035                 intel_uncore_write(uncore,
2036                                    RING_FORCE_TO_NONPRIV(base, i),
2037                                    i915_mmio_reg_offset(wa->reg));
2038
2039         /* And clear the rest just in case of garbage */
2040         for (; i < RING_MAX_NONPRIV_SLOTS; i++)
2041                 intel_uncore_write(uncore,
2042                                    RING_FORCE_TO_NONPRIV(base, i),
2043                                    i915_mmio_reg_offset(RING_NOPID(base)));
2044 }
2045
2046 /*
2047  * engine_fake_wa_init(), a place holder to program the registers
2048  * which are not part of an official workaround defined by the
2049  * hardware team.
2050  * Adding programming of those register inside workaround will
2051  * allow utilizing wa framework to proper application and verification.
2052  */
2053 static void
2054 engine_fake_wa_init(struct intel_engine_cs *engine, struct i915_wa_list *wal)
2055 {
2056         u8 mocs_w, mocs_r;
2057
2058         /*
2059          * RING_CMD_CCTL specifies the default MOCS entry that will be used
2060          * by the command streamer when executing commands that don't have
2061          * a way to explicitly specify a MOCS setting.  The default should
2062          * usually reference whichever MOCS entry corresponds to uncached
2063          * behavior, although use of a WB cached entry is recommended by the
2064          * spec in certain circumstances on specific platforms.
2065          */
2066         if (GRAPHICS_VER(engine->i915) >= 12) {
2067                 mocs_r = engine->gt->mocs.uc_index;
2068                 mocs_w = engine->gt->mocs.uc_index;
2069
2070                 if (HAS_L3_CCS_READ(engine->i915) &&
2071                     engine->class == COMPUTE_CLASS) {
2072                         mocs_r = engine->gt->mocs.wb_index;
2073
2074                         /*
2075                          * Even on the few platforms where MOCS 0 is a
2076                          * legitimate table entry, it's never the correct
2077                          * setting to use here; we can assume the MOCS init
2078                          * just forgot to initialize wb_index.
2079                          */
2080                         drm_WARN_ON(&engine->i915->drm, mocs_r == 0);
2081                 }
2082
2083                 wa_masked_field_set(wal,
2084                                     RING_CMD_CCTL(engine->mmio_base),
2085                                     CMD_CCTL_MOCS_MASK,
2086                                     CMD_CCTL_MOCS_OVERRIDE(mocs_w, mocs_r));
2087         }
2088 }
2089
2090 static bool needs_wa_1308578152(struct intel_engine_cs *engine)
2091 {
2092         return intel_sseu_find_first_xehp_dss(&engine->gt->info.sseu, 0, 0) >=
2093                 GEN_DSS_PER_GSLICE;
2094 }
2095
2096 static void
2097 rcs_engine_wa_init(struct intel_engine_cs *engine, struct i915_wa_list *wal)
2098 {
2099         struct drm_i915_private *i915 = engine->i915;
2100
2101         if (IS_DG2(i915)) {
2102                 /* Wa_1509235366:dg2 */
2103                 wa_write_or(wal, GEN12_GAMCNTRL_CTRL, INVALIDATION_BROADCAST_MODE_DIS |
2104                             GLOBAL_INVALIDATION_MODE);
2105
2106                 /*
2107                  * The following are not actually "workarounds" but rather
2108                  * recommended tuning settings documented in the bspec's
2109                  * performance guide section.
2110                  */
2111                 wa_write_or(wal, XEHP_L3SCQREG7, BLEND_FILL_CACHING_OPT_DIS);
2112         }
2113
2114         if (IS_DG2_GRAPHICS_STEP(i915, G11, STEP_A0, STEP_B0)) {
2115                 /* Wa_14013392000:dg2_g11 */
2116                 wa_masked_en(wal, GEN7_ROW_CHICKEN2, GEN12_ENABLE_LARGE_GRF_MODE);
2117
2118                 /* Wa_16011620976:dg2_g11 */
2119                 wa_write_or(wal, LSC_CHICKEN_BIT_0_UDW, DIS_CHAIN_2XSIMD8);
2120         }
2121
2122         if (IS_DG2_GRAPHICS_STEP(i915, G10, STEP_A0, STEP_B0) ||
2123             IS_DG2_GRAPHICS_STEP(i915, G11, STEP_A0, STEP_B0)) {
2124                 /* Wa_14012419201:dg2 */
2125                 wa_masked_en(wal, GEN9_ROW_CHICKEN4,
2126                              GEN12_DISABLE_HDR_PAST_PAYLOAD_HOLD_FIX);
2127         }
2128
2129         if (IS_DG2_GRAPHICS_STEP(i915, G10, STEP_B0, STEP_C0) ||
2130             IS_DG2_G11(i915)) {
2131                 /*
2132                  * Wa_22012826095:dg2
2133                  * Wa_22013059131:dg2
2134                  */
2135                 wa_write_clr_set(wal, LSC_CHICKEN_BIT_0_UDW,
2136                                  MAXREQS_PER_BANK,
2137                                  REG_FIELD_PREP(MAXREQS_PER_BANK, 2));
2138
2139                 /* Wa_22013059131:dg2 */
2140                 wa_write_or(wal, LSC_CHICKEN_BIT_0,
2141                             FORCE_1_SUB_MESSAGE_PER_FRAGMENT);
2142         }
2143
2144         /* Wa_1308578152:dg2_g10 when first gslice is fused off */
2145         if (IS_DG2_GRAPHICS_STEP(i915, G10, STEP_B0, STEP_C0) &&
2146             needs_wa_1308578152(engine)) {
2147                 wa_masked_dis(wal, GEN12_CS_DEBUG_MODE1_CCCSUNIT_BE_COMMON,
2148                               GEN12_REPLAY_MODE_GRANULARITY);
2149         }
2150
2151         if (IS_DG2_GRAPHICS_STEP(i915, G10, STEP_B0, STEP_FOREVER) ||
2152             IS_DG2_G11(i915) || IS_DG2_G12(i915)) {
2153                 /* Wa_22013037850:dg2 */
2154                 wa_write_or(wal, LSC_CHICKEN_BIT_0_UDW,
2155                             DISABLE_128B_EVICTION_COMMAND_UDW);
2156
2157                 /* Wa_22012856258:dg2 */
2158                 wa_masked_en(wal, GEN7_ROW_CHICKEN2,
2159                              GEN12_DISABLE_READ_SUPPRESSION);
2160
2161                 /*
2162                  * Wa_22010960976:dg2
2163                  * Wa_14013347512:dg2
2164                  */
2165                 wa_masked_dis(wal, GEN12_HDC_CHICKEN0,
2166                               LSC_L1_FLUSH_CTL_3D_DATAPORT_FLUSH_EVENTS_MASK);
2167         }
2168
2169         if (IS_DG2_GRAPHICS_STEP(i915, G10, STEP_A0, STEP_B0)) {
2170                 /*
2171                  * Wa_1608949956:dg2_g10
2172                  * Wa_14010198302:dg2_g10
2173                  */
2174                 wa_masked_en(wal, GEN8_ROW_CHICKEN,
2175                              MDQ_ARBITRATION_MODE | UGM_BACKUP_MODE);
2176
2177                 /*
2178                  * Wa_14010918519:dg2_g10
2179                  *
2180                  * LSC_CHICKEN_BIT_0 always reads back as 0 is this stepping,
2181                  * so ignoring verification.
2182                  */
2183                 wa_add(wal, LSC_CHICKEN_BIT_0_UDW, 0,
2184                        FORCE_SLM_FENCE_SCOPE_TO_TILE | FORCE_UGM_FENCE_SCOPE_TO_TILE,
2185                        0, false);
2186         }
2187
2188         if (IS_DG2_GRAPHICS_STEP(i915, G10, STEP_A0, STEP_B0)) {
2189                 /* Wa_22010430635:dg2 */
2190                 wa_masked_en(wal,
2191                              GEN9_ROW_CHICKEN4,
2192                              GEN12_DISABLE_GRF_CLEAR);
2193
2194                 /* Wa_14010648519:dg2 */
2195                 wa_write_or(wal, XEHP_L3NODEARBCFG, XEHP_LNESPARE);
2196         }
2197
2198         if (IS_DG2_GRAPHICS_STEP(i915, G10, STEP_A0, STEP_C0) ||
2199             IS_DG2_G11(i915)) {
2200                 /* Wa_22012654132:dg2 */
2201                 wa_add(wal, GEN10_CACHE_MODE_SS, 0,
2202                        _MASKED_BIT_ENABLE(ENABLE_PREFETCH_INTO_IC),
2203                        0 /* write-only, so skip validation */,
2204                        true);
2205         }
2206
2207         /* Wa_14013202645:dg2 */
2208         if (IS_DG2_GRAPHICS_STEP(i915, G10, STEP_B0, STEP_C0) ||
2209             IS_DG2_GRAPHICS_STEP(i915, G11, STEP_A0, STEP_B0))
2210                 wa_write_or(wal, RT_CTRL, DIS_NULL_QUERY);
2211
2212         /* Wa_22012532006:dg2 */
2213         if (IS_DG2_GRAPHICS_STEP(engine->i915, G10, STEP_A0, STEP_C0) ||
2214             IS_DG2_GRAPHICS_STEP(engine->i915, G11, STEP_A0, STEP_B0))
2215                 wa_masked_en(wal, GEN9_HALF_SLICE_CHICKEN7,
2216                              DG2_DISABLE_ROUND_ENABLE_ALLOW_FOR_SSLA);
2217
2218         if (IS_DG2_GRAPHICS_STEP(engine->i915, G10, STEP_A0, STEP_B0)) {
2219                 /* Wa_14010680813:dg2_g10 */
2220                 wa_write_or(wal, GEN12_GAMSTLB_CTRL, CONTROL_BLOCK_CLKGATE_DIS |
2221                             EGRESS_BLOCK_CLKGATE_DIS | TAG_BLOCK_CLKGATE_DIS);
2222         }
2223
2224         if (IS_DG2_GRAPHICS_STEP(engine->i915, G10, STEP_A0, STEP_B0) ||
2225             IS_DG2_GRAPHICS_STEP(engine->i915, G11, STEP_A0, STEP_B0)) {
2226                 /* Wa_14012362059:dg2 */
2227                 wa_write_or(wal, GEN12_MERT_MOD_CTRL, FORCE_MISS_FTLB);
2228         }
2229
2230         if (IS_DG2_GRAPHICS_STEP(i915, G11, STEP_B0, STEP_FOREVER) ||
2231             IS_DG2_G10(i915)) {
2232                 /* Wa_22014600077:dg2 */
2233                 wa_add(wal, GEN10_CACHE_MODE_SS, 0,
2234                        _MASKED_BIT_ENABLE(ENABLE_EU_COUNT_FOR_TDL_FLUSH),
2235                        0 /* Wa_14012342262 :write-only reg, so skip
2236                             verification */,
2237                        true);
2238         }
2239
2240         if (IS_DG1_GRAPHICS_STEP(i915, STEP_A0, STEP_B0) ||
2241             IS_TGL_UY_GRAPHICS_STEP(i915, STEP_A0, STEP_B0)) {
2242                 /*
2243                  * Wa_1607138336:tgl[a0],dg1[a0]
2244                  * Wa_1607063988:tgl[a0],dg1[a0]
2245                  */
2246                 wa_write_or(wal,
2247                             GEN9_CTX_PREEMPT_REG,
2248                             GEN12_DISABLE_POSH_BUSY_FF_DOP_CG);
2249         }
2250
2251         if (IS_TGL_UY_GRAPHICS_STEP(i915, STEP_A0, STEP_B0)) {
2252                 /*
2253                  * Wa_1606679103:tgl
2254                  * (see also Wa_1606682166:icl)
2255                  */
2256                 wa_write_or(wal,
2257                             GEN7_SARCHKMD,
2258                             GEN7_DISABLE_SAMPLER_PREFETCH);
2259         }
2260
2261         if (IS_ALDERLAKE_P(i915) || IS_ALDERLAKE_S(i915) || IS_DG1(i915) ||
2262             IS_ROCKETLAKE(i915) || IS_TIGERLAKE(i915)) {
2263                 /* Wa_1606931601:tgl,rkl,dg1,adl-s,adl-p */
2264                 wa_masked_en(wal, GEN7_ROW_CHICKEN2, GEN12_DISABLE_EARLY_READ);
2265
2266                 /*
2267                  * Wa_1407928979:tgl A*
2268                  * Wa_18011464164:tgl[B0+],dg1[B0+]
2269                  * Wa_22010931296:tgl[B0+],dg1[B0+]
2270                  * Wa_14010919138:rkl,dg1,adl-s,adl-p
2271                  */
2272                 wa_write_or(wal, GEN7_FF_THREAD_MODE,
2273                             GEN12_FF_TESSELATION_DOP_GATE_DISABLE);
2274         }
2275
2276         if (IS_ALDERLAKE_P(i915) || IS_DG2(i915) || IS_ALDERLAKE_S(i915) ||
2277             IS_DG1(i915) || IS_ROCKETLAKE(i915) || IS_TIGERLAKE(i915)) {
2278                 /*
2279                  * Wa_1606700617:tgl,dg1,adl-p
2280                  * Wa_22010271021:tgl,rkl,dg1,adl-s,adl-p
2281                  * Wa_14010826681:tgl,dg1,rkl,adl-p
2282                  * Wa_18019627453:dg2
2283                  */
2284                 wa_masked_en(wal,
2285                              GEN9_CS_DEBUG_MODE1,
2286                              FF_DOP_CLOCK_GATE_DISABLE);
2287         }
2288
2289         if (IS_ALDERLAKE_P(i915) || IS_ALDERLAKE_S(i915) ||
2290             IS_DG1_GRAPHICS_STEP(i915, STEP_A0, STEP_B0) ||
2291             IS_ROCKETLAKE(i915) || IS_TIGERLAKE(i915)) {
2292                 /* Wa_1409804808:tgl,rkl,dg1[a0],adl-s,adl-p */
2293                 wa_masked_en(wal, GEN7_ROW_CHICKEN2,
2294                              GEN12_PUSH_CONST_DEREF_HOLD_DIS);
2295
2296                 /*
2297                  * Wa_1409085225:tgl
2298                  * Wa_14010229206:tgl,rkl,dg1[a0],adl-s,adl-p
2299                  */
2300                 wa_masked_en(wal, GEN9_ROW_CHICKEN4, GEN12_DISABLE_TDL_PUSH);
2301         }
2302
2303         if (IS_DG1_GRAPHICS_STEP(i915, STEP_A0, STEP_B0) ||
2304             IS_ROCKETLAKE(i915) || IS_TIGERLAKE(i915)) {
2305                 /*
2306                  * Wa_1607030317:tgl
2307                  * Wa_1607186500:tgl
2308                  * Wa_1607297627:tgl,rkl,dg1[a0]
2309                  *
2310                  * On TGL and RKL there are multiple entries for this WA in the
2311                  * BSpec; some indicate this is an A0-only WA, others indicate
2312                  * it applies to all steppings so we trust the "all steppings."
2313                  * For DG1 this only applies to A0.
2314                  */
2315                 wa_masked_en(wal,
2316                              RING_PSMI_CTL(RENDER_RING_BASE),
2317                              GEN12_WAIT_FOR_EVENT_POWER_DOWN_DISABLE |
2318                              GEN8_RC_SEMA_IDLE_MSG_DISABLE);
2319         }
2320
2321         if (IS_DG1(i915) || IS_ROCKETLAKE(i915) || IS_TIGERLAKE(i915) ||
2322             IS_ALDERLAKE_S(i915) || IS_ALDERLAKE_P(i915)) {
2323                 /* Wa_1406941453:tgl,rkl,dg1,adl-s,adl-p */
2324                 wa_masked_en(wal,
2325                              GEN10_SAMPLER_MODE,
2326                              ENABLE_SMALLPL);
2327         }
2328
2329         if (GRAPHICS_VER(i915) == 11) {
2330                 /* This is not an Wa. Enable for better image quality */
2331                 wa_masked_en(wal,
2332                              _3D_CHICKEN3,
2333                              _3D_CHICKEN3_AA_LINE_QUALITY_FIX_ENABLE);
2334
2335                 /*
2336                  * Wa_1405543622:icl
2337                  * Formerly known as WaGAPZPriorityScheme
2338                  */
2339                 wa_write_or(wal,
2340                             GEN8_GARBCNTL,
2341                             GEN11_ARBITRATION_PRIO_ORDER_MASK);
2342
2343                 /*
2344                  * Wa_1604223664:icl
2345                  * Formerly known as WaL3BankAddressHashing
2346                  */
2347                 wa_write_clr_set(wal,
2348                                  GEN8_GARBCNTL,
2349                                  GEN11_HASH_CTRL_EXCL_MASK,
2350                                  GEN11_HASH_CTRL_EXCL_BIT0);
2351                 wa_write_clr_set(wal,
2352                                  GEN11_GLBLINVL,
2353                                  GEN11_BANK_HASH_ADDR_EXCL_MASK,
2354                                  GEN11_BANK_HASH_ADDR_EXCL_BIT0);
2355
2356                 /*
2357                  * Wa_1405733216:icl
2358                  * Formerly known as WaDisableCleanEvicts
2359                  */
2360                 wa_write_or(wal,
2361                             GEN8_L3SQCREG4,
2362                             GEN11_LQSC_CLEAN_EVICT_DISABLE);
2363
2364                 /* Wa_1606682166:icl */
2365                 wa_write_or(wal,
2366                             GEN7_SARCHKMD,
2367                             GEN7_DISABLE_SAMPLER_PREFETCH);
2368
2369                 /* Wa_1409178092:icl */
2370                 wa_write_clr_set(wal,
2371                                  GEN11_SCRATCH2,
2372                                  GEN11_COHERENT_PARTIAL_WRITE_MERGE_ENABLE,
2373                                  0);
2374
2375                 /* WaEnable32PlaneMode:icl */
2376                 wa_masked_en(wal, GEN9_CSFE_CHICKEN1_RCS,
2377                              GEN11_ENABLE_32_PLANE_MODE);
2378
2379                 /*
2380                  * Wa_1408615072:icl,ehl  (vsunit)
2381                  * Wa_1407596294:icl,ehl  (hsunit)
2382                  */
2383                 wa_write_or(wal, UNSLICE_UNIT_LEVEL_CLKGATE,
2384                             VSUNIT_CLKGATE_DIS | HSUNIT_CLKGATE_DIS);
2385
2386                 /*
2387                  * Wa_1408767742:icl[a2..forever],ehl[all]
2388                  * Wa_1605460711:icl[a0..c0]
2389                  */
2390                 wa_write_or(wal,
2391                             GEN7_FF_THREAD_MODE,
2392                             GEN12_FF_TESSELATION_DOP_GATE_DISABLE);
2393
2394                 /* Wa_22010271021 */
2395                 wa_masked_en(wal,
2396                              GEN9_CS_DEBUG_MODE1,
2397                              FF_DOP_CLOCK_GATE_DISABLE);
2398         }
2399
2400         if (HAS_PERCTX_PREEMPT_CTRL(i915)) {
2401                 /* FtrPerCtxtPreemptionGranularityControl:skl,bxt,kbl,cfl,cnl,icl,tgl */
2402                 wa_masked_en(wal,
2403                              GEN7_FF_SLICE_CS_CHICKEN1,
2404                              GEN9_FFSC_PERCTX_PREEMPT_CTRL);
2405         }
2406
2407         if (IS_SKYLAKE(i915) ||
2408             IS_KABYLAKE(i915) ||
2409             IS_COFFEELAKE(i915) ||
2410             IS_COMETLAKE(i915)) {
2411                 /* WaEnableGapsTsvCreditFix:skl,kbl,cfl */
2412                 wa_write_or(wal,
2413                             GEN8_GARBCNTL,
2414                             GEN9_GAPS_TSV_CREDIT_DISABLE);
2415         }
2416
2417         if (IS_BROXTON(i915)) {
2418                 /* WaDisablePooledEuLoadBalancingFix:bxt */
2419                 wa_masked_en(wal,
2420                              FF_SLICE_CS_CHICKEN2,
2421                              GEN9_POOLED_EU_LOAD_BALANCING_FIX_DISABLE);
2422         }
2423
2424         if (GRAPHICS_VER(i915) == 9) {
2425                 /* WaContextSwitchWithConcurrentTLBInvalidate:skl,bxt,kbl,glk,cfl */
2426                 wa_masked_en(wal,
2427                              GEN9_CSFE_CHICKEN1_RCS,
2428                              GEN9_PREEMPT_GPGPU_SYNC_SWITCH_DISABLE);
2429
2430                 /* WaEnableLbsSlaRetryTimerDecrement:skl,bxt,kbl,glk,cfl */
2431                 wa_write_or(wal,
2432                             BDW_SCRATCH1,
2433                             GEN9_LBS_SLA_RETRY_TIMER_DECREMENT_ENABLE);
2434
2435                 /* WaProgramL3SqcReg1DefaultForPerf:bxt,glk */
2436                 if (IS_GEN9_LP(i915))
2437                         wa_write_clr_set(wal,
2438                                          GEN8_L3SQCREG1,
2439                                          L3_PRIO_CREDITS_MASK,
2440                                          L3_GENERAL_PRIO_CREDITS(62) |
2441                                          L3_HIGH_PRIO_CREDITS(2));
2442
2443                 /* WaOCLCoherentLineFlush:skl,bxt,kbl,cfl */
2444                 wa_write_or(wal,
2445                             GEN8_L3SQCREG4,
2446                             GEN8_LQSC_FLUSH_COHERENT_LINES);
2447
2448                 /* Disable atomics in L3 to prevent unrecoverable hangs */
2449                 wa_write_clr_set(wal, GEN9_SCRATCH_LNCF1,
2450                                  GEN9_LNCF_NONIA_COHERENT_ATOMICS_ENABLE, 0);
2451                 wa_write_clr_set(wal, GEN8_L3SQCREG4,
2452                                  GEN8_LQSQ_NONIA_COHERENT_ATOMICS_ENABLE, 0);
2453                 wa_write_clr_set(wal, GEN9_SCRATCH1,
2454                                  EVICTION_PERF_FIX_ENABLE, 0);
2455         }
2456
2457         if (IS_HASWELL(i915)) {
2458                 /* WaSampleCChickenBitEnable:hsw */
2459                 wa_masked_en(wal,
2460                              HALF_SLICE_CHICKEN3, HSW_SAMPLE_C_PERFORMANCE);
2461
2462                 wa_masked_dis(wal,
2463                               CACHE_MODE_0_GEN7,
2464                               /* enable HiZ Raw Stall Optimization */
2465                               HIZ_RAW_STALL_OPT_DISABLE);
2466         }
2467
2468         if (IS_VALLEYVIEW(i915)) {
2469                 /* WaDisableEarlyCull:vlv */
2470                 wa_masked_en(wal,
2471                              _3D_CHICKEN3,
2472                              _3D_CHICKEN_SF_DISABLE_OBJEND_CULL);
2473
2474                 /*
2475                  * WaVSThreadDispatchOverride:ivb,vlv
2476                  *
2477                  * This actually overrides the dispatch
2478                  * mode for all thread types.
2479                  */
2480                 wa_write_clr_set(wal,
2481                                  GEN7_FF_THREAD_MODE,
2482                                  GEN7_FF_SCHED_MASK,
2483                                  GEN7_FF_TS_SCHED_HW |
2484                                  GEN7_FF_VS_SCHED_HW |
2485                                  GEN7_FF_DS_SCHED_HW);
2486
2487                 /* WaPsdDispatchEnable:vlv */
2488                 /* WaDisablePSDDualDispatchEnable:vlv */
2489                 wa_masked_en(wal,
2490                              GEN7_HALF_SLICE_CHICKEN1,
2491                              GEN7_MAX_PS_THREAD_DEP |
2492                              GEN7_PSD_SINGLE_PORT_DISPATCH_ENABLE);
2493         }
2494
2495         if (IS_IVYBRIDGE(i915)) {
2496                 /* WaDisableEarlyCull:ivb */
2497                 wa_masked_en(wal,
2498                              _3D_CHICKEN3,
2499                              _3D_CHICKEN_SF_DISABLE_OBJEND_CULL);
2500
2501                 if (0) { /* causes HiZ corruption on ivb:gt1 */
2502                         /* enable HiZ Raw Stall Optimization */
2503                         wa_masked_dis(wal,
2504                                       CACHE_MODE_0_GEN7,
2505                                       HIZ_RAW_STALL_OPT_DISABLE);
2506                 }
2507
2508                 /*
2509                  * WaVSThreadDispatchOverride:ivb,vlv
2510                  *
2511                  * This actually overrides the dispatch
2512                  * mode for all thread types.
2513                  */
2514                 wa_write_clr_set(wal,
2515                                  GEN7_FF_THREAD_MODE,
2516                                  GEN7_FF_SCHED_MASK,
2517                                  GEN7_FF_TS_SCHED_HW |
2518                                  GEN7_FF_VS_SCHED_HW |
2519                                  GEN7_FF_DS_SCHED_HW);
2520
2521                 /* WaDisablePSDDualDispatchEnable:ivb */
2522                 if (IS_IVB_GT1(i915))
2523                         wa_masked_en(wal,
2524                                      GEN7_HALF_SLICE_CHICKEN1,
2525                                      GEN7_PSD_SINGLE_PORT_DISPATCH_ENABLE);
2526         }
2527
2528         if (GRAPHICS_VER(i915) == 7) {
2529                 /* WaBCSVCSTlbInvalidationMode:ivb,vlv,hsw */
2530                 wa_masked_en(wal,
2531                              RING_MODE_GEN7(RENDER_RING_BASE),
2532                              GFX_TLB_INVALIDATE_EXPLICIT | GFX_REPLAY_MODE);
2533
2534                 /* WaDisable_RenderCache_OperationalFlush:ivb,vlv,hsw */
2535                 wa_masked_dis(wal, CACHE_MODE_0_GEN7, RC_OP_FLUSH_ENABLE);
2536
2537                 /*
2538                  * BSpec says this must be set, even though
2539                  * WaDisable4x2SubspanOptimization:ivb,hsw
2540                  * WaDisable4x2SubspanOptimization isn't listed for VLV.
2541                  */
2542                 wa_masked_en(wal,
2543                              CACHE_MODE_1,
2544                              PIXEL_SUBSPAN_COLLECT_OPT_DISABLE);
2545
2546                 /*
2547                  * BSpec recommends 8x4 when MSAA is used,
2548                  * however in practice 16x4 seems fastest.
2549                  *
2550                  * Note that PS/WM thread counts depend on the WIZ hashing
2551                  * disable bit, which we don't touch here, but it's good
2552                  * to keep in mind (see 3DSTATE_PS and 3DSTATE_WM).
2553                  */
2554                 wa_masked_field_set(wal,
2555                                     GEN7_GT_MODE,
2556                                     GEN6_WIZ_HASHING_MASK,
2557                                     GEN6_WIZ_HASHING_16x4);
2558         }
2559
2560         if (IS_GRAPHICS_VER(i915, 6, 7))
2561                 /*
2562                  * We need to disable the AsyncFlip performance optimisations in
2563                  * order to use MI_WAIT_FOR_EVENT within the CS. It should
2564                  * already be programmed to '1' on all products.
2565                  *
2566                  * WaDisableAsyncFlipPerfMode:snb,ivb,hsw,vlv
2567                  */
2568                 wa_masked_en(wal,
2569                              RING_MI_MODE(RENDER_RING_BASE),
2570                              ASYNC_FLIP_PERF_DISABLE);
2571
2572         if (GRAPHICS_VER(i915) == 6) {
2573                 /*
2574                  * Required for the hardware to program scanline values for
2575                  * waiting
2576                  * WaEnableFlushTlbInvalidationMode:snb
2577                  */
2578                 wa_masked_en(wal,
2579                              GFX_MODE,
2580                              GFX_TLB_INVALIDATE_EXPLICIT);
2581
2582                 /* WaDisableHiZPlanesWhenMSAAEnabled:snb */
2583                 wa_masked_en(wal,
2584                              _3D_CHICKEN,
2585                              _3D_CHICKEN_HIZ_PLANE_DISABLE_MSAA_4X_SNB);
2586
2587                 wa_masked_en(wal,
2588                              _3D_CHICKEN3,
2589                              /* WaStripsFansDisableFastClipPerformanceFix:snb */
2590                              _3D_CHICKEN3_SF_DISABLE_FASTCLIP_CULL |
2591                              /*
2592                               * Bspec says:
2593                               * "This bit must be set if 3DSTATE_CLIP clip mode is set
2594                               * to normal and 3DSTATE_SF number of SF output attributes
2595                               * is more than 16."
2596                               */
2597                              _3D_CHICKEN3_SF_DISABLE_PIPELINED_ATTR_FETCH);
2598
2599                 /*
2600                  * BSpec recommends 8x4 when MSAA is used,
2601                  * however in practice 16x4 seems fastest.
2602                  *
2603                  * Note that PS/WM thread counts depend on the WIZ hashing
2604                  * disable bit, which we don't touch here, but it's good
2605                  * to keep in mind (see 3DSTATE_PS and 3DSTATE_WM).
2606                  */
2607                 wa_masked_field_set(wal,
2608                                     GEN6_GT_MODE,
2609                                     GEN6_WIZ_HASHING_MASK,
2610                                     GEN6_WIZ_HASHING_16x4);
2611
2612                 /* WaDisable_RenderCache_OperationalFlush:snb */
2613                 wa_masked_dis(wal, CACHE_MODE_0, RC_OP_FLUSH_ENABLE);
2614
2615                 /*
2616                  * From the Sandybridge PRM, volume 1 part 3, page 24:
2617                  * "If this bit is set, STCunit will have LRA as replacement
2618                  *  policy. [...] This bit must be reset. LRA replacement
2619                  *  policy is not supported."
2620                  */
2621                 wa_masked_dis(wal,
2622                               CACHE_MODE_0,
2623                               CM0_STC_EVICT_DISABLE_LRA_SNB);
2624         }
2625
2626         if (IS_GRAPHICS_VER(i915, 4, 6))
2627                 /* WaTimedSingleVertexDispatch:cl,bw,ctg,elk,ilk,snb */
2628                 wa_add(wal, RING_MI_MODE(RENDER_RING_BASE),
2629                        0, _MASKED_BIT_ENABLE(VS_TIMER_DISPATCH),
2630                        /* XXX bit doesn't stick on Broadwater */
2631                        IS_I965G(i915) ? 0 : VS_TIMER_DISPATCH, true);
2632
2633         if (GRAPHICS_VER(i915) == 4)
2634                 /*
2635                  * Disable CONSTANT_BUFFER before it is loaded from the context
2636                  * image. For as it is loaded, it is executed and the stored
2637                  * address may no longer be valid, leading to a GPU hang.
2638                  *
2639                  * This imposes the requirement that userspace reload their
2640                  * CONSTANT_BUFFER on every batch, fortunately a requirement
2641                  * they are already accustomed to from before contexts were
2642                  * enabled.
2643                  */
2644                 wa_add(wal, ECOSKPD(RENDER_RING_BASE),
2645                        0, _MASKED_BIT_ENABLE(ECO_CONSTANT_BUFFER_SR_DISABLE),
2646                        0 /* XXX bit doesn't stick on Broadwater */,
2647                        true);
2648 }
2649
2650 static void
2651 xcs_engine_wa_init(struct intel_engine_cs *engine, struct i915_wa_list *wal)
2652 {
2653         struct drm_i915_private *i915 = engine->i915;
2654
2655         /* WaKBLVECSSemaphoreWaitPoll:kbl */
2656         if (IS_KBL_GRAPHICS_STEP(i915, STEP_A0, STEP_F0)) {
2657                 wa_write(wal,
2658                          RING_SEMA_WAIT_POLL(engine->mmio_base),
2659                          1);
2660         }
2661 }
2662
2663 static void
2664 ccs_engine_wa_init(struct intel_engine_cs *engine, struct i915_wa_list *wal)
2665 {
2666         if (IS_PVC_CT_STEP(engine->i915, STEP_A0, STEP_C0)) {
2667                 /* Wa_14014999345:pvc */
2668                 wa_masked_en(wal, GEN10_CACHE_MODE_SS, DISABLE_ECC);
2669         }
2670 }
2671
2672 /*
2673  * The workarounds in this function apply to shared registers in
2674  * the general render reset domain that aren't tied to a
2675  * specific engine.  Since all render+compute engines get reset
2676  * together, and the contents of these registers are lost during
2677  * the shared render domain reset, we'll define such workarounds
2678  * here and then add them to just a single RCS or CCS engine's
2679  * workaround list (whichever engine has the XXXX flag).
2680  */
2681 static void
2682 general_render_compute_wa_init(struct intel_engine_cs *engine, struct i915_wa_list *wal)
2683 {
2684         struct drm_i915_private *i915 = engine->i915;
2685
2686         if (IS_PONTEVECCHIO(i915)) {
2687                 /*
2688                  * The following is not actually a "workaround" but rather
2689                  * a recommended tuning setting documented in the bspec's
2690                  * performance guide section.
2691                  */
2692                 wa_write(wal, XEHPC_L3SCRUB, SCRUB_CL_DWNGRADE_SHARED | SCRUB_RATE_4B_PER_CLK);
2693
2694                 /* Wa_16016694945 */
2695                 wa_masked_en(wal, XEHPC_LNCFMISCCFGREG0, XEHPC_OVRLSCCC);
2696         }
2697
2698         if (IS_XEHPSDV(i915)) {
2699                 /* Wa_1409954639 */
2700                 wa_masked_en(wal,
2701                              GEN8_ROW_CHICKEN,
2702                              SYSTOLIC_DOP_CLOCK_GATING_DIS);
2703
2704                 /* Wa_1607196519 */
2705                 wa_masked_en(wal,
2706                              GEN9_ROW_CHICKEN4,
2707                              GEN12_DISABLE_GRF_CLEAR);
2708
2709                 /* Wa_14010670810:xehpsdv */
2710                 wa_write_or(wal, XEHP_L3NODEARBCFG, XEHP_LNESPARE);
2711
2712                 /* Wa_14010449647:xehpsdv */
2713                 wa_masked_en(wal, GEN7_HALF_SLICE_CHICKEN1,
2714                              GEN7_PSD_SINGLE_PORT_DISPATCH_ENABLE);
2715
2716                 /* Wa_18011725039:xehpsdv */
2717                 if (IS_XEHPSDV_GRAPHICS_STEP(i915, STEP_A1, STEP_B0)) {
2718                         wa_masked_dis(wal, MLTICTXCTL, TDONRENDER);
2719                         wa_write_or(wal, L3SQCREG1_CCS0, FLUSHALLNONCOH);
2720                 }
2721
2722                 /* Wa_14012362059:xehpsdv */
2723                 wa_write_or(wal, GEN12_MERT_MOD_CTRL, FORCE_MISS_FTLB);
2724
2725                 /* Wa_14014368820:xehpsdv */
2726                 wa_write_or(wal, GEN12_GAMCNTRL_CTRL, INVALIDATION_BROADCAST_MODE_DIS |
2727                                 GLOBAL_INVALIDATION_MODE);
2728         }
2729
2730         if (IS_DG2(i915) || IS_PONTEVECCHIO(i915)) {
2731                 /* Wa_14015227452:dg2,pvc */
2732                 wa_masked_en(wal, GEN9_ROW_CHICKEN4, XEHP_DIS_BBL_SYSPIPE);
2733
2734                 /* Wa_22014226127:dg2,pvc */
2735                 wa_write_or(wal, LSC_CHICKEN_BIT_0, DISABLE_D8_D16_COASLESCE);
2736
2737                 /* Wa_16015675438:dg2,pvc */
2738                 wa_masked_en(wal, FF_SLICE_CS_CHICKEN2, GEN12_PERF_FIX_BALANCING_CFE_DISABLE);
2739
2740                 /* Wa_18018781329:dg2,pvc */
2741                 wa_write_or(wal, RENDER_MOD_CTRL, FORCE_MISS_FTLB);
2742                 wa_write_or(wal, COMP_MOD_CTRL, FORCE_MISS_FTLB);
2743                 wa_write_or(wal, VDBX_MOD_CTRL, FORCE_MISS_FTLB);
2744                 wa_write_or(wal, VEBX_MOD_CTRL, FORCE_MISS_FTLB);
2745         }
2746 }
2747
2748 static void
2749 engine_init_workarounds(struct intel_engine_cs *engine, struct i915_wa_list *wal)
2750 {
2751         if (I915_SELFTEST_ONLY(GRAPHICS_VER(engine->i915) < 4))
2752                 return;
2753
2754         engine_fake_wa_init(engine, wal);
2755
2756         /*
2757          * These are common workarounds that just need to applied
2758          * to a single RCS/CCS engine's workaround list since
2759          * they're reset as part of the general render domain reset.
2760          */
2761         if (engine->flags & I915_ENGINE_FIRST_RENDER_COMPUTE)
2762                 general_render_compute_wa_init(engine, wal);
2763
2764         if (engine->class == COMPUTE_CLASS)
2765                 ccs_engine_wa_init(engine, wal);
2766         else if (engine->class == RENDER_CLASS)
2767                 rcs_engine_wa_init(engine, wal);
2768         else
2769                 xcs_engine_wa_init(engine, wal);
2770 }
2771
2772 void intel_engine_init_workarounds(struct intel_engine_cs *engine)
2773 {
2774         struct i915_wa_list *wal = &engine->wa_list;
2775
2776         if (GRAPHICS_VER(engine->i915) < 4)
2777                 return;
2778
2779         wa_init_start(wal, "engine", engine->name);
2780         engine_init_workarounds(engine, wal);
2781         wa_init_finish(wal);
2782 }
2783
2784 void intel_engine_apply_workarounds(struct intel_engine_cs *engine)
2785 {
2786         wa_list_apply(engine->gt, &engine->wa_list);
2787 }
2788
2789 static const struct i915_range mcr_ranges_gen8[] = {
2790         { .start = 0x5500, .end = 0x55ff },
2791         { .start = 0x7000, .end = 0x7fff },
2792         { .start = 0x9400, .end = 0x97ff },
2793         { .start = 0xb000, .end = 0xb3ff },
2794         { .start = 0xe000, .end = 0xe7ff },
2795         {},
2796 };
2797
2798 static const struct i915_range mcr_ranges_gen12[] = {
2799         { .start =  0x8150, .end =  0x815f },
2800         { .start =  0x9520, .end =  0x955f },
2801         { .start =  0xb100, .end =  0xb3ff },
2802         { .start =  0xde80, .end =  0xe8ff },
2803         { .start = 0x24a00, .end = 0x24a7f },
2804         {},
2805 };
2806
2807 static const struct i915_range mcr_ranges_xehp[] = {
2808         { .start =  0x4000, .end =  0x4aff },
2809         { .start =  0x5200, .end =  0x52ff },
2810         { .start =  0x5400, .end =  0x7fff },
2811         { .start =  0x8140, .end =  0x815f },
2812         { .start =  0x8c80, .end =  0x8dff },
2813         { .start =  0x94d0, .end =  0x955f },
2814         { .start =  0x9680, .end =  0x96ff },
2815         { .start =  0xb000, .end =  0xb3ff },
2816         { .start =  0xc800, .end =  0xcfff },
2817         { .start =  0xd800, .end =  0xd8ff },
2818         { .start =  0xdc00, .end =  0xffff },
2819         { .start = 0x17000, .end = 0x17fff },
2820         { .start = 0x24a00, .end = 0x24a7f },
2821         {},
2822 };
2823
2824 static bool mcr_range(struct drm_i915_private *i915, u32 offset)
2825 {
2826         const struct i915_range *mcr_ranges;
2827         int i;
2828
2829         if (GRAPHICS_VER_FULL(i915) >= IP_VER(12, 50))
2830                 mcr_ranges = mcr_ranges_xehp;
2831         else if (GRAPHICS_VER(i915) >= 12)
2832                 mcr_ranges = mcr_ranges_gen12;
2833         else if (GRAPHICS_VER(i915) >= 8)
2834                 mcr_ranges = mcr_ranges_gen8;
2835         else
2836                 return false;
2837
2838         /*
2839          * Registers in these ranges are affected by the MCR selector
2840          * which only controls CPU initiated MMIO. Routing does not
2841          * work for CS access so we cannot verify them on this path.
2842          */
2843         for (i = 0; mcr_ranges[i].start; i++)
2844                 if (offset >= mcr_ranges[i].start &&
2845                     offset <= mcr_ranges[i].end)
2846                         return true;
2847
2848         return false;
2849 }
2850
2851 static int
2852 wa_list_srm(struct i915_request *rq,
2853             const struct i915_wa_list *wal,
2854             struct i915_vma *vma)
2855 {
2856         struct drm_i915_private *i915 = rq->engine->i915;
2857         unsigned int i, count = 0;
2858         const struct i915_wa *wa;
2859         u32 srm, *cs;
2860
2861         srm = MI_STORE_REGISTER_MEM | MI_SRM_LRM_GLOBAL_GTT;
2862         if (GRAPHICS_VER(i915) >= 8)
2863                 srm++;
2864
2865         for (i = 0, wa = wal->list; i < wal->count; i++, wa++) {
2866                 if (!mcr_range(i915, i915_mmio_reg_offset(wa->reg)))
2867                         count++;
2868         }
2869
2870         cs = intel_ring_begin(rq, 4 * count);
2871         if (IS_ERR(cs))
2872                 return PTR_ERR(cs);
2873
2874         for (i = 0, wa = wal->list; i < wal->count; i++, wa++) {
2875                 u32 offset = i915_mmio_reg_offset(wa->reg);
2876
2877                 if (mcr_range(i915, offset))
2878                         continue;
2879
2880                 *cs++ = srm;
2881                 *cs++ = offset;
2882                 *cs++ = i915_ggtt_offset(vma) + sizeof(u32) * i;
2883                 *cs++ = 0;
2884         }
2885         intel_ring_advance(rq, cs);
2886
2887         return 0;
2888 }
2889
2890 static int engine_wa_list_verify(struct intel_context *ce,
2891                                  const struct i915_wa_list * const wal,
2892                                  const char *from)
2893 {
2894         const struct i915_wa *wa;
2895         struct i915_request *rq;
2896         struct i915_vma *vma;
2897         struct i915_gem_ww_ctx ww;
2898         unsigned int i;
2899         u32 *results;
2900         int err;
2901
2902         if (!wal->count)
2903                 return 0;
2904
2905         vma = __vm_create_scratch_for_read(&ce->engine->gt->ggtt->vm,
2906                                            wal->count * sizeof(u32));
2907         if (IS_ERR(vma))
2908                 return PTR_ERR(vma);
2909
2910         intel_engine_pm_get(ce->engine);
2911         i915_gem_ww_ctx_init(&ww, false);
2912 retry:
2913         err = i915_gem_object_lock(vma->obj, &ww);
2914         if (err == 0)
2915                 err = intel_context_pin_ww(ce, &ww);
2916         if (err)
2917                 goto err_pm;
2918
2919         err = i915_vma_pin_ww(vma, &ww, 0, 0,
2920                            i915_vma_is_ggtt(vma) ? PIN_GLOBAL : PIN_USER);
2921         if (err)
2922                 goto err_unpin;
2923
2924         rq = i915_request_create(ce);
2925         if (IS_ERR(rq)) {
2926                 err = PTR_ERR(rq);
2927                 goto err_vma;
2928         }
2929
2930         err = i915_request_await_object(rq, vma->obj, true);
2931         if (err == 0)
2932                 err = i915_vma_move_to_active(vma, rq, EXEC_OBJECT_WRITE);
2933         if (err == 0)
2934                 err = wa_list_srm(rq, wal, vma);
2935
2936         i915_request_get(rq);
2937         if (err)
2938                 i915_request_set_error_once(rq, err);
2939         i915_request_add(rq);
2940
2941         if (err)
2942                 goto err_rq;
2943
2944         if (i915_request_wait(rq, 0, HZ / 5) < 0) {
2945                 err = -ETIME;
2946                 goto err_rq;
2947         }
2948
2949         results = i915_gem_object_pin_map(vma->obj, I915_MAP_WB);
2950         if (IS_ERR(results)) {
2951                 err = PTR_ERR(results);
2952                 goto err_rq;
2953         }
2954
2955         err = 0;
2956         for (i = 0, wa = wal->list; i < wal->count; i++, wa++) {
2957                 if (mcr_range(rq->engine->i915, i915_mmio_reg_offset(wa->reg)))
2958                         continue;
2959
2960                 if (!wa_verify(wa, results[i], wal->name, from))
2961                         err = -ENXIO;
2962         }
2963
2964         i915_gem_object_unpin_map(vma->obj);
2965
2966 err_rq:
2967         i915_request_put(rq);
2968 err_vma:
2969         i915_vma_unpin(vma);
2970 err_unpin:
2971         intel_context_unpin(ce);
2972 err_pm:
2973         if (err == -EDEADLK) {
2974                 err = i915_gem_ww_ctx_backoff(&ww);
2975                 if (!err)
2976                         goto retry;
2977         }
2978         i915_gem_ww_ctx_fini(&ww);
2979         intel_engine_pm_put(ce->engine);
2980         i915_vma_put(vma);
2981         return err;
2982 }
2983
2984 int intel_engine_verify_workarounds(struct intel_engine_cs *engine,
2985                                     const char *from)
2986 {
2987         return engine_wa_list_verify(engine->kernel_context,
2988                                      &engine->wa_list,
2989                                      from);
2990 }
2991
2992 #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
2993 #include "selftest_workarounds.c"
2994 #endif