drivers/gpu/drm/i915/intel_uncore.c

   1 /*
   2  * Copyright © 2013 Intel Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21  * IN THE SOFTWARE.
  22  */
  23
  24 #include <drm/drm_managed.h>
  25 #include <linux/pm_runtime.h>
  26
  27 #include "gt/intel_gt.h"
  28 #include "gt/intel_engine_regs.h"
  29 #include "gt/intel_gt_regs.h"
  30
  31 #include "i915_drv.h"
  32 #include "i915_iosf_mbi.h"
  33 #include "i915_reg.h"
  34 #include "i915_vgpu.h"
  35 #include "intel_uncore_trace.h"
  36
  37 #define FORCEWAKE_ACK_TIMEOUT_MS 50
  38 #define GT_FIFO_TIMEOUT_MS       10
  39
  40 struct intel_uncore *to_intel_uncore(struct drm_device *drm)
  41 {
  42         return &to_i915(drm)->uncore;
  43 }
  44
  45 #define __raw_posting_read(...) ((void)__raw_uncore_read32(__VA_ARGS__))
  46
  47 static void
  48 fw_domains_get(struct intel_uncore *uncore, enum forcewake_domains fw_domains)
  49 {
  50         uncore->fw_get_funcs->force_wake_get(uncore, fw_domains);
  51 }
  52
  53 void
  54 intel_uncore_mmio_debug_init_early(struct drm_i915_private *i915)
  55 {
  56         spin_lock_init(&i915->mmio_debug.lock);
  57         i915->mmio_debug.unclaimed_mmio_check = 1;
  58
  59         i915->uncore.debug = &i915->mmio_debug;
  60 }
  61
  62 static void mmio_debug_suspend(struct intel_uncore *uncore)
  63 {
  64         if (!uncore->debug)
  65                 return;
  66
  67         spin_lock(&uncore->debug->lock);
  68
  69         /* Save and disable mmio debugging for the user bypass */
  70         if (!uncore->debug->suspend_count++) {
  71                 uncore->debug->saved_mmio_check = uncore->debug->unclaimed_mmio_check;
  72                 uncore->debug->unclaimed_mmio_check = 0;
  73         }
  74
  75         spin_unlock(&uncore->debug->lock);
  76 }
  77
  78 static bool check_for_unclaimed_mmio(struct intel_uncore *uncore);
  79
  80 static void mmio_debug_resume(struct intel_uncore *uncore)
  81 {
  82         if (!uncore->debug)
  83                 return;
  84
  85         spin_lock(&uncore->debug->lock);
  86
  87         if (!--uncore->debug->suspend_count)
  88                 uncore->debug->unclaimed_mmio_check = uncore->debug->saved_mmio_check;
  89
  90         if (check_for_unclaimed_mmio(uncore))
  91                 drm_info(&uncore->i915->drm,
  92                          "Invalid mmio detected during user access\n");
  93
  94         spin_unlock(&uncore->debug->lock);
  95 }
  96
  97 static const char * const forcewake_domain_names[] = {
  98         "render",
  99         "gt",
 100         "media",
 101         "vdbox0",
 102         "vdbox1",
 103         "vdbox2",
 104         "vdbox3",
 105         "vdbox4",
 106         "vdbox5",
 107         "vdbox6",
 108         "vdbox7",
 109         "vebox0",
 110         "vebox1",
 111         "vebox2",
 112         "vebox3",
 113         "gsc",
 114 };
 115
 116 const char *
 117 intel_uncore_forcewake_domain_to_str(const enum forcewake_domain_id id)
 118 {
 119         BUILD_BUG_ON(ARRAY_SIZE(forcewake_domain_names) != FW_DOMAIN_ID_COUNT);
 120
 121         if (id >= 0 && id < FW_DOMAIN_ID_COUNT)
 122                 return forcewake_domain_names[id];
 123
 124         WARN_ON(id);
 125
 126         return "unknown";
 127 }
 128
 129 #define fw_ack(d) readl((d)->reg_ack)
 130 #define fw_set(d, val) writel(_MASKED_BIT_ENABLE((val)), (d)->reg_set)
 131 #define fw_clear(d, val) writel(_MASKED_BIT_DISABLE((val)), (d)->reg_set)
 132
 133 static inline void
 134 fw_domain_reset(const struct intel_uncore_forcewake_domain *d)
 135 {
 136         /*
 137          * We don't really know if the powerwell for the forcewake domain we are
 138          * trying to reset here does exist at this point (engines could be fused
 139          * off in ICL+), so no waiting for acks
 140          */
 141         /* WaRsClearFWBitsAtReset */
 142         if (GRAPHICS_VER(d->uncore->i915) >= 12)
 143                 fw_clear(d, 0xefff);
 144         else
 145                 fw_clear(d, 0xffff);
 146 }
 147
 148 static inline void
 149 fw_domain_arm_timer(struct intel_uncore_forcewake_domain *d)
 150 {
 151         GEM_BUG_ON(d->uncore->fw_domains_timer & d->mask);
 152         d->uncore->fw_domains_timer |= d->mask;
 153         d->wake_count++;
 154         hrtimer_start_range_ns(&d->timer,
 155                                NSEC_PER_MSEC,
 156                                NSEC_PER_MSEC,
 157                                HRTIMER_MODE_REL);
 158 }
 159
 160 static inline int
 161 __wait_for_ack(const struct intel_uncore_forcewake_domain *d,
 162                const u32 ack,
 163                const u32 value)
 164 {
 165         return wait_for_atomic((fw_ack(d) & ack) == value,
 166                                FORCEWAKE_ACK_TIMEOUT_MS);
 167 }
 168
 169 static inline int
 170 wait_ack_clear(const struct intel_uncore_forcewake_domain *d,
 171                const u32 ack)
 172 {
 173         return __wait_for_ack(d, ack, 0);
 174 }
 175
 176 static inline int
 177 wait_ack_set(const struct intel_uncore_forcewake_domain *d,
 178              const u32 ack)
 179 {
 180         return __wait_for_ack(d, ack, ack);
 181 }
 182
 183 static inline void
 184 fw_domain_wait_ack_clear(const struct intel_uncore_forcewake_domain *d)
 185 {
 186         if (!wait_ack_clear(d, FORCEWAKE_KERNEL))
 187                 return;
 188
 189         if (fw_ack(d) == ~0) {
 190                 drm_err(&d->uncore->i915->drm,
 191                         "%s: MMIO unreliable (forcewake register returns 0xFFFFFFFF)!\n",
 192                         intel_uncore_forcewake_domain_to_str(d->id));
 193                 intel_gt_set_wedged_async(d->uncore->gt);
 194         } else {
 195                 drm_err(&d->uncore->i915->drm,
 196                         "%s: timed out waiting for forcewake ack to clear.\n",
 197                         intel_uncore_forcewake_domain_to_str(d->id));
 198         }
 199
 200         add_taint_for_CI(d->uncore->i915, TAINT_WARN); /* CI now unreliable */
 201 }
 202
 203 enum ack_type {
 204         ACK_CLEAR = 0,
 205         ACK_SET
 206 };
 207
 208 static int
 209 fw_domain_wait_ack_with_fallback(const struct intel_uncore_forcewake_domain *d,
 210                                  const enum ack_type type)
 211 {
 212         const u32 ack_bit = FORCEWAKE_KERNEL;
 213         const u32 value = type == ACK_SET ? ack_bit : 0;
 214         unsigned int pass;
 215         bool ack_detected;
 216
 217         /*
 218          * There is a possibility of driver's wake request colliding
 219          * with hardware's own wake requests and that can cause
 220          * hardware to not deliver the driver's ack message.
 221          *
 222          * Use a fallback bit toggle to kick the gpu state machine
 223          * in the hope that the original ack will be delivered along with
 224          * the fallback ack.
 225          *
 226          * This workaround is described in HSDES #1604254524 and it's known as:
 227          * WaRsForcewakeAddDelayForAck:skl,bxt,kbl,glk,cfl,cnl,icl
 228          * although the name is a bit misleading.
 229          */
 230
 231         pass = 1;
 232         do {
 233                 wait_ack_clear(d, FORCEWAKE_KERNEL_FALLBACK);
 234
 235                 fw_set(d, FORCEWAKE_KERNEL_FALLBACK);
 236                 /* Give gt some time to relax before the polling frenzy */
 237                 udelay(10 * pass);
 238                 wait_ack_set(d, FORCEWAKE_KERNEL_FALLBACK);
 239
 240                 ack_detected = (fw_ack(d) & ack_bit) == value;
 241
 242                 fw_clear(d, FORCEWAKE_KERNEL_FALLBACK);
 243         } while (!ack_detected && pass++ < 10);
 244
 245         drm_dbg(&d->uncore->i915->drm,
 246                 "%s had to use fallback to %s ack, 0x%x (passes %u)\n",
 247                 intel_uncore_forcewake_domain_to_str(d->id),
 248                 type == ACK_SET ? "set" : "clear",
 249                 fw_ack(d),
 250                 pass);
 251
 252         return ack_detected ? 0 : -ETIMEDOUT;
 253 }
 254
 255 static inline void
 256 fw_domain_wait_ack_clear_fallback(const struct intel_uncore_forcewake_domain *d)
 257 {
 258         if (likely(!wait_ack_clear(d, FORCEWAKE_KERNEL)))
 259                 return;
 260
 261         if (fw_domain_wait_ack_with_fallback(d, ACK_CLEAR))
 262                 fw_domain_wait_ack_clear(d);
 263 }
 264
 265 static inline void
 266 fw_domain_get(const struct intel_uncore_forcewake_domain *d)
 267 {
 268         fw_set(d, FORCEWAKE_KERNEL);
 269 }
 270
 271 static inline void
 272 fw_domain_wait_ack_set(const struct intel_uncore_forcewake_domain *d)
 273 {
 274         if (wait_ack_set(d, FORCEWAKE_KERNEL)) {
 275                 drm_err(&d->uncore->i915->drm,
 276                         "%s: timed out waiting for forcewake ack request.\n",
 277                         intel_uncore_forcewake_domain_to_str(d->id));
 278                 add_taint_for_CI(d->uncore->i915, TAINT_WARN); /* CI now unreliable */
 279         }
 280 }
 281
 282 static inline void
 283 fw_domain_wait_ack_set_fallback(const struct intel_uncore_forcewake_domain *d)
 284 {
 285         if (likely(!wait_ack_set(d, FORCEWAKE_KERNEL)))
 286                 return;
 287
 288         if (fw_domain_wait_ack_with_fallback(d, ACK_SET))
 289                 fw_domain_wait_ack_set(d);
 290 }
 291
 292 static inline void
 293 fw_domain_put(const struct intel_uncore_forcewake_domain *d)
 294 {
 295         fw_clear(d, FORCEWAKE_KERNEL);
 296 }
 297
 298 static void
 299 fw_domains_get_normal(struct intel_uncore *uncore, enum forcewake_domains fw_domains)
 300 {
 301         struct intel_uncore_forcewake_domain *d;
 302         unsigned int tmp;
 303
 304         GEM_BUG_ON(fw_domains & ~uncore->fw_domains);
 305
 306         for_each_fw_domain_masked(d, fw_domains, uncore, tmp) {
 307                 fw_domain_wait_ack_clear(d);
 308                 fw_domain_get(d);
 309         }
 310
 311         for_each_fw_domain_masked(d, fw_domains, uncore, tmp)
 312                 fw_domain_wait_ack_set(d);
 313
 314         uncore->fw_domains_active |= fw_domains;
 315 }
 316
 317 static void
 318 fw_domains_get_with_fallback(struct intel_uncore *uncore,
 319                              enum forcewake_domains fw_domains)
 320 {
 321         struct intel_uncore_forcewake_domain *d;
 322         unsigned int tmp;
 323
 324         GEM_BUG_ON(fw_domains & ~uncore->fw_domains);
 325
 326         for_each_fw_domain_masked(d, fw_domains, uncore, tmp) {
 327                 fw_domain_wait_ack_clear_fallback(d);
 328                 fw_domain_get(d);
 329         }
 330
 331         for_each_fw_domain_masked(d, fw_domains, uncore, tmp)
 332                 fw_domain_wait_ack_set_fallback(d);
 333
 334         uncore->fw_domains_active |= fw_domains;
 335 }
 336
 337 static void
 338 fw_domains_put(struct intel_uncore *uncore, enum forcewake_domains fw_domains)
 339 {
 340         struct intel_uncore_forcewake_domain *d;
 341         unsigned int tmp;
 342
 343         GEM_BUG_ON(fw_domains & ~uncore->fw_domains);
 344
 345         for_each_fw_domain_masked(d, fw_domains, uncore, tmp)
 346                 fw_domain_put(d);
 347
 348         uncore->fw_domains_active &= ~fw_domains;
 349 }
 350
 351 static void
 352 fw_domains_reset(struct intel_uncore *uncore,
 353                  enum forcewake_domains fw_domains)
 354 {
 355         struct intel_uncore_forcewake_domain *d;
 356         unsigned int tmp;
 357
 358         if (!fw_domains)
 359                 return;
 360
 361         GEM_BUG_ON(fw_domains & ~uncore->fw_domains);
 362
 363         for_each_fw_domain_masked(d, fw_domains, uncore, tmp)
 364                 fw_domain_reset(d);
 365 }
 366
 367 static inline u32 gt_thread_status(struct intel_uncore *uncore)
 368 {
 369         u32 val;
 370
 371         val = __raw_uncore_read32(uncore, GEN6_GT_THREAD_STATUS_REG);
 372         val &= GEN6_GT_THREAD_STATUS_CORE_MASK;
 373
 374         return val;
 375 }
 376
 377 static void __gen6_gt_wait_for_thread_c0(struct intel_uncore *uncore)
 378 {
 379         /*
 380          * w/a for a sporadic read returning 0 by waiting for the GT
 381          * thread to wake up.
 382          */
 383         drm_WARN_ONCE(&uncore->i915->drm,
 384                       wait_for_atomic_us(gt_thread_status(uncore) == 0, 5000),
 385                       "GT thread status wait timed out\n");
 386 }
 387
 388 static void fw_domains_get_with_thread_status(struct intel_uncore *uncore,
 389                                               enum forcewake_domains fw_domains)
 390 {
 391         fw_domains_get_normal(uncore, fw_domains);
 392
 393         /* WaRsForcewakeWaitTC0:snb,ivb,hsw,bdw,vlv */
 394         __gen6_gt_wait_for_thread_c0(uncore);
 395 }
 396
 397 static inline u32 fifo_free_entries(struct intel_uncore *uncore)
 398 {
 399         u32 count = __raw_uncore_read32(uncore, GTFIFOCTL);
 400
 401         return count & GT_FIFO_FREE_ENTRIES_MASK;
 402 }
 403
 404 static void __gen6_gt_wait_for_fifo(struct intel_uncore *uncore)
 405 {
 406         u32 n;
 407
 408         /* On VLV, FIFO will be shared by both SW and HW.
 409          * So, we need to read the FREE_ENTRIES everytime */
 410         if (IS_VALLEYVIEW(uncore->i915))
 411                 n = fifo_free_entries(uncore);
 412         else
 413                 n = uncore->fifo_count;
 414
 415         if (n <= GT_FIFO_NUM_RESERVED_ENTRIES) {
 416                 if (wait_for_atomic((n = fifo_free_entries(uncore)) >
 417                                     GT_FIFO_NUM_RESERVED_ENTRIES,
 418                                     GT_FIFO_TIMEOUT_MS)) {
 419                         drm_dbg(&uncore->i915->drm,
 420                                 "GT_FIFO timeout, entries: %u\n", n);
 421                         return;
 422                 }
 423         }
 424
 425         uncore->fifo_count = n - 1;
 426 }
 427
 428 static enum hrtimer_restart
 429 intel_uncore_fw_release_timer(struct hrtimer *timer)
 430 {
 431         struct intel_uncore_forcewake_domain *domain =
 432                container_of(timer, struct intel_uncore_forcewake_domain, timer);
 433         struct intel_uncore *uncore = domain->uncore;
 434         unsigned long irqflags;
 435
 436         assert_rpm_device_not_suspended(uncore->rpm);
 437
 438         if (xchg(&domain->active, false))
 439                 return HRTIMER_RESTART;
 440
 441         spin_lock_irqsave(&uncore->lock, irqflags);
 442
 443         uncore->fw_domains_timer &= ~domain->mask;
 444
 445         GEM_BUG_ON(!domain->wake_count);
 446         if (--domain->wake_count == 0)
 447                 fw_domains_put(uncore, domain->mask);
 448
 449         spin_unlock_irqrestore(&uncore->lock, irqflags);
 450
 451         return HRTIMER_NORESTART;
 452 }
 453
 454 /* Note callers must have acquired the PUNIT->PMIC bus, before calling this. */
 455 static unsigned int
 456 intel_uncore_forcewake_reset(struct intel_uncore *uncore)
 457 {
 458         unsigned long irqflags;
 459         struct intel_uncore_forcewake_domain *domain;
 460         int retry_count = 100;
 461         enum forcewake_domains fw, active_domains;
 462
 463         iosf_mbi_assert_punit_acquired();
 464
 465         /* Hold uncore.lock across reset to prevent any register access
 466          * with forcewake not set correctly. Wait until all pending
 467          * timers are run before holding.
 468          */
 469         while (1) {
 470                 unsigned int tmp;
 471
 472                 active_domains = 0;
 473
 474                 for_each_fw_domain(domain, uncore, tmp) {
 475                         smp_store_mb(domain->active, false);
 476                         if (hrtimer_cancel(&domain->timer) == 0)
 477                                 continue;
 478
 479                         intel_uncore_fw_release_timer(&domain->timer);
 480                 }
 481
 482                 spin_lock_irqsave(&uncore->lock, irqflags);
 483
 484                 for_each_fw_domain(domain, uncore, tmp) {
 485                         if (hrtimer_active(&domain->timer))
 486                                 active_domains |= domain->mask;
 487                 }
 488
 489                 if (active_domains == 0)
 490                         break;
 491
 492                 if (--retry_count == 0) {
 493                         drm_err(&uncore->i915->drm, "Timed out waiting for forcewake timers to finish\n");
 494                         break;
 495                 }
 496
 497                 spin_unlock_irqrestore(&uncore->lock, irqflags);
 498                 cond_resched();
 499         }
 500
 501         drm_WARN_ON(&uncore->i915->drm, active_domains);
 502
 503         fw = uncore->fw_domains_active;
 504         if (fw)
 505                 fw_domains_put(uncore, fw);
 506
 507         fw_domains_reset(uncore, uncore->fw_domains);
 508         assert_forcewakes_inactive(uncore);
 509
 510         spin_unlock_irqrestore(&uncore->lock, irqflags);
 511
 512         return fw; /* track the lost user forcewake domains */
 513 }
 514
 515 static bool
 516 fpga_check_for_unclaimed_mmio(struct intel_uncore *uncore)
 517 {
 518         u32 dbg;
 519
 520         dbg = __raw_uncore_read32(uncore, FPGA_DBG);
 521         if (likely(!(dbg & FPGA_DBG_RM_NOCLAIM)))
 522                 return false;
 523
 524         /*
 525          * Bugs in PCI programming (or failing hardware) can occasionally cause
 526          * us to lose access to the MMIO BAR.  When this happens, register
 527          * reads will come back with 0xFFFFFFFF for every register and things
 528          * go bad very quickly.  Let's try to detect that special case and at
 529          * least try to print a more informative message about what has
 530          * happened.
 531          *
 532          * During normal operation the FPGA_DBG register has several unused
 533          * bits that will always read back as 0's so we can use them as canaries
 534          * to recognize when MMIO accesses are just busted.
 535          */
 536         if (unlikely(dbg == ~0))
 537                 drm_err(&uncore->i915->drm,
 538                         "Lost access to MMIO BAR; all registers now read back as 0xFFFFFFFF!\n");
 539
 540         __raw_uncore_write32(uncore, FPGA_DBG, FPGA_DBG_RM_NOCLAIM);
 541
 542         return true;
 543 }
 544
 545 static bool
 546 vlv_check_for_unclaimed_mmio(struct intel_uncore *uncore)
 547 {
 548         u32 cer;
 549
 550         cer = __raw_uncore_read32(uncore, CLAIM_ER);
 551         if (likely(!(cer & (CLAIM_ER_OVERFLOW | CLAIM_ER_CTR_MASK))))
 552                 return false;
 553
 554         __raw_uncore_write32(uncore, CLAIM_ER, CLAIM_ER_CLR);
 555
 556         return true;
 557 }
 558
 559 static bool
 560 gen6_check_for_fifo_debug(struct intel_uncore *uncore)
 561 {
 562         u32 fifodbg;
 563
 564         fifodbg = __raw_uncore_read32(uncore, GTFIFODBG);
 565
 566         if (unlikely(fifodbg)) {
 567                 drm_dbg(&uncore->i915->drm, "GTFIFODBG = 0x08%x\n", fifodbg);
 568                 __raw_uncore_write32(uncore, GTFIFODBG, fifodbg);
 569         }
 570
 571         return fifodbg;
 572 }
 573
 574 static bool
 575 check_for_unclaimed_mmio(struct intel_uncore *uncore)
 576 {
 577         bool ret = false;
 578
 579         lockdep_assert_held(&uncore->debug->lock);
 580
 581         if (uncore->debug->suspend_count)
 582                 return false;
 583
 584         if (intel_uncore_has_fpga_dbg_unclaimed(uncore))
 585                 ret |= fpga_check_for_unclaimed_mmio(uncore);
 586
 587         if (intel_uncore_has_dbg_unclaimed(uncore))
 588                 ret |= vlv_check_for_unclaimed_mmio(uncore);
 589
 590         if (intel_uncore_has_fifo(uncore))
 591                 ret |= gen6_check_for_fifo_debug(uncore);
 592
 593         return ret;
 594 }
 595
 596 static void forcewake_early_sanitize(struct intel_uncore *uncore,
 597                                      unsigned int restore_forcewake)
 598 {
 599         GEM_BUG_ON(!intel_uncore_has_forcewake(uncore));
 600
 601         /* WaDisableShadowRegForCpd:chv */
 602         if (IS_CHERRYVIEW(uncore->i915)) {
 603                 __raw_uncore_write32(uncore, GTFIFOCTL,
 604                                      __raw_uncore_read32(uncore, GTFIFOCTL) |
 605                                      GT_FIFO_CTL_BLOCK_ALL_POLICY_STALL |
 606                                      GT_FIFO_CTL_RC6_POLICY_STALL);
 607         }
 608
 609         iosf_mbi_punit_acquire();
 610         intel_uncore_forcewake_reset(uncore);
 611         if (restore_forcewake) {
 612                 spin_lock_irq(&uncore->lock);
 613                 fw_domains_get(uncore, restore_forcewake);
 614
 615                 if (intel_uncore_has_fifo(uncore))
 616                         uncore->fifo_count = fifo_free_entries(uncore);
 617                 spin_unlock_irq(&uncore->lock);
 618         }
 619         iosf_mbi_punit_release();
 620 }
 621
 622 void intel_uncore_suspend(struct intel_uncore *uncore)
 623 {
 624         if (!intel_uncore_has_forcewake(uncore))
 625                 return;
 626
 627         iosf_mbi_punit_acquire();
 628         iosf_mbi_unregister_pmic_bus_access_notifier_unlocked(
 629                 &uncore->pmic_bus_access_nb);
 630         uncore->fw_domains_saved = intel_uncore_forcewake_reset(uncore);
 631         iosf_mbi_punit_release();
 632 }
 633
 634 void intel_uncore_resume_early(struct intel_uncore *uncore)
 635 {
 636         unsigned int restore_forcewake;
 637
 638         if (intel_uncore_unclaimed_mmio(uncore))
 639                 drm_dbg(&uncore->i915->drm, "unclaimed mmio detected on resume, clearing\n");
 640
 641         if (!intel_uncore_has_forcewake(uncore))
 642                 return;
 643
 644         restore_forcewake = fetch_and_zero(&uncore->fw_domains_saved);
 645         forcewake_early_sanitize(uncore, restore_forcewake);
 646
 647         iosf_mbi_register_pmic_bus_access_notifier(&uncore->pmic_bus_access_nb);
 648 }
 649
 650 void intel_uncore_runtime_resume(struct intel_uncore *uncore)
 651 {
 652         if (!intel_uncore_has_forcewake(uncore))
 653                 return;
 654
 655         iosf_mbi_register_pmic_bus_access_notifier(&uncore->pmic_bus_access_nb);
 656 }
 657
 658 static void __intel_uncore_forcewake_get(struct intel_uncore *uncore,
 659                                          enum forcewake_domains fw_domains)
 660 {
 661         struct intel_uncore_forcewake_domain *domain;
 662         unsigned int tmp;
 663
 664         fw_domains &= uncore->fw_domains;
 665
 666         for_each_fw_domain_masked(domain, fw_domains, uncore, tmp) {
 667                 if (domain->wake_count++) {
 668                         fw_domains &= ~domain->mask;
 669                         domain->active = true;
 670                 }
 671         }
 672
 673         if (fw_domains)
 674                 fw_domains_get(uncore, fw_domains);
 675 }
 676
 677 /**
 678  * intel_uncore_forcewake_get - grab forcewake domain references
 679  * @uncore: the intel_uncore structure
 680  * @fw_domains: forcewake domains to get reference on
 681  *
 682  * This function can be used get GT's forcewake domain references.
 683  * Normal register access will handle the forcewake domains automatically.
 684  * However if some sequence requires the GT to not power down a particular
 685  * forcewake domains this function should be called at the beginning of the
 686  * sequence. And subsequently the reference should be dropped by symmetric
 687  * call to intel_unforce_forcewake_put(). Usually caller wants all the domains
 688  * to be kept awake so the @fw_domains would be then FORCEWAKE_ALL.
 689  */
 690 void intel_uncore_forcewake_get(struct intel_uncore *uncore,
 691                                 enum forcewake_domains fw_domains)
 692 {
 693         unsigned long irqflags;
 694
 695         if (!uncore->fw_get_funcs)
 696                 return;
 697
 698         assert_rpm_wakelock_held(uncore->rpm);
 699
 700         spin_lock_irqsave(&uncore->lock, irqflags);
 701         __intel_uncore_forcewake_get(uncore, fw_domains);
 702         spin_unlock_irqrestore(&uncore->lock, irqflags);
 703 }
 704
 705 /**
 706  * intel_uncore_forcewake_user_get - claim forcewake on behalf of userspace
 707  * @uncore: the intel_uncore structure
 708  *
 709  * This function is a wrapper around intel_uncore_forcewake_get() to acquire
 710  * the GT powerwell and in the process disable our debugging for the
 711  * duration of userspace's bypass.
 712  */
 713 void intel_uncore_forcewake_user_get(struct intel_uncore *uncore)
 714 {
 715         spin_lock_irq(&uncore->lock);
 716         if (!uncore->user_forcewake_count++) {
 717                 intel_uncore_forcewake_get__locked(uncore, FORCEWAKE_ALL);
 718                 mmio_debug_suspend(uncore);
 719         }
 720         spin_unlock_irq(&uncore->lock);
 721 }
 722
 723 /**
 724  * intel_uncore_forcewake_user_put - release forcewake on behalf of userspace
 725  * @uncore: the intel_uncore structure
 726  *
 727  * This function complements intel_uncore_forcewake_user_get() and releases
 728  * the GT powerwell taken on behalf of the userspace bypass.
 729  */
 730 void intel_uncore_forcewake_user_put(struct intel_uncore *uncore)
 731 {
 732         spin_lock_irq(&uncore->lock);
 733         if (!--uncore->user_forcewake_count) {
 734                 mmio_debug_resume(uncore);
 735                 intel_uncore_forcewake_put__locked(uncore, FORCEWAKE_ALL);
 736         }
 737         spin_unlock_irq(&uncore->lock);
 738 }
 739
 740 /**
 741  * intel_uncore_forcewake_get__locked - grab forcewake domain references
 742  * @uncore: the intel_uncore structure
 743  * @fw_domains: forcewake domains to get reference on
 744  *
 745  * See intel_uncore_forcewake_get(). This variant places the onus
 746  * on the caller to explicitly handle the dev_priv->uncore.lock spinlock.
 747  */
 748 void intel_uncore_forcewake_get__locked(struct intel_uncore *uncore,
 749                                         enum forcewake_domains fw_domains)
 750 {
 751         lockdep_assert_held(&uncore->lock);
 752
 753         if (!uncore->fw_get_funcs)
 754                 return;
 755
 756         __intel_uncore_forcewake_get(uncore, fw_domains);
 757 }
 758
 759 static void __intel_uncore_forcewake_put(struct intel_uncore *uncore,
 760                                          enum forcewake_domains fw_domains,
 761                                          bool delayed)
 762 {
 763         struct intel_uncore_forcewake_domain *domain;
 764         unsigned int tmp;
 765
 766         fw_domains &= uncore->fw_domains;
 767
 768         for_each_fw_domain_masked(domain, fw_domains, uncore, tmp) {
 769                 GEM_BUG_ON(!domain->wake_count);
 770
 771                 if (--domain->wake_count) {
 772                         domain->active = true;
 773                         continue;
 774                 }
 775
 776                 if (delayed &&
 777                     !(domain->uncore->fw_domains_timer & domain->mask))
 778                         fw_domain_arm_timer(domain);
 779                 else
 780                         fw_domains_put(uncore, domain->mask);
 781         }
 782 }
 783
 784 /**
 785  * intel_uncore_forcewake_put - release a forcewake domain reference
 786  * @uncore: the intel_uncore structure
 787  * @fw_domains: forcewake domains to put references
 788  *
 789  * This function drops the device-level forcewakes for specified
 790  * domains obtained by intel_uncore_forcewake_get().
 791  */
 792 void intel_uncore_forcewake_put(struct intel_uncore *uncore,
 793                                 enum forcewake_domains fw_domains)
 794 {
 795         unsigned long irqflags;
 796
 797         if (!uncore->fw_get_funcs)
 798                 return;
 799
 800         spin_lock_irqsave(&uncore->lock, irqflags);
 801         __intel_uncore_forcewake_put(uncore, fw_domains, false);
 802         spin_unlock_irqrestore(&uncore->lock, irqflags);
 803 }
 804
 805 void intel_uncore_forcewake_put_delayed(struct intel_uncore *uncore,
 806                                         enum forcewake_domains fw_domains)
 807 {
 808         unsigned long irqflags;
 809
 810         if (!uncore->fw_get_funcs)
 811                 return;
 812
 813         spin_lock_irqsave(&uncore->lock, irqflags);
 814         __intel_uncore_forcewake_put(uncore, fw_domains, true);
 815         spin_unlock_irqrestore(&uncore->lock, irqflags);
 816 }
 817
 818 /**
 819  * intel_uncore_forcewake_flush - flush the delayed release
 820  * @uncore: the intel_uncore structure
 821  * @fw_domains: forcewake domains to flush
 822  */
 823 void intel_uncore_forcewake_flush(struct intel_uncore *uncore,
 824                                   enum forcewake_domains fw_domains)
 825 {
 826         struct intel_uncore_forcewake_domain *domain;
 827         unsigned int tmp;
 828
 829         if (!uncore->fw_get_funcs)
 830                 return;
 831
 832         fw_domains &= uncore->fw_domains;
 833         for_each_fw_domain_masked(domain, fw_domains, uncore, tmp) {
 834                 WRITE_ONCE(domain->active, false);
 835                 if (hrtimer_cancel(&domain->timer))
 836                         intel_uncore_fw_release_timer(&domain->timer);
 837         }
 838 }
 839
 840 /**
 841  * intel_uncore_forcewake_put__locked - release forcewake domain references
 842  * @uncore: the intel_uncore structure
 843  * @fw_domains: forcewake domains to put references
 844  *
 845  * See intel_uncore_forcewake_put(). This variant places the onus
 846  * on the caller to explicitly handle the dev_priv->uncore.lock spinlock.
 847  */
 848 void intel_uncore_forcewake_put__locked(struct intel_uncore *uncore,
 849                                         enum forcewake_domains fw_domains)
 850 {
 851         lockdep_assert_held(&uncore->lock);
 852
 853         if (!uncore->fw_get_funcs)
 854                 return;
 855
 856         __intel_uncore_forcewake_put(uncore, fw_domains, false);
 857 }
 858
 859 void assert_forcewakes_inactive(struct intel_uncore *uncore)
 860 {
 861         if (!uncore->fw_get_funcs)
 862                 return;
 863
 864         drm_WARN(&uncore->i915->drm, uncore->fw_domains_active,
 865                  "Expected all fw_domains to be inactive, but %08x are still on\n",
 866                  uncore->fw_domains_active);
 867 }
 868
 869 void assert_forcewakes_active(struct intel_uncore *uncore,
 870                               enum forcewake_domains fw_domains)
 871 {
 872         struct intel_uncore_forcewake_domain *domain;
 873         unsigned int tmp;
 874
 875         if (!IS_ENABLED(CONFIG_DRM_I915_DEBUG_RUNTIME_PM))
 876                 return;
 877
 878         if (!uncore->fw_get_funcs)
 879                 return;
 880
 881         spin_lock_irq(&uncore->lock);
 882
 883         assert_rpm_wakelock_held(uncore->rpm);
 884
 885         fw_domains &= uncore->fw_domains;
 886         drm_WARN(&uncore->i915->drm, fw_domains & ~uncore->fw_domains_active,
 887                  "Expected %08x fw_domains to be active, but %08x are off\n",
 888                  fw_domains, fw_domains & ~uncore->fw_domains_active);
 889
 890         /*
 891          * Check that the caller has an explicit wakeref and we don't mistake
 892          * it for the auto wakeref.
 893          */
 894         for_each_fw_domain_masked(domain, fw_domains, uncore, tmp) {
 895                 unsigned int actual = READ_ONCE(domain->wake_count);
 896                 unsigned int expect = 1;
 897
 898                 if (uncore->fw_domains_timer & domain->mask)
 899                         expect++; /* pending automatic release */
 900
 901                 if (drm_WARN(&uncore->i915->drm, actual < expect,
 902                              "Expected domain %d to be held awake by caller, count=%d\n",
 903                              domain->id, actual))
 904                         break;
 905         }
 906
 907         spin_unlock_irq(&uncore->lock);
 908 }
 909
 910 /*
 911  * We give fast paths for the really cool registers.  The second range includes
 912  * media domains (and the GSC starting from Xe_LPM+)
 913  */
 914 #define NEEDS_FORCE_WAKE(reg) ({ \
 915         u32 __reg = (reg); \
 916         __reg < 0x40000 || __reg >= 0x116000; \
 917 })
 918
 919 static int fw_range_cmp(u32 offset, const struct intel_forcewake_range *entry)
 920 {
 921         if (offset < entry->start)
 922                 return -1;
 923         else if (offset > entry->end)
 924                 return 1;
 925         else
 926                 return 0;
 927 }
 928
 929 /* Copied and "macroized" from lib/bsearch.c */
 930 #define BSEARCH(key, base, num, cmp) ({                                 \
 931         unsigned int start__ = 0, end__ = (num);                        \
 932         typeof(base) result__ = NULL;                                   \
 933         while (start__ < end__) {                                       \
 934                 unsigned int mid__ = start__ + (end__ - start__) / 2;   \
 935                 int ret__ = (cmp)((key), (base) + mid__);               \
 936                 if (ret__ < 0) {                                        \
 937                         end__ = mid__;                                  \
 938                 } else if (ret__ > 0) {                                 \
 939                         start__ = mid__ + 1;                            \
 940                 } else {                                                \
 941                         result__ = (base) + mid__;                      \
 942                         break;                                          \
 943                 }                                                       \
 944         }                                                               \
 945         result__;                                                       \
 946 })
 947
 948 static enum forcewake_domains
 949 find_fw_domain(struct intel_uncore *uncore, u32 offset)
 950 {
 951         const struct intel_forcewake_range *entry;
 952
 953         if (IS_GSI_REG(offset))
 954                 offset += uncore->gsi_offset;
 955
 956         entry = BSEARCH(offset,
 957                         uncore->fw_domains_table,
 958                         uncore->fw_domains_table_entries,
 959                         fw_range_cmp);
 960
 961         if (!entry)
 962                 return 0;
 963
 964         /*
 965          * The list of FW domains depends on the SKU in gen11+ so we
 966          * can't determine it statically. We use FORCEWAKE_ALL and
 967          * translate it here to the list of available domains.
 968          */
 969         if (entry->domains == FORCEWAKE_ALL)
 970                 return uncore->fw_domains;
 971
 972         drm_WARN(&uncore->i915->drm, entry->domains & ~uncore->fw_domains,
 973                  "Uninitialized forcewake domain(s) 0x%x accessed at 0x%x\n",
 974                  entry->domains & ~uncore->fw_domains, offset);
 975
 976         return entry->domains;
 977 }
 978
 979 /*
 980  * Shadowed register tables describe special register ranges that i915 is
 981  * allowed to write to without acquiring forcewake.  If these registers' power
 982  * wells are down, the hardware will save values written by i915 to a shadow
 983  * copy and automatically transfer them into the real register the next time
 984  * the power well is woken up.  Shadowing only applies to writes; forcewake
 985  * must still be acquired when reading from registers in these ranges.
 986  *
 987  * The documentation for shadowed registers is somewhat spotty on older
 988  * platforms.  However missing registers from these lists is non-fatal; it just
 989  * means we'll wake up the hardware for some register accesses where we didn't
 990  * really need to.
 991  *
 992  * The ranges listed in these tables must be sorted by offset.
 993  *
 994  * When adding new tables here, please also add them to
 995  * intel_shadow_table_check() in selftests/intel_uncore.c so that they will be
 996  * scanned for obvious mistakes or typos by the selftests.
 997  */
 998
 999 static const struct i915_range gen8_shadowed_regs[] = {
1000         { .start =  0x2030, .end =  0x2030 },
1001         { .start =  0xA008, .end =  0xA00C },
1002         { .start = 0x12030, .end = 0x12030 },
1003         { .start = 0x1a030, .end = 0x1a030 },
1004         { .start = 0x22030, .end = 0x22030 },
1005 };
1006
1007 static const struct i915_range gen11_shadowed_regs[] = {
1008         { .start =   0x2030, .end =   0x2030 },
1009         { .start =   0x2550, .end =   0x2550 },
1010         { .start =   0xA008, .end =   0xA00C },
1011         { .start =  0x22030, .end =  0x22030 },
1012         { .start =  0x22230, .end =  0x22230 },
1013         { .start =  0x22510, .end =  0x22550 },
1014         { .start = 0x1C0030, .end = 0x1C0030 },
1015         { .start = 0x1C0230, .end = 0x1C0230 },
1016         { .start = 0x1C0510, .end = 0x1C0550 },
1017         { .start = 0x1C4030, .end = 0x1C4030 },
1018         { .start = 0x1C4230, .end = 0x1C4230 },
1019         { .start = 0x1C4510, .end = 0x1C4550 },
1020         { .start = 0x1C8030, .end = 0x1C8030 },
1021         { .start = 0x1C8230, .end = 0x1C8230 },
1022         { .start = 0x1C8510, .end = 0x1C8550 },
1023         { .start = 0x1D0030, .end = 0x1D0030 },
1024         { .start = 0x1D0230, .end = 0x1D0230 },
1025         { .start = 0x1D0510, .end = 0x1D0550 },
1026         { .start = 0x1D4030, .end = 0x1D4030 },
1027         { .start = 0x1D4230, .end = 0x1D4230 },
1028         { .start = 0x1D4510, .end = 0x1D4550 },
1029         { .start = 0x1D8030, .end = 0x1D8030 },
1030         { .start = 0x1D8230, .end = 0x1D8230 },
1031         { .start = 0x1D8510, .end = 0x1D8550 },
1032 };
1033
1034 static const struct i915_range gen12_shadowed_regs[] = {
1035         { .start =   0x2030, .end =   0x2030 },
1036         { .start =   0x2510, .end =   0x2550 },
1037         { .start =   0xA008, .end =   0xA00C },
1038         { .start =   0xA188, .end =   0xA188 },
1039         { .start =   0xA278, .end =   0xA278 },
1040         { .start =   0xA540, .end =   0xA56C },
1041         { .start =   0xC4C8, .end =   0xC4C8 },
1042         { .start =   0xC4D4, .end =   0xC4D4 },
1043         { .start =   0xC600, .end =   0xC600 },
1044         { .start =  0x22030, .end =  0x22030 },
1045         { .start =  0x22510, .end =  0x22550 },
1046         { .start = 0x1C0030, .end = 0x1C0030 },
1047         { .start = 0x1C0510, .end = 0x1C0550 },
1048         { .start = 0x1C4030, .end = 0x1C4030 },
1049         { .start = 0x1C4510, .end = 0x1C4550 },
1050         { .start = 0x1C8030, .end = 0x1C8030 },
1051         { .start = 0x1C8510, .end = 0x1C8550 },
1052         { .start = 0x1D0030, .end = 0x1D0030 },
1053         { .start = 0x1D0510, .end = 0x1D0550 },
1054         { .start = 0x1D4030, .end = 0x1D4030 },
1055         { .start = 0x1D4510, .end = 0x1D4550 },
1056         { .start = 0x1D8030, .end = 0x1D8030 },
1057         { .start = 0x1D8510, .end = 0x1D8550 },
1058
1059         /*
1060          * The rest of these ranges are specific to Xe_HP and beyond, but
1061          * are reserved/unused ranges on earlier gen12 platforms, so they can
1062          * be safely added to the gen12 table.
1063          */
1064         { .start = 0x1E0030, .end = 0x1E0030 },
1065         { .start = 0x1E0510, .end = 0x1E0550 },
1066         { .start = 0x1E4030, .end = 0x1E4030 },
1067         { .start = 0x1E4510, .end = 0x1E4550 },
1068         { .start = 0x1E8030, .end = 0x1E8030 },
1069         { .start = 0x1E8510, .end = 0x1E8550 },
1070         { .start = 0x1F0030, .end = 0x1F0030 },
1071         { .start = 0x1F0510, .end = 0x1F0550 },
1072         { .start = 0x1F4030, .end = 0x1F4030 },
1073         { .start = 0x1F4510, .end = 0x1F4550 },
1074         { .start = 0x1F8030, .end = 0x1F8030 },
1075         { .start = 0x1F8510, .end = 0x1F8550 },
1076 };
1077
1078 static const struct i915_range dg2_shadowed_regs[] = {
1079         { .start =   0x2030, .end =   0x2030 },
1080         { .start =   0x2510, .end =   0x2550 },
1081         { .start =   0xA008, .end =   0xA00C },
1082         { .start =   0xA188, .end =   0xA188 },
1083         { .start =   0xA278, .end =   0xA278 },
1084         { .start =   0xA540, .end =   0xA56C },
1085         { .start =   0xC4C8, .end =   0xC4C8 },
1086         { .start =   0xC4E0, .end =   0xC4E0 },
1087         { .start =   0xC600, .end =   0xC600 },
1088         { .start =   0xC658, .end =   0xC658 },
1089         { .start =  0x22030, .end =  0x22030 },
1090         { .start =  0x22510, .end =  0x22550 },
1091         { .start = 0x1C0030, .end = 0x1C0030 },
1092         { .start = 0x1C0510, .end = 0x1C0550 },
1093         { .start = 0x1C4030, .end = 0x1C4030 },
1094         { .start = 0x1C4510, .end = 0x1C4550 },
1095         { .start = 0x1C8030, .end = 0x1C8030 },
1096         { .start = 0x1C8510, .end = 0x1C8550 },
1097         { .start = 0x1D0030, .end = 0x1D0030 },
1098         { .start = 0x1D0510, .end = 0x1D0550 },
1099         { .start = 0x1D4030, .end = 0x1D4030 },
1100         { .start = 0x1D4510, .end = 0x1D4550 },
1101         { .start = 0x1D8030, .end = 0x1D8030 },
1102         { .start = 0x1D8510, .end = 0x1D8550 },
1103         { .start = 0x1E0030, .end = 0x1E0030 },
1104         { .start = 0x1E0510, .end = 0x1E0550 },
1105         { .start = 0x1E4030, .end = 0x1E4030 },
1106         { .start = 0x1E4510, .end = 0x1E4550 },
1107         { .start = 0x1E8030, .end = 0x1E8030 },
1108         { .start = 0x1E8510, .end = 0x1E8550 },
1109         { .start = 0x1F0030, .end = 0x1F0030 },
1110         { .start = 0x1F0510, .end = 0x1F0550 },
1111         { .start = 0x1F4030, .end = 0x1F4030 },
1112         { .start = 0x1F4510, .end = 0x1F4550 },
1113         { .start = 0x1F8030, .end = 0x1F8030 },
1114         { .start = 0x1F8510, .end = 0x1F8550 },
1115 };
1116
1117 static const struct i915_range mtl_shadowed_regs[] = {
1118         { .start =   0x2030, .end =   0x2030 },
1119         { .start =   0x2510, .end =   0x2550 },
1120         { .start =   0xA008, .end =   0xA00C },
1121         { .start =   0xA188, .end =   0xA188 },
1122         { .start =   0xA278, .end =   0xA278 },
1123         { .start =   0xA540, .end =   0xA56C },
1124         { .start =   0xC050, .end =   0xC050 },
1125         { .start =   0xC340, .end =   0xC340 },
1126         { .start =   0xC4C8, .end =   0xC4C8 },
1127         { .start =   0xC4E0, .end =   0xC4E0 },
1128         { .start =   0xC600, .end =   0xC600 },
1129         { .start =   0xC658, .end =   0xC658 },
1130         { .start =   0xCFD4, .end =   0xCFDC },
1131         { .start =  0x22030, .end =  0x22030 },
1132         { .start =  0x22510, .end =  0x22550 },
1133 };
1134
1135 static const struct i915_range xelpmp_shadowed_regs[] = {
1136         { .start = 0x1C0030, .end = 0x1C0030 },
1137         { .start = 0x1C0510, .end = 0x1C0550 },
1138         { .start = 0x1C8030, .end = 0x1C8030 },
1139         { .start = 0x1C8510, .end = 0x1C8550 },
1140         { .start = 0x1D0030, .end = 0x1D0030 },
1141         { .start = 0x1D0510, .end = 0x1D0550 },
1142         { .start = 0x38A008, .end = 0x38A00C },
1143         { .start = 0x38A188, .end = 0x38A188 },
1144         { .start = 0x38A278, .end = 0x38A278 },
1145         { .start = 0x38A540, .end = 0x38A56C },
1146         { .start = 0x38A618, .end = 0x38A618 },
1147         { .start = 0x38C050, .end = 0x38C050 },
1148         { .start = 0x38C340, .end = 0x38C340 },
1149         { .start = 0x38C4C8, .end = 0x38C4C8 },
1150         { .start = 0x38C4E0, .end = 0x38C4E4 },
1151         { .start = 0x38C600, .end = 0x38C600 },
1152         { .start = 0x38C658, .end = 0x38C658 },
1153         { .start = 0x38CFD4, .end = 0x38CFDC },
1154 };
1155
1156 static int mmio_range_cmp(u32 key, const struct i915_range *range)
1157 {
1158         if (key < range->start)
1159                 return -1;
1160         else if (key > range->end)
1161                 return 1;
1162         else
1163                 return 0;
1164 }
1165
1166 static bool is_shadowed(struct intel_uncore *uncore, u32 offset)
1167 {
1168         if (drm_WARN_ON(&uncore->i915->drm, !uncore->shadowed_reg_table))
1169                 return false;
1170
1171         if (IS_GSI_REG(offset))
1172                 offset += uncore->gsi_offset;
1173
1174         return BSEARCH(offset,
1175                        uncore->shadowed_reg_table,
1176                        uncore->shadowed_reg_table_entries,
1177                        mmio_range_cmp);
1178 }
1179
1180 static enum forcewake_domains
1181 gen6_reg_write_fw_domains(struct intel_uncore *uncore, i915_reg_t reg)
1182 {
1183         return FORCEWAKE_RENDER;
1184 }
1185
1186 #define __fwtable_reg_read_fw_domains(uncore, offset) \
1187 ({ \
1188         enum forcewake_domains __fwd = 0; \
1189         if (NEEDS_FORCE_WAKE((offset))) \
1190                 __fwd = find_fw_domain(uncore, offset); \
1191         __fwd; \
1192 })
1193
1194 #define __fwtable_reg_write_fw_domains(uncore, offset) \
1195 ({ \
1196         enum forcewake_domains __fwd = 0; \
1197         const u32 __offset = (offset); \
1198         if (NEEDS_FORCE_WAKE((__offset)) && !is_shadowed(uncore, __offset)) \
1199                 __fwd = find_fw_domain(uncore, __offset); \
1200         __fwd; \
1201 })
1202
1203 #define GEN_FW_RANGE(s, e, d) \
1204         { .start = (s), .end = (e), .domains = (d) }
1205
1206 /*
1207  * All platforms' forcewake tables below must be sorted by offset ranges.
1208  * Furthermore, new forcewake tables added should be "watertight" and have
1209  * no gaps between ranges.
1210  *
1211  * When there are multiple consecutive ranges listed in the bspec with
1212  * the same forcewake domain, it is customary to combine them into a single
1213  * row in the tables below to keep the tables small and lookups fast.
1214  * Likewise, reserved/unused ranges may be combined with the preceding and/or
1215  * following ranges since the driver will never be making MMIO accesses in
1216  * those ranges.
1217  *
1218  * For example, if the bspec were to list:
1219  *
1220  *    ...
1221  *    0x1000 - 0x1fff:  GT
1222  *    0x2000 - 0x2cff:  GT
1223  *    0x2d00 - 0x2fff:  unused/reserved
1224  *    0x3000 - 0xffff:  GT
1225  *    ...
1226  *
1227  * these could all be represented by a single line in the code:
1228  *
1229  *   GEN_FW_RANGE(0x1000, 0xffff, FORCEWAKE_GT)
1230  *
1231  * When adding new forcewake tables here, please also add them to
1232  * intel_uncore_mock_selftests in selftests/intel_uncore.c so that they will be
1233  * scanned for obvious mistakes or typos by the selftests.
1234  */
1235
1236 static const struct intel_forcewake_range __gen6_fw_ranges[] = {
1237         GEN_FW_RANGE(0x0, 0x3ffff, FORCEWAKE_RENDER),
1238 };
1239
1240 static const struct intel_forcewake_range __vlv_fw_ranges[] = {
1241         GEN_FW_RANGE(0x2000, 0x3fff, FORCEWAKE_RENDER),
1242         GEN_FW_RANGE(0x5000, 0x7fff, FORCEWAKE_RENDER),
1243         GEN_FW_RANGE(0xb000, 0x11fff, FORCEWAKE_RENDER),
1244         GEN_FW_RANGE(0x12000, 0x13fff, FORCEWAKE_MEDIA),
1245         GEN_FW_RANGE(0x22000, 0x23fff, FORCEWAKE_MEDIA),
1246         GEN_FW_RANGE(0x2e000, 0x2ffff, FORCEWAKE_RENDER),
1247         GEN_FW_RANGE(0x30000, 0x3ffff, FORCEWAKE_MEDIA),
1248 };
1249
1250 static const struct intel_forcewake_range __chv_fw_ranges[] = {
1251         GEN_FW_RANGE(0x2000, 0x3fff, FORCEWAKE_RENDER),
1252         GEN_FW_RANGE(0x4000, 0x4fff, FORCEWAKE_RENDER | FORCEWAKE_MEDIA),
1253         GEN_FW_RANGE(0x5200, 0x7fff, FORCEWAKE_RENDER),
1254         GEN_FW_RANGE(0x8000, 0x82ff, FORCEWAKE_RENDER | FORCEWAKE_MEDIA),
1255         GEN_FW_RANGE(0x8300, 0x84ff, FORCEWAKE_RENDER),
1256         GEN_FW_RANGE(0x8500, 0x85ff, FORCEWAKE_RENDER | FORCEWAKE_MEDIA),
1257         GEN_FW_RANGE(0x8800, 0x88ff, FORCEWAKE_MEDIA),
1258         GEN_FW_RANGE(0x9000, 0xafff, FORCEWAKE_RENDER | FORCEWAKE_MEDIA),
1259         GEN_FW_RANGE(0xb000, 0xb47f, FORCEWAKE_RENDER),
1260         GEN_FW_RANGE(0xd000, 0xd7ff, FORCEWAKE_MEDIA),
1261         GEN_FW_RANGE(0xe000, 0xe7ff, FORCEWAKE_RENDER),
1262         GEN_FW_RANGE(0xf000, 0xffff, FORCEWAKE_RENDER | FORCEWAKE_MEDIA),
1263         GEN_FW_RANGE(0x12000, 0x13fff, FORCEWAKE_MEDIA),
1264         GEN_FW_RANGE(0x1a000, 0x1bfff, FORCEWAKE_MEDIA),
1265         GEN_FW_RANGE(0x1e800, 0x1e9ff, FORCEWAKE_MEDIA),
1266         GEN_FW_RANGE(0x30000, 0x37fff, FORCEWAKE_MEDIA),
1267 };
1268
1269 static const struct intel_forcewake_range __gen9_fw_ranges[] = {
1270         GEN_FW_RANGE(0x0, 0xaff, FORCEWAKE_GT),
1271         GEN_FW_RANGE(0xb00, 0x1fff, 0), /* uncore range */
1272         GEN_FW_RANGE(0x2000, 0x26ff, FORCEWAKE_RENDER),
1273         GEN_FW_RANGE(0x2700, 0x2fff, FORCEWAKE_GT),
1274         GEN_FW_RANGE(0x3000, 0x3fff, FORCEWAKE_RENDER),
1275         GEN_FW_RANGE(0x4000, 0x51ff, FORCEWAKE_GT),
1276         GEN_FW_RANGE(0x5200, 0x7fff, FORCEWAKE_RENDER),
1277         GEN_FW_RANGE(0x8000, 0x812f, FORCEWAKE_GT),
1278         GEN_FW_RANGE(0x8130, 0x813f, FORCEWAKE_MEDIA),
1279         GEN_FW_RANGE(0x8140, 0x815f, FORCEWAKE_RENDER),
1280         GEN_FW_RANGE(0x8160, 0x82ff, FORCEWAKE_GT),
1281         GEN_FW_RANGE(0x8300, 0x84ff, FORCEWAKE_RENDER),
1282         GEN_FW_RANGE(0x8500, 0x87ff, FORCEWAKE_GT),
1283         GEN_FW_RANGE(0x8800, 0x89ff, FORCEWAKE_MEDIA),
1284         GEN_FW_RANGE(0x8a00, 0x8bff, FORCEWAKE_GT),
1285         GEN_FW_RANGE(0x8c00, 0x8cff, FORCEWAKE_RENDER),
1286         GEN_FW_RANGE(0x8d00, 0x93ff, FORCEWAKE_GT),
1287         GEN_FW_RANGE(0x9400, 0x97ff, FORCEWAKE_RENDER | FORCEWAKE_MEDIA),
1288         GEN_FW_RANGE(0x9800, 0xafff, FORCEWAKE_GT),
1289         GEN_FW_RANGE(0xb000, 0xb47f, FORCEWAKE_RENDER),
1290         GEN_FW_RANGE(0xb480, 0xcfff, FORCEWAKE_GT),
1291         GEN_FW_RANGE(0xd000, 0xd7ff, FORCEWAKE_MEDIA),
1292         GEN_FW_RANGE(0xd800, 0xdfff, FORCEWAKE_GT),
1293         GEN_FW_RANGE(0xe000, 0xe8ff, FORCEWAKE_RENDER),
1294         GEN_FW_RANGE(0xe900, 0x11fff, FORCEWAKE_GT),
1295         GEN_FW_RANGE(0x12000, 0x13fff, FORCEWAKE_MEDIA),
1296         GEN_FW_RANGE(0x14000, 0x19fff, FORCEWAKE_GT),
1297         GEN_FW_RANGE(0x1a000, 0x1e9ff, FORCEWAKE_MEDIA),
1298         GEN_FW_RANGE(0x1ea00, 0x243ff, FORCEWAKE_GT),
1299         GEN_FW_RANGE(0x24400, 0x247ff, FORCEWAKE_RENDER),
1300         GEN_FW_RANGE(0x24800, 0x2ffff, FORCEWAKE_GT),
1301         GEN_FW_RANGE(0x30000, 0x3ffff, FORCEWAKE_MEDIA),
1302 };
1303
1304 static const struct intel_forcewake_range __gen11_fw_ranges[] = {
1305         GEN_FW_RANGE(0x0, 0x1fff, 0), /* uncore range */
1306         GEN_FW_RANGE(0x2000, 0x26ff, FORCEWAKE_RENDER),
1307         GEN_FW_RANGE(0x2700, 0x2fff, FORCEWAKE_GT),
1308         GEN_FW_RANGE(0x3000, 0x3fff, FORCEWAKE_RENDER),
1309         GEN_FW_RANGE(0x4000, 0x51ff, FORCEWAKE_GT),
1310         GEN_FW_RANGE(0x5200, 0x7fff, FORCEWAKE_RENDER),
1311         GEN_FW_RANGE(0x8000, 0x813f, FORCEWAKE_GT),
1312         GEN_FW_RANGE(0x8140, 0x815f, FORCEWAKE_RENDER),
1313         GEN_FW_RANGE(0x8160, 0x82ff, FORCEWAKE_GT),
1314         GEN_FW_RANGE(0x8300, 0x84ff, FORCEWAKE_RENDER),
1315         GEN_FW_RANGE(0x8500, 0x87ff, FORCEWAKE_GT),
1316         GEN_FW_RANGE(0x8800, 0x8bff, 0),
1317         GEN_FW_RANGE(0x8c00, 0x8cff, FORCEWAKE_RENDER),
1318         GEN_FW_RANGE(0x8d00, 0x94cf, FORCEWAKE_GT),
1319         GEN_FW_RANGE(0x94d0, 0x955f, FORCEWAKE_RENDER),
1320         GEN_FW_RANGE(0x9560, 0x95ff, 0),
1321         GEN_FW_RANGE(0x9600, 0xafff, FORCEWAKE_GT),
1322         GEN_FW_RANGE(0xb000, 0xb47f, FORCEWAKE_RENDER),
1323         GEN_FW_RANGE(0xb480, 0xdeff, FORCEWAKE_GT),
1324         GEN_FW_RANGE(0xdf00, 0xe8ff, FORCEWAKE_RENDER),
1325         GEN_FW_RANGE(0xe900, 0x16dff, FORCEWAKE_GT),
1326         GEN_FW_RANGE(0x16e00, 0x19fff, FORCEWAKE_RENDER),
1327         GEN_FW_RANGE(0x1a000, 0x23fff, FORCEWAKE_GT),
1328         GEN_FW_RANGE(0x24000, 0x2407f, 0),
1329         GEN_FW_RANGE(0x24080, 0x2417f, FORCEWAKE_GT),
1330         GEN_FW_RANGE(0x24180, 0x242ff, FORCEWAKE_RENDER),
1331         GEN_FW_RANGE(0x24300, 0x243ff, FORCEWAKE_GT),
1332         GEN_FW_RANGE(0x24400, 0x24fff, FORCEWAKE_RENDER),
1333         GEN_FW_RANGE(0x25000, 0x3ffff, FORCEWAKE_GT),
1334         GEN_FW_RANGE(0x40000, 0x1bffff, 0),
1335         GEN_FW_RANGE(0x1c0000, 0x1c3fff, FORCEWAKE_MEDIA_VDBOX0),
1336         GEN_FW_RANGE(0x1c4000, 0x1c7fff, 0),
1337         GEN_FW_RANGE(0x1c8000, 0x1cffff, FORCEWAKE_MEDIA_VEBOX0),
1338         GEN_FW_RANGE(0x1d0000, 0x1d3fff, FORCEWAKE_MEDIA_VDBOX2),
1339         GEN_FW_RANGE(0x1d4000, 0x1dbfff, 0)
1340 };
1341
1342 static const struct intel_forcewake_range __gen12_fw_ranges[] = {
1343         GEN_FW_RANGE(0x0, 0x1fff, 0), /*
1344                 0x0   -  0xaff: reserved
1345                 0xb00 - 0x1fff: always on */
1346         GEN_FW_RANGE(0x2000, 0x26ff, FORCEWAKE_RENDER),
1347         GEN_FW_RANGE(0x2700, 0x27ff, FORCEWAKE_GT),
1348         GEN_FW_RANGE(0x2800, 0x2aff, FORCEWAKE_RENDER),
1349         GEN_FW_RANGE(0x2b00, 0x2fff, FORCEWAKE_GT),
1350         GEN_FW_RANGE(0x3000, 0x3fff, FORCEWAKE_RENDER),
1351         GEN_FW_RANGE(0x4000, 0x51ff, FORCEWAKE_GT), /*
1352                 0x4000 - 0x48ff: gt
1353                 0x4900 - 0x51ff: reserved */
1354         GEN_FW_RANGE(0x5200, 0x7fff, FORCEWAKE_RENDER), /*
1355                 0x5200 - 0x53ff: render
1356                 0x5400 - 0x54ff: reserved
1357                 0x5500 - 0x7fff: render */
1358         GEN_FW_RANGE(0x8000, 0x813f, FORCEWAKE_GT),
1359         GEN_FW_RANGE(0x8140, 0x815f, FORCEWAKE_RENDER),
1360         GEN_FW_RANGE(0x8160, 0x81ff, 0), /*
1361                 0x8160 - 0x817f: reserved
1362                 0x8180 - 0x81ff: always on */
1363         GEN_FW_RANGE(0x8200, 0x82ff, FORCEWAKE_GT),
1364         GEN_FW_RANGE(0x8300, 0x84ff, FORCEWAKE_RENDER),
1365         GEN_FW_RANGE(0x8500, 0x94cf, FORCEWAKE_GT), /*
1366                 0x8500 - 0x87ff: gt
1367                 0x8800 - 0x8fff: reserved
1368                 0x9000 - 0x947f: gt
1369                 0x9480 - 0x94cf: reserved */
1370         GEN_FW_RANGE(0x94d0, 0x955f, FORCEWAKE_RENDER),
1371         GEN_FW_RANGE(0x9560, 0x97ff, 0), /*
1372                 0x9560 - 0x95ff: always on
1373                 0x9600 - 0x97ff: reserved */
1374         GEN_FW_RANGE(0x9800, 0xafff, FORCEWAKE_GT),
1375         GEN_FW_RANGE(0xb000, 0xb3ff, FORCEWAKE_RENDER),
1376         GEN_FW_RANGE(0xb400, 0xcfff, FORCEWAKE_GT), /*
1377                 0xb400 - 0xbf7f: gt
1378                 0xb480 - 0xbfff: reserved
1379                 0xc000 - 0xcfff: gt */
1380         GEN_FW_RANGE(0xd000, 0xd7ff, 0),
1381         GEN_FW_RANGE(0xd800, 0xd8ff, FORCEWAKE_RENDER),
1382         GEN_FW_RANGE(0xd900, 0xdbff, FORCEWAKE_GT),
1383         GEN_FW_RANGE(0xdc00, 0xefff, FORCEWAKE_RENDER), /*
1384                 0xdc00 - 0xddff: render
1385                 0xde00 - 0xde7f: reserved
1386                 0xde80 - 0xe8ff: render
1387                 0xe900 - 0xefff: reserved */
1388         GEN_FW_RANGE(0xf000, 0x147ff, FORCEWAKE_GT), /*
1389                  0xf000 - 0xffff: gt
1390                 0x10000 - 0x147ff: reserved */
1391         GEN_FW_RANGE(0x14800, 0x1ffff, FORCEWAKE_RENDER), /*
1392                 0x14800 - 0x14fff: render
1393                 0x15000 - 0x16dff: reserved
1394                 0x16e00 - 0x1bfff: render
1395                 0x1c000 - 0x1ffff: reserved */
1396         GEN_FW_RANGE(0x20000, 0x20fff, FORCEWAKE_MEDIA_VDBOX0),
1397         GEN_FW_RANGE(0x21000, 0x21fff, FORCEWAKE_MEDIA_VDBOX2),
1398         GEN_FW_RANGE(0x22000, 0x23fff, FORCEWAKE_GT),
1399         GEN_FW_RANGE(0x24000, 0x2417f, 0), /*
1400                 0x24000 - 0x2407f: always on
1401                 0x24080 - 0x2417f: reserved */
1402         GEN_FW_RANGE(0x24180, 0x249ff, FORCEWAKE_GT), /*
1403                 0x24180 - 0x241ff: gt
1404                 0x24200 - 0x249ff: reserved */
1405         GEN_FW_RANGE(0x24a00, 0x251ff, FORCEWAKE_RENDER), /*
1406                 0x24a00 - 0x24a7f: render
1407                 0x24a80 - 0x251ff: reserved */
1408         GEN_FW_RANGE(0x25200, 0x255ff, FORCEWAKE_GT), /*
1409                 0x25200 - 0x252ff: gt
1410                 0x25300 - 0x255ff: reserved */
1411         GEN_FW_RANGE(0x25600, 0x2567f, FORCEWAKE_MEDIA_VDBOX0),
1412         GEN_FW_RANGE(0x25680, 0x259ff, FORCEWAKE_MEDIA_VDBOX2), /*
1413                 0x25680 - 0x256ff: VD2
1414                 0x25700 - 0x259ff: reserved */
1415         GEN_FW_RANGE(0x25a00, 0x25a7f, FORCEWAKE_MEDIA_VDBOX0),
1416         GEN_FW_RANGE(0x25a80, 0x2ffff, FORCEWAKE_MEDIA_VDBOX2), /*
1417                 0x25a80 - 0x25aff: VD2
1418                 0x25b00 - 0x2ffff: reserved */
1419         GEN_FW_RANGE(0x30000, 0x3ffff, FORCEWAKE_GT),
1420         GEN_FW_RANGE(0x40000, 0x1bffff, 0),
1421         GEN_FW_RANGE(0x1c0000, 0x1c3fff, FORCEWAKE_MEDIA_VDBOX0), /*
1422                 0x1c0000 - 0x1c2bff: VD0
1423                 0x1c2c00 - 0x1c2cff: reserved
1424                 0x1c2d00 - 0x1c2dff: VD0
1425                 0x1c2e00 - 0x1c3eff: reserved
1426                 0x1c3f00 - 0x1c3fff: VD0 */
1427         GEN_FW_RANGE(0x1c4000, 0x1c7fff, 0),
1428         GEN_FW_RANGE(0x1c8000, 0x1cbfff, FORCEWAKE_MEDIA_VEBOX0), /*
1429                 0x1c8000 - 0x1ca0ff: VE0
1430                 0x1ca100 - 0x1cbeff: reserved
1431                 0x1cbf00 - 0x1cbfff: VE0 */
1432         GEN_FW_RANGE(0x1cc000, 0x1cffff, FORCEWAKE_MEDIA_VDBOX0), /*
1433                 0x1cc000 - 0x1ccfff: VD0
1434                 0x1cd000 - 0x1cffff: reserved */
1435         GEN_FW_RANGE(0x1d0000, 0x1d3fff, FORCEWAKE_MEDIA_VDBOX2), /*
1436                 0x1d0000 - 0x1d2bff: VD2
1437                 0x1d2c00 - 0x1d2cff: reserved
1438                 0x1d2d00 - 0x1d2dff: VD2
1439                 0x1d2e00 - 0x1d3eff: reserved
1440                 0x1d3f00 - 0x1d3fff: VD2 */
1441 };
1442
1443 static const struct intel_forcewake_range __dg2_fw_ranges[] = {
1444         GEN_FW_RANGE(0x0, 0x1fff, 0), /*
1445                   0x0 -  0xaff: reserved
1446                 0xb00 - 0x1fff: always on */
1447         GEN_FW_RANGE(0x2000, 0x26ff, FORCEWAKE_RENDER),
1448         GEN_FW_RANGE(0x2700, 0x4aff, FORCEWAKE_GT),
1449         GEN_FW_RANGE(0x4b00, 0x51ff, 0), /*
1450                 0x4b00 - 0x4fff: reserved
1451                 0x5000 - 0x51ff: always on */
1452         GEN_FW_RANGE(0x5200, 0x7fff, FORCEWAKE_RENDER),
1453         GEN_FW_RANGE(0x8000, 0x813f, FORCEWAKE_GT),
1454         GEN_FW_RANGE(0x8140, 0x815f, FORCEWAKE_RENDER),
1455         GEN_FW_RANGE(0x8160, 0x81ff, 0), /*
1456                 0x8160 - 0x817f: reserved
1457                 0x8180 - 0x81ff: always on */
1458         GEN_FW_RANGE(0x8200, 0x82ff, FORCEWAKE_GT),
1459         GEN_FW_RANGE(0x8300, 0x84ff, FORCEWAKE_RENDER),
1460         GEN_FW_RANGE(0x8500, 0x8cff, FORCEWAKE_GT), /*
1461                 0x8500 - 0x87ff: gt
1462                 0x8800 - 0x8c7f: reserved
1463                 0x8c80 - 0x8cff: gt (DG2 only) */
1464         GEN_FW_RANGE(0x8d00, 0x8fff, FORCEWAKE_RENDER), /*
1465                 0x8d00 - 0x8dff: render (DG2 only)
1466                 0x8e00 - 0x8fff: reserved */
1467         GEN_FW_RANGE(0x9000, 0x94cf, FORCEWAKE_GT), /*
1468                 0x9000 - 0x947f: gt
1469                 0x9480 - 0x94cf: reserved */
1470         GEN_FW_RANGE(0x94d0, 0x955f, FORCEWAKE_RENDER),
1471         GEN_FW_RANGE(0x9560, 0x967f, 0), /*
1472                 0x9560 - 0x95ff: always on
1473                 0x9600 - 0x967f: reserved */
1474         GEN_FW_RANGE(0x9680, 0x97ff, FORCEWAKE_RENDER), /*
1475                 0x9680 - 0x96ff: render
1476                 0x9700 - 0x97ff: reserved */
1477         GEN_FW_RANGE(0x9800, 0xcfff, FORCEWAKE_GT), /*
1478                 0x9800 - 0xb4ff: gt
1479                 0xb500 - 0xbfff: reserved
1480                 0xc000 - 0xcfff: gt */
1481         GEN_FW_RANGE(0xd000, 0xd7ff, 0),
1482         GEN_FW_RANGE(0xd800, 0xd87f, FORCEWAKE_RENDER),
1483         GEN_FW_RANGE(0xd880, 0xdbff, FORCEWAKE_GT),
1484         GEN_FW_RANGE(0xdc00, 0xdcff, FORCEWAKE_RENDER),
1485         GEN_FW_RANGE(0xdd00, 0xde7f, FORCEWAKE_GT), /*
1486                 0xdd00 - 0xddff: gt
1487                 0xde00 - 0xde7f: reserved */
1488         GEN_FW_RANGE(0xde80, 0xe8ff, FORCEWAKE_RENDER), /*
1489                 0xde80 - 0xdfff: render
1490                 0xe000 - 0xe0ff: reserved
1491                 0xe100 - 0xe8ff: render */
1492         GEN_FW_RANGE(0xe900, 0xffff, FORCEWAKE_GT), /*
1493                 0xe900 - 0xe9ff: gt
1494                 0xea00 - 0xefff: reserved
1495                 0xf000 - 0xffff: gt */
1496         GEN_FW_RANGE(0x10000, 0x12fff, 0), /*
1497                 0x10000 - 0x11fff: reserved
1498                 0x12000 - 0x127ff: always on
1499                 0x12800 - 0x12fff: reserved */
1500         GEN_FW_RANGE(0x13000, 0x131ff, FORCEWAKE_MEDIA_VDBOX0),
1501         GEN_FW_RANGE(0x13200, 0x147ff, FORCEWAKE_MEDIA_VDBOX2), /*
1502                 0x13200 - 0x133ff: VD2 (DG2 only)
1503                 0x13400 - 0x147ff: reserved */
1504         GEN_FW_RANGE(0x14800, 0x14fff, FORCEWAKE_RENDER),
1505         GEN_FW_RANGE(0x15000, 0x16dff, FORCEWAKE_GT), /*
1506                 0x15000 - 0x15fff: gt (DG2 only)
1507                 0x16000 - 0x16dff: reserved */
1508         GEN_FW_RANGE(0x16e00, 0x21fff, FORCEWAKE_RENDER), /*
1509                 0x16e00 - 0x1ffff: render
1510                 0x20000 - 0x21fff: reserved */
1511         GEN_FW_RANGE(0x22000, 0x23fff, FORCEWAKE_GT),
1512         GEN_FW_RANGE(0x24000, 0x2417f, 0), /*
1513                 0x24000 - 0x2407f: always on
1514                 0x24080 - 0x2417f: reserved */
1515         GEN_FW_RANGE(0x24180, 0x249ff, FORCEWAKE_GT), /*
1516                 0x24180 - 0x241ff: gt
1517                 0x24200 - 0x249ff: reserved */
1518         GEN_FW_RANGE(0x24a00, 0x251ff, FORCEWAKE_RENDER), /*
1519                 0x24a00 - 0x24a7f: render
1520                 0x24a80 - 0x251ff: reserved */
1521         GEN_FW_RANGE(0x25200, 0x25fff, FORCEWAKE_GT), /*
1522                 0x25200 - 0x252ff: gt
1523                 0x25300 - 0x25fff: reserved */
1524         GEN_FW_RANGE(0x26000, 0x2ffff, FORCEWAKE_RENDER), /*
1525                 0x26000 - 0x27fff: render
1526                 0x28000 - 0x29fff: reserved
1527                 0x2a000 - 0x2ffff: undocumented */
1528         GEN_FW_RANGE(0x30000, 0x3ffff, FORCEWAKE_GT),
1529         GEN_FW_RANGE(0x40000, 0x1bffff, 0),
1530         GEN_FW_RANGE(0x1c0000, 0x1c3fff, FORCEWAKE_MEDIA_VDBOX0), /*
1531                 0x1c0000 - 0x1c2bff: VD0
1532                 0x1c2c00 - 0x1c2cff: reserved
1533                 0x1c2d00 - 0x1c2dff: VD0
1534                 0x1c2e00 - 0x1c3eff: VD0
1535                 0x1c3f00 - 0x1c3fff: VD0 */
1536         GEN_FW_RANGE(0x1c4000, 0x1c7fff, FORCEWAKE_MEDIA_VDBOX1), /*
1537                 0x1c4000 - 0x1c6bff: VD1
1538                 0x1c6c00 - 0x1c6cff: reserved
1539                 0x1c6d00 - 0x1c6dff: VD1
1540                 0x1c6e00 - 0x1c7fff: reserved */
1541         GEN_FW_RANGE(0x1c8000, 0x1cbfff, FORCEWAKE_MEDIA_VEBOX0), /*
1542                 0x1c8000 - 0x1ca0ff: VE0
1543                 0x1ca100 - 0x1cbfff: reserved */
1544         GEN_FW_RANGE(0x1cc000, 0x1ccfff, FORCEWAKE_MEDIA_VDBOX0),
1545         GEN_FW_RANGE(0x1cd000, 0x1cdfff, FORCEWAKE_MEDIA_VDBOX2),
1546         GEN_FW_RANGE(0x1ce000, 0x1cefff, FORCEWAKE_MEDIA_VDBOX4),
1547         GEN_FW_RANGE(0x1cf000, 0x1cffff, FORCEWAKE_MEDIA_VDBOX6),
1548         GEN_FW_RANGE(0x1d0000, 0x1d3fff, FORCEWAKE_MEDIA_VDBOX2), /*
1549                 0x1d0000 - 0x1d2bff: VD2
1550                 0x1d2c00 - 0x1d2cff: reserved
1551                 0x1d2d00 - 0x1d2dff: VD2
1552                 0x1d2e00 - 0x1d3dff: VD2
1553                 0x1d3e00 - 0x1d3eff: reserved
1554                 0x1d3f00 - 0x1d3fff: VD2 */
1555         GEN_FW_RANGE(0x1d4000, 0x1d7fff, FORCEWAKE_MEDIA_VDBOX3), /*
1556                 0x1d4000 - 0x1d6bff: VD3
1557                 0x1d6c00 - 0x1d6cff: reserved
1558                 0x1d6d00 - 0x1d6dff: VD3
1559                 0x1d6e00 - 0x1d7fff: reserved */
1560         GEN_FW_RANGE(0x1d8000, 0x1dffff, FORCEWAKE_MEDIA_VEBOX1), /*
1561                 0x1d8000 - 0x1da0ff: VE1
1562                 0x1da100 - 0x1dffff: reserved */
1563         GEN_FW_RANGE(0x1e0000, 0x1e3fff, FORCEWAKE_MEDIA_VDBOX4), /*
1564                 0x1e0000 - 0x1e2bff: VD4
1565                 0x1e2c00 - 0x1e2cff: reserved
1566                 0x1e2d00 - 0x1e2dff: VD4
1567                 0x1e2e00 - 0x1e3eff: reserved
1568                 0x1e3f00 - 0x1e3fff: VD4 */
1569         GEN_FW_RANGE(0x1e4000, 0x1e7fff, FORCEWAKE_MEDIA_VDBOX5), /*
1570                 0x1e4000 - 0x1e6bff: VD5
1571                 0x1e6c00 - 0x1e6cff: reserved
1572                 0x1e6d00 - 0x1e6dff: VD5
1573                 0x1e6e00 - 0x1e7fff: reserved */
1574         GEN_FW_RANGE(0x1e8000, 0x1effff, FORCEWAKE_MEDIA_VEBOX2), /*
1575                 0x1e8000 - 0x1ea0ff: VE2
1576                 0x1ea100 - 0x1effff: reserved */
1577         GEN_FW_RANGE(0x1f0000, 0x1f3fff, FORCEWAKE_MEDIA_VDBOX6), /*
1578                 0x1f0000 - 0x1f2bff: VD6
1579                 0x1f2c00 - 0x1f2cff: reserved
1580                 0x1f2d00 - 0x1f2dff: VD6
1581                 0x1f2e00 - 0x1f3eff: reserved
1582                 0x1f3f00 - 0x1f3fff: VD6 */
1583         GEN_FW_RANGE(0x1f4000, 0x1f7fff, FORCEWAKE_MEDIA_VDBOX7), /*
1584                 0x1f4000 - 0x1f6bff: VD7
1585                 0x1f6c00 - 0x1f6cff: reserved
1586                 0x1f6d00 - 0x1f6dff: VD7
1587                 0x1f6e00 - 0x1f7fff: reserved */
1588         GEN_FW_RANGE(0x1f8000, 0x1fa0ff, FORCEWAKE_MEDIA_VEBOX3),
1589 };
1590
1591 static const struct intel_forcewake_range __mtl_fw_ranges[] = {
1592         GEN_FW_RANGE(0x0, 0xaff, 0),
1593         GEN_FW_RANGE(0xb00, 0xbff, FORCEWAKE_GT),
1594         GEN_FW_RANGE(0xc00, 0xfff, 0),
1595         GEN_FW_RANGE(0x1000, 0x1fff, FORCEWAKE_GT),
1596         GEN_FW_RANGE(0x2000, 0x26ff, FORCEWAKE_RENDER),
1597         GEN_FW_RANGE(0x2700, 0x2fff, FORCEWAKE_GT),
1598         GEN_FW_RANGE(0x3000, 0x3fff, FORCEWAKE_RENDER),
1599         GEN_FW_RANGE(0x4000, 0x51ff, FORCEWAKE_GT), /*
1600                 0x4000 - 0x48ff: render
1601                 0x4900 - 0x51ff: reserved */
1602         GEN_FW_RANGE(0x5200, 0x7fff, FORCEWAKE_RENDER), /*
1603                 0x5200 - 0x53ff: render
1604                 0x5400 - 0x54ff: reserved
1605                 0x5500 - 0x7fff: render */
1606         GEN_FW_RANGE(0x8000, 0x813f, FORCEWAKE_GT),
1607         GEN_FW_RANGE(0x8140, 0x817f, FORCEWAKE_RENDER), /*
1608                 0x8140 - 0x815f: render
1609                 0x8160 - 0x817f: reserved */
1610         GEN_FW_RANGE(0x8180, 0x81ff, 0),
1611         GEN_FW_RANGE(0x8200, 0x94cf, FORCEWAKE_GT), /*
1612                 0x8200 - 0x87ff: gt
1613                 0x8800 - 0x8dff: reserved
1614                 0x8e00 - 0x8f7f: gt
1615                 0x8f80 - 0x8fff: reserved
1616                 0x9000 - 0x947f: gt
1617                 0x9480 - 0x94cf: reserved */
1618         GEN_FW_RANGE(0x94d0, 0x955f, FORCEWAKE_RENDER),
1619         GEN_FW_RANGE(0x9560, 0x967f, 0), /*
1620                 0x9560 - 0x95ff: always on
1621                 0x9600 - 0x967f: reserved */
1622         GEN_FW_RANGE(0x9680, 0x97ff, FORCEWAKE_RENDER), /*
1623                 0x9680 - 0x96ff: render
1624                 0x9700 - 0x97ff: reserved */
1625         GEN_FW_RANGE(0x9800, 0xcfff, FORCEWAKE_GT), /*
1626                 0x9800 - 0xb4ff: gt
1627                 0xb500 - 0xbfff: reserved
1628                 0xc000 - 0xcfff: gt */
1629         GEN_FW_RANGE(0xd000, 0xd7ff, 0), /*
1630                 0xd000 - 0xd3ff: always on
1631                 0xd400 - 0xd7ff: reserved */
1632         GEN_FW_RANGE(0xd800, 0xd87f, FORCEWAKE_RENDER),
1633         GEN_FW_RANGE(0xd880, 0xdbff, FORCEWAKE_GT),
1634         GEN_FW_RANGE(0xdc00, 0xdcff, FORCEWAKE_RENDER),
1635         GEN_FW_RANGE(0xdd00, 0xde7f, FORCEWAKE_GT), /*
1636                 0xdd00 - 0xddff: gt
1637                 0xde00 - 0xde7f: reserved */
1638         GEN_FW_RANGE(0xde80, 0xe8ff, FORCEWAKE_RENDER), /*
1639                 0xde80 - 0xdfff: render
1640                 0xe000 - 0xe0ff: reserved
1641                 0xe100 - 0xe8ff: render */
1642         GEN_FW_RANGE(0xe900, 0xe9ff, FORCEWAKE_GT),
1643         GEN_FW_RANGE(0xea00, 0x147ff, 0), /*
1644                  0xea00 - 0x11fff: reserved
1645                 0x12000 - 0x127ff: always on
1646                 0x12800 - 0x147ff: reserved */
1647         GEN_FW_RANGE(0x14800, 0x19fff, FORCEWAKE_GT), /*
1648                 0x14800 - 0x153ff: gt
1649                 0x15400 - 0x19fff: reserved */
1650         GEN_FW_RANGE(0x1a000, 0x21fff, FORCEWAKE_RENDER), /*
1651                 0x1a000 - 0x1bfff: render
1652                 0x1c000 - 0x21fff: reserved */
1653         GEN_FW_RANGE(0x22000, 0x23fff, FORCEWAKE_GT),
1654         GEN_FW_RANGE(0x24000, 0x2ffff, 0), /*
1655                 0x24000 - 0x2407f: always on
1656                 0x24080 - 0x2ffff: reserved */
1657         GEN_FW_RANGE(0x30000, 0x3ffff, FORCEWAKE_GT),
1658         GEN_FW_RANGE(0x40000, 0x1901ef, 0),
1659         GEN_FW_RANGE(0x1901f0, 0x1901f3, FORCEWAKE_GT)
1660                 /* FIXME: WA to wake GT while triggering H2G */
1661 };
1662
1663 /*
1664  * Note that the register ranges here are the final offsets after
1665  * translation of the GSI block to the 0x380000 offset.
1666  *
1667  * NOTE:  There are a couple MCR ranges near the bottom of this table
1668  * that need to power up either VD0 or VD2 depending on which replicated
1669  * instance of the register we're trying to access.  Our forcewake logic
1670  * at the moment doesn't have a good way to take steering into consideration,
1671  * and the driver doesn't even access any registers in those ranges today,
1672  * so for now we just mark those ranges as FORCEWAKE_ALL.  That will ensure
1673  * proper operation if we do start using the ranges in the future, and we
1674  * can determine at that time whether it's worth adding extra complexity to
1675  * the forcewake handling to take steering into consideration.
1676  */
1677 static const struct intel_forcewake_range __xelpmp_fw_ranges[] = {
1678         GEN_FW_RANGE(0x0, 0x115fff, 0), /* render GT range */
1679         GEN_FW_RANGE(0x116000, 0x11ffff, FORCEWAKE_GSC), /*
1680                 0x116000 - 0x117fff: gsc
1681                 0x118000 - 0x119fff: reserved
1682                 0x11a000 - 0x11efff: gsc
1683                 0x11f000 - 0x11ffff: reserved */
1684         GEN_FW_RANGE(0x120000, 0x1bffff, 0), /* non-GT range */
1685         GEN_FW_RANGE(0x1c0000, 0x1c7fff, FORCEWAKE_MEDIA_VDBOX0), /*
1686                 0x1c0000 - 0x1c3dff: VD0
1687                 0x1c3e00 - 0x1c3eff: reserved
1688                 0x1c3f00 - 0x1c3fff: VD0
1689                 0x1c4000 - 0x1c7fff: reserved */
1690         GEN_FW_RANGE(0x1c8000, 0x1cbfff, FORCEWAKE_MEDIA_VEBOX0), /*
1691                 0x1c8000 - 0x1ca0ff: VE0
1692                 0x1ca100 - 0x1cbfff: reserved */
1693         GEN_FW_RANGE(0x1cc000, 0x1cffff, FORCEWAKE_MEDIA_VDBOX0), /*
1694                 0x1cc000 - 0x1cdfff: VD0
1695                 0x1ce000 - 0x1cffff: reserved */
1696         GEN_FW_RANGE(0x1d0000, 0x1d7fff, FORCEWAKE_MEDIA_VDBOX2), /*
1697                 0x1d0000 - 0x1d3dff: VD2
1698                 0x1d3e00 - 0x1d3eff: reserved
1699                 0x1d4000 - 0x1d7fff: VD2 */
1700         GEN_FW_RANGE(0x1d8000, 0x1da0ff, FORCEWAKE_MEDIA_VEBOX1),
1701         GEN_FW_RANGE(0x1da100, 0x380aff, 0), /*
1702                 0x1da100 - 0x23ffff: reserved
1703                 0x240000 - 0x37ffff: non-GT range
1704                 0x380000 - 0x380aff: reserved */
1705         GEN_FW_RANGE(0x380b00, 0x380bff, FORCEWAKE_GT),
1706         GEN_FW_RANGE(0x380c00, 0x380fff, 0),
1707         GEN_FW_RANGE(0x381000, 0x38817f, FORCEWAKE_GT), /*
1708                 0x381000 - 0x381fff: gt
1709                 0x382000 - 0x383fff: reserved
1710                 0x384000 - 0x384aff: gt
1711                 0x384b00 - 0x3851ff: reserved
1712                 0x385200 - 0x3871ff: gt
1713                 0x387200 - 0x387fff: reserved
1714                 0x388000 - 0x38813f: gt
1715                 0x388140 - 0x38817f: reserved */
1716         GEN_FW_RANGE(0x388180, 0x3882ff, 0), /*
1717                 0x388180 - 0x3881ff: always on
1718                 0x388200 - 0x3882ff: reserved */
1719         GEN_FW_RANGE(0x388300, 0x38955f, FORCEWAKE_GT), /*
1720                 0x388300 - 0x38887f: gt
1721                 0x388880 - 0x388fff: reserved
1722                 0x389000 - 0x38947f: gt
1723                 0x389480 - 0x38955f: reserved */
1724         GEN_FW_RANGE(0x389560, 0x389fff, 0), /*
1725                 0x389560 - 0x3895ff: always on
1726                 0x389600 - 0x389fff: reserved */
1727         GEN_FW_RANGE(0x38a000, 0x38cfff, FORCEWAKE_GT), /*
1728                 0x38a000 - 0x38afff: gt
1729                 0x38b000 - 0x38bfff: reserved
1730                 0x38c000 - 0x38cfff: gt */
1731         GEN_FW_RANGE(0x38d000, 0x38d11f, 0),
1732         GEN_FW_RANGE(0x38d120, 0x391fff, FORCEWAKE_GT), /*
1733                 0x38d120 - 0x38dfff: gt
1734                 0x38e000 - 0x38efff: reserved
1735                 0x38f000 - 0x38ffff: gt
1736                 0x389000 - 0x391fff: reserved */
1737         GEN_FW_RANGE(0x392000, 0x392fff, 0), /*
1738                 0x392000 - 0x3927ff: always on
1739                 0x392800 - 0x292fff: reserved */
1740         GEN_FW_RANGE(0x393000, 0x3931ff, FORCEWAKE_GT),
1741         GEN_FW_RANGE(0x393200, 0x39323f, FORCEWAKE_ALL), /* instance-based, see note above */
1742         GEN_FW_RANGE(0x393240, 0x3933ff, FORCEWAKE_GT),
1743         GEN_FW_RANGE(0x393400, 0x3934ff, FORCEWAKE_ALL), /* instance-based, see note above */
1744         GEN_FW_RANGE(0x393500, 0x393c7f, 0), /*
1745                 0x393500 - 0x393bff: reserved
1746                 0x393c00 - 0x393c7f: always on */
1747         GEN_FW_RANGE(0x393c80, 0x393dff, FORCEWAKE_GT),
1748 };
1749
1750 static void
1751 ilk_dummy_write(struct intel_uncore *uncore)
1752 {
1753         /* WaIssueDummyWriteToWakeupFromRC6:ilk Issue a dummy write to wake up
1754          * the chip from rc6 before touching it for real. MI_MODE is masked,
1755          * hence harmless to write 0 into. */
1756         __raw_uncore_write32(uncore, RING_MI_MODE(RENDER_RING_BASE), 0);
1757 }
1758
1759 static void
1760 __unclaimed_reg_debug(struct intel_uncore *uncore,
1761                       const i915_reg_t reg,
1762                       const bool read)
1763 {
1764         if (drm_WARN(&uncore->i915->drm,
1765                      check_for_unclaimed_mmio(uncore),
1766                      "Unclaimed %s register 0x%x\n",
1767                      read ? "read from" : "write to",
1768                      i915_mmio_reg_offset(reg)))
1769                 /* Only report the first N failures */
1770                 uncore->i915->params.mmio_debug--;
1771 }
1772
1773 static void
1774 __unclaimed_previous_reg_debug(struct intel_uncore *uncore,
1775                                const i915_reg_t reg,
1776                                const bool read)
1777 {
1778         if (check_for_unclaimed_mmio(uncore))
1779                 drm_dbg(&uncore->i915->drm,
1780                         "Unclaimed access detected before %s register 0x%x\n",
1781                         read ? "read from" : "write to",
1782                         i915_mmio_reg_offset(reg));
1783 }
1784
1785 static inline bool __must_check
1786 unclaimed_reg_debug_header(struct intel_uncore *uncore,
1787                            const i915_reg_t reg, const bool read)
1788 {
1789         if (likely(!uncore->i915->params.mmio_debug) || !uncore->debug)
1790                 return false;
1791
1792         /* interrupts are disabled and re-enabled around uncore->lock usage */
1793         lockdep_assert_held(&uncore->lock);
1794
1795         spin_lock(&uncore->debug->lock);
1796         __unclaimed_previous_reg_debug(uncore, reg, read);
1797
1798         return true;
1799 }
1800
1801 static inline void
1802 unclaimed_reg_debug_footer(struct intel_uncore *uncore,
1803                            const i915_reg_t reg, const bool read)
1804 {
1805         /* interrupts are disabled and re-enabled around uncore->lock usage */
1806         lockdep_assert_held(&uncore->lock);
1807
1808         __unclaimed_reg_debug(uncore, reg, read);
1809         spin_unlock(&uncore->debug->lock);
1810 }
1811
1812 #define __vgpu_read(x) \
1813 static u##x \
1814 vgpu_read##x(struct intel_uncore *uncore, i915_reg_t reg, bool trace) { \
1815         u##x val = __raw_uncore_read##x(uncore, reg); \
1816         trace_i915_reg_rw(false, reg, val, sizeof(val), trace); \
1817         return val; \
1818 }
1819 __vgpu_read(8)
1820 __vgpu_read(16)
1821 __vgpu_read(32)
1822 __vgpu_read(64)
1823
1824 #define GEN2_READ_HEADER(x) \
1825         u##x val = 0; \
1826         assert_rpm_wakelock_held(uncore->rpm);
1827
1828 #define GEN2_READ_FOOTER \
1829         trace_i915_reg_rw(false, reg, val, sizeof(val), trace); \
1830         return val
1831
1832 #define __gen2_read(x) \
1833 static u##x \
1834 gen2_read##x(struct intel_uncore *uncore, i915_reg_t reg, bool trace) { \
1835         GEN2_READ_HEADER(x); \
1836         val = __raw_uncore_read##x(uncore, reg); \
1837         GEN2_READ_FOOTER; \
1838 }
1839
1840 #define __gen5_read(x) \
1841 static u##x \
1842 gen5_read##x(struct intel_uncore *uncore, i915_reg_t reg, bool trace) { \
1843         GEN2_READ_HEADER(x); \
1844         ilk_dummy_write(uncore); \
1845         val = __raw_uncore_read##x(uncore, reg); \
1846         GEN2_READ_FOOTER; \
1847 }
1848
1849 __gen5_read(8)
1850 __gen5_read(16)
1851 __gen5_read(32)
1852 __gen5_read(64)
1853 __gen2_read(8)
1854 __gen2_read(16)
1855 __gen2_read(32)
1856 __gen2_read(64)
1857
1858 #undef __gen5_read
1859 #undef __gen2_read
1860
1861 #undef GEN2_READ_FOOTER
1862 #undef GEN2_READ_HEADER
1863
1864 #define GEN6_READ_HEADER(x) \
1865         u32 offset = i915_mmio_reg_offset(reg); \
1866         unsigned long irqflags; \
1867         bool unclaimed_reg_debug; \
1868         u##x val = 0; \
1869         assert_rpm_wakelock_held(uncore->rpm); \
1870         spin_lock_irqsave(&uncore->lock, irqflags); \
1871         unclaimed_reg_debug = unclaimed_reg_debug_header(uncore, reg, true)
1872
1873 #define GEN6_READ_FOOTER \
1874         if (unclaimed_reg_debug) \
1875                 unclaimed_reg_debug_footer(uncore, reg, true);  \
1876         spin_unlock_irqrestore(&uncore->lock, irqflags); \
1877         trace_i915_reg_rw(false, reg, val, sizeof(val), trace); \
1878         return val
1879
1880 static noinline void ___force_wake_auto(struct intel_uncore *uncore,
1881                                         enum forcewake_domains fw_domains)
1882 {
1883         struct intel_uncore_forcewake_domain *domain;
1884         unsigned int tmp;
1885
1886         GEM_BUG_ON(fw_domains & ~uncore->fw_domains);
1887
1888         for_each_fw_domain_masked(domain, fw_domains, uncore, tmp)
1889                 fw_domain_arm_timer(domain);
1890
1891         fw_domains_get(uncore, fw_domains);
1892 }
1893
1894 static inline void __force_wake_auto(struct intel_uncore *uncore,
1895                                      enum forcewake_domains fw_domains)
1896 {
1897         GEM_BUG_ON(!fw_domains);
1898
1899         /* Turn on all requested but inactive supported forcewake domains. */
1900         fw_domains &= uncore->fw_domains;
1901         fw_domains &= ~uncore->fw_domains_active;
1902
1903         if (fw_domains)
1904                 ___force_wake_auto(uncore, fw_domains);
1905 }
1906
1907 #define __gen_fwtable_read(x) \
1908 static u##x \
1909 fwtable_read##x(struct intel_uncore *uncore, i915_reg_t reg, bool trace) \
1910 { \
1911         enum forcewake_domains fw_engine; \
1912         GEN6_READ_HEADER(x); \
1913         fw_engine = __fwtable_reg_read_fw_domains(uncore, offset); \
1914         if (fw_engine) \
1915                 __force_wake_auto(uncore, fw_engine); \
1916         val = __raw_uncore_read##x(uncore, reg); \
1917         GEN6_READ_FOOTER; \
1918 }
1919
1920 static enum forcewake_domains
1921 fwtable_reg_read_fw_domains(struct intel_uncore *uncore, i915_reg_t reg) {
1922         return __fwtable_reg_read_fw_domains(uncore, i915_mmio_reg_offset(reg));
1923 }
1924
1925 __gen_fwtable_read(8)
1926 __gen_fwtable_read(16)
1927 __gen_fwtable_read(32)
1928 __gen_fwtable_read(64)
1929
1930 #undef __gen_fwtable_read
1931 #undef GEN6_READ_FOOTER
1932 #undef GEN6_READ_HEADER
1933
1934 #define GEN2_WRITE_HEADER \
1935         trace_i915_reg_rw(true, reg, val, sizeof(val), trace); \
1936         assert_rpm_wakelock_held(uncore->rpm); \
1937
1938 #define GEN2_WRITE_FOOTER
1939
1940 #define __gen2_write(x) \
1941 static void \
1942 gen2_write##x(struct intel_uncore *uncore, i915_reg_t reg, u##x val, bool trace) { \
1943         GEN2_WRITE_HEADER; \
1944         __raw_uncore_write##x(uncore, reg, val); \
1945         GEN2_WRITE_FOOTER; \
1946 }
1947
1948 #define __gen5_write(x) \
1949 static void \
1950 gen5_write##x(struct intel_uncore *uncore, i915_reg_t reg, u##x val, bool trace) { \
1951         GEN2_WRITE_HEADER; \
1952         ilk_dummy_write(uncore); \
1953         __raw_uncore_write##x(uncore, reg, val); \
1954         GEN2_WRITE_FOOTER; \
1955 }
1956
1957 __gen5_write(8)
1958 __gen5_write(16)
1959 __gen5_write(32)
1960 __gen2_write(8)
1961 __gen2_write(16)
1962 __gen2_write(32)
1963
1964 #undef __gen5_write
1965 #undef __gen2_write
1966
1967 #undef GEN2_WRITE_FOOTER
1968 #undef GEN2_WRITE_HEADER
1969
1970 #define GEN6_WRITE_HEADER \
1971         u32 offset = i915_mmio_reg_offset(reg); \
1972         unsigned long irqflags; \
1973         bool unclaimed_reg_debug; \
1974         trace_i915_reg_rw(true, reg, val, sizeof(val), trace); \
1975         assert_rpm_wakelock_held(uncore->rpm); \
1976         spin_lock_irqsave(&uncore->lock, irqflags); \
1977         unclaimed_reg_debug = unclaimed_reg_debug_header(uncore, reg, false)
1978
1979 #define GEN6_WRITE_FOOTER \
1980         if (unclaimed_reg_debug) \
1981                 unclaimed_reg_debug_footer(uncore, reg, false); \
1982         spin_unlock_irqrestore(&uncore->lock, irqflags)
1983
1984 #define __gen6_write(x) \
1985 static void \
1986 gen6_write##x(struct intel_uncore *uncore, i915_reg_t reg, u##x val, bool trace) { \
1987         GEN6_WRITE_HEADER; \
1988         if (NEEDS_FORCE_WAKE(offset)) \
1989                 __gen6_gt_wait_for_fifo(uncore); \
1990         __raw_uncore_write##x(uncore, reg, val); \
1991         GEN6_WRITE_FOOTER; \
1992 }
1993 __gen6_write(8)
1994 __gen6_write(16)
1995 __gen6_write(32)
1996
1997 #define __gen_fwtable_write(x) \
1998 static void \
1999 fwtable_write##x(struct intel_uncore *uncore, i915_reg_t reg, u##x val, bool trace) { \
2000         enum forcewake_domains fw_engine; \
2001         GEN6_WRITE_HEADER; \
2002         fw_engine = __fwtable_reg_write_fw_domains(uncore, offset); \
2003         if (fw_engine) \
2004                 __force_wake_auto(uncore, fw_engine); \
2005         __raw_uncore_write##x(uncore, reg, val); \
2006         GEN6_WRITE_FOOTER; \
2007 }
2008
2009 static enum forcewake_domains
2010 fwtable_reg_write_fw_domains(struct intel_uncore *uncore, i915_reg_t reg)
2011 {
2012         return __fwtable_reg_write_fw_domains(uncore, i915_mmio_reg_offset(reg));
2013 }
2014
2015 __gen_fwtable_write(8)
2016 __gen_fwtable_write(16)
2017 __gen_fwtable_write(32)
2018
2019 #undef __gen_fwtable_write
2020 #undef GEN6_WRITE_FOOTER
2021 #undef GEN6_WRITE_HEADER
2022
2023 #define __vgpu_write(x) \
2024 static void \
2025 vgpu_write##x(struct intel_uncore *uncore, i915_reg_t reg, u##x val, bool trace) { \
2026         trace_i915_reg_rw(true, reg, val, sizeof(val), trace); \
2027         __raw_uncore_write##x(uncore, reg, val); \
2028 }
2029 __vgpu_write(8)
2030 __vgpu_write(16)
2031 __vgpu_write(32)
2032
2033 #define ASSIGN_RAW_WRITE_MMIO_VFUNCS(uncore, x) \
2034 do { \
2035         (uncore)->funcs.mmio_writeb = x##_write8; \
2036         (uncore)->funcs.mmio_writew = x##_write16; \
2037         (uncore)->funcs.mmio_writel = x##_write32; \
2038 } while (0)
2039
2040 #define ASSIGN_RAW_READ_MMIO_VFUNCS(uncore, x) \
2041 do { \
2042         (uncore)->funcs.mmio_readb = x##_read8; \
2043         (uncore)->funcs.mmio_readw = x##_read16; \
2044         (uncore)->funcs.mmio_readl = x##_read32; \
2045         (uncore)->funcs.mmio_readq = x##_read64; \
2046 } while (0)
2047
2048 #define ASSIGN_WRITE_MMIO_VFUNCS(uncore, x) \
2049 do { \
2050         ASSIGN_RAW_WRITE_MMIO_VFUNCS((uncore), x); \
2051         (uncore)->funcs.write_fw_domains = x##_reg_write_fw_domains; \
2052 } while (0)
2053
2054 #define ASSIGN_READ_MMIO_VFUNCS(uncore, x) \
2055 do { \
2056         ASSIGN_RAW_READ_MMIO_VFUNCS(uncore, x); \
2057         (uncore)->funcs.read_fw_domains = x##_reg_read_fw_domains; \
2058 } while (0)
2059
2060 static int __fw_domain_init(struct intel_uncore *uncore,
2061                             enum forcewake_domain_id domain_id,
2062                             i915_reg_t reg_set,
2063                             i915_reg_t reg_ack)
2064 {
2065         struct intel_uncore_forcewake_domain *d;
2066
2067         GEM_BUG_ON(domain_id >= FW_DOMAIN_ID_COUNT);
2068         GEM_BUG_ON(uncore->fw_domain[domain_id]);
2069
2070         if (i915_inject_probe_failure(uncore->i915))
2071                 return -ENOMEM;
2072
2073         d = kzalloc(sizeof(*d), GFP_KERNEL);
2074         if (!d)
2075                 return -ENOMEM;
2076
2077         drm_WARN_ON(&uncore->i915->drm, !i915_mmio_reg_valid(reg_set));
2078         drm_WARN_ON(&uncore->i915->drm, !i915_mmio_reg_valid(reg_ack));
2079
2080         d->uncore = uncore;
2081         d->wake_count = 0;
2082         d->reg_set = uncore->regs + i915_mmio_reg_offset(reg_set) + uncore->gsi_offset;
2083         d->reg_ack = uncore->regs + i915_mmio_reg_offset(reg_ack) + uncore->gsi_offset;
2084
2085         d->id = domain_id;
2086
2087         BUILD_BUG_ON(FORCEWAKE_RENDER != (1 << FW_DOMAIN_ID_RENDER));
2088         BUILD_BUG_ON(FORCEWAKE_GT != (1 << FW_DOMAIN_ID_GT));
2089         BUILD_BUG_ON(FORCEWAKE_MEDIA != (1 << FW_DOMAIN_ID_MEDIA));
2090         BUILD_BUG_ON(FORCEWAKE_MEDIA_VDBOX0 != (1 << FW_DOMAIN_ID_MEDIA_VDBOX0));
2091         BUILD_BUG_ON(FORCEWAKE_MEDIA_VDBOX1 != (1 << FW_DOMAIN_ID_MEDIA_VDBOX1));
2092         BUILD_BUG_ON(FORCEWAKE_MEDIA_VDBOX2 != (1 << FW_DOMAIN_ID_MEDIA_VDBOX2));
2093         BUILD_BUG_ON(FORCEWAKE_MEDIA_VDBOX3 != (1 << FW_DOMAIN_ID_MEDIA_VDBOX3));
2094         BUILD_BUG_ON(FORCEWAKE_MEDIA_VDBOX4 != (1 << FW_DOMAIN_ID_MEDIA_VDBOX4));
2095         BUILD_BUG_ON(FORCEWAKE_MEDIA_VDBOX5 != (1 << FW_DOMAIN_ID_MEDIA_VDBOX5));
2096         BUILD_BUG_ON(FORCEWAKE_MEDIA_VDBOX6 != (1 << FW_DOMAIN_ID_MEDIA_VDBOX6));
2097         BUILD_BUG_ON(FORCEWAKE_MEDIA_VDBOX7 != (1 << FW_DOMAIN_ID_MEDIA_VDBOX7));
2098         BUILD_BUG_ON(FORCEWAKE_MEDIA_VEBOX0 != (1 << FW_DOMAIN_ID_MEDIA_VEBOX0));
2099         BUILD_BUG_ON(FORCEWAKE_MEDIA_VEBOX1 != (1 << FW_DOMAIN_ID_MEDIA_VEBOX1));
2100         BUILD_BUG_ON(FORCEWAKE_MEDIA_VEBOX2 != (1 << FW_DOMAIN_ID_MEDIA_VEBOX2));
2101         BUILD_BUG_ON(FORCEWAKE_MEDIA_VEBOX3 != (1 << FW_DOMAIN_ID_MEDIA_VEBOX3));
2102         BUILD_BUG_ON(FORCEWAKE_GSC != (1 << FW_DOMAIN_ID_GSC));
2103
2104         d->mask = BIT(domain_id);
2105
2106         hrtimer_init(&d->timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
2107         d->timer.function = intel_uncore_fw_release_timer;
2108
2109         uncore->fw_domains |= BIT(domain_id);
2110
2111         fw_domain_reset(d);
2112
2113         uncore->fw_domain[domain_id] = d;
2114
2115         return 0;
2116 }
2117
2118 static void fw_domain_fini(struct intel_uncore *uncore,
2119                            enum forcewake_domain_id domain_id)
2120 {
2121         struct intel_uncore_forcewake_domain *d;
2122
2123         GEM_BUG_ON(domain_id >= FW_DOMAIN_ID_COUNT);
2124
2125         d = fetch_and_zero(&uncore->fw_domain[domain_id]);
2126         if (!d)
2127                 return;
2128
2129         uncore->fw_domains &= ~BIT(domain_id);
2130         drm_WARN_ON(&uncore->i915->drm, d->wake_count);
2131         drm_WARN_ON(&uncore->i915->drm, hrtimer_cancel(&d->timer));
2132         kfree(d);
2133 }
2134
2135 static void intel_uncore_fw_domains_fini(struct intel_uncore *uncore)
2136 {
2137         struct intel_uncore_forcewake_domain *d;
2138         int tmp;
2139
2140         for_each_fw_domain(d, uncore, tmp)
2141                 fw_domain_fini(uncore, d->id);
2142 }
2143
2144 static const struct intel_uncore_fw_get uncore_get_fallback = {
2145         .force_wake_get = fw_domains_get_with_fallback
2146 };
2147
2148 static const struct intel_uncore_fw_get uncore_get_normal = {
2149         .force_wake_get = fw_domains_get_normal,
2150 };
2151
2152 static const struct intel_uncore_fw_get uncore_get_thread_status = {
2153         .force_wake_get = fw_domains_get_with_thread_status
2154 };
2155
2156 static int intel_uncore_fw_domains_init(struct intel_uncore *uncore)
2157 {
2158         struct drm_i915_private *i915 = uncore->i915;
2159         int ret = 0;
2160
2161         GEM_BUG_ON(!intel_uncore_has_forcewake(uncore));
2162
2163 #define fw_domain_init(uncore__, id__, set__, ack__) \
2164         (ret ?: (ret = __fw_domain_init((uncore__), (id__), (set__), (ack__))))
2165
2166         if (GRAPHICS_VER(i915) >= 11) {
2167                 intel_engine_mask_t emask;
2168                 int i;
2169
2170                 /* we'll prune the domains of missing engines later */
2171                 emask = uncore->gt->info.engine_mask;
2172
2173                 uncore->fw_get_funcs = &uncore_get_fallback;
2174                 if (GRAPHICS_VER_FULL(i915) >= IP_VER(12, 70))
2175                         fw_domain_init(uncore, FW_DOMAIN_ID_GT,
2176                                        FORCEWAKE_GT_GEN9,
2177                                        FORCEWAKE_ACK_GT_MTL);
2178                 else
2179                         fw_domain_init(uncore, FW_DOMAIN_ID_GT,
2180                                        FORCEWAKE_GT_GEN9,
2181                                        FORCEWAKE_ACK_GT_GEN9);
2182
2183                 if (RCS_MASK(uncore->gt) || CCS_MASK(uncore->gt))
2184                         fw_domain_init(uncore, FW_DOMAIN_ID_RENDER,
2185                                        FORCEWAKE_RENDER_GEN9,
2186                                        FORCEWAKE_ACK_RENDER_GEN9);
2187
2188                 for (i = 0; i < I915_MAX_VCS; i++) {
2189                         if (!__HAS_ENGINE(emask, _VCS(i)))
2190                                 continue;
2191
2192                         fw_domain_init(uncore, FW_DOMAIN_ID_MEDIA_VDBOX0 + i,
2193                                        FORCEWAKE_MEDIA_VDBOX_GEN11(i),
2194                                        FORCEWAKE_ACK_MEDIA_VDBOX_GEN11(i));
2195                 }
2196                 for (i = 0; i < I915_MAX_VECS; i++) {
2197                         if (!__HAS_ENGINE(emask, _VECS(i)))
2198                                 continue;
2199
2200                         fw_domain_init(uncore, FW_DOMAIN_ID_MEDIA_VEBOX0 + i,
2201                                        FORCEWAKE_MEDIA_VEBOX_GEN11(i),
2202                                        FORCEWAKE_ACK_MEDIA_VEBOX_GEN11(i));
2203                 }
2204
2205                 if (uncore->gt->type == GT_MEDIA)
2206                         fw_domain_init(uncore, FW_DOMAIN_ID_GSC,
2207                                        FORCEWAKE_REQ_GSC, FORCEWAKE_ACK_GSC);
2208         } else if (IS_GRAPHICS_VER(i915, 9, 10)) {
2209                 uncore->fw_get_funcs = &uncore_get_fallback;
2210                 fw_domain_init(uncore, FW_DOMAIN_ID_RENDER,
2211                                FORCEWAKE_RENDER_GEN9,
2212                                FORCEWAKE_ACK_RENDER_GEN9);
2213                 fw_domain_init(uncore, FW_DOMAIN_ID_GT,
2214                                FORCEWAKE_GT_GEN9,
2215                                FORCEWAKE_ACK_GT_GEN9);
2216                 fw_domain_init(uncore, FW_DOMAIN_ID_MEDIA,
2217                                FORCEWAKE_MEDIA_GEN9, FORCEWAKE_ACK_MEDIA_GEN9);
2218         } else if (IS_VALLEYVIEW(i915) || IS_CHERRYVIEW(i915)) {
2219                 uncore->fw_get_funcs = &uncore_get_normal;
2220                 fw_domain_init(uncore, FW_DOMAIN_ID_RENDER,
2221                                FORCEWAKE_VLV, FORCEWAKE_ACK_VLV);
2222                 fw_domain_init(uncore, FW_DOMAIN_ID_MEDIA,
2223                                FORCEWAKE_MEDIA_VLV, FORCEWAKE_ACK_MEDIA_VLV);
2224         } else if (IS_HASWELL(i915) || IS_BROADWELL(i915)) {
2225                 uncore->fw_get_funcs = &uncore_get_thread_status;
2226                 fw_domain_init(uncore, FW_DOMAIN_ID_RENDER,
2227                                FORCEWAKE_MT, FORCEWAKE_ACK_HSW);
2228         } else if (IS_IVYBRIDGE(i915)) {
2229                 u32 ecobus;
2230
2231                 /* IVB configs may use multi-threaded forcewake */
2232
2233                 /* A small trick here - if the bios hasn't configured
2234                  * MT forcewake, and if the device is in RC6, then
2235                  * force_wake_mt_get will not wake the device and the
2236                  * ECOBUS read will return zero. Which will be
2237                  * (correctly) interpreted by the test below as MT
2238                  * forcewake being disabled.
2239                  */
2240                 uncore->fw_get_funcs = &uncore_get_thread_status;
2241
2242                 /* We need to init first for ECOBUS access and then
2243                  * determine later if we want to reinit, in case of MT access is
2244                  * not working. In this stage we don't know which flavour this
2245                  * ivb is, so it is better to reset also the gen6 fw registers
2246                  * before the ecobus check.
2247                  */
2248
2249                 __raw_uncore_write32(uncore, FORCEWAKE, 0);
2250                 __raw_posting_read(uncore, ECOBUS);
2251
2252                 ret = __fw_domain_init(uncore, FW_DOMAIN_ID_RENDER,
2253                                        FORCEWAKE_MT, FORCEWAKE_MT_ACK);
2254                 if (ret)
2255                         goto out;
2256
2257                 spin_lock_irq(&uncore->lock);
2258                 fw_domains_get_with_thread_status(uncore, FORCEWAKE_RENDER);
2259                 ecobus = __raw_uncore_read32(uncore, ECOBUS);
2260                 fw_domains_put(uncore, FORCEWAKE_RENDER);
2261                 spin_unlock_irq(&uncore->lock);
2262
2263                 if (!(ecobus & FORCEWAKE_MT_ENABLE)) {
2264                         drm_info(&i915->drm, "No MT forcewake available on Ivybridge, this can result in issues\n");
2265                         drm_info(&i915->drm, "when using vblank-synced partial screen updates.\n");
2266                         fw_domain_fini(uncore, FW_DOMAIN_ID_RENDER);
2267                         fw_domain_init(uncore, FW_DOMAIN_ID_RENDER,
2268                                        FORCEWAKE, FORCEWAKE_ACK);
2269                 }
2270         } else if (GRAPHICS_VER(i915) == 6) {
2271                 uncore->fw_get_funcs = &uncore_get_thread_status;
2272                 fw_domain_init(uncore, FW_DOMAIN_ID_RENDER,
2273                                FORCEWAKE, FORCEWAKE_ACK);
2274         }
2275
2276 #undef fw_domain_init
2277
2278         /* All future platforms are expected to require complex power gating */
2279         drm_WARN_ON(&i915->drm, !ret && uncore->fw_domains == 0);
2280
2281 out:
2282         if (ret)
2283                 intel_uncore_fw_domains_fini(uncore);
2284
2285         return ret;
2286 }
2287
2288 #define ASSIGN_FW_DOMAINS_TABLE(uncore, d) \
2289 { \
2290         (uncore)->fw_domains_table = \
2291                         (struct intel_forcewake_range *)(d); \
2292         (uncore)->fw_domains_table_entries = ARRAY_SIZE((d)); \
2293 }
2294
2295 #define ASSIGN_SHADOW_TABLE(uncore, d) \
2296 { \
2297         (uncore)->shadowed_reg_table = d; \
2298         (uncore)->shadowed_reg_table_entries = ARRAY_SIZE((d)); \
2299 }
2300
2301 static int i915_pmic_bus_access_notifier(struct notifier_block *nb,
2302                                          unsigned long action, void *data)
2303 {
2304         struct intel_uncore *uncore = container_of(nb,
2305                         struct intel_uncore, pmic_bus_access_nb);
2306
2307         switch (action) {
2308         case MBI_PMIC_BUS_ACCESS_BEGIN:
2309                 /*
2310                  * forcewake all now to make sure that we don't need to do a
2311                  * forcewake later which on systems where this notifier gets
2312                  * called requires the punit to access to the shared pmic i2c
2313                  * bus, which will be busy after this notification, leading to:
2314                  * "render: timed out waiting for forcewake ack request."
2315                  * errors.
2316                  *
2317                  * The notifier is unregistered during intel_runtime_suspend(),
2318                  * so it's ok to access the HW here without holding a RPM
2319                  * wake reference -> disable wakeref asserts for the time of
2320                  * the access.
2321                  */
2322                 disable_rpm_wakeref_asserts(uncore->rpm);
2323                 intel_uncore_forcewake_get(uncore, FORCEWAKE_ALL);
2324                 enable_rpm_wakeref_asserts(uncore->rpm);
2325                 break;
2326         case MBI_PMIC_BUS_ACCESS_END:
2327                 intel_uncore_forcewake_put(uncore, FORCEWAKE_ALL);
2328                 break;
2329         }
2330
2331         return NOTIFY_OK;
2332 }
2333
2334 static void uncore_unmap_mmio(struct drm_device *drm, void *regs)
2335 {
2336         iounmap((void __iomem *)regs);
2337 }
2338
2339 int intel_uncore_setup_mmio(struct intel_uncore *uncore, phys_addr_t phys_addr)
2340 {
2341         struct drm_i915_private *i915 = uncore->i915;
2342         int mmio_size;
2343
2344         /*
2345          * Before gen4, the registers and the GTT are behind different BARs.
2346          * However, from gen4 onwards, the registers and the GTT are shared
2347          * in the same BAR, so we want to restrict this ioremap from
2348          * clobbering the GTT which we want ioremap_wc instead. Fortunately,
2349          * the register BAR remains the same size for all the earlier
2350          * generations up to Ironlake.
2351          * For dgfx chips register range is expanded to 4MB, and this larger
2352          * range is also used for integrated gpus beginning with Meteor Lake.
2353          */
2354         if (IS_DGFX(i915) || GRAPHICS_VER_FULL(i915) >= IP_VER(12, 70))
2355                 mmio_size = 4 * 1024 * 1024;
2356         else if (GRAPHICS_VER(i915) >= 5)
2357                 mmio_size = 2 * 1024 * 1024;
2358         else
2359                 mmio_size = 512 * 1024;
2360
2361         uncore->regs = ioremap(phys_addr, mmio_size);
2362         if (uncore->regs == NULL) {
2363                 drm_err(&i915->drm, "failed to map registers\n");
2364                 return -EIO;
2365         }
2366
2367         return drmm_add_action_or_reset(&i915->drm, uncore_unmap_mmio,
2368                                         (void __force *)uncore->regs);
2369 }
2370
2371 void intel_uncore_init_early(struct intel_uncore *uncore,
2372                              struct intel_gt *gt)
2373 {
2374         spin_lock_init(&uncore->lock);
2375         uncore->i915 = gt->i915;
2376         uncore->gt = gt;
2377         uncore->rpm = &gt->i915->runtime_pm;
2378 }
2379
2380 static void uncore_raw_init(struct intel_uncore *uncore)
2381 {
2382         GEM_BUG_ON(intel_uncore_has_forcewake(uncore));
2383
2384         if (intel_vgpu_active(uncore->i915)) {
2385                 ASSIGN_RAW_WRITE_MMIO_VFUNCS(uncore, vgpu);
2386                 ASSIGN_RAW_READ_MMIO_VFUNCS(uncore, vgpu);
2387         } else if (GRAPHICS_VER(uncore->i915) == 5) {
2388                 ASSIGN_RAW_WRITE_MMIO_VFUNCS(uncore, gen5);
2389                 ASSIGN_RAW_READ_MMIO_VFUNCS(uncore, gen5);
2390         } else {
2391                 ASSIGN_RAW_WRITE_MMIO_VFUNCS(uncore, gen2);
2392                 ASSIGN_RAW_READ_MMIO_VFUNCS(uncore, gen2);
2393         }
2394 }
2395
2396 static int uncore_media_forcewake_init(struct intel_uncore *uncore)
2397 {
2398         struct drm_i915_private *i915 = uncore->i915;
2399
2400         if (MEDIA_VER(i915) >= 13) {
2401                 ASSIGN_FW_DOMAINS_TABLE(uncore, __xelpmp_fw_ranges);
2402                 ASSIGN_SHADOW_TABLE(uncore, xelpmp_shadowed_regs);
2403                 ASSIGN_WRITE_MMIO_VFUNCS(uncore, fwtable);
2404         } else {
2405                 MISSING_CASE(MEDIA_VER(i915));
2406                 return -ENODEV;
2407         }
2408
2409         return 0;
2410 }
2411
2412 static int uncore_forcewake_init(struct intel_uncore *uncore)
2413 {
2414         struct drm_i915_private *i915 = uncore->i915;
2415         int ret;
2416
2417         GEM_BUG_ON(!intel_uncore_has_forcewake(uncore));
2418
2419         ret = intel_uncore_fw_domains_init(uncore);
2420         if (ret)
2421                 return ret;
2422         forcewake_early_sanitize(uncore, 0);
2423
2424         ASSIGN_READ_MMIO_VFUNCS(uncore, fwtable);
2425
2426         if (uncore->gt->type == GT_MEDIA)
2427                 return uncore_media_forcewake_init(uncore);
2428
2429         if (GRAPHICS_VER_FULL(i915) >= IP_VER(12, 70)) {
2430                 ASSIGN_FW_DOMAINS_TABLE(uncore, __mtl_fw_ranges);
2431                 ASSIGN_SHADOW_TABLE(uncore, mtl_shadowed_regs);
2432                 ASSIGN_WRITE_MMIO_VFUNCS(uncore, fwtable);
2433         } else if (GRAPHICS_VER_FULL(i915) >= IP_VER(12, 55)) {
2434                 ASSIGN_FW_DOMAINS_TABLE(uncore, __dg2_fw_ranges);
2435                 ASSIGN_SHADOW_TABLE(uncore, dg2_shadowed_regs);
2436                 ASSIGN_WRITE_MMIO_VFUNCS(uncore, fwtable);
2437         } else if (GRAPHICS_VER(i915) >= 12) {
2438                 ASSIGN_FW_DOMAINS_TABLE(uncore, __gen12_fw_ranges);
2439                 ASSIGN_SHADOW_TABLE(uncore, gen12_shadowed_regs);
2440                 ASSIGN_WRITE_MMIO_VFUNCS(uncore, fwtable);
2441         } else if (GRAPHICS_VER(i915) == 11) {
2442                 ASSIGN_FW_DOMAINS_TABLE(uncore, __gen11_fw_ranges);
2443                 ASSIGN_SHADOW_TABLE(uncore, gen11_shadowed_regs);
2444                 ASSIGN_WRITE_MMIO_VFUNCS(uncore, fwtable);
2445         } else if (IS_GRAPHICS_VER(i915, 9, 10)) {
2446                 ASSIGN_FW_DOMAINS_TABLE(uncore, __gen9_fw_ranges);
2447                 ASSIGN_SHADOW_TABLE(uncore, gen8_shadowed_regs);
2448                 ASSIGN_WRITE_MMIO_VFUNCS(uncore, fwtable);
2449         } else if (IS_CHERRYVIEW(i915)) {
2450                 ASSIGN_FW_DOMAINS_TABLE(uncore, __chv_fw_ranges);
2451                 ASSIGN_SHADOW_TABLE(uncore, gen8_shadowed_regs);
2452                 ASSIGN_WRITE_MMIO_VFUNCS(uncore, fwtable);
2453         } else if (GRAPHICS_VER(i915) == 8) {
2454                 ASSIGN_FW_DOMAINS_TABLE(uncore, __gen6_fw_ranges);
2455                 ASSIGN_SHADOW_TABLE(uncore, gen8_shadowed_regs);
2456                 ASSIGN_WRITE_MMIO_VFUNCS(uncore, fwtable);
2457         } else if (IS_VALLEYVIEW(i915)) {
2458                 ASSIGN_FW_DOMAINS_TABLE(uncore, __vlv_fw_ranges);
2459                 ASSIGN_WRITE_MMIO_VFUNCS(uncore, gen6);
2460         } else if (IS_GRAPHICS_VER(i915, 6, 7)) {
2461                 ASSIGN_FW_DOMAINS_TABLE(uncore, __gen6_fw_ranges);
2462                 ASSIGN_WRITE_MMIO_VFUNCS(uncore, gen6);
2463         }
2464
2465         uncore->pmic_bus_access_nb.notifier_call = i915_pmic_bus_access_notifier;
2466         iosf_mbi_register_pmic_bus_access_notifier(&uncore->pmic_bus_access_nb);
2467
2468         return 0;
2469 }
2470
2471 static int sanity_check_mmio_access(struct intel_uncore *uncore)
2472 {
2473         struct drm_i915_private *i915 = uncore->i915;
2474
2475         if (GRAPHICS_VER(i915) < 8)
2476                 return 0;
2477
2478         /*
2479          * Sanitycheck that MMIO access to the device is working properly.  If
2480          * the CPU is unable to communcate with a PCI device, BAR reads will
2481          * return 0xFFFFFFFF.  Let's make sure the device isn't in this state
2482          * before we start trying to access registers.
2483          *
2484          * We use the primary GT's forcewake register as our guinea pig since
2485          * it's been around since HSW and it's a masked register so the upper
2486          * 16 bits can never read back as 1's if device access is operating
2487          * properly.
2488          *
2489          * If MMIO isn't working, we'll wait up to 2 seconds to see if it
2490          * recovers, then give up.
2491          */
2492 #define COND (__raw_uncore_read32(uncore, FORCEWAKE_MT) != ~0)
2493         if (wait_for(COND, 2000) == -ETIMEDOUT) {
2494                 drm_err(&i915->drm, "Device is non-operational; MMIO access returns 0xFFFFFFFF!\n");
2495                 return -EIO;
2496         }
2497
2498         return 0;
2499 }
2500
2501 int intel_uncore_init_mmio(struct intel_uncore *uncore)
2502 {
2503         struct drm_i915_private *i915 = uncore->i915;
2504         int ret;
2505
2506         ret = sanity_check_mmio_access(uncore);
2507         if (ret)
2508                 return ret;
2509
2510         /*
2511          * The boot firmware initializes local memory and assesses its health.
2512          * If memory training fails, the punit will have been instructed to
2513          * keep the GT powered down; we won't be able to communicate with it
2514          * and we should not continue with driver initialization.
2515          */
2516         if (IS_DGFX(i915) &&
2517             !(__raw_uncore_read32(uncore, GU_CNTL) & LMEM_INIT)) {
2518                 drm_err(&i915->drm, "LMEM not initialized by firmware\n");
2519                 return -ENODEV;
2520         }
2521
2522         if (GRAPHICS_VER(i915) > 5 && !intel_vgpu_active(i915))
2523                 uncore->flags |= UNCORE_HAS_FORCEWAKE;
2524
2525         if (!intel_uncore_has_forcewake(uncore)) {
2526                 uncore_raw_init(uncore);
2527         } else {
2528                 ret = uncore_forcewake_init(uncore);
2529                 if (ret)
2530                         return ret;
2531         }
2532
2533         /* make sure fw funcs are set if and only if we have fw*/
2534         GEM_BUG_ON(intel_uncore_has_forcewake(uncore) != !!uncore->fw_get_funcs);
2535         GEM_BUG_ON(intel_uncore_has_forcewake(uncore) != !!uncore->funcs.read_fw_domains);
2536         GEM_BUG_ON(intel_uncore_has_forcewake(uncore) != !!uncore->funcs.write_fw_domains);
2537
2538         if (HAS_FPGA_DBG_UNCLAIMED(i915))
2539                 uncore->flags |= UNCORE_HAS_FPGA_DBG_UNCLAIMED;
2540
2541         if (IS_VALLEYVIEW(i915) || IS_CHERRYVIEW(i915))
2542                 uncore->flags |= UNCORE_HAS_DBG_UNCLAIMED;
2543
2544         if (IS_GRAPHICS_VER(i915, 6, 7))
2545                 uncore->flags |= UNCORE_HAS_FIFO;
2546
2547         /* clear out unclaimed reg detection bit */
2548         if (intel_uncore_unclaimed_mmio(uncore))
2549                 drm_dbg(&i915->drm, "unclaimed mmio detected on uncore init, clearing\n");
2550
2551         return 0;
2552 }
2553
2554 /*
2555  * We might have detected that some engines are fused off after we initialized
2556  * the forcewake domains. Prune them, to make sure they only reference existing
2557  * engines.
2558  */
2559 void intel_uncore_prune_engine_fw_domains(struct intel_uncore *uncore,
2560                                           struct intel_gt *gt)
2561 {
2562         enum forcewake_domains fw_domains = uncore->fw_domains;
2563         enum forcewake_domain_id domain_id;
2564         int i;
2565
2566         if (!intel_uncore_has_forcewake(uncore) || GRAPHICS_VER(uncore->i915) < 11)
2567                 return;
2568
2569         for (i = 0; i < I915_MAX_VCS; i++) {
2570                 domain_id = FW_DOMAIN_ID_MEDIA_VDBOX0 + i;
2571
2572                 if (HAS_ENGINE(gt, _VCS(i)))
2573                         continue;
2574
2575                 /*
2576                  * Starting with XeHP, the power well for an even-numbered
2577                  * VDBOX is also used for shared units within the
2578                  * media slice such as SFC.  So even if the engine
2579                  * itself is fused off, we still need to initialize
2580                  * the forcewake domain if any of the other engines
2581                  * in the same media slice are present.
2582                  */
2583                 if (GRAPHICS_VER_FULL(uncore->i915) >= IP_VER(12, 55) && i % 2 == 0) {
2584                         if ((i + 1 < I915_MAX_VCS) && HAS_ENGINE(gt, _VCS(i + 1)))
2585                                 continue;
2586
2587                         if (HAS_ENGINE(gt, _VECS(i / 2)))
2588                                 continue;
2589                 }
2590
2591                 if (fw_domains & BIT(domain_id))
2592                         fw_domain_fini(uncore, domain_id);
2593         }
2594
2595         for (i = 0; i < I915_MAX_VECS; i++) {
2596                 domain_id = FW_DOMAIN_ID_MEDIA_VEBOX0 + i;
2597
2598                 if (HAS_ENGINE(gt, _VECS(i)))
2599                         continue;
2600
2601                 if (fw_domains & BIT(domain_id))
2602                         fw_domain_fini(uncore, domain_id);
2603         }
2604
2605         if ((fw_domains & BIT(FW_DOMAIN_ID_GSC)) && !HAS_ENGINE(gt, GSC0))
2606                 fw_domain_fini(uncore, FW_DOMAIN_ID_GSC);
2607 }
2608
2609 /*
2610  * The driver-initiated FLR is the highest level of reset that we can trigger
2611  * from within the driver. It is different from the PCI FLR in that it doesn't
2612  * fully reset the SGUnit and doesn't modify the PCI config space and therefore
2613  * it doesn't require a re-enumeration of the PCI BARs. However, the
2614  * driver-initiated FLR does still cause a reset of both GT and display and a
2615  * memory wipe of local and stolen memory, so recovery would require a full HW
2616  * re-init and saving/restoring (or re-populating) the wiped memory. Since we
2617  * perform the FLR as the very last action before releasing access to the HW
2618  * during the driver release flow, we don't attempt recovery at all, because
2619  * if/when a new instance of i915 is bound to the device it will do a full
2620  * re-init anyway.
2621  */
2622 static void driver_initiated_flr(struct intel_uncore *uncore)
2623 {
2624         struct drm_i915_private *i915 = uncore->i915;
2625         unsigned int flr_timeout_ms;
2626         int ret;
2627
2628         drm_dbg(&i915->drm, "Triggering Driver-FLR\n");
2629
2630         /*
2631          * The specification recommends a 3 seconds FLR reset timeout. To be
2632          * cautious, we will extend this to 9 seconds, three times the specified
2633          * timeout.
2634          */
2635         flr_timeout_ms = 9000;
2636
2637         /*
2638          * Make sure any pending FLR requests have cleared by waiting for the
2639          * FLR trigger bit to go to zero. Also clear GU_DEBUG's DRIVERFLR_STATUS
2640          * to make sure it's not still set from a prior attempt (it's a write to
2641          * clear bit).
2642          * Note that we should never be in a situation where a previous attempt
2643          * is still pending (unless the HW is totally dead), but better to be
2644          * safe in case something unexpected happens
2645          */
2646         ret = intel_wait_for_register_fw(uncore, GU_CNTL, DRIVERFLR, 0, flr_timeout_ms);
2647         if (ret) {
2648                 drm_err(&i915->drm,
2649                         "Failed to wait for Driver-FLR bit to clear! %d\n",
2650                         ret);
2651                 return;
2652         }
2653         intel_uncore_write_fw(uncore, GU_DEBUG, DRIVERFLR_STATUS);
2654
2655         /* Trigger the actual Driver-FLR */
2656         intel_uncore_rmw_fw(uncore, GU_CNTL, 0, DRIVERFLR);
2657
2658         /* Wait for hardware teardown to complete */
2659         ret = intel_wait_for_register_fw(uncore, GU_CNTL,
2660                                          DRIVERFLR, 0,
2661                                          flr_timeout_ms);
2662         if (ret) {
2663                 drm_err(&i915->drm, "Driver-FLR-teardown wait completion failed! %d\n", ret);
2664                 return;
2665         }
2666
2667         /* Wait for hardware/firmware re-init to complete */
2668         ret = intel_wait_for_register_fw(uncore, GU_DEBUG,
2669                                          DRIVERFLR_STATUS, DRIVERFLR_STATUS,
2670                                          flr_timeout_ms);
2671         if (ret) {
2672                 drm_err(&i915->drm, "Driver-FLR-reinit wait completion failed! %d\n", ret);
2673                 return;
2674         }
2675
2676         /* Clear sticky completion status */
2677         intel_uncore_write_fw(uncore, GU_DEBUG, DRIVERFLR_STATUS);
2678 }
2679
2680 /* Called via drm-managed action */
2681 void intel_uncore_fini_mmio(struct drm_device *dev, void *data)
2682 {
2683         struct intel_uncore *uncore = data;
2684
2685         if (intel_uncore_has_forcewake(uncore)) {
2686                 iosf_mbi_punit_acquire();
2687                 iosf_mbi_unregister_pmic_bus_access_notifier_unlocked(
2688                         &uncore->pmic_bus_access_nb);
2689                 intel_uncore_forcewake_reset(uncore);
2690                 intel_uncore_fw_domains_fini(uncore);
2691                 iosf_mbi_punit_release();
2692         }
2693
2694         if (intel_uncore_needs_flr_on_fini(uncore))
2695                 driver_initiated_flr(uncore);
2696 }
2697
2698 /**
2699  * __intel_wait_for_register_fw - wait until register matches expected state
2700  * @uncore: the struct intel_uncore
2701  * @reg: the register to read
2702  * @mask: mask to apply to register value
2703  * @value: expected value
2704  * @fast_timeout_us: fast timeout in microsecond for atomic/tight wait
2705  * @slow_timeout_ms: slow timeout in millisecond
2706  * @out_value: optional placeholder to hold registry value
2707  *
2708  * This routine waits until the target register @reg contains the expected
2709  * @value after applying the @mask, i.e. it waits until ::
2710  *
2711  *     (intel_uncore_read_fw(uncore, reg) & mask) == value
2712  *
2713  * Otherwise, the wait will timeout after @slow_timeout_ms milliseconds.
2714  * For atomic context @slow_timeout_ms must be zero and @fast_timeout_us
2715  * must be not larger than 20,0000 microseconds.
2716  *
2717  * Note that this routine assumes the caller holds forcewake asserted, it is
2718  * not suitable for very long waits. See intel_wait_for_register() if you
2719  * wish to wait without holding forcewake for the duration (i.e. you expect
2720  * the wait to be slow).
2721  *
2722  * Return: 0 if the register matches the desired condition, or -ETIMEDOUT.
2723  */
2724 int __intel_wait_for_register_fw(struct intel_uncore *uncore,
2725                                  i915_reg_t reg,
2726                                  u32 mask,
2727                                  u32 value,
2728                                  unsigned int fast_timeout_us,
2729                                  unsigned int slow_timeout_ms,
2730                                  u32 *out_value)
2731 {
2732         u32 reg_value = 0;
2733 #define done (((reg_value = intel_uncore_read_fw(uncore, reg)) & mask) == value)
2734         int ret;
2735
2736         /* Catch any overuse of this function */
2737         might_sleep_if(slow_timeout_ms);
2738         GEM_BUG_ON(fast_timeout_us > 20000);
2739         GEM_BUG_ON(!fast_timeout_us && !slow_timeout_ms);
2740
2741         ret = -ETIMEDOUT;
2742         if (fast_timeout_us && fast_timeout_us <= 20000)
2743                 ret = _wait_for_atomic(done, fast_timeout_us, 0);
2744         if (ret && slow_timeout_ms)
2745                 ret = wait_for(done, slow_timeout_ms);
2746
2747         if (out_value)
2748                 *out_value = reg_value;
2749
2750         return ret;
2751 #undef done
2752 }
2753
2754 /**
2755  * __intel_wait_for_register - wait until register matches expected state
2756  * @uncore: the struct intel_uncore
2757  * @reg: the register to read
2758  * @mask: mask to apply to register value
2759  * @value: expected value
2760  * @fast_timeout_us: fast timeout in microsecond for atomic/tight wait
2761  * @slow_timeout_ms: slow timeout in millisecond
2762  * @out_value: optional placeholder to hold registry value
2763  *
2764  * This routine waits until the target register @reg contains the expected
2765  * @value after applying the @mask, i.e. it waits until ::
2766  *
2767  *     (intel_uncore_read(uncore, reg) & mask) == value
2768  *
2769  * Otherwise, the wait will timeout after @timeout_ms milliseconds.
2770  *
2771  * Return: 0 if the register matches the desired condition, or -ETIMEDOUT.
2772  */
2773 int __intel_wait_for_register(struct intel_uncore *uncore,
2774                               i915_reg_t reg,
2775                               u32 mask,
2776                               u32 value,
2777                               unsigned int fast_timeout_us,
2778                               unsigned int slow_timeout_ms,
2779                               u32 *out_value)
2780 {
2781         unsigned fw =
2782                 intel_uncore_forcewake_for_reg(uncore, reg, FW_REG_READ);
2783         u32 reg_value;
2784         int ret;
2785
2786         might_sleep_if(slow_timeout_ms);
2787
2788         spin_lock_irq(&uncore->lock);
2789         intel_uncore_forcewake_get__locked(uncore, fw);
2790
2791         ret = __intel_wait_for_register_fw(uncore,
2792                                            reg, mask, value,
2793                                            fast_timeout_us, 0, &reg_value);
2794
2795         intel_uncore_forcewake_put__locked(uncore, fw);
2796         spin_unlock_irq(&uncore->lock);
2797
2798         if (ret && slow_timeout_ms)
2799                 ret = __wait_for(reg_value = intel_uncore_read_notrace(uncore,
2800                                                                        reg),
2801                                  (reg_value & mask) == value,
2802                                  slow_timeout_ms * 1000, 10, 1000);
2803
2804         /* just trace the final value */
2805         trace_i915_reg_rw(false, reg, reg_value, sizeof(reg_value), true);
2806
2807         if (out_value)
2808                 *out_value = reg_value;
2809
2810         return ret;
2811 }
2812
2813 bool intel_uncore_unclaimed_mmio(struct intel_uncore *uncore)
2814 {
2815         bool ret;
2816
2817         if (!uncore->debug)
2818                 return false;
2819
2820         spin_lock_irq(&uncore->debug->lock);
2821         ret = check_for_unclaimed_mmio(uncore);
2822         spin_unlock_irq(&uncore->debug->lock);
2823
2824         return ret;
2825 }
2826
2827 bool
2828 intel_uncore_arm_unclaimed_mmio_detection(struct intel_uncore *uncore)
2829 {
2830         bool ret = false;
2831
2832         if (drm_WARN_ON(&uncore->i915->drm, !uncore->debug))
2833                 return false;
2834
2835         spin_lock_irq(&uncore->debug->lock);
2836
2837         if (unlikely(uncore->debug->unclaimed_mmio_check <= 0))
2838                 goto out;
2839
2840         if (unlikely(check_for_unclaimed_mmio(uncore))) {
2841                 if (!uncore->i915->params.mmio_debug) {
2842                         drm_dbg(&uncore->i915->drm,
2843                                 "Unclaimed register detected, "
2844                                 "enabling oneshot unclaimed register reporting. "
2845                                 "Please use i915.mmio_debug=N for more information.\n");
2846                         uncore->i915->params.mmio_debug++;
2847                 }
2848                 uncore->debug->unclaimed_mmio_check--;
2849                 ret = true;
2850         }
2851
2852 out:
2853         spin_unlock_irq(&uncore->debug->lock);
2854
2855         return ret;
2856 }
2857
2858 /**
2859  * intel_uncore_forcewake_for_reg - which forcewake domains are needed to access
2860  *                                  a register
2861  * @uncore: pointer to struct intel_uncore
2862  * @reg: register in question
2863  * @op: operation bitmask of FW_REG_READ and/or FW_REG_WRITE
2864  *
2865  * Returns a set of forcewake domains required to be taken with for example
2866  * intel_uncore_forcewake_get for the specified register to be accessible in the
2867  * specified mode (read, write or read/write) with raw mmio accessors.
2868  *
2869  * NOTE: On Gen6 and Gen7 write forcewake domain (FORCEWAKE_RENDER) requires the
2870  * callers to do FIFO management on their own or risk losing writes.
2871  */
2872 enum forcewake_domains
2873 intel_uncore_forcewake_for_reg(struct intel_uncore *uncore,
2874                                i915_reg_t reg, unsigned int op)
2875 {
2876         enum forcewake_domains fw_domains = 0;
2877
2878         drm_WARN_ON(&uncore->i915->drm, !op);
2879
2880         if (!intel_uncore_has_forcewake(uncore))
2881                 return 0;
2882
2883         if (op & FW_REG_READ)
2884                 fw_domains = uncore->funcs.read_fw_domains(uncore, reg);
2885
2886         if (op & FW_REG_WRITE)
2887                 fw_domains |= uncore->funcs.write_fw_domains(uncore, reg);
2888
2889         drm_WARN_ON(&uncore->i915->drm, fw_domains & ~uncore->fw_domains);
2890
2891         return fw_domains;
2892 }
2893
2894 #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
2895 #include "selftests/mock_uncore.c"
2896 #include "selftests/intel_uncore.c"
2897 #endif