drivers/gpu/drm/i915/intel_uncore.c

   1 /*
   2  * Copyright © 2013 Intel Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21  * IN THE SOFTWARE.
  22  */
  23
  24 #include <drm/drm_managed.h>
  25 #include <linux/pm_runtime.h>
  26
  27 #include "gt/intel_gt.h"
  28 #include "gt/intel_engine_regs.h"
  29 #include "gt/intel_gt_regs.h"
  30
  31 #include "i915_drv.h"
  32 #include "i915_iosf_mbi.h"
  33 #include "i915_reg.h"
  34 #include "i915_trace.h"
  35 #include "i915_vgpu.h"
  36
  37 #define FORCEWAKE_ACK_TIMEOUT_MS 50
  38 #define GT_FIFO_TIMEOUT_MS       10
  39
  40 #define __raw_posting_read(...) ((void)__raw_uncore_read32(__VA_ARGS__))
  41
  42 static void
  43 fw_domains_get(struct intel_uncore *uncore, enum forcewake_domains fw_domains)
  44 {
  45         uncore->fw_get_funcs->force_wake_get(uncore, fw_domains);
  46 }
  47
  48 void
  49 intel_uncore_mmio_debug_init_early(struct drm_i915_private *i915)
  50 {
  51         spin_lock_init(&i915->mmio_debug.lock);
  52         i915->mmio_debug.unclaimed_mmio_check = 1;
  53
  54         i915->uncore.debug = &i915->mmio_debug;
  55 }
  56
  57 static void mmio_debug_suspend(struct intel_uncore *uncore)
  58 {
  59         if (!uncore->debug)
  60                 return;
  61
  62         spin_lock(&uncore->debug->lock);
  63
  64         /* Save and disable mmio debugging for the user bypass */
  65         if (!uncore->debug->suspend_count++) {
  66                 uncore->debug->saved_mmio_check = uncore->debug->unclaimed_mmio_check;
  67                 uncore->debug->unclaimed_mmio_check = 0;
  68         }
  69
  70         spin_unlock(&uncore->debug->lock);
  71 }
  72
  73 static bool check_for_unclaimed_mmio(struct intel_uncore *uncore);
  74
  75 static void mmio_debug_resume(struct intel_uncore *uncore)
  76 {
  77         if (!uncore->debug)
  78                 return;
  79
  80         spin_lock(&uncore->debug->lock);
  81
  82         if (!--uncore->debug->suspend_count)
  83                 uncore->debug->unclaimed_mmio_check = uncore->debug->saved_mmio_check;
  84
  85         if (check_for_unclaimed_mmio(uncore))
  86                 drm_info(&uncore->i915->drm,
  87                          "Invalid mmio detected during user access\n");
  88
  89         spin_unlock(&uncore->debug->lock);
  90 }
  91
  92 static const char * const forcewake_domain_names[] = {
  93         "render",
  94         "gt",
  95         "media",
  96         "vdbox0",
  97         "vdbox1",
  98         "vdbox2",
  99         "vdbox3",
 100         "vdbox4",
 101         "vdbox5",
 102         "vdbox6",
 103         "vdbox7",
 104         "vebox0",
 105         "vebox1",
 106         "vebox2",
 107         "vebox3",
 108         "gsc",
 109 };
 110
 111 const char *
 112 intel_uncore_forcewake_domain_to_str(const enum forcewake_domain_id id)
 113 {
 114         BUILD_BUG_ON(ARRAY_SIZE(forcewake_domain_names) != FW_DOMAIN_ID_COUNT);
 115
 116         if (id >= 0 && id < FW_DOMAIN_ID_COUNT)
 117                 return forcewake_domain_names[id];
 118
 119         WARN_ON(id);
 120
 121         return "unknown";
 122 }
 123
 124 #define fw_ack(d) readl((d)->reg_ack)
 125 #define fw_set(d, val) writel(_MASKED_BIT_ENABLE((val)), (d)->reg_set)
 126 #define fw_clear(d, val) writel(_MASKED_BIT_DISABLE((val)), (d)->reg_set)
 127
 128 static inline void
 129 fw_domain_reset(const struct intel_uncore_forcewake_domain *d)
 130 {
 131         /*
 132          * We don't really know if the powerwell for the forcewake domain we are
 133          * trying to reset here does exist at this point (engines could be fused
 134          * off in ICL+), so no waiting for acks
 135          */
 136         /* WaRsClearFWBitsAtReset */
 137         if (GRAPHICS_VER(d->uncore->i915) >= 12)
 138                 fw_clear(d, 0xefff);
 139         else
 140                 fw_clear(d, 0xffff);
 141 }
 142
 143 static inline void
 144 fw_domain_arm_timer(struct intel_uncore_forcewake_domain *d)
 145 {
 146         GEM_BUG_ON(d->uncore->fw_domains_timer & d->mask);
 147         d->uncore->fw_domains_timer |= d->mask;
 148         d->wake_count++;
 149         hrtimer_start_range_ns(&d->timer,
 150                                NSEC_PER_MSEC,
 151                                NSEC_PER_MSEC,
 152                                HRTIMER_MODE_REL);
 153 }
 154
 155 static inline int
 156 __wait_for_ack(const struct intel_uncore_forcewake_domain *d,
 157                const u32 ack,
 158                const u32 value)
 159 {
 160         return wait_for_atomic((fw_ack(d) & ack) == value,
 161                                FORCEWAKE_ACK_TIMEOUT_MS);
 162 }
 163
 164 static inline int
 165 wait_ack_clear(const struct intel_uncore_forcewake_domain *d,
 166                const u32 ack)
 167 {
 168         return __wait_for_ack(d, ack, 0);
 169 }
 170
 171 static inline int
 172 wait_ack_set(const struct intel_uncore_forcewake_domain *d,
 173              const u32 ack)
 174 {
 175         return __wait_for_ack(d, ack, ack);
 176 }
 177
 178 static inline void
 179 fw_domain_wait_ack_clear(const struct intel_uncore_forcewake_domain *d)
 180 {
 181         if (!wait_ack_clear(d, FORCEWAKE_KERNEL))
 182                 return;
 183
 184         if (fw_ack(d) == ~0) {
 185                 drm_err(&d->uncore->i915->drm,
 186                         "%s: MMIO unreliable (forcewake register returns 0xFFFFFFFF)!\n",
 187                         intel_uncore_forcewake_domain_to_str(d->id));
 188                 intel_gt_set_wedged_async(d->uncore->gt);
 189         } else {
 190                 drm_err(&d->uncore->i915->drm,
 191                         "%s: timed out waiting for forcewake ack to clear.\n",
 192                         intel_uncore_forcewake_domain_to_str(d->id));
 193         }
 194
 195         add_taint_for_CI(d->uncore->i915, TAINT_WARN); /* CI now unreliable */
 196 }
 197
 198 enum ack_type {
 199         ACK_CLEAR = 0,
 200         ACK_SET
 201 };
 202
 203 static int
 204 fw_domain_wait_ack_with_fallback(const struct intel_uncore_forcewake_domain *d,
 205                                  const enum ack_type type)
 206 {
 207         const u32 ack_bit = FORCEWAKE_KERNEL;
 208         const u32 value = type == ACK_SET ? ack_bit : 0;
 209         unsigned int pass;
 210         bool ack_detected;
 211
 212         /*
 213          * There is a possibility of driver's wake request colliding
 214          * with hardware's own wake requests and that can cause
 215          * hardware to not deliver the driver's ack message.
 216          *
 217          * Use a fallback bit toggle to kick the gpu state machine
 218          * in the hope that the original ack will be delivered along with
 219          * the fallback ack.
 220          *
 221          * This workaround is described in HSDES #1604254524 and it's known as:
 222          * WaRsForcewakeAddDelayForAck:skl,bxt,kbl,glk,cfl,cnl,icl
 223          * although the name is a bit misleading.
 224          */
 225
 226         pass = 1;
 227         do {
 228                 wait_ack_clear(d, FORCEWAKE_KERNEL_FALLBACK);
 229
 230                 fw_set(d, FORCEWAKE_KERNEL_FALLBACK);
 231                 /* Give gt some time to relax before the polling frenzy */
 232                 udelay(10 * pass);
 233                 wait_ack_set(d, FORCEWAKE_KERNEL_FALLBACK);
 234
 235                 ack_detected = (fw_ack(d) & ack_bit) == value;
 236
 237                 fw_clear(d, FORCEWAKE_KERNEL_FALLBACK);
 238         } while (!ack_detected && pass++ < 10);
 239
 240         drm_dbg(&d->uncore->i915->drm,
 241                 "%s had to use fallback to %s ack, 0x%x (passes %u)\n",
 242                 intel_uncore_forcewake_domain_to_str(d->id),
 243                 type == ACK_SET ? "set" : "clear",
 244                 fw_ack(d),
 245                 pass);
 246
 247         return ack_detected ? 0 : -ETIMEDOUT;
 248 }
 249
 250 static inline void
 251 fw_domain_wait_ack_clear_fallback(const struct intel_uncore_forcewake_domain *d)
 252 {
 253         if (likely(!wait_ack_clear(d, FORCEWAKE_KERNEL)))
 254                 return;
 255
 256         if (fw_domain_wait_ack_with_fallback(d, ACK_CLEAR))
 257                 fw_domain_wait_ack_clear(d);
 258 }
 259
 260 static inline void
 261 fw_domain_get(const struct intel_uncore_forcewake_domain *d)
 262 {
 263         fw_set(d, FORCEWAKE_KERNEL);
 264 }
 265
 266 static inline void
 267 fw_domain_wait_ack_set(const struct intel_uncore_forcewake_domain *d)
 268 {
 269         if (wait_ack_set(d, FORCEWAKE_KERNEL)) {
 270                 drm_err(&d->uncore->i915->drm,
 271                         "%s: timed out waiting for forcewake ack request.\n",
 272                         intel_uncore_forcewake_domain_to_str(d->id));
 273                 add_taint_for_CI(d->uncore->i915, TAINT_WARN); /* CI now unreliable */
 274         }
 275 }
 276
 277 static inline void
 278 fw_domain_wait_ack_set_fallback(const struct intel_uncore_forcewake_domain *d)
 279 {
 280         if (likely(!wait_ack_set(d, FORCEWAKE_KERNEL)))
 281                 return;
 282
 283         if (fw_domain_wait_ack_with_fallback(d, ACK_SET))
 284                 fw_domain_wait_ack_set(d);
 285 }
 286
 287 static inline void
 288 fw_domain_put(const struct intel_uncore_forcewake_domain *d)
 289 {
 290         fw_clear(d, FORCEWAKE_KERNEL);
 291 }
 292
 293 static void
 294 fw_domains_get_normal(struct intel_uncore *uncore, enum forcewake_domains fw_domains)
 295 {
 296         struct intel_uncore_forcewake_domain *d;
 297         unsigned int tmp;
 298
 299         GEM_BUG_ON(fw_domains & ~uncore->fw_domains);
 300
 301         for_each_fw_domain_masked(d, fw_domains, uncore, tmp) {
 302                 fw_domain_wait_ack_clear(d);
 303                 fw_domain_get(d);
 304         }
 305
 306         for_each_fw_domain_masked(d, fw_domains, uncore, tmp)
 307                 fw_domain_wait_ack_set(d);
 308
 309         uncore->fw_domains_active |= fw_domains;
 310 }
 311
 312 static void
 313 fw_domains_get_with_fallback(struct intel_uncore *uncore,
 314                              enum forcewake_domains fw_domains)
 315 {
 316         struct intel_uncore_forcewake_domain *d;
 317         unsigned int tmp;
 318
 319         GEM_BUG_ON(fw_domains & ~uncore->fw_domains);
 320
 321         for_each_fw_domain_masked(d, fw_domains, uncore, tmp) {
 322                 fw_domain_wait_ack_clear_fallback(d);
 323                 fw_domain_get(d);
 324         }
 325
 326         for_each_fw_domain_masked(d, fw_domains, uncore, tmp)
 327                 fw_domain_wait_ack_set_fallback(d);
 328
 329         uncore->fw_domains_active |= fw_domains;
 330 }
 331
 332 static void
 333 fw_domains_put(struct intel_uncore *uncore, enum forcewake_domains fw_domains)
 334 {
 335         struct intel_uncore_forcewake_domain *d;
 336         unsigned int tmp;
 337
 338         GEM_BUG_ON(fw_domains & ~uncore->fw_domains);
 339
 340         for_each_fw_domain_masked(d, fw_domains, uncore, tmp)
 341                 fw_domain_put(d);
 342
 343         uncore->fw_domains_active &= ~fw_domains;
 344 }
 345
 346 static void
 347 fw_domains_reset(struct intel_uncore *uncore,
 348                  enum forcewake_domains fw_domains)
 349 {
 350         struct intel_uncore_forcewake_domain *d;
 351         unsigned int tmp;
 352
 353         if (!fw_domains)
 354                 return;
 355
 356         GEM_BUG_ON(fw_domains & ~uncore->fw_domains);
 357
 358         for_each_fw_domain_masked(d, fw_domains, uncore, tmp)
 359                 fw_domain_reset(d);
 360 }
 361
 362 static inline u32 gt_thread_status(struct intel_uncore *uncore)
 363 {
 364         u32 val;
 365
 366         val = __raw_uncore_read32(uncore, GEN6_GT_THREAD_STATUS_REG);
 367         val &= GEN6_GT_THREAD_STATUS_CORE_MASK;
 368
 369         return val;
 370 }
 371
 372 static void __gen6_gt_wait_for_thread_c0(struct intel_uncore *uncore)
 373 {
 374         /*
 375          * w/a for a sporadic read returning 0 by waiting for the GT
 376          * thread to wake up.
 377          */
 378         drm_WARN_ONCE(&uncore->i915->drm,
 379                       wait_for_atomic_us(gt_thread_status(uncore) == 0, 5000),
 380                       "GT thread status wait timed out\n");
 381 }
 382
 383 static void fw_domains_get_with_thread_status(struct intel_uncore *uncore,
 384                                               enum forcewake_domains fw_domains)
 385 {
 386         fw_domains_get_normal(uncore, fw_domains);
 387
 388         /* WaRsForcewakeWaitTC0:snb,ivb,hsw,bdw,vlv */
 389         __gen6_gt_wait_for_thread_c0(uncore);
 390 }
 391
 392 static inline u32 fifo_free_entries(struct intel_uncore *uncore)
 393 {
 394         u32 count = __raw_uncore_read32(uncore, GTFIFOCTL);
 395
 396         return count & GT_FIFO_FREE_ENTRIES_MASK;
 397 }
 398
 399 static void __gen6_gt_wait_for_fifo(struct intel_uncore *uncore)
 400 {
 401         u32 n;
 402
 403         /* On VLV, FIFO will be shared by both SW and HW.
 404          * So, we need to read the FREE_ENTRIES everytime */
 405         if (IS_VALLEYVIEW(uncore->i915))
 406                 n = fifo_free_entries(uncore);
 407         else
 408                 n = uncore->fifo_count;
 409
 410         if (n <= GT_FIFO_NUM_RESERVED_ENTRIES) {
 411                 if (wait_for_atomic((n = fifo_free_entries(uncore)) >
 412                                     GT_FIFO_NUM_RESERVED_ENTRIES,
 413                                     GT_FIFO_TIMEOUT_MS)) {
 414                         drm_dbg(&uncore->i915->drm,
 415                                 "GT_FIFO timeout, entries: %u\n", n);
 416                         return;
 417                 }
 418         }
 419
 420         uncore->fifo_count = n - 1;
 421 }
 422
 423 static enum hrtimer_restart
 424 intel_uncore_fw_release_timer(struct hrtimer *timer)
 425 {
 426         struct intel_uncore_forcewake_domain *domain =
 427                container_of(timer, struct intel_uncore_forcewake_domain, timer);
 428         struct intel_uncore *uncore = domain->uncore;
 429         unsigned long irqflags;
 430
 431         assert_rpm_device_not_suspended(uncore->rpm);
 432
 433         if (xchg(&domain->active, false))
 434                 return HRTIMER_RESTART;
 435
 436         spin_lock_irqsave(&uncore->lock, irqflags);
 437
 438         uncore->fw_domains_timer &= ~domain->mask;
 439
 440         GEM_BUG_ON(!domain->wake_count);
 441         if (--domain->wake_count == 0)
 442                 fw_domains_put(uncore, domain->mask);
 443
 444         spin_unlock_irqrestore(&uncore->lock, irqflags);
 445
 446         return HRTIMER_NORESTART;
 447 }
 448
 449 /* Note callers must have acquired the PUNIT->PMIC bus, before calling this. */
 450 static unsigned int
 451 intel_uncore_forcewake_reset(struct intel_uncore *uncore)
 452 {
 453         unsigned long irqflags;
 454         struct intel_uncore_forcewake_domain *domain;
 455         int retry_count = 100;
 456         enum forcewake_domains fw, active_domains;
 457
 458         iosf_mbi_assert_punit_acquired();
 459
 460         /* Hold uncore.lock across reset to prevent any register access
 461          * with forcewake not set correctly. Wait until all pending
 462          * timers are run before holding.
 463          */
 464         while (1) {
 465                 unsigned int tmp;
 466
 467                 active_domains = 0;
 468
 469                 for_each_fw_domain(domain, uncore, tmp) {
 470                         smp_store_mb(domain->active, false);
 471                         if (hrtimer_cancel(&domain->timer) == 0)
 472                                 continue;
 473
 474                         intel_uncore_fw_release_timer(&domain->timer);
 475                 }
 476
 477                 spin_lock_irqsave(&uncore->lock, irqflags);
 478
 479                 for_each_fw_domain(domain, uncore, tmp) {
 480                         if (hrtimer_active(&domain->timer))
 481                                 active_domains |= domain->mask;
 482                 }
 483
 484                 if (active_domains == 0)
 485                         break;
 486
 487                 if (--retry_count == 0) {
 488                         drm_err(&uncore->i915->drm, "Timed out waiting for forcewake timers to finish\n");
 489                         break;
 490                 }
 491
 492                 spin_unlock_irqrestore(&uncore->lock, irqflags);
 493                 cond_resched();
 494         }
 495
 496         drm_WARN_ON(&uncore->i915->drm, active_domains);
 497
 498         fw = uncore->fw_domains_active;
 499         if (fw)
 500                 fw_domains_put(uncore, fw);
 501
 502         fw_domains_reset(uncore, uncore->fw_domains);
 503         assert_forcewakes_inactive(uncore);
 504
 505         spin_unlock_irqrestore(&uncore->lock, irqflags);
 506
 507         return fw; /* track the lost user forcewake domains */
 508 }
 509
 510 static bool
 511 fpga_check_for_unclaimed_mmio(struct intel_uncore *uncore)
 512 {
 513         u32 dbg;
 514
 515         dbg = __raw_uncore_read32(uncore, FPGA_DBG);
 516         if (likely(!(dbg & FPGA_DBG_RM_NOCLAIM)))
 517                 return false;
 518
 519         /*
 520          * Bugs in PCI programming (or failing hardware) can occasionally cause
 521          * us to lose access to the MMIO BAR.  When this happens, register
 522          * reads will come back with 0xFFFFFFFF for every register and things
 523          * go bad very quickly.  Let's try to detect that special case and at
 524          * least try to print a more informative message about what has
 525          * happened.
 526          *
 527          * During normal operation the FPGA_DBG register has several unused
 528          * bits that will always read back as 0's so we can use them as canaries
 529          * to recognize when MMIO accesses are just busted.
 530          */
 531         if (unlikely(dbg == ~0))
 532                 drm_err(&uncore->i915->drm,
 533                         "Lost access to MMIO BAR; all registers now read back as 0xFFFFFFFF!\n");
 534
 535         __raw_uncore_write32(uncore, FPGA_DBG, FPGA_DBG_RM_NOCLAIM);
 536
 537         return true;
 538 }
 539
 540 static bool
 541 vlv_check_for_unclaimed_mmio(struct intel_uncore *uncore)
 542 {
 543         u32 cer;
 544
 545         cer = __raw_uncore_read32(uncore, CLAIM_ER);
 546         if (likely(!(cer & (CLAIM_ER_OVERFLOW | CLAIM_ER_CTR_MASK))))
 547                 return false;
 548
 549         __raw_uncore_write32(uncore, CLAIM_ER, CLAIM_ER_CLR);
 550
 551         return true;
 552 }
 553
 554 static bool
 555 gen6_check_for_fifo_debug(struct intel_uncore *uncore)
 556 {
 557         u32 fifodbg;
 558
 559         fifodbg = __raw_uncore_read32(uncore, GTFIFODBG);
 560
 561         if (unlikely(fifodbg)) {
 562                 drm_dbg(&uncore->i915->drm, "GTFIFODBG = 0x08%x\n", fifodbg);
 563                 __raw_uncore_write32(uncore, GTFIFODBG, fifodbg);
 564         }
 565
 566         return fifodbg;
 567 }
 568
 569 static bool
 570 check_for_unclaimed_mmio(struct intel_uncore *uncore)
 571 {
 572         bool ret = false;
 573
 574         lockdep_assert_held(&uncore->debug->lock);
 575
 576         if (uncore->debug->suspend_count)
 577                 return false;
 578
 579         if (intel_uncore_has_fpga_dbg_unclaimed(uncore))
 580                 ret |= fpga_check_for_unclaimed_mmio(uncore);
 581
 582         if (intel_uncore_has_dbg_unclaimed(uncore))
 583                 ret |= vlv_check_for_unclaimed_mmio(uncore);
 584
 585         if (intel_uncore_has_fifo(uncore))
 586                 ret |= gen6_check_for_fifo_debug(uncore);
 587
 588         return ret;
 589 }
 590
 591 static void forcewake_early_sanitize(struct intel_uncore *uncore,
 592                                      unsigned int restore_forcewake)
 593 {
 594         GEM_BUG_ON(!intel_uncore_has_forcewake(uncore));
 595
 596         /* WaDisableShadowRegForCpd:chv */
 597         if (IS_CHERRYVIEW(uncore->i915)) {
 598                 __raw_uncore_write32(uncore, GTFIFOCTL,
 599                                      __raw_uncore_read32(uncore, GTFIFOCTL) |
 600                                      GT_FIFO_CTL_BLOCK_ALL_POLICY_STALL |
 601                                      GT_FIFO_CTL_RC6_POLICY_STALL);
 602         }
 603
 604         iosf_mbi_punit_acquire();
 605         intel_uncore_forcewake_reset(uncore);
 606         if (restore_forcewake) {
 607                 spin_lock_irq(&uncore->lock);
 608                 fw_domains_get(uncore, restore_forcewake);
 609
 610                 if (intel_uncore_has_fifo(uncore))
 611                         uncore->fifo_count = fifo_free_entries(uncore);
 612                 spin_unlock_irq(&uncore->lock);
 613         }
 614         iosf_mbi_punit_release();
 615 }
 616
 617 void intel_uncore_suspend(struct intel_uncore *uncore)
 618 {
 619         if (!intel_uncore_has_forcewake(uncore))
 620                 return;
 621
 622         iosf_mbi_punit_acquire();
 623         iosf_mbi_unregister_pmic_bus_access_notifier_unlocked(
 624                 &uncore->pmic_bus_access_nb);
 625         uncore->fw_domains_saved = intel_uncore_forcewake_reset(uncore);
 626         iosf_mbi_punit_release();
 627 }
 628
 629 void intel_uncore_resume_early(struct intel_uncore *uncore)
 630 {
 631         unsigned int restore_forcewake;
 632
 633         if (intel_uncore_unclaimed_mmio(uncore))
 634                 drm_dbg(&uncore->i915->drm, "unclaimed mmio detected on resume, clearing\n");
 635
 636         if (!intel_uncore_has_forcewake(uncore))
 637                 return;
 638
 639         restore_forcewake = fetch_and_zero(&uncore->fw_domains_saved);
 640         forcewake_early_sanitize(uncore, restore_forcewake);
 641
 642         iosf_mbi_register_pmic_bus_access_notifier(&uncore->pmic_bus_access_nb);
 643 }
 644
 645 void intel_uncore_runtime_resume(struct intel_uncore *uncore)
 646 {
 647         if (!intel_uncore_has_forcewake(uncore))
 648                 return;
 649
 650         iosf_mbi_register_pmic_bus_access_notifier(&uncore->pmic_bus_access_nb);
 651 }
 652
 653 static void __intel_uncore_forcewake_get(struct intel_uncore *uncore,
 654                                          enum forcewake_domains fw_domains)
 655 {
 656         struct intel_uncore_forcewake_domain *domain;
 657         unsigned int tmp;
 658
 659         fw_domains &= uncore->fw_domains;
 660
 661         for_each_fw_domain_masked(domain, fw_domains, uncore, tmp) {
 662                 if (domain->wake_count++) {
 663                         fw_domains &= ~domain->mask;
 664                         domain->active = true;
 665                 }
 666         }
 667
 668         if (fw_domains)
 669                 fw_domains_get(uncore, fw_domains);
 670 }
 671
 672 /**
 673  * intel_uncore_forcewake_get - grab forcewake domain references
 674  * @uncore: the intel_uncore structure
 675  * @fw_domains: forcewake domains to get reference on
 676  *
 677  * This function can be used get GT's forcewake domain references.
 678  * Normal register access will handle the forcewake domains automatically.
 679  * However if some sequence requires the GT to not power down a particular
 680  * forcewake domains this function should be called at the beginning of the
 681  * sequence. And subsequently the reference should be dropped by symmetric
 682  * call to intel_unforce_forcewake_put(). Usually caller wants all the domains
 683  * to be kept awake so the @fw_domains would be then FORCEWAKE_ALL.
 684  */
 685 void intel_uncore_forcewake_get(struct intel_uncore *uncore,
 686                                 enum forcewake_domains fw_domains)
 687 {
 688         unsigned long irqflags;
 689
 690         if (!uncore->fw_get_funcs)
 691                 return;
 692
 693         assert_rpm_wakelock_held(uncore->rpm);
 694
 695         spin_lock_irqsave(&uncore->lock, irqflags);
 696         __intel_uncore_forcewake_get(uncore, fw_domains);
 697         spin_unlock_irqrestore(&uncore->lock, irqflags);
 698 }
 699
 700 /**
 701  * intel_uncore_forcewake_user_get - claim forcewake on behalf of userspace
 702  * @uncore: the intel_uncore structure
 703  *
 704  * This function is a wrapper around intel_uncore_forcewake_get() to acquire
 705  * the GT powerwell and in the process disable our debugging for the
 706  * duration of userspace's bypass.
 707  */
 708 void intel_uncore_forcewake_user_get(struct intel_uncore *uncore)
 709 {
 710         spin_lock_irq(&uncore->lock);
 711         if (!uncore->user_forcewake_count++) {
 712                 intel_uncore_forcewake_get__locked(uncore, FORCEWAKE_ALL);
 713                 mmio_debug_suspend(uncore);
 714         }
 715         spin_unlock_irq(&uncore->lock);
 716 }
 717
 718 /**
 719  * intel_uncore_forcewake_user_put - release forcewake on behalf of userspace
 720  * @uncore: the intel_uncore structure
 721  *
 722  * This function complements intel_uncore_forcewake_user_get() and releases
 723  * the GT powerwell taken on behalf of the userspace bypass.
 724  */
 725 void intel_uncore_forcewake_user_put(struct intel_uncore *uncore)
 726 {
 727         spin_lock_irq(&uncore->lock);
 728         if (!--uncore->user_forcewake_count) {
 729                 mmio_debug_resume(uncore);
 730                 intel_uncore_forcewake_put__locked(uncore, FORCEWAKE_ALL);
 731         }
 732         spin_unlock_irq(&uncore->lock);
 733 }
 734
 735 /**
 736  * intel_uncore_forcewake_get__locked - grab forcewake domain references
 737  * @uncore: the intel_uncore structure
 738  * @fw_domains: forcewake domains to get reference on
 739  *
 740  * See intel_uncore_forcewake_get(). This variant places the onus
 741  * on the caller to explicitly handle the dev_priv->uncore.lock spinlock.
 742  */
 743 void intel_uncore_forcewake_get__locked(struct intel_uncore *uncore,
 744                                         enum forcewake_domains fw_domains)
 745 {
 746         lockdep_assert_held(&uncore->lock);
 747
 748         if (!uncore->fw_get_funcs)
 749                 return;
 750
 751         __intel_uncore_forcewake_get(uncore, fw_domains);
 752 }
 753
 754 static void __intel_uncore_forcewake_put(struct intel_uncore *uncore,
 755                                          enum forcewake_domains fw_domains,
 756                                          bool delayed)
 757 {
 758         struct intel_uncore_forcewake_domain *domain;
 759         unsigned int tmp;
 760
 761         fw_domains &= uncore->fw_domains;
 762
 763         for_each_fw_domain_masked(domain, fw_domains, uncore, tmp) {
 764                 GEM_BUG_ON(!domain->wake_count);
 765
 766                 if (--domain->wake_count) {
 767                         domain->active = true;
 768                         continue;
 769                 }
 770
 771                 if (delayed &&
 772                     !(domain->uncore->fw_domains_timer & domain->mask))
 773                         fw_domain_arm_timer(domain);
 774                 else
 775                         fw_domains_put(uncore, domain->mask);
 776         }
 777 }
 778
 779 /**
 780  * intel_uncore_forcewake_put - release a forcewake domain reference
 781  * @uncore: the intel_uncore structure
 782  * @fw_domains: forcewake domains to put references
 783  *
 784  * This function drops the device-level forcewakes for specified
 785  * domains obtained by intel_uncore_forcewake_get().
 786  */
 787 void intel_uncore_forcewake_put(struct intel_uncore *uncore,
 788                                 enum forcewake_domains fw_domains)
 789 {
 790         unsigned long irqflags;
 791
 792         if (!uncore->fw_get_funcs)
 793                 return;
 794
 795         spin_lock_irqsave(&uncore->lock, irqflags);
 796         __intel_uncore_forcewake_put(uncore, fw_domains, false);
 797         spin_unlock_irqrestore(&uncore->lock, irqflags);
 798 }
 799
 800 void intel_uncore_forcewake_put_delayed(struct intel_uncore *uncore,
 801                                         enum forcewake_domains fw_domains)
 802 {
 803         unsigned long irqflags;
 804
 805         if (!uncore->fw_get_funcs)
 806                 return;
 807
 808         spin_lock_irqsave(&uncore->lock, irqflags);
 809         __intel_uncore_forcewake_put(uncore, fw_domains, true);
 810         spin_unlock_irqrestore(&uncore->lock, irqflags);
 811 }
 812
 813 /**
 814  * intel_uncore_forcewake_flush - flush the delayed release
 815  * @uncore: the intel_uncore structure
 816  * @fw_domains: forcewake domains to flush
 817  */
 818 void intel_uncore_forcewake_flush(struct intel_uncore *uncore,
 819                                   enum forcewake_domains fw_domains)
 820 {
 821         struct intel_uncore_forcewake_domain *domain;
 822         unsigned int tmp;
 823
 824         if (!uncore->fw_get_funcs)
 825                 return;
 826
 827         fw_domains &= uncore->fw_domains;
 828         for_each_fw_domain_masked(domain, fw_domains, uncore, tmp) {
 829                 WRITE_ONCE(domain->active, false);
 830                 if (hrtimer_cancel(&domain->timer))
 831                         intel_uncore_fw_release_timer(&domain->timer);
 832         }
 833 }
 834
 835 /**
 836  * intel_uncore_forcewake_put__locked - release forcewake domain references
 837  * @uncore: the intel_uncore structure
 838  * @fw_domains: forcewake domains to put references
 839  *
 840  * See intel_uncore_forcewake_put(). This variant places the onus
 841  * on the caller to explicitly handle the dev_priv->uncore.lock spinlock.
 842  */
 843 void intel_uncore_forcewake_put__locked(struct intel_uncore *uncore,
 844                                         enum forcewake_domains fw_domains)
 845 {
 846         lockdep_assert_held(&uncore->lock);
 847
 848         if (!uncore->fw_get_funcs)
 849                 return;
 850
 851         __intel_uncore_forcewake_put(uncore, fw_domains, false);
 852 }
 853
 854 void assert_forcewakes_inactive(struct intel_uncore *uncore)
 855 {
 856         if (!uncore->fw_get_funcs)
 857                 return;
 858
 859         drm_WARN(&uncore->i915->drm, uncore->fw_domains_active,
 860                  "Expected all fw_domains to be inactive, but %08x are still on\n",
 861                  uncore->fw_domains_active);
 862 }
 863
 864 void assert_forcewakes_active(struct intel_uncore *uncore,
 865                               enum forcewake_domains fw_domains)
 866 {
 867         struct intel_uncore_forcewake_domain *domain;
 868         unsigned int tmp;
 869
 870         if (!IS_ENABLED(CONFIG_DRM_I915_DEBUG_RUNTIME_PM))
 871                 return;
 872
 873         if (!uncore->fw_get_funcs)
 874                 return;
 875
 876         spin_lock_irq(&uncore->lock);
 877
 878         assert_rpm_wakelock_held(uncore->rpm);
 879
 880         fw_domains &= uncore->fw_domains;
 881         drm_WARN(&uncore->i915->drm, fw_domains & ~uncore->fw_domains_active,
 882                  "Expected %08x fw_domains to be active, but %08x are off\n",
 883                  fw_domains, fw_domains & ~uncore->fw_domains_active);
 884
 885         /*
 886          * Check that the caller has an explicit wakeref and we don't mistake
 887          * it for the auto wakeref.
 888          */
 889         for_each_fw_domain_masked(domain, fw_domains, uncore, tmp) {
 890                 unsigned int actual = READ_ONCE(domain->wake_count);
 891                 unsigned int expect = 1;
 892
 893                 if (uncore->fw_domains_timer & domain->mask)
 894                         expect++; /* pending automatic release */
 895
 896                 if (drm_WARN(&uncore->i915->drm, actual < expect,
 897                              "Expected domain %d to be held awake by caller, count=%d\n",
 898                              domain->id, actual))
 899                         break;
 900         }
 901
 902         spin_unlock_irq(&uncore->lock);
 903 }
 904
 905 /*
 906  * We give fast paths for the really cool registers.  The second range includes
 907  * media domains (and the GSC starting from Xe_LPM+)
 908  */
 909 #define NEEDS_FORCE_WAKE(reg) ({ \
 910         u32 __reg = (reg); \
 911         __reg < 0x40000 || __reg >= 0x116000; \
 912 })
 913
 914 static int fw_range_cmp(u32 offset, const struct intel_forcewake_range *entry)
 915 {
 916         if (offset < entry->start)
 917                 return -1;
 918         else if (offset > entry->end)
 919                 return 1;
 920         else
 921                 return 0;
 922 }
 923
 924 /* Copied and "macroized" from lib/bsearch.c */
 925 #define BSEARCH(key, base, num, cmp) ({                                 \
 926         unsigned int start__ = 0, end__ = (num);                        \
 927         typeof(base) result__ = NULL;                                   \
 928         while (start__ < end__) {                                       \
 929                 unsigned int mid__ = start__ + (end__ - start__) / 2;   \
 930                 int ret__ = (cmp)((key), (base) + mid__);               \
 931                 if (ret__ < 0) {                                        \
 932                         end__ = mid__;                                  \
 933                 } else if (ret__ > 0) {                                 \
 934                         start__ = mid__ + 1;                            \
 935                 } else {                                                \
 936                         result__ = (base) + mid__;                      \
 937                         break;                                          \
 938                 }                                                       \
 939         }                                                               \
 940         result__;                                                       \
 941 })
 942
 943 static enum forcewake_domains
 944 find_fw_domain(struct intel_uncore *uncore, u32 offset)
 945 {
 946         const struct intel_forcewake_range *entry;
 947
 948         if (IS_GSI_REG(offset))
 949                 offset += uncore->gsi_offset;
 950
 951         entry = BSEARCH(offset,
 952                         uncore->fw_domains_table,
 953                         uncore->fw_domains_table_entries,
 954                         fw_range_cmp);
 955
 956         if (!entry)
 957                 return 0;
 958
 959         /*
 960          * The list of FW domains depends on the SKU in gen11+ so we
 961          * can't determine it statically. We use FORCEWAKE_ALL and
 962          * translate it here to the list of available domains.
 963          */
 964         if (entry->domains == FORCEWAKE_ALL)
 965                 return uncore->fw_domains;
 966
 967         drm_WARN(&uncore->i915->drm, entry->domains & ~uncore->fw_domains,
 968                  "Uninitialized forcewake domain(s) 0x%x accessed at 0x%x\n",
 969                  entry->domains & ~uncore->fw_domains, offset);
 970
 971         return entry->domains;
 972 }
 973
 974 /*
 975  * Shadowed register tables describe special register ranges that i915 is
 976  * allowed to write to without acquiring forcewake.  If these registers' power
 977  * wells are down, the hardware will save values written by i915 to a shadow
 978  * copy and automatically transfer them into the real register the next time
 979  * the power well is woken up.  Shadowing only applies to writes; forcewake
 980  * must still be acquired when reading from registers in these ranges.
 981  *
 982  * The documentation for shadowed registers is somewhat spotty on older
 983  * platforms.  However missing registers from these lists is non-fatal; it just
 984  * means we'll wake up the hardware for some register accesses where we didn't
 985  * really need to.
 986  *
 987  * The ranges listed in these tables must be sorted by offset.
 988  *
 989  * When adding new tables here, please also add them to
 990  * intel_shadow_table_check() in selftests/intel_uncore.c so that they will be
 991  * scanned for obvious mistakes or typos by the selftests.
 992  */
 993
 994 static const struct i915_range gen8_shadowed_regs[] = {
 995         { .start =  0x2030, .end =  0x2030 },
 996         { .start =  0xA008, .end =  0xA00C },
 997         { .start = 0x12030, .end = 0x12030 },
 998         { .start = 0x1a030, .end = 0x1a030 },
 999         { .start = 0x22030, .end = 0x22030 },
1000 };
1001
1002 static const struct i915_range gen11_shadowed_regs[] = {
1003         { .start =   0x2030, .end =   0x2030 },
1004         { .start =   0x2550, .end =   0x2550 },
1005         { .start =   0xA008, .end =   0xA00C },
1006         { .start =  0x22030, .end =  0x22030 },
1007         { .start =  0x22230, .end =  0x22230 },
1008         { .start =  0x22510, .end =  0x22550 },
1009         { .start = 0x1C0030, .end = 0x1C0030 },
1010         { .start = 0x1C0230, .end = 0x1C0230 },
1011         { .start = 0x1C0510, .end = 0x1C0550 },
1012         { .start = 0x1C4030, .end = 0x1C4030 },
1013         { .start = 0x1C4230, .end = 0x1C4230 },
1014         { .start = 0x1C4510, .end = 0x1C4550 },
1015         { .start = 0x1C8030, .end = 0x1C8030 },
1016         { .start = 0x1C8230, .end = 0x1C8230 },
1017         { .start = 0x1C8510, .end = 0x1C8550 },
1018         { .start = 0x1D0030, .end = 0x1D0030 },
1019         { .start = 0x1D0230, .end = 0x1D0230 },
1020         { .start = 0x1D0510, .end = 0x1D0550 },
1021         { .start = 0x1D4030, .end = 0x1D4030 },
1022         { .start = 0x1D4230, .end = 0x1D4230 },
1023         { .start = 0x1D4510, .end = 0x1D4550 },
1024         { .start = 0x1D8030, .end = 0x1D8030 },
1025         { .start = 0x1D8230, .end = 0x1D8230 },
1026         { .start = 0x1D8510, .end = 0x1D8550 },
1027 };
1028
1029 static const struct i915_range gen12_shadowed_regs[] = {
1030         { .start =   0x2030, .end =   0x2030 },
1031         { .start =   0x2510, .end =   0x2550 },
1032         { .start =   0xA008, .end =   0xA00C },
1033         { .start =   0xA188, .end =   0xA188 },
1034         { .start =   0xA278, .end =   0xA278 },
1035         { .start =   0xA540, .end =   0xA56C },
1036         { .start =   0xC4C8, .end =   0xC4C8 },
1037         { .start =   0xC4D4, .end =   0xC4D4 },
1038         { .start =   0xC600, .end =   0xC600 },
1039         { .start =  0x22030, .end =  0x22030 },
1040         { .start =  0x22510, .end =  0x22550 },
1041         { .start = 0x1C0030, .end = 0x1C0030 },
1042         { .start = 0x1C0510, .end = 0x1C0550 },
1043         { .start = 0x1C4030, .end = 0x1C4030 },
1044         { .start = 0x1C4510, .end = 0x1C4550 },
1045         { .start = 0x1C8030, .end = 0x1C8030 },
1046         { .start = 0x1C8510, .end = 0x1C8550 },
1047         { .start = 0x1D0030, .end = 0x1D0030 },
1048         { .start = 0x1D0510, .end = 0x1D0550 },
1049         { .start = 0x1D4030, .end = 0x1D4030 },
1050         { .start = 0x1D4510, .end = 0x1D4550 },
1051         { .start = 0x1D8030, .end = 0x1D8030 },
1052         { .start = 0x1D8510, .end = 0x1D8550 },
1053
1054         /*
1055          * The rest of these ranges are specific to Xe_HP and beyond, but
1056          * are reserved/unused ranges on earlier gen12 platforms, so they can
1057          * be safely added to the gen12 table.
1058          */
1059         { .start = 0x1E0030, .end = 0x1E0030 },
1060         { .start = 0x1E0510, .end = 0x1E0550 },
1061         { .start = 0x1E4030, .end = 0x1E4030 },
1062         { .start = 0x1E4510, .end = 0x1E4550 },
1063         { .start = 0x1E8030, .end = 0x1E8030 },
1064         { .start = 0x1E8510, .end = 0x1E8550 },
1065         { .start = 0x1F0030, .end = 0x1F0030 },
1066         { .start = 0x1F0510, .end = 0x1F0550 },
1067         { .start = 0x1F4030, .end = 0x1F4030 },
1068         { .start = 0x1F4510, .end = 0x1F4550 },
1069         { .start = 0x1F8030, .end = 0x1F8030 },
1070         { .start = 0x1F8510, .end = 0x1F8550 },
1071 };
1072
1073 static const struct i915_range dg2_shadowed_regs[] = {
1074         { .start =   0x2030, .end =   0x2030 },
1075         { .start =   0x2510, .end =   0x2550 },
1076         { .start =   0xA008, .end =   0xA00C },
1077         { .start =   0xA188, .end =   0xA188 },
1078         { .start =   0xA278, .end =   0xA278 },
1079         { .start =   0xA540, .end =   0xA56C },
1080         { .start =   0xC4C8, .end =   0xC4C8 },
1081         { .start =   0xC4E0, .end =   0xC4E0 },
1082         { .start =   0xC600, .end =   0xC600 },
1083         { .start =   0xC658, .end =   0xC658 },
1084         { .start =  0x22030, .end =  0x22030 },
1085         { .start =  0x22510, .end =  0x22550 },
1086         { .start = 0x1C0030, .end = 0x1C0030 },
1087         { .start = 0x1C0510, .end = 0x1C0550 },
1088         { .start = 0x1C4030, .end = 0x1C4030 },
1089         { .start = 0x1C4510, .end = 0x1C4550 },
1090         { .start = 0x1C8030, .end = 0x1C8030 },
1091         { .start = 0x1C8510, .end = 0x1C8550 },
1092         { .start = 0x1D0030, .end = 0x1D0030 },
1093         { .start = 0x1D0510, .end = 0x1D0550 },
1094         { .start = 0x1D4030, .end = 0x1D4030 },
1095         { .start = 0x1D4510, .end = 0x1D4550 },
1096         { .start = 0x1D8030, .end = 0x1D8030 },
1097         { .start = 0x1D8510, .end = 0x1D8550 },
1098         { .start = 0x1E0030, .end = 0x1E0030 },
1099         { .start = 0x1E0510, .end = 0x1E0550 },
1100         { .start = 0x1E4030, .end = 0x1E4030 },
1101         { .start = 0x1E4510, .end = 0x1E4550 },
1102         { .start = 0x1E8030, .end = 0x1E8030 },
1103         { .start = 0x1E8510, .end = 0x1E8550 },
1104         { .start = 0x1F0030, .end = 0x1F0030 },
1105         { .start = 0x1F0510, .end = 0x1F0550 },
1106         { .start = 0x1F4030, .end = 0x1F4030 },
1107         { .start = 0x1F4510, .end = 0x1F4550 },
1108         { .start = 0x1F8030, .end = 0x1F8030 },
1109         { .start = 0x1F8510, .end = 0x1F8550 },
1110 };
1111
1112 static const struct i915_range mtl_shadowed_regs[] = {
1113         { .start =   0x2030, .end =   0x2030 },
1114         { .start =   0x2510, .end =   0x2550 },
1115         { .start =   0xA008, .end =   0xA00C },
1116         { .start =   0xA188, .end =   0xA188 },
1117         { .start =   0xA278, .end =   0xA278 },
1118         { .start =   0xA540, .end =   0xA56C },
1119         { .start =   0xC050, .end =   0xC050 },
1120         { .start =   0xC340, .end =   0xC340 },
1121         { .start =   0xC4C8, .end =   0xC4C8 },
1122         { .start =   0xC4E0, .end =   0xC4E0 },
1123         { .start =   0xC600, .end =   0xC600 },
1124         { .start =   0xC658, .end =   0xC658 },
1125         { .start =   0xCFD4, .end =   0xCFDC },
1126         { .start =  0x22030, .end =  0x22030 },
1127         { .start =  0x22510, .end =  0x22550 },
1128 };
1129
1130 static const struct i915_range xelpmp_shadowed_regs[] = {
1131         { .start = 0x1C0030, .end = 0x1C0030 },
1132         { .start = 0x1C0510, .end = 0x1C0550 },
1133         { .start = 0x1C8030, .end = 0x1C8030 },
1134         { .start = 0x1C8510, .end = 0x1C8550 },
1135         { .start = 0x1D0030, .end = 0x1D0030 },
1136         { .start = 0x1D0510, .end = 0x1D0550 },
1137         { .start = 0x38A008, .end = 0x38A00C },
1138         { .start = 0x38A188, .end = 0x38A188 },
1139         { .start = 0x38A278, .end = 0x38A278 },
1140         { .start = 0x38A540, .end = 0x38A56C },
1141         { .start = 0x38A618, .end = 0x38A618 },
1142         { .start = 0x38C050, .end = 0x38C050 },
1143         { .start = 0x38C340, .end = 0x38C340 },
1144         { .start = 0x38C4C8, .end = 0x38C4C8 },
1145         { .start = 0x38C4E0, .end = 0x38C4E4 },
1146         { .start = 0x38C600, .end = 0x38C600 },
1147         { .start = 0x38C658, .end = 0x38C658 },
1148         { .start = 0x38CFD4, .end = 0x38CFDC },
1149 };
1150
1151 static int mmio_range_cmp(u32 key, const struct i915_range *range)
1152 {
1153         if (key < range->start)
1154                 return -1;
1155         else if (key > range->end)
1156                 return 1;
1157         else
1158                 return 0;
1159 }
1160
1161 static bool is_shadowed(struct intel_uncore *uncore, u32 offset)
1162 {
1163         if (drm_WARN_ON(&uncore->i915->drm, !uncore->shadowed_reg_table))
1164                 return false;
1165
1166         if (IS_GSI_REG(offset))
1167                 offset += uncore->gsi_offset;
1168
1169         return BSEARCH(offset,
1170                        uncore->shadowed_reg_table,
1171                        uncore->shadowed_reg_table_entries,
1172                        mmio_range_cmp);
1173 }
1174
1175 static enum forcewake_domains
1176 gen6_reg_write_fw_domains(struct intel_uncore *uncore, i915_reg_t reg)
1177 {
1178         return FORCEWAKE_RENDER;
1179 }
1180
1181 #define __fwtable_reg_read_fw_domains(uncore, offset) \
1182 ({ \
1183         enum forcewake_domains __fwd = 0; \
1184         if (NEEDS_FORCE_WAKE((offset))) \
1185                 __fwd = find_fw_domain(uncore, offset); \
1186         __fwd; \
1187 })
1188
1189 #define __fwtable_reg_write_fw_domains(uncore, offset) \
1190 ({ \
1191         enum forcewake_domains __fwd = 0; \
1192         const u32 __offset = (offset); \
1193         if (NEEDS_FORCE_WAKE((__offset)) && !is_shadowed(uncore, __offset)) \
1194                 __fwd = find_fw_domain(uncore, __offset); \
1195         __fwd; \
1196 })
1197
1198 #define GEN_FW_RANGE(s, e, d) \
1199         { .start = (s), .end = (e), .domains = (d) }
1200
1201 /*
1202  * All platforms' forcewake tables below must be sorted by offset ranges.
1203  * Furthermore, new forcewake tables added should be "watertight" and have
1204  * no gaps between ranges.
1205  *
1206  * When there are multiple consecutive ranges listed in the bspec with
1207  * the same forcewake domain, it is customary to combine them into a single
1208  * row in the tables below to keep the tables small and lookups fast.
1209  * Likewise, reserved/unused ranges may be combined with the preceding and/or
1210  * following ranges since the driver will never be making MMIO accesses in
1211  * those ranges.
1212  *
1213  * For example, if the bspec were to list:
1214  *
1215  *    ...
1216  *    0x1000 - 0x1fff:  GT
1217  *    0x2000 - 0x2cff:  GT
1218  *    0x2d00 - 0x2fff:  unused/reserved
1219  *    0x3000 - 0xffff:  GT
1220  *    ...
1221  *
1222  * these could all be represented by a single line in the code:
1223  *
1224  *   GEN_FW_RANGE(0x1000, 0xffff, FORCEWAKE_GT)
1225  *
1226  * When adding new forcewake tables here, please also add them to
1227  * intel_uncore_mock_selftests in selftests/intel_uncore.c so that they will be
1228  * scanned for obvious mistakes or typos by the selftests.
1229  */
1230
1231 static const struct intel_forcewake_range __gen6_fw_ranges[] = {
1232         GEN_FW_RANGE(0x0, 0x3ffff, FORCEWAKE_RENDER),
1233 };
1234
1235 static const struct intel_forcewake_range __vlv_fw_ranges[] = {
1236         GEN_FW_RANGE(0x2000, 0x3fff, FORCEWAKE_RENDER),
1237         GEN_FW_RANGE(0x5000, 0x7fff, FORCEWAKE_RENDER),
1238         GEN_FW_RANGE(0xb000, 0x11fff, FORCEWAKE_RENDER),
1239         GEN_FW_RANGE(0x12000, 0x13fff, FORCEWAKE_MEDIA),
1240         GEN_FW_RANGE(0x22000, 0x23fff, FORCEWAKE_MEDIA),
1241         GEN_FW_RANGE(0x2e000, 0x2ffff, FORCEWAKE_RENDER),
1242         GEN_FW_RANGE(0x30000, 0x3ffff, FORCEWAKE_MEDIA),
1243 };
1244
1245 static const struct intel_forcewake_range __chv_fw_ranges[] = {
1246         GEN_FW_RANGE(0x2000, 0x3fff, FORCEWAKE_RENDER),
1247         GEN_FW_RANGE(0x4000, 0x4fff, FORCEWAKE_RENDER | FORCEWAKE_MEDIA),
1248         GEN_FW_RANGE(0x5200, 0x7fff, FORCEWAKE_RENDER),
1249         GEN_FW_RANGE(0x8000, 0x82ff, FORCEWAKE_RENDER | FORCEWAKE_MEDIA),
1250         GEN_FW_RANGE(0x8300, 0x84ff, FORCEWAKE_RENDER),
1251         GEN_FW_RANGE(0x8500, 0x85ff, FORCEWAKE_RENDER | FORCEWAKE_MEDIA),
1252         GEN_FW_RANGE(0x8800, 0x88ff, FORCEWAKE_MEDIA),
1253         GEN_FW_RANGE(0x9000, 0xafff, FORCEWAKE_RENDER | FORCEWAKE_MEDIA),
1254         GEN_FW_RANGE(0xb000, 0xb47f, FORCEWAKE_RENDER),
1255         GEN_FW_RANGE(0xd000, 0xd7ff, FORCEWAKE_MEDIA),
1256         GEN_FW_RANGE(0xe000, 0xe7ff, FORCEWAKE_RENDER),
1257         GEN_FW_RANGE(0xf000, 0xffff, FORCEWAKE_RENDER | FORCEWAKE_MEDIA),
1258         GEN_FW_RANGE(0x12000, 0x13fff, FORCEWAKE_MEDIA),
1259         GEN_FW_RANGE(0x1a000, 0x1bfff, FORCEWAKE_MEDIA),
1260         GEN_FW_RANGE(0x1e800, 0x1e9ff, FORCEWAKE_MEDIA),
1261         GEN_FW_RANGE(0x30000, 0x37fff, FORCEWAKE_MEDIA),
1262 };
1263
1264 static const struct intel_forcewake_range __gen9_fw_ranges[] = {
1265         GEN_FW_RANGE(0x0, 0xaff, FORCEWAKE_GT),
1266         GEN_FW_RANGE(0xb00, 0x1fff, 0), /* uncore range */
1267         GEN_FW_RANGE(0x2000, 0x26ff, FORCEWAKE_RENDER),
1268         GEN_FW_RANGE(0x2700, 0x2fff, FORCEWAKE_GT),
1269         GEN_FW_RANGE(0x3000, 0x3fff, FORCEWAKE_RENDER),
1270         GEN_FW_RANGE(0x4000, 0x51ff, FORCEWAKE_GT),
1271         GEN_FW_RANGE(0x5200, 0x7fff, FORCEWAKE_RENDER),
1272         GEN_FW_RANGE(0x8000, 0x812f, FORCEWAKE_GT),
1273         GEN_FW_RANGE(0x8130, 0x813f, FORCEWAKE_MEDIA),
1274         GEN_FW_RANGE(0x8140, 0x815f, FORCEWAKE_RENDER),
1275         GEN_FW_RANGE(0x8160, 0x82ff, FORCEWAKE_GT),
1276         GEN_FW_RANGE(0x8300, 0x84ff, FORCEWAKE_RENDER),
1277         GEN_FW_RANGE(0x8500, 0x87ff, FORCEWAKE_GT),
1278         GEN_FW_RANGE(0x8800, 0x89ff, FORCEWAKE_MEDIA),
1279         GEN_FW_RANGE(0x8a00, 0x8bff, FORCEWAKE_GT),
1280         GEN_FW_RANGE(0x8c00, 0x8cff, FORCEWAKE_RENDER),
1281         GEN_FW_RANGE(0x8d00, 0x93ff, FORCEWAKE_GT),
1282         GEN_FW_RANGE(0x9400, 0x97ff, FORCEWAKE_RENDER | FORCEWAKE_MEDIA),
1283         GEN_FW_RANGE(0x9800, 0xafff, FORCEWAKE_GT),
1284         GEN_FW_RANGE(0xb000, 0xb47f, FORCEWAKE_RENDER),
1285         GEN_FW_RANGE(0xb480, 0xcfff, FORCEWAKE_GT),
1286         GEN_FW_RANGE(0xd000, 0xd7ff, FORCEWAKE_MEDIA),
1287         GEN_FW_RANGE(0xd800, 0xdfff, FORCEWAKE_GT),
1288         GEN_FW_RANGE(0xe000, 0xe8ff, FORCEWAKE_RENDER),
1289         GEN_FW_RANGE(0xe900, 0x11fff, FORCEWAKE_GT),
1290         GEN_FW_RANGE(0x12000, 0x13fff, FORCEWAKE_MEDIA),
1291         GEN_FW_RANGE(0x14000, 0x19fff, FORCEWAKE_GT),
1292         GEN_FW_RANGE(0x1a000, 0x1e9ff, FORCEWAKE_MEDIA),
1293         GEN_FW_RANGE(0x1ea00, 0x243ff, FORCEWAKE_GT),
1294         GEN_FW_RANGE(0x24400, 0x247ff, FORCEWAKE_RENDER),
1295         GEN_FW_RANGE(0x24800, 0x2ffff, FORCEWAKE_GT),
1296         GEN_FW_RANGE(0x30000, 0x3ffff, FORCEWAKE_MEDIA),
1297 };
1298
1299 static const struct intel_forcewake_range __gen11_fw_ranges[] = {
1300         GEN_FW_RANGE(0x0, 0x1fff, 0), /* uncore range */
1301         GEN_FW_RANGE(0x2000, 0x26ff, FORCEWAKE_RENDER),
1302         GEN_FW_RANGE(0x2700, 0x2fff, FORCEWAKE_GT),
1303         GEN_FW_RANGE(0x3000, 0x3fff, FORCEWAKE_RENDER),
1304         GEN_FW_RANGE(0x4000, 0x51ff, FORCEWAKE_GT),
1305         GEN_FW_RANGE(0x5200, 0x7fff, FORCEWAKE_RENDER),
1306         GEN_FW_RANGE(0x8000, 0x813f, FORCEWAKE_GT),
1307         GEN_FW_RANGE(0x8140, 0x815f, FORCEWAKE_RENDER),
1308         GEN_FW_RANGE(0x8160, 0x82ff, FORCEWAKE_GT),
1309         GEN_FW_RANGE(0x8300, 0x84ff, FORCEWAKE_RENDER),
1310         GEN_FW_RANGE(0x8500, 0x87ff, FORCEWAKE_GT),
1311         GEN_FW_RANGE(0x8800, 0x8bff, 0),
1312         GEN_FW_RANGE(0x8c00, 0x8cff, FORCEWAKE_RENDER),
1313         GEN_FW_RANGE(0x8d00, 0x94cf, FORCEWAKE_GT),
1314         GEN_FW_RANGE(0x94d0, 0x955f, FORCEWAKE_RENDER),
1315         GEN_FW_RANGE(0x9560, 0x95ff, 0),
1316         GEN_FW_RANGE(0x9600, 0xafff, FORCEWAKE_GT),
1317         GEN_FW_RANGE(0xb000, 0xb47f, FORCEWAKE_RENDER),
1318         GEN_FW_RANGE(0xb480, 0xdeff, FORCEWAKE_GT),
1319         GEN_FW_RANGE(0xdf00, 0xe8ff, FORCEWAKE_RENDER),
1320         GEN_FW_RANGE(0xe900, 0x16dff, FORCEWAKE_GT),
1321         GEN_FW_RANGE(0x16e00, 0x19fff, FORCEWAKE_RENDER),
1322         GEN_FW_RANGE(0x1a000, 0x23fff, FORCEWAKE_GT),
1323         GEN_FW_RANGE(0x24000, 0x2407f, 0),
1324         GEN_FW_RANGE(0x24080, 0x2417f, FORCEWAKE_GT),
1325         GEN_FW_RANGE(0x24180, 0x242ff, FORCEWAKE_RENDER),
1326         GEN_FW_RANGE(0x24300, 0x243ff, FORCEWAKE_GT),
1327         GEN_FW_RANGE(0x24400, 0x24fff, FORCEWAKE_RENDER),
1328         GEN_FW_RANGE(0x25000, 0x3ffff, FORCEWAKE_GT),
1329         GEN_FW_RANGE(0x40000, 0x1bffff, 0),
1330         GEN_FW_RANGE(0x1c0000, 0x1c3fff, FORCEWAKE_MEDIA_VDBOX0),
1331         GEN_FW_RANGE(0x1c4000, 0x1c7fff, 0),
1332         GEN_FW_RANGE(0x1c8000, 0x1cffff, FORCEWAKE_MEDIA_VEBOX0),
1333         GEN_FW_RANGE(0x1d0000, 0x1d3fff, FORCEWAKE_MEDIA_VDBOX2),
1334         GEN_FW_RANGE(0x1d4000, 0x1dbfff, 0)
1335 };
1336
1337 static const struct intel_forcewake_range __gen12_fw_ranges[] = {
1338         GEN_FW_RANGE(0x0, 0x1fff, 0), /*
1339                 0x0   -  0xaff: reserved
1340                 0xb00 - 0x1fff: always on */
1341         GEN_FW_RANGE(0x2000, 0x26ff, FORCEWAKE_RENDER),
1342         GEN_FW_RANGE(0x2700, 0x27ff, FORCEWAKE_GT),
1343         GEN_FW_RANGE(0x2800, 0x2aff, FORCEWAKE_RENDER),
1344         GEN_FW_RANGE(0x2b00, 0x2fff, FORCEWAKE_GT),
1345         GEN_FW_RANGE(0x3000, 0x3fff, FORCEWAKE_RENDER),
1346         GEN_FW_RANGE(0x4000, 0x51ff, FORCEWAKE_GT), /*
1347                 0x4000 - 0x48ff: gt
1348                 0x4900 - 0x51ff: reserved */
1349         GEN_FW_RANGE(0x5200, 0x7fff, FORCEWAKE_RENDER), /*
1350                 0x5200 - 0x53ff: render
1351                 0x5400 - 0x54ff: reserved
1352                 0x5500 - 0x7fff: render */
1353         GEN_FW_RANGE(0x8000, 0x813f, FORCEWAKE_GT),
1354         GEN_FW_RANGE(0x8140, 0x815f, FORCEWAKE_RENDER),
1355         GEN_FW_RANGE(0x8160, 0x81ff, 0), /*
1356                 0x8160 - 0x817f: reserved
1357                 0x8180 - 0x81ff: always on */
1358         GEN_FW_RANGE(0x8200, 0x82ff, FORCEWAKE_GT),
1359         GEN_FW_RANGE(0x8300, 0x84ff, FORCEWAKE_RENDER),
1360         GEN_FW_RANGE(0x8500, 0x94cf, FORCEWAKE_GT), /*
1361                 0x8500 - 0x87ff: gt
1362                 0x8800 - 0x8fff: reserved
1363                 0x9000 - 0x947f: gt
1364                 0x9480 - 0x94cf: reserved */
1365         GEN_FW_RANGE(0x94d0, 0x955f, FORCEWAKE_RENDER),
1366         GEN_FW_RANGE(0x9560, 0x97ff, 0), /*
1367                 0x9560 - 0x95ff: always on
1368                 0x9600 - 0x97ff: reserved */
1369         GEN_FW_RANGE(0x9800, 0xafff, FORCEWAKE_GT),
1370         GEN_FW_RANGE(0xb000, 0xb3ff, FORCEWAKE_RENDER),
1371         GEN_FW_RANGE(0xb400, 0xcfff, FORCEWAKE_GT), /*
1372                 0xb400 - 0xbf7f: gt
1373                 0xb480 - 0xbfff: reserved
1374                 0xc000 - 0xcfff: gt */
1375         GEN_FW_RANGE(0xd000, 0xd7ff, 0),
1376         GEN_FW_RANGE(0xd800, 0xd8ff, FORCEWAKE_RENDER),
1377         GEN_FW_RANGE(0xd900, 0xdbff, FORCEWAKE_GT),
1378         GEN_FW_RANGE(0xdc00, 0xefff, FORCEWAKE_RENDER), /*
1379                 0xdc00 - 0xddff: render
1380                 0xde00 - 0xde7f: reserved
1381                 0xde80 - 0xe8ff: render
1382                 0xe900 - 0xefff: reserved */
1383         GEN_FW_RANGE(0xf000, 0x147ff, FORCEWAKE_GT), /*
1384                  0xf000 - 0xffff: gt
1385                 0x10000 - 0x147ff: reserved */
1386         GEN_FW_RANGE(0x14800, 0x1ffff, FORCEWAKE_RENDER), /*
1387                 0x14800 - 0x14fff: render
1388                 0x15000 - 0x16dff: reserved
1389                 0x16e00 - 0x1bfff: render
1390                 0x1c000 - 0x1ffff: reserved */
1391         GEN_FW_RANGE(0x20000, 0x20fff, FORCEWAKE_MEDIA_VDBOX0),
1392         GEN_FW_RANGE(0x21000, 0x21fff, FORCEWAKE_MEDIA_VDBOX2),
1393         GEN_FW_RANGE(0x22000, 0x23fff, FORCEWAKE_GT),
1394         GEN_FW_RANGE(0x24000, 0x2417f, 0), /*
1395                 0x24000 - 0x2407f: always on
1396                 0x24080 - 0x2417f: reserved */
1397         GEN_FW_RANGE(0x24180, 0x249ff, FORCEWAKE_GT), /*
1398                 0x24180 - 0x241ff: gt
1399                 0x24200 - 0x249ff: reserved */
1400         GEN_FW_RANGE(0x24a00, 0x251ff, FORCEWAKE_RENDER), /*
1401                 0x24a00 - 0x24a7f: render
1402                 0x24a80 - 0x251ff: reserved */
1403         GEN_FW_RANGE(0x25200, 0x255ff, FORCEWAKE_GT), /*
1404                 0x25200 - 0x252ff: gt
1405                 0x25300 - 0x255ff: reserved */
1406         GEN_FW_RANGE(0x25600, 0x2567f, FORCEWAKE_MEDIA_VDBOX0),
1407         GEN_FW_RANGE(0x25680, 0x259ff, FORCEWAKE_MEDIA_VDBOX2), /*
1408                 0x25680 - 0x256ff: VD2
1409                 0x25700 - 0x259ff: reserved */
1410         GEN_FW_RANGE(0x25a00, 0x25a7f, FORCEWAKE_MEDIA_VDBOX0),
1411         GEN_FW_RANGE(0x25a80, 0x2ffff, FORCEWAKE_MEDIA_VDBOX2), /*
1412                 0x25a80 - 0x25aff: VD2
1413                 0x25b00 - 0x2ffff: reserved */
1414         GEN_FW_RANGE(0x30000, 0x3ffff, FORCEWAKE_GT),
1415         GEN_FW_RANGE(0x40000, 0x1bffff, 0),
1416         GEN_FW_RANGE(0x1c0000, 0x1c3fff, FORCEWAKE_MEDIA_VDBOX0), /*
1417                 0x1c0000 - 0x1c2bff: VD0
1418                 0x1c2c00 - 0x1c2cff: reserved
1419                 0x1c2d00 - 0x1c2dff: VD0
1420                 0x1c2e00 - 0x1c3eff: reserved
1421                 0x1c3f00 - 0x1c3fff: VD0 */
1422         GEN_FW_RANGE(0x1c4000, 0x1c7fff, 0),
1423         GEN_FW_RANGE(0x1c8000, 0x1cbfff, FORCEWAKE_MEDIA_VEBOX0), /*
1424                 0x1c8000 - 0x1ca0ff: VE0
1425                 0x1ca100 - 0x1cbeff: reserved
1426                 0x1cbf00 - 0x1cbfff: VE0 */
1427         GEN_FW_RANGE(0x1cc000, 0x1cffff, FORCEWAKE_MEDIA_VDBOX0), /*
1428                 0x1cc000 - 0x1ccfff: VD0
1429                 0x1cd000 - 0x1cffff: reserved */
1430         GEN_FW_RANGE(0x1d0000, 0x1d3fff, FORCEWAKE_MEDIA_VDBOX2), /*
1431                 0x1d0000 - 0x1d2bff: VD2
1432                 0x1d2c00 - 0x1d2cff: reserved
1433                 0x1d2d00 - 0x1d2dff: VD2
1434                 0x1d2e00 - 0x1d3eff: reserved
1435                 0x1d3f00 - 0x1d3fff: VD2 */
1436 };
1437
1438 static const struct intel_forcewake_range __dg2_fw_ranges[] = {
1439         GEN_FW_RANGE(0x0, 0x1fff, 0), /*
1440                   0x0 -  0xaff: reserved
1441                 0xb00 - 0x1fff: always on */
1442         GEN_FW_RANGE(0x2000, 0x26ff, FORCEWAKE_RENDER),
1443         GEN_FW_RANGE(0x2700, 0x4aff, FORCEWAKE_GT),
1444         GEN_FW_RANGE(0x4b00, 0x51ff, 0), /*
1445                 0x4b00 - 0x4fff: reserved
1446                 0x5000 - 0x51ff: always on */
1447         GEN_FW_RANGE(0x5200, 0x7fff, FORCEWAKE_RENDER),
1448         GEN_FW_RANGE(0x8000, 0x813f, FORCEWAKE_GT),
1449         GEN_FW_RANGE(0x8140, 0x815f, FORCEWAKE_RENDER),
1450         GEN_FW_RANGE(0x8160, 0x81ff, 0), /*
1451                 0x8160 - 0x817f: reserved
1452                 0x8180 - 0x81ff: always on */
1453         GEN_FW_RANGE(0x8200, 0x82ff, FORCEWAKE_GT),
1454         GEN_FW_RANGE(0x8300, 0x84ff, FORCEWAKE_RENDER),
1455         GEN_FW_RANGE(0x8500, 0x8cff, FORCEWAKE_GT), /*
1456                 0x8500 - 0x87ff: gt
1457                 0x8800 - 0x8c7f: reserved
1458                 0x8c80 - 0x8cff: gt (DG2 only) */
1459         GEN_FW_RANGE(0x8d00, 0x8fff, FORCEWAKE_RENDER), /*
1460                 0x8d00 - 0x8dff: render (DG2 only)
1461                 0x8e00 - 0x8fff: reserved */
1462         GEN_FW_RANGE(0x9000, 0x94cf, FORCEWAKE_GT), /*
1463                 0x9000 - 0x947f: gt
1464                 0x9480 - 0x94cf: reserved */
1465         GEN_FW_RANGE(0x94d0, 0x955f, FORCEWAKE_RENDER),
1466         GEN_FW_RANGE(0x9560, 0x967f, 0), /*
1467                 0x9560 - 0x95ff: always on
1468                 0x9600 - 0x967f: reserved */
1469         GEN_FW_RANGE(0x9680, 0x97ff, FORCEWAKE_RENDER), /*
1470                 0x9680 - 0x96ff: render
1471                 0x9700 - 0x97ff: reserved */
1472         GEN_FW_RANGE(0x9800, 0xcfff, FORCEWAKE_GT), /*
1473                 0x9800 - 0xb4ff: gt
1474                 0xb500 - 0xbfff: reserved
1475                 0xc000 - 0xcfff: gt */
1476         GEN_FW_RANGE(0xd000, 0xd7ff, 0),
1477         GEN_FW_RANGE(0xd800, 0xd87f, FORCEWAKE_RENDER),
1478         GEN_FW_RANGE(0xd880, 0xdbff, FORCEWAKE_GT),
1479         GEN_FW_RANGE(0xdc00, 0xdcff, FORCEWAKE_RENDER),
1480         GEN_FW_RANGE(0xdd00, 0xde7f, FORCEWAKE_GT), /*
1481                 0xdd00 - 0xddff: gt
1482                 0xde00 - 0xde7f: reserved */
1483         GEN_FW_RANGE(0xde80, 0xe8ff, FORCEWAKE_RENDER), /*
1484                 0xde80 - 0xdfff: render
1485                 0xe000 - 0xe0ff: reserved
1486                 0xe100 - 0xe8ff: render */
1487         GEN_FW_RANGE(0xe900, 0xffff, FORCEWAKE_GT), /*
1488                 0xe900 - 0xe9ff: gt
1489                 0xea00 - 0xefff: reserved
1490                 0xf000 - 0xffff: gt */
1491         GEN_FW_RANGE(0x10000, 0x12fff, 0), /*
1492                 0x10000 - 0x11fff: reserved
1493                 0x12000 - 0x127ff: always on
1494                 0x12800 - 0x12fff: reserved */
1495         GEN_FW_RANGE(0x13000, 0x131ff, FORCEWAKE_MEDIA_VDBOX0),
1496         GEN_FW_RANGE(0x13200, 0x147ff, FORCEWAKE_MEDIA_VDBOX2), /*
1497                 0x13200 - 0x133ff: VD2 (DG2 only)
1498                 0x13400 - 0x147ff: reserved */
1499         GEN_FW_RANGE(0x14800, 0x14fff, FORCEWAKE_RENDER),
1500         GEN_FW_RANGE(0x15000, 0x16dff, FORCEWAKE_GT), /*
1501                 0x15000 - 0x15fff: gt (DG2 only)
1502                 0x16000 - 0x16dff: reserved */
1503         GEN_FW_RANGE(0x16e00, 0x21fff, FORCEWAKE_RENDER), /*
1504                 0x16e00 - 0x1ffff: render
1505                 0x20000 - 0x21fff: reserved */
1506         GEN_FW_RANGE(0x22000, 0x23fff, FORCEWAKE_GT),
1507         GEN_FW_RANGE(0x24000, 0x2417f, 0), /*
1508                 0x24000 - 0x2407f: always on
1509                 0x24080 - 0x2417f: reserved */
1510         GEN_FW_RANGE(0x24180, 0x249ff, FORCEWAKE_GT), /*
1511                 0x24180 - 0x241ff: gt
1512                 0x24200 - 0x249ff: reserved */
1513         GEN_FW_RANGE(0x24a00, 0x251ff, FORCEWAKE_RENDER), /*
1514                 0x24a00 - 0x24a7f: render
1515                 0x24a80 - 0x251ff: reserved */
1516         GEN_FW_RANGE(0x25200, 0x25fff, FORCEWAKE_GT), /*
1517                 0x25200 - 0x252ff: gt
1518                 0x25300 - 0x25fff: reserved */
1519         GEN_FW_RANGE(0x26000, 0x2ffff, FORCEWAKE_RENDER), /*
1520                 0x26000 - 0x27fff: render
1521                 0x28000 - 0x29fff: reserved
1522                 0x2a000 - 0x2ffff: undocumented */
1523         GEN_FW_RANGE(0x30000, 0x3ffff, FORCEWAKE_GT),
1524         GEN_FW_RANGE(0x40000, 0x1bffff, 0),
1525         GEN_FW_RANGE(0x1c0000, 0x1c3fff, FORCEWAKE_MEDIA_VDBOX0), /*
1526                 0x1c0000 - 0x1c2bff: VD0
1527                 0x1c2c00 - 0x1c2cff: reserved
1528                 0x1c2d00 - 0x1c2dff: VD0
1529                 0x1c2e00 - 0x1c3eff: VD0
1530                 0x1c3f00 - 0x1c3fff: VD0 */
1531         GEN_FW_RANGE(0x1c4000, 0x1c7fff, FORCEWAKE_MEDIA_VDBOX1), /*
1532                 0x1c4000 - 0x1c6bff: VD1
1533                 0x1c6c00 - 0x1c6cff: reserved
1534                 0x1c6d00 - 0x1c6dff: VD1
1535                 0x1c6e00 - 0x1c7fff: reserved */
1536         GEN_FW_RANGE(0x1c8000, 0x1cbfff, FORCEWAKE_MEDIA_VEBOX0), /*
1537                 0x1c8000 - 0x1ca0ff: VE0
1538                 0x1ca100 - 0x1cbfff: reserved */
1539         GEN_FW_RANGE(0x1cc000, 0x1ccfff, FORCEWAKE_MEDIA_VDBOX0),
1540         GEN_FW_RANGE(0x1cd000, 0x1cdfff, FORCEWAKE_MEDIA_VDBOX2),
1541         GEN_FW_RANGE(0x1ce000, 0x1cefff, FORCEWAKE_MEDIA_VDBOX4),
1542         GEN_FW_RANGE(0x1cf000, 0x1cffff, FORCEWAKE_MEDIA_VDBOX6),
1543         GEN_FW_RANGE(0x1d0000, 0x1d3fff, FORCEWAKE_MEDIA_VDBOX2), /*
1544                 0x1d0000 - 0x1d2bff: VD2
1545                 0x1d2c00 - 0x1d2cff: reserved
1546                 0x1d2d00 - 0x1d2dff: VD2
1547                 0x1d2e00 - 0x1d3dff: VD2
1548                 0x1d3e00 - 0x1d3eff: reserved
1549                 0x1d3f00 - 0x1d3fff: VD2 */
1550         GEN_FW_RANGE(0x1d4000, 0x1d7fff, FORCEWAKE_MEDIA_VDBOX3), /*
1551                 0x1d4000 - 0x1d6bff: VD3
1552                 0x1d6c00 - 0x1d6cff: reserved
1553                 0x1d6d00 - 0x1d6dff: VD3
1554                 0x1d6e00 - 0x1d7fff: reserved */
1555         GEN_FW_RANGE(0x1d8000, 0x1dffff, FORCEWAKE_MEDIA_VEBOX1), /*
1556                 0x1d8000 - 0x1da0ff: VE1
1557                 0x1da100 - 0x1dffff: reserved */
1558         GEN_FW_RANGE(0x1e0000, 0x1e3fff, FORCEWAKE_MEDIA_VDBOX4), /*
1559                 0x1e0000 - 0x1e2bff: VD4
1560                 0x1e2c00 - 0x1e2cff: reserved
1561                 0x1e2d00 - 0x1e2dff: VD4
1562                 0x1e2e00 - 0x1e3eff: reserved
1563                 0x1e3f00 - 0x1e3fff: VD4 */
1564         GEN_FW_RANGE(0x1e4000, 0x1e7fff, FORCEWAKE_MEDIA_VDBOX5), /*
1565                 0x1e4000 - 0x1e6bff: VD5
1566                 0x1e6c00 - 0x1e6cff: reserved
1567                 0x1e6d00 - 0x1e6dff: VD5
1568                 0x1e6e00 - 0x1e7fff: reserved */
1569         GEN_FW_RANGE(0x1e8000, 0x1effff, FORCEWAKE_MEDIA_VEBOX2), /*
1570                 0x1e8000 - 0x1ea0ff: VE2
1571                 0x1ea100 - 0x1effff: reserved */
1572         GEN_FW_RANGE(0x1f0000, 0x1f3fff, FORCEWAKE_MEDIA_VDBOX6), /*
1573                 0x1f0000 - 0x1f2bff: VD6
1574                 0x1f2c00 - 0x1f2cff: reserved
1575                 0x1f2d00 - 0x1f2dff: VD6
1576                 0x1f2e00 - 0x1f3eff: reserved
1577                 0x1f3f00 - 0x1f3fff: VD6 */
1578         GEN_FW_RANGE(0x1f4000, 0x1f7fff, FORCEWAKE_MEDIA_VDBOX7), /*
1579                 0x1f4000 - 0x1f6bff: VD7
1580                 0x1f6c00 - 0x1f6cff: reserved
1581                 0x1f6d00 - 0x1f6dff: VD7
1582                 0x1f6e00 - 0x1f7fff: reserved */
1583         GEN_FW_RANGE(0x1f8000, 0x1fa0ff, FORCEWAKE_MEDIA_VEBOX3),
1584 };
1585
1586 static const struct intel_forcewake_range __mtl_fw_ranges[] = {
1587         GEN_FW_RANGE(0x0, 0xaff, 0),
1588         GEN_FW_RANGE(0xb00, 0xbff, FORCEWAKE_GT),
1589         GEN_FW_RANGE(0xc00, 0xfff, 0),
1590         GEN_FW_RANGE(0x1000, 0x1fff, FORCEWAKE_GT),
1591         GEN_FW_RANGE(0x2000, 0x26ff, FORCEWAKE_RENDER),
1592         GEN_FW_RANGE(0x2700, 0x2fff, FORCEWAKE_GT),
1593         GEN_FW_RANGE(0x3000, 0x3fff, FORCEWAKE_RENDER),
1594         GEN_FW_RANGE(0x4000, 0x51ff, FORCEWAKE_GT), /*
1595                 0x4000 - 0x48ff: render
1596                 0x4900 - 0x51ff: reserved */
1597         GEN_FW_RANGE(0x5200, 0x7fff, FORCEWAKE_RENDER), /*
1598                 0x5200 - 0x53ff: render
1599                 0x5400 - 0x54ff: reserved
1600                 0x5500 - 0x7fff: render */
1601         GEN_FW_RANGE(0x8000, 0x813f, FORCEWAKE_GT),
1602         GEN_FW_RANGE(0x8140, 0x817f, FORCEWAKE_RENDER), /*
1603                 0x8140 - 0x815f: render
1604                 0x8160 - 0x817f: reserved */
1605         GEN_FW_RANGE(0x8180, 0x81ff, 0),
1606         GEN_FW_RANGE(0x8200, 0x94cf, FORCEWAKE_GT), /*
1607                 0x8200 - 0x87ff: gt
1608                 0x8800 - 0x8dff: reserved
1609                 0x8e00 - 0x8f7f: gt
1610                 0x8f80 - 0x8fff: reserved
1611                 0x9000 - 0x947f: gt
1612                 0x9480 - 0x94cf: reserved */
1613         GEN_FW_RANGE(0x94d0, 0x955f, FORCEWAKE_RENDER),
1614         GEN_FW_RANGE(0x9560, 0x967f, 0), /*
1615                 0x9560 - 0x95ff: always on
1616                 0x9600 - 0x967f: reserved */
1617         GEN_FW_RANGE(0x9680, 0x97ff, FORCEWAKE_RENDER), /*
1618                 0x9680 - 0x96ff: render
1619                 0x9700 - 0x97ff: reserved */
1620         GEN_FW_RANGE(0x9800, 0xcfff, FORCEWAKE_GT), /*
1621                 0x9800 - 0xb4ff: gt
1622                 0xb500 - 0xbfff: reserved
1623                 0xc000 - 0xcfff: gt */
1624         GEN_FW_RANGE(0xd000, 0xd7ff, 0), /*
1625                 0xd000 - 0xd3ff: always on
1626                 0xd400 - 0xd7ff: reserved */
1627         GEN_FW_RANGE(0xd800, 0xd87f, FORCEWAKE_RENDER),
1628         GEN_FW_RANGE(0xd880, 0xdbff, FORCEWAKE_GT),
1629         GEN_FW_RANGE(0xdc00, 0xdcff, FORCEWAKE_RENDER),
1630         GEN_FW_RANGE(0xdd00, 0xde7f, FORCEWAKE_GT), /*
1631                 0xdd00 - 0xddff: gt
1632                 0xde00 - 0xde7f: reserved */
1633         GEN_FW_RANGE(0xde80, 0xe8ff, FORCEWAKE_RENDER), /*
1634                 0xde80 - 0xdfff: render
1635                 0xe000 - 0xe0ff: reserved
1636                 0xe100 - 0xe8ff: render */
1637         GEN_FW_RANGE(0xe900, 0xe9ff, FORCEWAKE_GT),
1638         GEN_FW_RANGE(0xea00, 0x147ff, 0), /*
1639                  0xea00 - 0x11fff: reserved
1640                 0x12000 - 0x127ff: always on
1641                 0x12800 - 0x147ff: reserved */
1642         GEN_FW_RANGE(0x14800, 0x19fff, FORCEWAKE_GT), /*
1643                 0x14800 - 0x153ff: gt
1644                 0x15400 - 0x19fff: reserved */
1645         GEN_FW_RANGE(0x1a000, 0x21fff, FORCEWAKE_RENDER), /*
1646                 0x1a000 - 0x1bfff: render
1647                 0x1c000 - 0x21fff: reserved */
1648         GEN_FW_RANGE(0x22000, 0x23fff, FORCEWAKE_GT),
1649         GEN_FW_RANGE(0x24000, 0x2ffff, 0), /*
1650                 0x24000 - 0x2407f: always on
1651                 0x24080 - 0x2ffff: reserved */
1652         GEN_FW_RANGE(0x30000, 0x3ffff, FORCEWAKE_GT),
1653         GEN_FW_RANGE(0x40000, 0x1901ef, 0),
1654         GEN_FW_RANGE(0x1901f0, 0x1901f3, FORCEWAKE_GT)
1655                 /* FIXME: WA to wake GT while triggering H2G */
1656 };
1657
1658 /*
1659  * Note that the register ranges here are the final offsets after
1660  * translation of the GSI block to the 0x380000 offset.
1661  *
1662  * NOTE:  There are a couple MCR ranges near the bottom of this table
1663  * that need to power up either VD0 or VD2 depending on which replicated
1664  * instance of the register we're trying to access.  Our forcewake logic
1665  * at the moment doesn't have a good way to take steering into consideration,
1666  * and the driver doesn't even access any registers in those ranges today,
1667  * so for now we just mark those ranges as FORCEWAKE_ALL.  That will ensure
1668  * proper operation if we do start using the ranges in the future, and we
1669  * can determine at that time whether it's worth adding extra complexity to
1670  * the forcewake handling to take steering into consideration.
1671  */
1672 static const struct intel_forcewake_range __xelpmp_fw_ranges[] = {
1673         GEN_FW_RANGE(0x0, 0x115fff, 0), /* render GT range */
1674         GEN_FW_RANGE(0x116000, 0x11ffff, FORCEWAKE_GSC), /*
1675                 0x116000 - 0x117fff: gsc
1676                 0x118000 - 0x119fff: reserved
1677                 0x11a000 - 0x11efff: gsc
1678                 0x11f000 - 0x11ffff: reserved */
1679         GEN_FW_RANGE(0x120000, 0x1bffff, 0), /* non-GT range */
1680         GEN_FW_RANGE(0x1c0000, 0x1c7fff, FORCEWAKE_MEDIA_VDBOX0), /*
1681                 0x1c0000 - 0x1c3dff: VD0
1682                 0x1c3e00 - 0x1c3eff: reserved
1683                 0x1c3f00 - 0x1c3fff: VD0
1684                 0x1c4000 - 0x1c7fff: reserved */
1685         GEN_FW_RANGE(0x1c8000, 0x1cbfff, FORCEWAKE_MEDIA_VEBOX0), /*
1686                 0x1c8000 - 0x1ca0ff: VE0
1687                 0x1ca100 - 0x1cbfff: reserved */
1688         GEN_FW_RANGE(0x1cc000, 0x1cffff, FORCEWAKE_MEDIA_VDBOX0), /*
1689                 0x1cc000 - 0x1cdfff: VD0
1690                 0x1ce000 - 0x1cffff: reserved */
1691         GEN_FW_RANGE(0x1d0000, 0x1d7fff, FORCEWAKE_MEDIA_VDBOX2), /*
1692                 0x1d0000 - 0x1d3dff: VD2
1693                 0x1d3e00 - 0x1d3eff: reserved
1694                 0x1d4000 - 0x1d7fff: VD2 */
1695         GEN_FW_RANGE(0x1d8000, 0x1da0ff, FORCEWAKE_MEDIA_VEBOX1),
1696         GEN_FW_RANGE(0x1da100, 0x380aff, 0), /*
1697                 0x1da100 - 0x23ffff: reserved
1698                 0x240000 - 0x37ffff: non-GT range
1699                 0x380000 - 0x380aff: reserved */
1700         GEN_FW_RANGE(0x380b00, 0x380bff, FORCEWAKE_GT),
1701         GEN_FW_RANGE(0x380c00, 0x380fff, 0),
1702         GEN_FW_RANGE(0x381000, 0x38817f, FORCEWAKE_GT), /*
1703                 0x381000 - 0x381fff: gt
1704                 0x382000 - 0x383fff: reserved
1705                 0x384000 - 0x384aff: gt
1706                 0x384b00 - 0x3851ff: reserved
1707                 0x385200 - 0x3871ff: gt
1708                 0x387200 - 0x387fff: reserved
1709                 0x388000 - 0x38813f: gt
1710                 0x388140 - 0x38817f: reserved */
1711         GEN_FW_RANGE(0x388180, 0x3882ff, 0), /*
1712                 0x388180 - 0x3881ff: always on
1713                 0x388200 - 0x3882ff: reserved */
1714         GEN_FW_RANGE(0x388300, 0x38955f, FORCEWAKE_GT), /*
1715                 0x388300 - 0x38887f: gt
1716                 0x388880 - 0x388fff: reserved
1717                 0x389000 - 0x38947f: gt
1718                 0x389480 - 0x38955f: reserved */
1719         GEN_FW_RANGE(0x389560, 0x389fff, 0), /*
1720                 0x389560 - 0x3895ff: always on
1721                 0x389600 - 0x389fff: reserved */
1722         GEN_FW_RANGE(0x38a000, 0x38cfff, FORCEWAKE_GT), /*
1723                 0x38a000 - 0x38afff: gt
1724                 0x38b000 - 0x38bfff: reserved
1725                 0x38c000 - 0x38cfff: gt */
1726         GEN_FW_RANGE(0x38d000, 0x38d11f, 0),
1727         GEN_FW_RANGE(0x38d120, 0x391fff, FORCEWAKE_GT), /*
1728                 0x38d120 - 0x38dfff: gt
1729                 0x38e000 - 0x38efff: reserved
1730                 0x38f000 - 0x38ffff: gt
1731                 0x389000 - 0x391fff: reserved */
1732         GEN_FW_RANGE(0x392000, 0x392fff, 0), /*
1733                 0x392000 - 0x3927ff: always on
1734                 0x392800 - 0x292fff: reserved */
1735         GEN_FW_RANGE(0x393000, 0x3931ff, FORCEWAKE_GT),
1736         GEN_FW_RANGE(0x393200, 0x39323f, FORCEWAKE_ALL), /* instance-based, see note above */
1737         GEN_FW_RANGE(0x393240, 0x3933ff, FORCEWAKE_GT),
1738         GEN_FW_RANGE(0x393400, 0x3934ff, FORCEWAKE_ALL), /* instance-based, see note above */
1739         GEN_FW_RANGE(0x393500, 0x393c7f, 0), /*
1740                 0x393500 - 0x393bff: reserved
1741                 0x393c00 - 0x393c7f: always on */
1742         GEN_FW_RANGE(0x393c80, 0x393dff, FORCEWAKE_GT),
1743 };
1744
1745 static void
1746 ilk_dummy_write(struct intel_uncore *uncore)
1747 {
1748         /* WaIssueDummyWriteToWakeupFromRC6:ilk Issue a dummy write to wake up
1749          * the chip from rc6 before touching it for real. MI_MODE is masked,
1750          * hence harmless to write 0 into. */
1751         __raw_uncore_write32(uncore, RING_MI_MODE(RENDER_RING_BASE), 0);
1752 }
1753
1754 static void
1755 __unclaimed_reg_debug(struct intel_uncore *uncore,
1756                       const i915_reg_t reg,
1757                       const bool read)
1758 {
1759         if (drm_WARN(&uncore->i915->drm,
1760                      check_for_unclaimed_mmio(uncore),
1761                      "Unclaimed %s register 0x%x\n",
1762                      read ? "read from" : "write to",
1763                      i915_mmio_reg_offset(reg)))
1764                 /* Only report the first N failures */
1765                 uncore->i915->params.mmio_debug--;
1766 }
1767
1768 static void
1769 __unclaimed_previous_reg_debug(struct intel_uncore *uncore,
1770                                const i915_reg_t reg,
1771                                const bool read)
1772 {
1773         if (check_for_unclaimed_mmio(uncore))
1774                 drm_dbg(&uncore->i915->drm,
1775                         "Unclaimed access detected before %s register 0x%x\n",
1776                         read ? "read from" : "write to",
1777                         i915_mmio_reg_offset(reg));
1778 }
1779
1780 static inline bool __must_check
1781 unclaimed_reg_debug_header(struct intel_uncore *uncore,
1782                            const i915_reg_t reg, const bool read)
1783 {
1784         if (likely(!uncore->i915->params.mmio_debug) || !uncore->debug)
1785                 return false;
1786
1787         /* interrupts are disabled and re-enabled around uncore->lock usage */
1788         lockdep_assert_held(&uncore->lock);
1789
1790         spin_lock(&uncore->debug->lock);
1791         __unclaimed_previous_reg_debug(uncore, reg, read);
1792
1793         return true;
1794 }
1795
1796 static inline void
1797 unclaimed_reg_debug_footer(struct intel_uncore *uncore,
1798                            const i915_reg_t reg, const bool read)
1799 {
1800         /* interrupts are disabled and re-enabled around uncore->lock usage */
1801         lockdep_assert_held(&uncore->lock);
1802
1803         __unclaimed_reg_debug(uncore, reg, read);
1804         spin_unlock(&uncore->debug->lock);
1805 }
1806
1807 #define __vgpu_read(x) \
1808 static u##x \
1809 vgpu_read##x(struct intel_uncore *uncore, i915_reg_t reg, bool trace) { \
1810         u##x val = __raw_uncore_read##x(uncore, reg); \
1811         trace_i915_reg_rw(false, reg, val, sizeof(val), trace); \
1812         return val; \
1813 }
1814 __vgpu_read(8)
1815 __vgpu_read(16)
1816 __vgpu_read(32)
1817 __vgpu_read(64)
1818
1819 #define GEN2_READ_HEADER(x) \
1820         u##x val = 0; \
1821         assert_rpm_wakelock_held(uncore->rpm);
1822
1823 #define GEN2_READ_FOOTER \
1824         trace_i915_reg_rw(false, reg, val, sizeof(val), trace); \
1825         return val
1826
1827 #define __gen2_read(x) \
1828 static u##x \
1829 gen2_read##x(struct intel_uncore *uncore, i915_reg_t reg, bool trace) { \
1830         GEN2_READ_HEADER(x); \
1831         val = __raw_uncore_read##x(uncore, reg); \
1832         GEN2_READ_FOOTER; \
1833 }
1834
1835 #define __gen5_read(x) \
1836 static u##x \
1837 gen5_read##x(struct intel_uncore *uncore, i915_reg_t reg, bool trace) { \
1838         GEN2_READ_HEADER(x); \
1839         ilk_dummy_write(uncore); \
1840         val = __raw_uncore_read##x(uncore, reg); \
1841         GEN2_READ_FOOTER; \
1842 }
1843
1844 __gen5_read(8)
1845 __gen5_read(16)
1846 __gen5_read(32)
1847 __gen5_read(64)
1848 __gen2_read(8)
1849 __gen2_read(16)
1850 __gen2_read(32)
1851 __gen2_read(64)
1852
1853 #undef __gen5_read
1854 #undef __gen2_read
1855
1856 #undef GEN2_READ_FOOTER
1857 #undef GEN2_READ_HEADER
1858
1859 #define GEN6_READ_HEADER(x) \
1860         u32 offset = i915_mmio_reg_offset(reg); \
1861         unsigned long irqflags; \
1862         bool unclaimed_reg_debug; \
1863         u##x val = 0; \
1864         assert_rpm_wakelock_held(uncore->rpm); \
1865         spin_lock_irqsave(&uncore->lock, irqflags); \
1866         unclaimed_reg_debug = unclaimed_reg_debug_header(uncore, reg, true)
1867
1868 #define GEN6_READ_FOOTER \
1869         if (unclaimed_reg_debug) \
1870                 unclaimed_reg_debug_footer(uncore, reg, true);  \
1871         spin_unlock_irqrestore(&uncore->lock, irqflags); \
1872         trace_i915_reg_rw(false, reg, val, sizeof(val), trace); \
1873         return val
1874
1875 static noinline void ___force_wake_auto(struct intel_uncore *uncore,
1876                                         enum forcewake_domains fw_domains)
1877 {
1878         struct intel_uncore_forcewake_domain *domain;
1879         unsigned int tmp;
1880
1881         GEM_BUG_ON(fw_domains & ~uncore->fw_domains);
1882
1883         for_each_fw_domain_masked(domain, fw_domains, uncore, tmp)
1884                 fw_domain_arm_timer(domain);
1885
1886         fw_domains_get(uncore, fw_domains);
1887 }
1888
1889 static inline void __force_wake_auto(struct intel_uncore *uncore,
1890                                      enum forcewake_domains fw_domains)
1891 {
1892         GEM_BUG_ON(!fw_domains);
1893
1894         /* Turn on all requested but inactive supported forcewake domains. */
1895         fw_domains &= uncore->fw_domains;
1896         fw_domains &= ~uncore->fw_domains_active;
1897
1898         if (fw_domains)
1899                 ___force_wake_auto(uncore, fw_domains);
1900 }
1901
1902 #define __gen_fwtable_read(x) \
1903 static u##x \
1904 fwtable_read##x(struct intel_uncore *uncore, i915_reg_t reg, bool trace) \
1905 { \
1906         enum forcewake_domains fw_engine; \
1907         GEN6_READ_HEADER(x); \
1908         fw_engine = __fwtable_reg_read_fw_domains(uncore, offset); \
1909         if (fw_engine) \
1910                 __force_wake_auto(uncore, fw_engine); \
1911         val = __raw_uncore_read##x(uncore, reg); \
1912         GEN6_READ_FOOTER; \
1913 }
1914
1915 static enum forcewake_domains
1916 fwtable_reg_read_fw_domains(struct intel_uncore *uncore, i915_reg_t reg) {
1917         return __fwtable_reg_read_fw_domains(uncore, i915_mmio_reg_offset(reg));
1918 }
1919
1920 __gen_fwtable_read(8)
1921 __gen_fwtable_read(16)
1922 __gen_fwtable_read(32)
1923 __gen_fwtable_read(64)
1924
1925 #undef __gen_fwtable_read
1926 #undef GEN6_READ_FOOTER
1927 #undef GEN6_READ_HEADER
1928
1929 #define GEN2_WRITE_HEADER \
1930         trace_i915_reg_rw(true, reg, val, sizeof(val), trace); \
1931         assert_rpm_wakelock_held(uncore->rpm); \
1932
1933 #define GEN2_WRITE_FOOTER
1934
1935 #define __gen2_write(x) \
1936 static void \
1937 gen2_write##x(struct intel_uncore *uncore, i915_reg_t reg, u##x val, bool trace) { \
1938         GEN2_WRITE_HEADER; \
1939         __raw_uncore_write##x(uncore, reg, val); \
1940         GEN2_WRITE_FOOTER; \
1941 }
1942
1943 #define __gen5_write(x) \
1944 static void \
1945 gen5_write##x(struct intel_uncore *uncore, i915_reg_t reg, u##x val, bool trace) { \
1946         GEN2_WRITE_HEADER; \
1947         ilk_dummy_write(uncore); \
1948         __raw_uncore_write##x(uncore, reg, val); \
1949         GEN2_WRITE_FOOTER; \
1950 }
1951
1952 __gen5_write(8)
1953 __gen5_write(16)
1954 __gen5_write(32)
1955 __gen2_write(8)
1956 __gen2_write(16)
1957 __gen2_write(32)
1958
1959 #undef __gen5_write
1960 #undef __gen2_write
1961
1962 #undef GEN2_WRITE_FOOTER
1963 #undef GEN2_WRITE_HEADER
1964
1965 #define GEN6_WRITE_HEADER \
1966         u32 offset = i915_mmio_reg_offset(reg); \
1967         unsigned long irqflags; \
1968         bool unclaimed_reg_debug; \
1969         trace_i915_reg_rw(true, reg, val, sizeof(val), trace); \
1970         assert_rpm_wakelock_held(uncore->rpm); \
1971         spin_lock_irqsave(&uncore->lock, irqflags); \
1972         unclaimed_reg_debug = unclaimed_reg_debug_header(uncore, reg, false)
1973
1974 #define GEN6_WRITE_FOOTER \
1975         if (unclaimed_reg_debug) \
1976                 unclaimed_reg_debug_footer(uncore, reg, false); \
1977         spin_unlock_irqrestore(&uncore->lock, irqflags)
1978
1979 #define __gen6_write(x) \
1980 static void \
1981 gen6_write##x(struct intel_uncore *uncore, i915_reg_t reg, u##x val, bool trace) { \
1982         GEN6_WRITE_HEADER; \
1983         if (NEEDS_FORCE_WAKE(offset)) \
1984                 __gen6_gt_wait_for_fifo(uncore); \
1985         __raw_uncore_write##x(uncore, reg, val); \
1986         GEN6_WRITE_FOOTER; \
1987 }
1988 __gen6_write(8)
1989 __gen6_write(16)
1990 __gen6_write(32)
1991
1992 #define __gen_fwtable_write(x) \
1993 static void \
1994 fwtable_write##x(struct intel_uncore *uncore, i915_reg_t reg, u##x val, bool trace) { \
1995         enum forcewake_domains fw_engine; \
1996         GEN6_WRITE_HEADER; \
1997         fw_engine = __fwtable_reg_write_fw_domains(uncore, offset); \
1998         if (fw_engine) \
1999                 __force_wake_auto(uncore, fw_engine); \
2000         __raw_uncore_write##x(uncore, reg, val); \
2001         GEN6_WRITE_FOOTER; \
2002 }
2003
2004 static enum forcewake_domains
2005 fwtable_reg_write_fw_domains(struct intel_uncore *uncore, i915_reg_t reg)
2006 {
2007         return __fwtable_reg_write_fw_domains(uncore, i915_mmio_reg_offset(reg));
2008 }
2009
2010 __gen_fwtable_write(8)
2011 __gen_fwtable_write(16)
2012 __gen_fwtable_write(32)
2013
2014 #undef __gen_fwtable_write
2015 #undef GEN6_WRITE_FOOTER
2016 #undef GEN6_WRITE_HEADER
2017
2018 #define __vgpu_write(x) \
2019 static void \
2020 vgpu_write##x(struct intel_uncore *uncore, i915_reg_t reg, u##x val, bool trace) { \
2021         trace_i915_reg_rw(true, reg, val, sizeof(val), trace); \
2022         __raw_uncore_write##x(uncore, reg, val); \
2023 }
2024 __vgpu_write(8)
2025 __vgpu_write(16)
2026 __vgpu_write(32)
2027
2028 #define ASSIGN_RAW_WRITE_MMIO_VFUNCS(uncore, x) \
2029 do { \
2030         (uncore)->funcs.mmio_writeb = x##_write8; \
2031         (uncore)->funcs.mmio_writew = x##_write16; \
2032         (uncore)->funcs.mmio_writel = x##_write32; \
2033 } while (0)
2034
2035 #define ASSIGN_RAW_READ_MMIO_VFUNCS(uncore, x) \
2036 do { \
2037         (uncore)->funcs.mmio_readb = x##_read8; \
2038         (uncore)->funcs.mmio_readw = x##_read16; \
2039         (uncore)->funcs.mmio_readl = x##_read32; \
2040         (uncore)->funcs.mmio_readq = x##_read64; \
2041 } while (0)
2042
2043 #define ASSIGN_WRITE_MMIO_VFUNCS(uncore, x) \
2044 do { \
2045         ASSIGN_RAW_WRITE_MMIO_VFUNCS((uncore), x); \
2046         (uncore)->funcs.write_fw_domains = x##_reg_write_fw_domains; \
2047 } while (0)
2048
2049 #define ASSIGN_READ_MMIO_VFUNCS(uncore, x) \
2050 do { \
2051         ASSIGN_RAW_READ_MMIO_VFUNCS(uncore, x); \
2052         (uncore)->funcs.read_fw_domains = x##_reg_read_fw_domains; \
2053 } while (0)
2054
2055 static int __fw_domain_init(struct intel_uncore *uncore,
2056                             enum forcewake_domain_id domain_id,
2057                             i915_reg_t reg_set,
2058                             i915_reg_t reg_ack)
2059 {
2060         struct intel_uncore_forcewake_domain *d;
2061
2062         GEM_BUG_ON(domain_id >= FW_DOMAIN_ID_COUNT);
2063         GEM_BUG_ON(uncore->fw_domain[domain_id]);
2064
2065         if (i915_inject_probe_failure(uncore->i915))
2066                 return -ENOMEM;
2067
2068         d = kzalloc(sizeof(*d), GFP_KERNEL);
2069         if (!d)
2070                 return -ENOMEM;
2071
2072         drm_WARN_ON(&uncore->i915->drm, !i915_mmio_reg_valid(reg_set));
2073         drm_WARN_ON(&uncore->i915->drm, !i915_mmio_reg_valid(reg_ack));
2074
2075         d->uncore = uncore;
2076         d->wake_count = 0;
2077         d->reg_set = uncore->regs + i915_mmio_reg_offset(reg_set) + uncore->gsi_offset;
2078         d->reg_ack = uncore->regs + i915_mmio_reg_offset(reg_ack) + uncore->gsi_offset;
2079
2080         d->id = domain_id;
2081
2082         BUILD_BUG_ON(FORCEWAKE_RENDER != (1 << FW_DOMAIN_ID_RENDER));
2083         BUILD_BUG_ON(FORCEWAKE_GT != (1 << FW_DOMAIN_ID_GT));
2084         BUILD_BUG_ON(FORCEWAKE_MEDIA != (1 << FW_DOMAIN_ID_MEDIA));
2085         BUILD_BUG_ON(FORCEWAKE_MEDIA_VDBOX0 != (1 << FW_DOMAIN_ID_MEDIA_VDBOX0));
2086         BUILD_BUG_ON(FORCEWAKE_MEDIA_VDBOX1 != (1 << FW_DOMAIN_ID_MEDIA_VDBOX1));
2087         BUILD_BUG_ON(FORCEWAKE_MEDIA_VDBOX2 != (1 << FW_DOMAIN_ID_MEDIA_VDBOX2));
2088         BUILD_BUG_ON(FORCEWAKE_MEDIA_VDBOX3 != (1 << FW_DOMAIN_ID_MEDIA_VDBOX3));
2089         BUILD_BUG_ON(FORCEWAKE_MEDIA_VDBOX4 != (1 << FW_DOMAIN_ID_MEDIA_VDBOX4));
2090         BUILD_BUG_ON(FORCEWAKE_MEDIA_VDBOX5 != (1 << FW_DOMAIN_ID_MEDIA_VDBOX5));
2091         BUILD_BUG_ON(FORCEWAKE_MEDIA_VDBOX6 != (1 << FW_DOMAIN_ID_MEDIA_VDBOX6));
2092         BUILD_BUG_ON(FORCEWAKE_MEDIA_VDBOX7 != (1 << FW_DOMAIN_ID_MEDIA_VDBOX7));
2093         BUILD_BUG_ON(FORCEWAKE_MEDIA_VEBOX0 != (1 << FW_DOMAIN_ID_MEDIA_VEBOX0));
2094         BUILD_BUG_ON(FORCEWAKE_MEDIA_VEBOX1 != (1 << FW_DOMAIN_ID_MEDIA_VEBOX1));
2095         BUILD_BUG_ON(FORCEWAKE_MEDIA_VEBOX2 != (1 << FW_DOMAIN_ID_MEDIA_VEBOX2));
2096         BUILD_BUG_ON(FORCEWAKE_MEDIA_VEBOX3 != (1 << FW_DOMAIN_ID_MEDIA_VEBOX3));
2097         BUILD_BUG_ON(FORCEWAKE_GSC != (1 << FW_DOMAIN_ID_GSC));
2098
2099         d->mask = BIT(domain_id);
2100
2101         hrtimer_init(&d->timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
2102         d->timer.function = intel_uncore_fw_release_timer;
2103
2104         uncore->fw_domains |= BIT(domain_id);
2105
2106         fw_domain_reset(d);
2107
2108         uncore->fw_domain[domain_id] = d;
2109
2110         return 0;
2111 }
2112
2113 static void fw_domain_fini(struct intel_uncore *uncore,
2114                            enum forcewake_domain_id domain_id)
2115 {
2116         struct intel_uncore_forcewake_domain *d;
2117
2118         GEM_BUG_ON(domain_id >= FW_DOMAIN_ID_COUNT);
2119
2120         d = fetch_and_zero(&uncore->fw_domain[domain_id]);
2121         if (!d)
2122                 return;
2123
2124         uncore->fw_domains &= ~BIT(domain_id);
2125         drm_WARN_ON(&uncore->i915->drm, d->wake_count);
2126         drm_WARN_ON(&uncore->i915->drm, hrtimer_cancel(&d->timer));
2127         kfree(d);
2128 }
2129
2130 static void intel_uncore_fw_domains_fini(struct intel_uncore *uncore)
2131 {
2132         struct intel_uncore_forcewake_domain *d;
2133         int tmp;
2134
2135         for_each_fw_domain(d, uncore, tmp)
2136                 fw_domain_fini(uncore, d->id);
2137 }
2138
2139 static const struct intel_uncore_fw_get uncore_get_fallback = {
2140         .force_wake_get = fw_domains_get_with_fallback
2141 };
2142
2143 static const struct intel_uncore_fw_get uncore_get_normal = {
2144         .force_wake_get = fw_domains_get_normal,
2145 };
2146
2147 static const struct intel_uncore_fw_get uncore_get_thread_status = {
2148         .force_wake_get = fw_domains_get_with_thread_status
2149 };
2150
2151 static int intel_uncore_fw_domains_init(struct intel_uncore *uncore)
2152 {
2153         struct drm_i915_private *i915 = uncore->i915;
2154         int ret = 0;
2155
2156         GEM_BUG_ON(!intel_uncore_has_forcewake(uncore));
2157
2158 #define fw_domain_init(uncore__, id__, set__, ack__) \
2159         (ret ?: (ret = __fw_domain_init((uncore__), (id__), (set__), (ack__))))
2160
2161         if (GRAPHICS_VER(i915) >= 11) {
2162                 intel_engine_mask_t emask;
2163                 int i;
2164
2165                 /* we'll prune the domains of missing engines later */
2166                 emask = uncore->gt->info.engine_mask;
2167
2168                 uncore->fw_get_funcs = &uncore_get_fallback;
2169                 if (GRAPHICS_VER_FULL(i915) >= IP_VER(12, 70))
2170                         fw_domain_init(uncore, FW_DOMAIN_ID_GT,
2171                                        FORCEWAKE_GT_GEN9,
2172                                        FORCEWAKE_ACK_GT_MTL);
2173                 else
2174                         fw_domain_init(uncore, FW_DOMAIN_ID_GT,
2175                                        FORCEWAKE_GT_GEN9,
2176                                        FORCEWAKE_ACK_GT_GEN9);
2177
2178                 if (RCS_MASK(uncore->gt) || CCS_MASK(uncore->gt))
2179                         fw_domain_init(uncore, FW_DOMAIN_ID_RENDER,
2180                                        FORCEWAKE_RENDER_GEN9,
2181                                        FORCEWAKE_ACK_RENDER_GEN9);
2182
2183                 for (i = 0; i < I915_MAX_VCS; i++) {
2184                         if (!__HAS_ENGINE(emask, _VCS(i)))
2185                                 continue;
2186
2187                         fw_domain_init(uncore, FW_DOMAIN_ID_MEDIA_VDBOX0 + i,
2188                                        FORCEWAKE_MEDIA_VDBOX_GEN11(i),
2189                                        FORCEWAKE_ACK_MEDIA_VDBOX_GEN11(i));
2190                 }
2191                 for (i = 0; i < I915_MAX_VECS; i++) {
2192                         if (!__HAS_ENGINE(emask, _VECS(i)))
2193                                 continue;
2194
2195                         fw_domain_init(uncore, FW_DOMAIN_ID_MEDIA_VEBOX0 + i,
2196                                        FORCEWAKE_MEDIA_VEBOX_GEN11(i),
2197                                        FORCEWAKE_ACK_MEDIA_VEBOX_GEN11(i));
2198                 }
2199
2200                 if (uncore->gt->type == GT_MEDIA)
2201                         fw_domain_init(uncore, FW_DOMAIN_ID_GSC,
2202                                        FORCEWAKE_REQ_GSC, FORCEWAKE_ACK_GSC);
2203         } else if (IS_GRAPHICS_VER(i915, 9, 10)) {
2204                 uncore->fw_get_funcs = &uncore_get_fallback;
2205                 fw_domain_init(uncore, FW_DOMAIN_ID_RENDER,
2206                                FORCEWAKE_RENDER_GEN9,
2207                                FORCEWAKE_ACK_RENDER_GEN9);
2208                 fw_domain_init(uncore, FW_DOMAIN_ID_GT,
2209                                FORCEWAKE_GT_GEN9,
2210                                FORCEWAKE_ACK_GT_GEN9);
2211                 fw_domain_init(uncore, FW_DOMAIN_ID_MEDIA,
2212                                FORCEWAKE_MEDIA_GEN9, FORCEWAKE_ACK_MEDIA_GEN9);
2213         } else if (IS_VALLEYVIEW(i915) || IS_CHERRYVIEW(i915)) {
2214                 uncore->fw_get_funcs = &uncore_get_normal;
2215                 fw_domain_init(uncore, FW_DOMAIN_ID_RENDER,
2216                                FORCEWAKE_VLV, FORCEWAKE_ACK_VLV);
2217                 fw_domain_init(uncore, FW_DOMAIN_ID_MEDIA,
2218                                FORCEWAKE_MEDIA_VLV, FORCEWAKE_ACK_MEDIA_VLV);
2219         } else if (IS_HASWELL(i915) || IS_BROADWELL(i915)) {
2220                 uncore->fw_get_funcs = &uncore_get_thread_status;
2221                 fw_domain_init(uncore, FW_DOMAIN_ID_RENDER,
2222                                FORCEWAKE_MT, FORCEWAKE_ACK_HSW);
2223         } else if (IS_IVYBRIDGE(i915)) {
2224                 u32 ecobus;
2225
2226                 /* IVB configs may use multi-threaded forcewake */
2227
2228                 /* A small trick here - if the bios hasn't configured
2229                  * MT forcewake, and if the device is in RC6, then
2230                  * force_wake_mt_get will not wake the device and the
2231                  * ECOBUS read will return zero. Which will be
2232                  * (correctly) interpreted by the test below as MT
2233                  * forcewake being disabled.
2234                  */
2235                 uncore->fw_get_funcs = &uncore_get_thread_status;
2236
2237                 /* We need to init first for ECOBUS access and then
2238                  * determine later if we want to reinit, in case of MT access is
2239                  * not working. In this stage we don't know which flavour this
2240                  * ivb is, so it is better to reset also the gen6 fw registers
2241                  * before the ecobus check.
2242                  */
2243
2244                 __raw_uncore_write32(uncore, FORCEWAKE, 0);
2245                 __raw_posting_read(uncore, ECOBUS);
2246
2247                 ret = __fw_domain_init(uncore, FW_DOMAIN_ID_RENDER,
2248                                        FORCEWAKE_MT, FORCEWAKE_MT_ACK);
2249                 if (ret)
2250                         goto out;
2251
2252                 spin_lock_irq(&uncore->lock);
2253                 fw_domains_get_with_thread_status(uncore, FORCEWAKE_RENDER);
2254                 ecobus = __raw_uncore_read32(uncore, ECOBUS);
2255                 fw_domains_put(uncore, FORCEWAKE_RENDER);
2256                 spin_unlock_irq(&uncore->lock);
2257
2258                 if (!(ecobus & FORCEWAKE_MT_ENABLE)) {
2259                         drm_info(&i915->drm, "No MT forcewake available on Ivybridge, this can result in issues\n");
2260                         drm_info(&i915->drm, "when using vblank-synced partial screen updates.\n");
2261                         fw_domain_fini(uncore, FW_DOMAIN_ID_RENDER);
2262                         fw_domain_init(uncore, FW_DOMAIN_ID_RENDER,
2263                                        FORCEWAKE, FORCEWAKE_ACK);
2264                 }
2265         } else if (GRAPHICS_VER(i915) == 6) {
2266                 uncore->fw_get_funcs = &uncore_get_thread_status;
2267                 fw_domain_init(uncore, FW_DOMAIN_ID_RENDER,
2268                                FORCEWAKE, FORCEWAKE_ACK);
2269         }
2270
2271 #undef fw_domain_init
2272
2273         /* All future platforms are expected to require complex power gating */
2274         drm_WARN_ON(&i915->drm, !ret && uncore->fw_domains == 0);
2275
2276 out:
2277         if (ret)
2278                 intel_uncore_fw_domains_fini(uncore);
2279
2280         return ret;
2281 }
2282
2283 #define ASSIGN_FW_DOMAINS_TABLE(uncore, d) \
2284 { \
2285         (uncore)->fw_domains_table = \
2286                         (struct intel_forcewake_range *)(d); \
2287         (uncore)->fw_domains_table_entries = ARRAY_SIZE((d)); \
2288 }
2289
2290 #define ASSIGN_SHADOW_TABLE(uncore, d) \
2291 { \
2292         (uncore)->shadowed_reg_table = d; \
2293         (uncore)->shadowed_reg_table_entries = ARRAY_SIZE((d)); \
2294 }
2295
2296 static int i915_pmic_bus_access_notifier(struct notifier_block *nb,
2297                                          unsigned long action, void *data)
2298 {
2299         struct intel_uncore *uncore = container_of(nb,
2300                         struct intel_uncore, pmic_bus_access_nb);
2301
2302         switch (action) {
2303         case MBI_PMIC_BUS_ACCESS_BEGIN:
2304                 /*
2305                  * forcewake all now to make sure that we don't need to do a
2306                  * forcewake later which on systems where this notifier gets
2307                  * called requires the punit to access to the shared pmic i2c
2308                  * bus, which will be busy after this notification, leading to:
2309                  * "render: timed out waiting for forcewake ack request."
2310                  * errors.
2311                  *
2312                  * The notifier is unregistered during intel_runtime_suspend(),
2313                  * so it's ok to access the HW here without holding a RPM
2314                  * wake reference -> disable wakeref asserts for the time of
2315                  * the access.
2316                  */
2317                 disable_rpm_wakeref_asserts(uncore->rpm);
2318                 intel_uncore_forcewake_get(uncore, FORCEWAKE_ALL);
2319                 enable_rpm_wakeref_asserts(uncore->rpm);
2320                 break;
2321         case MBI_PMIC_BUS_ACCESS_END:
2322                 intel_uncore_forcewake_put(uncore, FORCEWAKE_ALL);
2323                 break;
2324         }
2325
2326         return NOTIFY_OK;
2327 }
2328
2329 static void uncore_unmap_mmio(struct drm_device *drm, void *regs)
2330 {
2331         iounmap((void __iomem *)regs);
2332 }
2333
2334 int intel_uncore_setup_mmio(struct intel_uncore *uncore, phys_addr_t phys_addr)
2335 {
2336         struct drm_i915_private *i915 = uncore->i915;
2337         int mmio_size;
2338
2339         /*
2340          * Before gen4, the registers and the GTT are behind different BARs.
2341          * However, from gen4 onwards, the registers and the GTT are shared
2342          * in the same BAR, so we want to restrict this ioremap from
2343          * clobbering the GTT which we want ioremap_wc instead. Fortunately,
2344          * the register BAR remains the same size for all the earlier
2345          * generations up to Ironlake.
2346          * For dgfx chips register range is expanded to 4MB, and this larger
2347          * range is also used for integrated gpus beginning with Meteor Lake.
2348          */
2349         if (IS_DGFX(i915) || GRAPHICS_VER_FULL(i915) >= IP_VER(12, 70))
2350                 mmio_size = 4 * 1024 * 1024;
2351         else if (GRAPHICS_VER(i915) >= 5)
2352                 mmio_size = 2 * 1024 * 1024;
2353         else
2354                 mmio_size = 512 * 1024;
2355
2356         uncore->regs = ioremap(phys_addr, mmio_size);
2357         if (uncore->regs == NULL) {
2358                 drm_err(&i915->drm, "failed to map registers\n");
2359                 return -EIO;
2360         }
2361
2362         return drmm_add_action_or_reset(&i915->drm, uncore_unmap_mmio,
2363                                         (void __force *)uncore->regs);
2364 }
2365
2366 void intel_uncore_init_early(struct intel_uncore *uncore,
2367                              struct intel_gt *gt)
2368 {
2369         spin_lock_init(&uncore->lock);
2370         uncore->i915 = gt->i915;
2371         uncore->gt = gt;
2372         uncore->rpm = &gt->i915->runtime_pm;
2373 }
2374
2375 static void uncore_raw_init(struct intel_uncore *uncore)
2376 {
2377         GEM_BUG_ON(intel_uncore_has_forcewake(uncore));
2378
2379         if (intel_vgpu_active(uncore->i915)) {
2380                 ASSIGN_RAW_WRITE_MMIO_VFUNCS(uncore, vgpu);
2381                 ASSIGN_RAW_READ_MMIO_VFUNCS(uncore, vgpu);
2382         } else if (GRAPHICS_VER(uncore->i915) == 5) {
2383                 ASSIGN_RAW_WRITE_MMIO_VFUNCS(uncore, gen5);
2384                 ASSIGN_RAW_READ_MMIO_VFUNCS(uncore, gen5);
2385         } else {
2386                 ASSIGN_RAW_WRITE_MMIO_VFUNCS(uncore, gen2);
2387                 ASSIGN_RAW_READ_MMIO_VFUNCS(uncore, gen2);
2388         }
2389 }
2390
2391 static int uncore_media_forcewake_init(struct intel_uncore *uncore)
2392 {
2393         struct drm_i915_private *i915 = uncore->i915;
2394
2395         if (MEDIA_VER(i915) >= 13) {
2396                 ASSIGN_FW_DOMAINS_TABLE(uncore, __xelpmp_fw_ranges);
2397                 ASSIGN_SHADOW_TABLE(uncore, xelpmp_shadowed_regs);
2398                 ASSIGN_WRITE_MMIO_VFUNCS(uncore, fwtable);
2399         } else {
2400                 MISSING_CASE(MEDIA_VER(i915));
2401                 return -ENODEV;
2402         }
2403
2404         return 0;
2405 }
2406
2407 static int uncore_forcewake_init(struct intel_uncore *uncore)
2408 {
2409         struct drm_i915_private *i915 = uncore->i915;
2410         int ret;
2411
2412         GEM_BUG_ON(!intel_uncore_has_forcewake(uncore));
2413
2414         ret = intel_uncore_fw_domains_init(uncore);
2415         if (ret)
2416                 return ret;
2417         forcewake_early_sanitize(uncore, 0);
2418
2419         ASSIGN_READ_MMIO_VFUNCS(uncore, fwtable);
2420
2421         if (uncore->gt->type == GT_MEDIA)
2422                 return uncore_media_forcewake_init(uncore);
2423
2424         if (GRAPHICS_VER_FULL(i915) >= IP_VER(12, 70)) {
2425                 ASSIGN_FW_DOMAINS_TABLE(uncore, __mtl_fw_ranges);
2426                 ASSIGN_SHADOW_TABLE(uncore, mtl_shadowed_regs);
2427                 ASSIGN_WRITE_MMIO_VFUNCS(uncore, fwtable);
2428         } else if (GRAPHICS_VER_FULL(i915) >= IP_VER(12, 55)) {
2429                 ASSIGN_FW_DOMAINS_TABLE(uncore, __dg2_fw_ranges);
2430                 ASSIGN_SHADOW_TABLE(uncore, dg2_shadowed_regs);
2431                 ASSIGN_WRITE_MMIO_VFUNCS(uncore, fwtable);
2432         } else if (GRAPHICS_VER(i915) >= 12) {
2433                 ASSIGN_FW_DOMAINS_TABLE(uncore, __gen12_fw_ranges);
2434                 ASSIGN_SHADOW_TABLE(uncore, gen12_shadowed_regs);
2435                 ASSIGN_WRITE_MMIO_VFUNCS(uncore, fwtable);
2436         } else if (GRAPHICS_VER(i915) == 11) {
2437                 ASSIGN_FW_DOMAINS_TABLE(uncore, __gen11_fw_ranges);
2438                 ASSIGN_SHADOW_TABLE(uncore, gen11_shadowed_regs);
2439                 ASSIGN_WRITE_MMIO_VFUNCS(uncore, fwtable);
2440         } else if (IS_GRAPHICS_VER(i915, 9, 10)) {
2441                 ASSIGN_FW_DOMAINS_TABLE(uncore, __gen9_fw_ranges);
2442                 ASSIGN_SHADOW_TABLE(uncore, gen8_shadowed_regs);
2443                 ASSIGN_WRITE_MMIO_VFUNCS(uncore, fwtable);
2444         } else if (IS_CHERRYVIEW(i915)) {
2445                 ASSIGN_FW_DOMAINS_TABLE(uncore, __chv_fw_ranges);
2446                 ASSIGN_SHADOW_TABLE(uncore, gen8_shadowed_regs);
2447                 ASSIGN_WRITE_MMIO_VFUNCS(uncore, fwtable);
2448         } else if (GRAPHICS_VER(i915) == 8) {
2449                 ASSIGN_FW_DOMAINS_TABLE(uncore, __gen6_fw_ranges);
2450                 ASSIGN_SHADOW_TABLE(uncore, gen8_shadowed_regs);
2451                 ASSIGN_WRITE_MMIO_VFUNCS(uncore, fwtable);
2452         } else if (IS_VALLEYVIEW(i915)) {
2453                 ASSIGN_FW_DOMAINS_TABLE(uncore, __vlv_fw_ranges);
2454                 ASSIGN_WRITE_MMIO_VFUNCS(uncore, gen6);
2455         } else if (IS_GRAPHICS_VER(i915, 6, 7)) {
2456                 ASSIGN_FW_DOMAINS_TABLE(uncore, __gen6_fw_ranges);
2457                 ASSIGN_WRITE_MMIO_VFUNCS(uncore, gen6);
2458         }
2459
2460         uncore->pmic_bus_access_nb.notifier_call = i915_pmic_bus_access_notifier;
2461         iosf_mbi_register_pmic_bus_access_notifier(&uncore->pmic_bus_access_nb);
2462
2463         return 0;
2464 }
2465
2466 static int sanity_check_mmio_access(struct intel_uncore *uncore)
2467 {
2468         struct drm_i915_private *i915 = uncore->i915;
2469
2470         if (GRAPHICS_VER(i915) < 8)
2471                 return 0;
2472
2473         /*
2474          * Sanitycheck that MMIO access to the device is working properly.  If
2475          * the CPU is unable to communcate with a PCI device, BAR reads will
2476          * return 0xFFFFFFFF.  Let's make sure the device isn't in this state
2477          * before we start trying to access registers.
2478          *
2479          * We use the primary GT's forcewake register as our guinea pig since
2480          * it's been around since HSW and it's a masked register so the upper
2481          * 16 bits can never read back as 1's if device access is operating
2482          * properly.
2483          *
2484          * If MMIO isn't working, we'll wait up to 2 seconds to see if it
2485          * recovers, then give up.
2486          */
2487 #define COND (__raw_uncore_read32(uncore, FORCEWAKE_MT) != ~0)
2488         if (wait_for(COND, 2000) == -ETIMEDOUT) {
2489                 drm_err(&i915->drm, "Device is non-operational; MMIO access returns 0xFFFFFFFF!\n");
2490                 return -EIO;
2491         }
2492
2493         return 0;
2494 }
2495
2496 int intel_uncore_init_mmio(struct intel_uncore *uncore)
2497 {
2498         struct drm_i915_private *i915 = uncore->i915;
2499         int ret;
2500
2501         ret = sanity_check_mmio_access(uncore);
2502         if (ret)
2503                 return ret;
2504
2505         /*
2506          * The boot firmware initializes local memory and assesses its health.
2507          * If memory training fails, the punit will have been instructed to
2508          * keep the GT powered down; we won't be able to communicate with it
2509          * and we should not continue with driver initialization.
2510          */
2511         if (IS_DGFX(i915) &&
2512             !(__raw_uncore_read32(uncore, GU_CNTL) & LMEM_INIT)) {
2513                 drm_err(&i915->drm, "LMEM not initialized by firmware\n");
2514                 return -ENODEV;
2515         }
2516
2517         if (GRAPHICS_VER(i915) > 5 && !intel_vgpu_active(i915))
2518                 uncore->flags |= UNCORE_HAS_FORCEWAKE;
2519
2520         if (!intel_uncore_has_forcewake(uncore)) {
2521                 uncore_raw_init(uncore);
2522         } else {
2523                 ret = uncore_forcewake_init(uncore);
2524                 if (ret)
2525                         return ret;
2526         }
2527
2528         /* make sure fw funcs are set if and only if we have fw*/
2529         GEM_BUG_ON(intel_uncore_has_forcewake(uncore) != !!uncore->fw_get_funcs);
2530         GEM_BUG_ON(intel_uncore_has_forcewake(uncore) != !!uncore->funcs.read_fw_domains);
2531         GEM_BUG_ON(intel_uncore_has_forcewake(uncore) != !!uncore->funcs.write_fw_domains);
2532
2533         if (HAS_FPGA_DBG_UNCLAIMED(i915))
2534                 uncore->flags |= UNCORE_HAS_FPGA_DBG_UNCLAIMED;
2535
2536         if (IS_VALLEYVIEW(i915) || IS_CHERRYVIEW(i915))
2537                 uncore->flags |= UNCORE_HAS_DBG_UNCLAIMED;
2538
2539         if (IS_GRAPHICS_VER(i915, 6, 7))
2540                 uncore->flags |= UNCORE_HAS_FIFO;
2541
2542         /* clear out unclaimed reg detection bit */
2543         if (intel_uncore_unclaimed_mmio(uncore))
2544                 drm_dbg(&i915->drm, "unclaimed mmio detected on uncore init, clearing\n");
2545
2546         return 0;
2547 }
2548
2549 /*
2550  * We might have detected that some engines are fused off after we initialized
2551  * the forcewake domains. Prune them, to make sure they only reference existing
2552  * engines.
2553  */
2554 void intel_uncore_prune_engine_fw_domains(struct intel_uncore *uncore,
2555                                           struct intel_gt *gt)
2556 {
2557         enum forcewake_domains fw_domains = uncore->fw_domains;
2558         enum forcewake_domain_id domain_id;
2559         int i;
2560
2561         if (!intel_uncore_has_forcewake(uncore) || GRAPHICS_VER(uncore->i915) < 11)
2562                 return;
2563
2564         for (i = 0; i < I915_MAX_VCS; i++) {
2565                 domain_id = FW_DOMAIN_ID_MEDIA_VDBOX0 + i;
2566
2567                 if (HAS_ENGINE(gt, _VCS(i)))
2568                         continue;
2569
2570                 /*
2571                  * Starting with XeHP, the power well for an even-numbered
2572                  * VDBOX is also used for shared units within the
2573                  * media slice such as SFC.  So even if the engine
2574                  * itself is fused off, we still need to initialize
2575                  * the forcewake domain if any of the other engines
2576                  * in the same media slice are present.
2577                  */
2578                 if (GRAPHICS_VER_FULL(uncore->i915) >= IP_VER(12, 55) && i % 2 == 0) {
2579                         if ((i + 1 < I915_MAX_VCS) && HAS_ENGINE(gt, _VCS(i + 1)))
2580                                 continue;
2581
2582                         if (HAS_ENGINE(gt, _VECS(i / 2)))
2583                                 continue;
2584                 }
2585
2586                 if (fw_domains & BIT(domain_id))
2587                         fw_domain_fini(uncore, domain_id);
2588         }
2589
2590         for (i = 0; i < I915_MAX_VECS; i++) {
2591                 domain_id = FW_DOMAIN_ID_MEDIA_VEBOX0 + i;
2592
2593                 if (HAS_ENGINE(gt, _VECS(i)))
2594                         continue;
2595
2596                 if (fw_domains & BIT(domain_id))
2597                         fw_domain_fini(uncore, domain_id);
2598         }
2599
2600         if ((fw_domains & BIT(FW_DOMAIN_ID_GSC)) && !HAS_ENGINE(gt, GSC0))
2601                 fw_domain_fini(uncore, FW_DOMAIN_ID_GSC);
2602 }
2603
2604 /*
2605  * The driver-initiated FLR is the highest level of reset that we can trigger
2606  * from within the driver. It is different from the PCI FLR in that it doesn't
2607  * fully reset the SGUnit and doesn't modify the PCI config space and therefore
2608  * it doesn't require a re-enumeration of the PCI BARs. However, the
2609  * driver-initiated FLR does still cause a reset of both GT and display and a
2610  * memory wipe of local and stolen memory, so recovery would require a full HW
2611  * re-init and saving/restoring (or re-populating) the wiped memory. Since we
2612  * perform the FLR as the very last action before releasing access to the HW
2613  * during the driver release flow, we don't attempt recovery at all, because
2614  * if/when a new instance of i915 is bound to the device it will do a full
2615  * re-init anyway.
2616  */
2617 static void driver_initiated_flr(struct intel_uncore *uncore)
2618 {
2619         struct drm_i915_private *i915 = uncore->i915;
2620         unsigned int flr_timeout_ms;
2621         int ret;
2622
2623         drm_dbg(&i915->drm, "Triggering Driver-FLR\n");
2624
2625         /*
2626          * The specification recommends a 3 seconds FLR reset timeout. To be
2627          * cautious, we will extend this to 9 seconds, three times the specified
2628          * timeout.
2629          */
2630         flr_timeout_ms = 9000;
2631
2632         /*
2633          * Make sure any pending FLR requests have cleared by waiting for the
2634          * FLR trigger bit to go to zero. Also clear GU_DEBUG's DRIVERFLR_STATUS
2635          * to make sure it's not still set from a prior attempt (it's a write to
2636          * clear bit).
2637          * Note that we should never be in a situation where a previous attempt
2638          * is still pending (unless the HW is totally dead), but better to be
2639          * safe in case something unexpected happens
2640          */
2641         ret = intel_wait_for_register_fw(uncore, GU_CNTL, DRIVERFLR, 0, flr_timeout_ms);
2642         if (ret) {
2643                 drm_err(&i915->drm,
2644                         "Failed to wait for Driver-FLR bit to clear! %d\n",
2645                         ret);
2646                 return;
2647         }
2648         intel_uncore_write_fw(uncore, GU_DEBUG, DRIVERFLR_STATUS);
2649
2650         /* Trigger the actual Driver-FLR */
2651         intel_uncore_rmw_fw(uncore, GU_CNTL, 0, DRIVERFLR);
2652
2653         /* Wait for hardware teardown to complete */
2654         ret = intel_wait_for_register_fw(uncore, GU_CNTL,
2655                                          DRIVERFLR, 0,
2656                                          flr_timeout_ms);
2657         if (ret) {
2658                 drm_err(&i915->drm, "Driver-FLR-teardown wait completion failed! %d\n", ret);
2659                 return;
2660         }
2661
2662         /* Wait for hardware/firmware re-init to complete */
2663         ret = intel_wait_for_register_fw(uncore, GU_DEBUG,
2664                                          DRIVERFLR_STATUS, DRIVERFLR_STATUS,
2665                                          flr_timeout_ms);
2666         if (ret) {
2667                 drm_err(&i915->drm, "Driver-FLR-reinit wait completion failed! %d\n", ret);
2668                 return;
2669         }
2670
2671         /* Clear sticky completion status */
2672         intel_uncore_write_fw(uncore, GU_DEBUG, DRIVERFLR_STATUS);
2673 }
2674
2675 /* Called via drm-managed action */
2676 void intel_uncore_fini_mmio(struct drm_device *dev, void *data)
2677 {
2678         struct intel_uncore *uncore = data;
2679
2680         if (intel_uncore_has_forcewake(uncore)) {
2681                 iosf_mbi_punit_acquire();
2682                 iosf_mbi_unregister_pmic_bus_access_notifier_unlocked(
2683                         &uncore->pmic_bus_access_nb);
2684                 intel_uncore_forcewake_reset(uncore);
2685                 intel_uncore_fw_domains_fini(uncore);
2686                 iosf_mbi_punit_release();
2687         }
2688
2689         if (intel_uncore_needs_flr_on_fini(uncore))
2690                 driver_initiated_flr(uncore);
2691 }
2692
2693 /**
2694  * __intel_wait_for_register_fw - wait until register matches expected state
2695  * @uncore: the struct intel_uncore
2696  * @reg: the register to read
2697  * @mask: mask to apply to register value
2698  * @value: expected value
2699  * @fast_timeout_us: fast timeout in microsecond for atomic/tight wait
2700  * @slow_timeout_ms: slow timeout in millisecond
2701  * @out_value: optional placeholder to hold registry value
2702  *
2703  * This routine waits until the target register @reg contains the expected
2704  * @value after applying the @mask, i.e. it waits until ::
2705  *
2706  *     (intel_uncore_read_fw(uncore, reg) & mask) == value
2707  *
2708  * Otherwise, the wait will timeout after @slow_timeout_ms milliseconds.
2709  * For atomic context @slow_timeout_ms must be zero and @fast_timeout_us
2710  * must be not larger than 20,0000 microseconds.
2711  *
2712  * Note that this routine assumes the caller holds forcewake asserted, it is
2713  * not suitable for very long waits. See intel_wait_for_register() if you
2714  * wish to wait without holding forcewake for the duration (i.e. you expect
2715  * the wait to be slow).
2716  *
2717  * Return: 0 if the register matches the desired condition, or -ETIMEDOUT.
2718  */
2719 int __intel_wait_for_register_fw(struct intel_uncore *uncore,
2720                                  i915_reg_t reg,
2721                                  u32 mask,
2722                                  u32 value,
2723                                  unsigned int fast_timeout_us,
2724                                  unsigned int slow_timeout_ms,
2725                                  u32 *out_value)
2726 {
2727         u32 reg_value = 0;
2728 #define done (((reg_value = intel_uncore_read_fw(uncore, reg)) & mask) == value)
2729         int ret;
2730
2731         /* Catch any overuse of this function */
2732         might_sleep_if(slow_timeout_ms);
2733         GEM_BUG_ON(fast_timeout_us > 20000);
2734         GEM_BUG_ON(!fast_timeout_us && !slow_timeout_ms);
2735
2736         ret = -ETIMEDOUT;
2737         if (fast_timeout_us && fast_timeout_us <= 20000)
2738                 ret = _wait_for_atomic(done, fast_timeout_us, 0);
2739         if (ret && slow_timeout_ms)
2740                 ret = wait_for(done, slow_timeout_ms);
2741
2742         if (out_value)
2743                 *out_value = reg_value;
2744
2745         return ret;
2746 #undef done
2747 }
2748
2749 /**
2750  * __intel_wait_for_register - wait until register matches expected state
2751  * @uncore: the struct intel_uncore
2752  * @reg: the register to read
2753  * @mask: mask to apply to register value
2754  * @value: expected value
2755  * @fast_timeout_us: fast timeout in microsecond for atomic/tight wait
2756  * @slow_timeout_ms: slow timeout in millisecond
2757  * @out_value: optional placeholder to hold registry value
2758  *
2759  * This routine waits until the target register @reg contains the expected
2760  * @value after applying the @mask, i.e. it waits until ::
2761  *
2762  *     (intel_uncore_read(uncore, reg) & mask) == value
2763  *
2764  * Otherwise, the wait will timeout after @timeout_ms milliseconds.
2765  *
2766  * Return: 0 if the register matches the desired condition, or -ETIMEDOUT.
2767  */
2768 int __intel_wait_for_register(struct intel_uncore *uncore,
2769                               i915_reg_t reg,
2770                               u32 mask,
2771                               u32 value,
2772                               unsigned int fast_timeout_us,
2773                               unsigned int slow_timeout_ms,
2774                               u32 *out_value)
2775 {
2776         unsigned fw =
2777                 intel_uncore_forcewake_for_reg(uncore, reg, FW_REG_READ);
2778         u32 reg_value;
2779         int ret;
2780
2781         might_sleep_if(slow_timeout_ms);
2782
2783         spin_lock_irq(&uncore->lock);
2784         intel_uncore_forcewake_get__locked(uncore, fw);
2785
2786         ret = __intel_wait_for_register_fw(uncore,
2787                                            reg, mask, value,
2788                                            fast_timeout_us, 0, &reg_value);
2789
2790         intel_uncore_forcewake_put__locked(uncore, fw);
2791         spin_unlock_irq(&uncore->lock);
2792
2793         if (ret && slow_timeout_ms)
2794                 ret = __wait_for(reg_value = intel_uncore_read_notrace(uncore,
2795                                                                        reg),
2796                                  (reg_value & mask) == value,
2797                                  slow_timeout_ms * 1000, 10, 1000);
2798
2799         /* just trace the final value */
2800         trace_i915_reg_rw(false, reg, reg_value, sizeof(reg_value), true);
2801
2802         if (out_value)
2803                 *out_value = reg_value;
2804
2805         return ret;
2806 }
2807
2808 bool intel_uncore_unclaimed_mmio(struct intel_uncore *uncore)
2809 {
2810         bool ret;
2811
2812         if (!uncore->debug)
2813                 return false;
2814
2815         spin_lock_irq(&uncore->debug->lock);
2816         ret = check_for_unclaimed_mmio(uncore);
2817         spin_unlock_irq(&uncore->debug->lock);
2818
2819         return ret;
2820 }
2821
2822 bool
2823 intel_uncore_arm_unclaimed_mmio_detection(struct intel_uncore *uncore)
2824 {
2825         bool ret = false;
2826
2827         if (drm_WARN_ON(&uncore->i915->drm, !uncore->debug))
2828                 return false;
2829
2830         spin_lock_irq(&uncore->debug->lock);
2831
2832         if (unlikely(uncore->debug->unclaimed_mmio_check <= 0))
2833                 goto out;
2834
2835         if (unlikely(check_for_unclaimed_mmio(uncore))) {
2836                 if (!uncore->i915->params.mmio_debug) {
2837                         drm_dbg(&uncore->i915->drm,
2838                                 "Unclaimed register detected, "
2839                                 "enabling oneshot unclaimed register reporting. "
2840                                 "Please use i915.mmio_debug=N for more information.\n");
2841                         uncore->i915->params.mmio_debug++;
2842                 }
2843                 uncore->debug->unclaimed_mmio_check--;
2844                 ret = true;
2845         }
2846
2847 out:
2848         spin_unlock_irq(&uncore->debug->lock);
2849
2850         return ret;
2851 }
2852
2853 /**
2854  * intel_uncore_forcewake_for_reg - which forcewake domains are needed to access
2855  *                                  a register
2856  * @uncore: pointer to struct intel_uncore
2857  * @reg: register in question
2858  * @op: operation bitmask of FW_REG_READ and/or FW_REG_WRITE
2859  *
2860  * Returns a set of forcewake domains required to be taken with for example
2861  * intel_uncore_forcewake_get for the specified register to be accessible in the
2862  * specified mode (read, write or read/write) with raw mmio accessors.
2863  *
2864  * NOTE: On Gen6 and Gen7 write forcewake domain (FORCEWAKE_RENDER) requires the
2865  * callers to do FIFO management on their own or risk losing writes.
2866  */
2867 enum forcewake_domains
2868 intel_uncore_forcewake_for_reg(struct intel_uncore *uncore,
2869                                i915_reg_t reg, unsigned int op)
2870 {
2871         enum forcewake_domains fw_domains = 0;
2872
2873         drm_WARN_ON(&uncore->i915->drm, !op);
2874
2875         if (!intel_uncore_has_forcewake(uncore))
2876                 return 0;
2877
2878         if (op & FW_REG_READ)
2879                 fw_domains = uncore->funcs.read_fw_domains(uncore, reg);
2880
2881         if (op & FW_REG_WRITE)
2882                 fw_domains |= uncore->funcs.write_fw_domains(uncore, reg);
2883
2884         drm_WARN_ON(&uncore->i915->drm, fw_domains & ~uncore->fw_domains);
2885
2886         return fw_domains;
2887 }
2888
2889 #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
2890 #include "selftests/mock_uncore.c"
2891 #include "selftests/intel_uncore.c"
2892 #endif