drivers/gpu/drm/i915/intel_uncore.c

   1 /*
   2  * Copyright © 2013 Intel Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21  * IN THE SOFTWARE.
  22  */
  23
  24 #include "i915_drv.h"
  25 #include "intel_drv.h"
  26
  27 #define FORCEWAKE_ACK_TIMEOUT_MS 2
  28
  29 #define __raw_i915_read8(dev_priv__, reg__) readb((dev_priv__)->regs + (reg__))
  30 #define __raw_i915_write8(dev_priv__, reg__, val__) writeb(val__, (dev_priv__)->regs + (reg__))
  31
  32 #define __raw_i915_read16(dev_priv__, reg__) readw((dev_priv__)->regs + (reg__))
  33 #define __raw_i915_write16(dev_priv__, reg__, val__) writew(val__, (dev_priv__)->regs + (reg__))
  34
  35 #define __raw_i915_read32(dev_priv__, reg__) readl((dev_priv__)->regs + (reg__))
  36 #define __raw_i915_write32(dev_priv__, reg__, val__) writel(val__, (dev_priv__)->regs + (reg__))
  37
  38 #define __raw_i915_read64(dev_priv__, reg__) readq((dev_priv__)->regs + (reg__))
  39 #define __raw_i915_write64(dev_priv__, reg__, val__) writeq(val__, (dev_priv__)->regs + (reg__))
  40
  41 #define __raw_posting_read(dev_priv__, reg__) (void)__raw_i915_read32(dev_priv__, reg__)
  42
  43
  44 static void __gen6_gt_wait_for_thread_c0(struct drm_i915_private *dev_priv)
  45 {
  46         u32 gt_thread_status_mask;
  47
  48         if (IS_HASWELL(dev_priv->dev))
  49                 gt_thread_status_mask = GEN6_GT_THREAD_STATUS_CORE_MASK_HSW;
  50         else
  51                 gt_thread_status_mask = GEN6_GT_THREAD_STATUS_CORE_MASK;
  52
  53         /* w/a for a sporadic read returning 0 by waiting for the GT
  54          * thread to wake up.
  55          */
  56         if (wait_for_atomic_us((__raw_i915_read32(dev_priv, GEN6_GT_THREAD_STATUS_REG) & gt_thread_status_mask) == 0, 500))
  57                 DRM_ERROR("GT thread status wait timed out\n");
  58 }
  59
  60 static void __gen6_gt_force_wake_reset(struct drm_i915_private *dev_priv)
  61 {
  62         __raw_i915_write32(dev_priv, FORCEWAKE, 0);
  63         /* something from same cacheline, but !FORCEWAKE */
  64         __raw_posting_read(dev_priv, ECOBUS);
  65 }
  66
  67 static void __gen6_gt_force_wake_get(struct drm_i915_private *dev_priv,
  68                                                         int fw_engine)
  69 {
  70         if (wait_for_atomic((__raw_i915_read32(dev_priv, FORCEWAKE_ACK) & 1) == 0,
  71                             FORCEWAKE_ACK_TIMEOUT_MS))
  72                 DRM_ERROR("Timed out waiting for forcewake old ack to clear.\n");
  73
  74         __raw_i915_write32(dev_priv, FORCEWAKE, 1);
  75         /* something from same cacheline, but !FORCEWAKE */
  76         __raw_posting_read(dev_priv, ECOBUS);
  77
  78         if (wait_for_atomic((__raw_i915_read32(dev_priv, FORCEWAKE_ACK) & 1),
  79                             FORCEWAKE_ACK_TIMEOUT_MS))
  80                 DRM_ERROR("Timed out waiting for forcewake to ack request.\n");
  81
  82         /* WaRsForcewakeWaitTC0:snb */
  83         __gen6_gt_wait_for_thread_c0(dev_priv);
  84 }
  85
  86 static void __gen6_gt_force_wake_mt_reset(struct drm_i915_private *dev_priv)
  87 {
  88         __raw_i915_write32(dev_priv, FORCEWAKE_MT, _MASKED_BIT_DISABLE(0xffff));
  89         /* something from same cacheline, but !FORCEWAKE_MT */
  90         __raw_posting_read(dev_priv, ECOBUS);
  91 }
  92
  93 static void __gen6_gt_force_wake_mt_get(struct drm_i915_private *dev_priv,
  94                                                         int fw_engine)
  95 {
  96         u32 forcewake_ack;
  97
  98         if (IS_HASWELL(dev_priv->dev) || IS_GEN8(dev_priv->dev))
  99                 forcewake_ack = FORCEWAKE_ACK_HSW;
 100         else
 101                 forcewake_ack = FORCEWAKE_MT_ACK;
 102
 103         if (wait_for_atomic((__raw_i915_read32(dev_priv, forcewake_ack) & FORCEWAKE_KERNEL) == 0,
 104                             FORCEWAKE_ACK_TIMEOUT_MS))
 105                 DRM_ERROR("Timed out waiting for forcewake old ack to clear.\n");
 106
 107         __raw_i915_write32(dev_priv, FORCEWAKE_MT,
 108                            _MASKED_BIT_ENABLE(FORCEWAKE_KERNEL));
 109         /* something from same cacheline, but !FORCEWAKE_MT */
 110         __raw_posting_read(dev_priv, ECOBUS);
 111
 112         if (wait_for_atomic((__raw_i915_read32(dev_priv, forcewake_ack) & FORCEWAKE_KERNEL),
 113                             FORCEWAKE_ACK_TIMEOUT_MS))
 114                 DRM_ERROR("Timed out waiting for forcewake to ack request.\n");
 115
 116         /* WaRsForcewakeWaitTC0:ivb,hsw */
 117         if (INTEL_INFO(dev_priv->dev)->gen < 8)
 118                 __gen6_gt_wait_for_thread_c0(dev_priv);
 119 }
 120
 121 static void gen6_gt_check_fifodbg(struct drm_i915_private *dev_priv)
 122 {
 123         u32 gtfifodbg;
 124
 125         gtfifodbg = __raw_i915_read32(dev_priv, GTFIFODBG);
 126         if (WARN(gtfifodbg, "GT wake FIFO error 0x%x\n", gtfifodbg))
 127                 __raw_i915_write32(dev_priv, GTFIFODBG, gtfifodbg);
 128 }
 129
 130 static void __gen6_gt_force_wake_put(struct drm_i915_private *dev_priv,
 131                                                         int fw_engine)
 132 {
 133         __raw_i915_write32(dev_priv, FORCEWAKE, 0);
 134         /* something from same cacheline, but !FORCEWAKE */
 135         __raw_posting_read(dev_priv, ECOBUS);
 136         gen6_gt_check_fifodbg(dev_priv);
 137 }
 138
 139 static void __gen6_gt_force_wake_mt_put(struct drm_i915_private *dev_priv,
 140                                                         int fw_engine)
 141 {
 142         __raw_i915_write32(dev_priv, FORCEWAKE_MT,
 143                            _MASKED_BIT_DISABLE(FORCEWAKE_KERNEL));
 144         /* something from same cacheline, but !FORCEWAKE_MT */
 145         __raw_posting_read(dev_priv, ECOBUS);
 146         gen6_gt_check_fifodbg(dev_priv);
 147 }
 148
 149 static int __gen6_gt_wait_for_fifo(struct drm_i915_private *dev_priv)
 150 {
 151         int ret = 0;
 152
 153         if (dev_priv->uncore.fifo_count < GT_FIFO_NUM_RESERVED_ENTRIES) {
 154                 int loop = 500;
 155                 u32 fifo = __raw_i915_read32(dev_priv, GTFIFOCTL) & GT_FIFO_FREE_ENTRIES_MASK;
 156                 while (fifo <= GT_FIFO_NUM_RESERVED_ENTRIES && loop--) {
 157                         udelay(10);
 158                         fifo = __raw_i915_read32(dev_priv, GTFIFOCTL) & GT_FIFO_FREE_ENTRIES_MASK;
 159                 }
 160                 if (WARN_ON(loop < 0 && fifo <= GT_FIFO_NUM_RESERVED_ENTRIES))
 161                         ++ret;
 162                 dev_priv->uncore.fifo_count = fifo;
 163         }
 164         dev_priv->uncore.fifo_count--;
 165
 166         return ret;
 167 }
 168
 169 static void vlv_force_wake_reset(struct drm_i915_private *dev_priv)
 170 {
 171         __raw_i915_write32(dev_priv, FORCEWAKE_VLV,
 172                            _MASKED_BIT_DISABLE(0xffff));
 173         /* something from same cacheline, but !FORCEWAKE_VLV */
 174         __raw_posting_read(dev_priv, FORCEWAKE_ACK_VLV);
 175 }
 176
 177 static void __vlv_force_wake_get(struct drm_i915_private *dev_priv,
 178                                                 int fw_engine)
 179 {
 180         /* Check for Render Engine */
 181         if (FORCEWAKE_RENDER & fw_engine) {
 182                 if (wait_for_atomic((__raw_i915_read32(dev_priv,
 183                                                 FORCEWAKE_ACK_VLV) &
 184                                                 FORCEWAKE_KERNEL) == 0,
 185                                         FORCEWAKE_ACK_TIMEOUT_MS))
 186                         DRM_ERROR("Timed out: Render forcewake old ack to clear.\n");
 187
 188                 __raw_i915_write32(dev_priv, FORCEWAKE_VLV,
 189                                    _MASKED_BIT_ENABLE(FORCEWAKE_KERNEL));
 190
 191                 if (wait_for_atomic((__raw_i915_read32(dev_priv,
 192                                                 FORCEWAKE_ACK_VLV) &
 193                                                 FORCEWAKE_KERNEL),
 194                                         FORCEWAKE_ACK_TIMEOUT_MS))
 195                         DRM_ERROR("Timed out: waiting for Render to ack.\n");
 196         }
 197
 198         /* Check for Media Engine */
 199         if (FORCEWAKE_MEDIA & fw_engine) {
 200                 if (wait_for_atomic((__raw_i915_read32(dev_priv,
 201                                                 FORCEWAKE_ACK_MEDIA_VLV) &
 202                                                 FORCEWAKE_KERNEL) == 0,
 203                                         FORCEWAKE_ACK_TIMEOUT_MS))
 204                         DRM_ERROR("Timed out: Media forcewake old ack to clear.\n");
 205
 206                 __raw_i915_write32(dev_priv, FORCEWAKE_MEDIA_VLV,
 207                                    _MASKED_BIT_ENABLE(FORCEWAKE_KERNEL));
 208
 209                 if (wait_for_atomic((__raw_i915_read32(dev_priv,
 210                                                 FORCEWAKE_ACK_MEDIA_VLV) &
 211                                                 FORCEWAKE_KERNEL),
 212                                         FORCEWAKE_ACK_TIMEOUT_MS))
 213                         DRM_ERROR("Timed out: waiting for media to ack.\n");
 214         }
 215
 216         /* WaRsForcewakeWaitTC0:vlv */
 217         __gen6_gt_wait_for_thread_c0(dev_priv);
 218
 219 }
 220
 221 static void __vlv_force_wake_put(struct drm_i915_private *dev_priv,
 222                                         int fw_engine)
 223 {
 224
 225         /* Check for Render Engine */
 226         if (FORCEWAKE_RENDER & fw_engine)
 227                 __raw_i915_write32(dev_priv, FORCEWAKE_VLV,
 228                                         _MASKED_BIT_DISABLE(FORCEWAKE_KERNEL));
 229
 230
 231         /* Check for Media Engine */
 232         if (FORCEWAKE_MEDIA & fw_engine)
 233                 __raw_i915_write32(dev_priv, FORCEWAKE_MEDIA_VLV,
 234                                 _MASKED_BIT_DISABLE(FORCEWAKE_KERNEL));
 235
 236         /* The below doubles as a POSTING_READ */
 237         gen6_gt_check_fifodbg(dev_priv);
 238
 239 }
 240
 241 void vlv_force_wake_get(struct drm_i915_private *dev_priv,
 242                                                 int fw_engine)
 243 {
 244         unsigned long irqflags;
 245
 246         spin_lock_irqsave(&dev_priv->uncore.lock, irqflags);
 247         if (FORCEWAKE_RENDER & fw_engine) {
 248                 if (dev_priv->uncore.fw_rendercount++ == 0)
 249                         dev_priv->uncore.funcs.force_wake_get(dev_priv,
 250                                                         FORCEWAKE_RENDER);
 251         }
 252         if (FORCEWAKE_MEDIA & fw_engine) {
 253                 if (dev_priv->uncore.fw_mediacount++ == 0)
 254                         dev_priv->uncore.funcs.force_wake_get(dev_priv,
 255                                                         FORCEWAKE_MEDIA);
 256         }
 257
 258         spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags);
 259 }
 260
 261 void vlv_force_wake_put(struct drm_i915_private *dev_priv,
 262                                                 int fw_engine)
 263 {
 264         unsigned long irqflags;
 265
 266         spin_lock_irqsave(&dev_priv->uncore.lock, irqflags);
 267
 268         if (FORCEWAKE_RENDER & fw_engine) {
 269                 WARN_ON(dev_priv->uncore.fw_rendercount == 0);
 270                 if (--dev_priv->uncore.fw_rendercount == 0)
 271                         dev_priv->uncore.funcs.force_wake_put(dev_priv,
 272                                                         FORCEWAKE_RENDER);
 273         }
 274
 275         if (FORCEWAKE_MEDIA & fw_engine) {
 276                 WARN_ON(dev_priv->uncore.fw_mediacount == 0);
 277                 if (--dev_priv->uncore.fw_mediacount == 0)
 278                         dev_priv->uncore.funcs.force_wake_put(dev_priv,
 279                                                         FORCEWAKE_MEDIA);
 280         }
 281
 282         spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags);
 283 }
 284
 285 static void gen6_force_wake_work(struct work_struct *work)
 286 {
 287         struct drm_i915_private *dev_priv =
 288                 container_of(work, typeof(*dev_priv), uncore.force_wake_work.work);
 289         unsigned long irqflags;
 290
 291         spin_lock_irqsave(&dev_priv->uncore.lock, irqflags);
 292         if (--dev_priv->uncore.forcewake_count == 0)
 293                 dev_priv->uncore.funcs.force_wake_put(dev_priv, FORCEWAKE_ALL);
 294         spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags);
 295 }
 296
 297 static void intel_uncore_forcewake_reset(struct drm_device *dev)
 298 {
 299         struct drm_i915_private *dev_priv = dev->dev_private;
 300
 301         if (IS_VALLEYVIEW(dev)) {
 302                 vlv_force_wake_reset(dev_priv);
 303         } else if (INTEL_INFO(dev)->gen >= 6) {
 304                 __gen6_gt_force_wake_reset(dev_priv);
 305                 if (IS_IVYBRIDGE(dev) || IS_HASWELL(dev))
 306                         __gen6_gt_force_wake_mt_reset(dev_priv);
 307         }
 308 }
 309
 310 void intel_uncore_early_sanitize(struct drm_device *dev)
 311 {
 312         struct drm_i915_private *dev_priv = dev->dev_private;
 313
 314         if (HAS_FPGA_DBG_UNCLAIMED(dev))
 315                 __raw_i915_write32(dev_priv, FPGA_DBG, FPGA_DBG_RM_NOCLAIM);
 316
 317         if (IS_HASWELL(dev) &&
 318             (__raw_i915_read32(dev_priv, HSW_EDRAM_PRESENT) == 1)) {
 319                 /* The docs do not explain exactly how the calculation can be
 320                  * made. It is somewhat guessable, but for now, it's always
 321                  * 128MB.
 322                  * NB: We can't write IDICR yet because we do not have gt funcs
 323                  * set up */
 324                 dev_priv->ellc_size = 128;
 325                 DRM_INFO("Found %zuMB of eLLC\n", dev_priv->ellc_size);
 326         }
 327
 328         intel_uncore_forcewake_reset(dev);
 329 }
 330
 331 void intel_uncore_sanitize(struct drm_device *dev)
 332 {
 333         struct drm_i915_private *dev_priv = dev->dev_private;
 334         u32 reg_val;
 335
 336         intel_uncore_forcewake_reset(dev);
 337
 338         /* BIOS often leaves RC6 enabled, but disable it for hw init */
 339         intel_disable_gt_powersave(dev);
 340
 341         /* Turn off power gate, require especially for the BIOS less system */
 342         if (IS_VALLEYVIEW(dev)) {
 343
 344                 mutex_lock(&dev_priv->rps.hw_lock);
 345                 reg_val = vlv_punit_read(dev_priv, PUNIT_REG_PWRGT_STATUS);
 346
 347                 if (reg_val & (RENDER_PWRGT | MEDIA_PWRGT | DISP2D_PWRGT))
 348                         vlv_punit_write(dev_priv, PUNIT_REG_PWRGT_CTRL, 0x0);
 349
 350                 mutex_unlock(&dev_priv->rps.hw_lock);
 351
 352         }
 353 }
 354
 355 /*
 356  * Generally this is called implicitly by the register read function. However,
 357  * if some sequence requires the GT to not power down then this function should
 358  * be called at the beginning of the sequence followed by a call to
 359  * gen6_gt_force_wake_put() at the end of the sequence.
 360  */
 361 void gen6_gt_force_wake_get(struct drm_i915_private *dev_priv, int fw_engine)
 362 {
 363         unsigned long irqflags;
 364
 365         if (!dev_priv->uncore.funcs.force_wake_get)
 366                 return;
 367
 368         /* Redirect to VLV specific routine */
 369         if (IS_VALLEYVIEW(dev_priv->dev))
 370                 return vlv_force_wake_get(dev_priv, fw_engine);
 371
 372         spin_lock_irqsave(&dev_priv->uncore.lock, irqflags);
 373         if (dev_priv->uncore.forcewake_count++ == 0)
 374                 dev_priv->uncore.funcs.force_wake_get(dev_priv, FORCEWAKE_ALL);
 375         spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags);
 376 }
 377
 378 /*
 379  * see gen6_gt_force_wake_get()
 380  */
 381 void gen6_gt_force_wake_put(struct drm_i915_private *dev_priv, int fw_engine)
 382 {
 383         unsigned long irqflags;
 384
 385         if (!dev_priv->uncore.funcs.force_wake_put)
 386                 return;
 387
 388         /* Redirect to VLV specific routine */
 389         if (IS_VALLEYVIEW(dev_priv->dev))
 390                 return vlv_force_wake_put(dev_priv, fw_engine);
 391
 392
 393         spin_lock_irqsave(&dev_priv->uncore.lock, irqflags);
 394         if (--dev_priv->uncore.forcewake_count == 0) {
 395                 dev_priv->uncore.forcewake_count++;
 396                 mod_delayed_work(dev_priv->wq,
 397                                  &dev_priv->uncore.force_wake_work,
 398                                  1);
 399         }
 400         spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags);
 401 }
 402
 403 /* We give fast paths for the really cool registers */
 404 #define NEEDS_FORCE_WAKE(dev_priv, reg) \
 405          ((reg) < 0x40000 && (reg) != FORCEWAKE)
 406
 407 static void
 408 ilk_dummy_write(struct drm_i915_private *dev_priv)
 409 {
 410         /* WaIssueDummyWriteToWakeupFromRC6:ilk Issue a dummy write to wake up
 411          * the chip from rc6 before touching it for real. MI_MODE is masked,
 412          * hence harmless to write 0 into. */
 413         __raw_i915_write32(dev_priv, MI_MODE, 0);
 414 }
 415
 416 static void
 417 hsw_unclaimed_reg_clear(struct drm_i915_private *dev_priv, u32 reg)
 418 {
 419         if (__raw_i915_read32(dev_priv, FPGA_DBG) & FPGA_DBG_RM_NOCLAIM) {
 420                 DRM_ERROR("Unknown unclaimed register before writing to %x\n",
 421                           reg);
 422                 __raw_i915_write32(dev_priv, FPGA_DBG, FPGA_DBG_RM_NOCLAIM);
 423         }
 424 }
 425
 426 static void
 427 hsw_unclaimed_reg_check(struct drm_i915_private *dev_priv, u32 reg)
 428 {
 429         if (__raw_i915_read32(dev_priv, FPGA_DBG) & FPGA_DBG_RM_NOCLAIM) {
 430                 DRM_ERROR("Unclaimed write to %x\n", reg);
 431                 __raw_i915_write32(dev_priv, FPGA_DBG, FPGA_DBG_RM_NOCLAIM);
 432         }
 433 }
 434
 435 #define REG_READ_HEADER(x) \
 436         unsigned long irqflags; \
 437         u##x val = 0; \
 438         spin_lock_irqsave(&dev_priv->uncore.lock, irqflags)
 439
 440 #define REG_READ_FOOTER \
 441         spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags); \
 442         trace_i915_reg_rw(false, reg, val, sizeof(val), trace); \
 443         return val
 444
 445 #define __gen4_read(x) \
 446 static u##x \
 447 gen4_read##x(struct drm_i915_private *dev_priv, off_t reg, bool trace) { \
 448         REG_READ_HEADER(x); \
 449         val = __raw_i915_read##x(dev_priv, reg); \
 450         REG_READ_FOOTER; \
 451 }
 452
 453 #define __gen5_read(x) \
 454 static u##x \
 455 gen5_read##x(struct drm_i915_private *dev_priv, off_t reg, bool trace) { \
 456         REG_READ_HEADER(x); \
 457         ilk_dummy_write(dev_priv); \
 458         val = __raw_i915_read##x(dev_priv, reg); \
 459         REG_READ_FOOTER; \
 460 }
 461
 462 #define __gen6_read(x) \
 463 static u##x \
 464 gen6_read##x(struct drm_i915_private *dev_priv, off_t reg, bool trace) { \
 465         REG_READ_HEADER(x); \
 466         if (NEEDS_FORCE_WAKE((dev_priv), (reg))) { \
 467                 if (dev_priv->uncore.forcewake_count == 0) \
 468                         dev_priv->uncore.funcs.force_wake_get(dev_priv, \
 469                                                         FORCEWAKE_ALL); \
 470                 val = __raw_i915_read##x(dev_priv, reg); \
 471                 if (dev_priv->uncore.forcewake_count == 0) \
 472                         dev_priv->uncore.funcs.force_wake_put(dev_priv, \
 473                                                         FORCEWAKE_ALL); \
 474         } else { \
 475                 val = __raw_i915_read##x(dev_priv, reg); \
 476         } \
 477         REG_READ_FOOTER; \
 478 }
 479
 480 #define __vlv_read(x) \
 481 static u##x \
 482 vlv_read##x(struct drm_i915_private *dev_priv, off_t reg, bool trace) { \
 483         unsigned fwengine = 0; \
 484         unsigned *fwcount; \
 485         REG_READ_HEADER(x); \
 486         if (FORCEWAKE_VLV_RENDER_RANGE_OFFSET(reg)) {   \
 487                 fwengine = FORCEWAKE_RENDER;            \
 488                 fwcount = &dev_priv->uncore.fw_rendercount;    \
 489         }                                               \
 490         else if (FORCEWAKE_VLV_MEDIA_RANGE_OFFSET(reg)) {       \
 491                 fwengine = FORCEWAKE_MEDIA;             \
 492                 fwcount = &dev_priv->uncore.fw_mediacount;     \
 493         }  \
 494         if (fwengine != 0) {            \
 495                 if ((*fwcount)++ == 0) \
 496                         (dev_priv)->uncore.funcs.force_wake_get(dev_priv, \
 497                                                                 fwengine); \
 498                 val = __raw_i915_read##x(dev_priv, reg); \
 499                 if (--(*fwcount) == 0) \
 500                         (dev_priv)->uncore.funcs.force_wake_put(dev_priv, \
 501                                                         fwengine); \
 502         } else { \
 503                 val = __raw_i915_read##x(dev_priv, reg); \
 504         } \
 505         REG_READ_FOOTER; \
 506 }
 507
 508
 509 __vlv_read(8)
 510 __vlv_read(16)
 511 __vlv_read(32)
 512 __vlv_read(64)
 513 __gen6_read(8)
 514 __gen6_read(16)
 515 __gen6_read(32)
 516 __gen6_read(64)
 517 __gen5_read(8)
 518 __gen5_read(16)
 519 __gen5_read(32)
 520 __gen5_read(64)
 521 __gen4_read(8)
 522 __gen4_read(16)
 523 __gen4_read(32)
 524 __gen4_read(64)
 525
 526 #undef __vlv_read
 527 #undef __gen6_read
 528 #undef __gen5_read
 529 #undef __gen4_read
 530 #undef REG_READ_FOOTER
 531 #undef REG_READ_HEADER
 532
 533 #define REG_WRITE_HEADER \
 534         unsigned long irqflags; \
 535         trace_i915_reg_rw(true, reg, val, sizeof(val), trace); \
 536         spin_lock_irqsave(&dev_priv->uncore.lock, irqflags)
 537
 538 #define __gen4_write(x) \
 539 static void \
 540 gen4_write##x(struct drm_i915_private *dev_priv, off_t reg, u##x val, bool trace) { \
 541         REG_WRITE_HEADER; \
 542         __raw_i915_write##x(dev_priv, reg, val); \
 543         spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags); \
 544 }
 545
 546 #define __gen5_write(x) \
 547 static void \
 548 gen5_write##x(struct drm_i915_private *dev_priv, off_t reg, u##x val, bool trace) { \
 549         REG_WRITE_HEADER; \
 550         ilk_dummy_write(dev_priv); \
 551         __raw_i915_write##x(dev_priv, reg, val); \
 552         spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags); \
 553 }
 554
 555 #define __gen6_write(x) \
 556 static void \
 557 gen6_write##x(struct drm_i915_private *dev_priv, off_t reg, u##x val, bool trace) { \
 558         u32 __fifo_ret = 0; \
 559         REG_WRITE_HEADER; \
 560         if (NEEDS_FORCE_WAKE((dev_priv), (reg))) { \
 561                 __fifo_ret = __gen6_gt_wait_for_fifo(dev_priv); \
 562         } \
 563         __raw_i915_write##x(dev_priv, reg, val); \
 564         if (unlikely(__fifo_ret)) { \
 565                 gen6_gt_check_fifodbg(dev_priv); \
 566         } \
 567         spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags); \
 568 }
 569
 570 #define __hsw_write(x) \
 571 static void \
 572 hsw_write##x(struct drm_i915_private *dev_priv, off_t reg, u##x val, bool trace) { \
 573         u32 __fifo_ret = 0; \
 574         REG_WRITE_HEADER; \
 575         if (NEEDS_FORCE_WAKE((dev_priv), (reg))) { \
 576                 __fifo_ret = __gen6_gt_wait_for_fifo(dev_priv); \
 577         } \
 578         hsw_unclaimed_reg_clear(dev_priv, reg); \
 579         __raw_i915_write##x(dev_priv, reg, val); \
 580         if (unlikely(__fifo_ret)) { \
 581                 gen6_gt_check_fifodbg(dev_priv); \
 582         } \
 583         hsw_unclaimed_reg_check(dev_priv, reg); \
 584         spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags); \
 585 }
 586
 587 static const u32 gen8_shadowed_regs[] = {
 588         FORCEWAKE_MT,
 589         GEN6_RPNSWREQ,
 590         GEN6_RC_VIDEO_FREQ,
 591         RING_TAIL(RENDER_RING_BASE),
 592         RING_TAIL(GEN6_BSD_RING_BASE),
 593         RING_TAIL(VEBOX_RING_BASE),
 594         RING_TAIL(BLT_RING_BASE),
 595         /* TODO: Other registers are not yet used */
 596 };
 597
 598 static bool is_gen8_shadowed(struct drm_i915_private *dev_priv, u32 reg)
 599 {
 600         int i;
 601         for (i = 0; i < ARRAY_SIZE(gen8_shadowed_regs); i++)
 602                 if (reg == gen8_shadowed_regs[i])
 603                         return true;
 604
 605         return false;
 606 }
 607
 608 #define __gen8_write(x) \
 609 static void \
 610 gen8_write##x(struct drm_i915_private *dev_priv, off_t reg, u##x val, bool trace) { \
 611         bool __needs_put = !is_gen8_shadowed(dev_priv, reg); \
 612         REG_WRITE_HEADER; \
 613         if (__needs_put) { \
 614                 dev_priv->uncore.funcs.force_wake_get(dev_priv, \
 615                                                         FORCEWAKE_ALL); \
 616         } \
 617         __raw_i915_write##x(dev_priv, reg, val); \
 618         if (__needs_put) { \
 619                 dev_priv->uncore.funcs.force_wake_put(dev_priv, \
 620                                                         FORCEWAKE_ALL); \
 621         } \
 622         spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags); \
 623 }
 624
 625 __gen8_write(8)
 626 __gen8_write(16)
 627 __gen8_write(32)
 628 __gen8_write(64)
 629 __hsw_write(8)
 630 __hsw_write(16)
 631 __hsw_write(32)
 632 __hsw_write(64)
 633 __gen6_write(8)
 634 __gen6_write(16)
 635 __gen6_write(32)
 636 __gen6_write(64)
 637 __gen5_write(8)
 638 __gen5_write(16)
 639 __gen5_write(32)
 640 __gen5_write(64)
 641 __gen4_write(8)
 642 __gen4_write(16)
 643 __gen4_write(32)
 644 __gen4_write(64)
 645
 646 #undef __gen8_write
 647 #undef __hsw_write
 648 #undef __gen6_write
 649 #undef __gen5_write
 650 #undef __gen4_write
 651 #undef REG_WRITE_HEADER
 652
 653 void intel_uncore_init(struct drm_device *dev)
 654 {
 655         struct drm_i915_private *dev_priv = dev->dev_private;
 656
 657         INIT_DELAYED_WORK(&dev_priv->uncore.force_wake_work,
 658                           gen6_force_wake_work);
 659
 660         if (IS_VALLEYVIEW(dev)) {
 661                 dev_priv->uncore.funcs.force_wake_get = __vlv_force_wake_get;
 662                 dev_priv->uncore.funcs.force_wake_put = __vlv_force_wake_put;
 663         } else if (IS_HASWELL(dev) || IS_GEN8(dev)) {
 664                 dev_priv->uncore.funcs.force_wake_get = __gen6_gt_force_wake_mt_get;
 665                 dev_priv->uncore.funcs.force_wake_put = __gen6_gt_force_wake_mt_put;
 666         } else if (IS_IVYBRIDGE(dev)) {
 667                 u32 ecobus;
 668
 669                 /* IVB configs may use multi-threaded forcewake */
 670
 671                 /* A small trick here - if the bios hasn't configured
 672                  * MT forcewake, and if the device is in RC6, then
 673                  * force_wake_mt_get will not wake the device and the
 674                  * ECOBUS read will return zero. Which will be
 675                  * (correctly) interpreted by the test below as MT
 676                  * forcewake being disabled.
 677                  */
 678                 mutex_lock(&dev->struct_mutex);
 679                 __gen6_gt_force_wake_mt_get(dev_priv, FORCEWAKE_ALL);
 680                 ecobus = __raw_i915_read32(dev_priv, ECOBUS);
 681                 __gen6_gt_force_wake_mt_put(dev_priv, FORCEWAKE_ALL);
 682                 mutex_unlock(&dev->struct_mutex);
 683
 684                 if (ecobus & FORCEWAKE_MT_ENABLE) {
 685                         dev_priv->uncore.funcs.force_wake_get =
 686                                 __gen6_gt_force_wake_mt_get;
 687                         dev_priv->uncore.funcs.force_wake_put =
 688                                 __gen6_gt_force_wake_mt_put;
 689                 } else {
 690                         DRM_INFO("No MT forcewake available on Ivybridge, this can result in issues\n");
 691                         DRM_INFO("when using vblank-synced partial screen updates.\n");
 692                         dev_priv->uncore.funcs.force_wake_get =
 693                                 __gen6_gt_force_wake_get;
 694                         dev_priv->uncore.funcs.force_wake_put =
 695                                 __gen6_gt_force_wake_put;
 696                 }
 697         } else if (IS_GEN6(dev)) {
 698                 dev_priv->uncore.funcs.force_wake_get =
 699                         __gen6_gt_force_wake_get;
 700                 dev_priv->uncore.funcs.force_wake_put =
 701                         __gen6_gt_force_wake_put;
 702         }
 703
 704         switch (INTEL_INFO(dev)->gen) {
 705         default:
 706                 dev_priv->uncore.funcs.mmio_writeb  = gen8_write8;
 707                 dev_priv->uncore.funcs.mmio_writew  = gen8_write16;
 708                 dev_priv->uncore.funcs.mmio_writel  = gen8_write32;
 709                 dev_priv->uncore.funcs.mmio_writeq  = gen8_write64;
 710                 dev_priv->uncore.funcs.mmio_readb  = gen6_read8;
 711                 dev_priv->uncore.funcs.mmio_readw  = gen6_read16;
 712                 dev_priv->uncore.funcs.mmio_readl  = gen6_read32;
 713                 dev_priv->uncore.funcs.mmio_readq  = gen6_read64;
 714                 break;
 715         case 7:
 716         case 6:
 717                 if (IS_HASWELL(dev)) {
 718                         dev_priv->uncore.funcs.mmio_writeb  = hsw_write8;
 719                         dev_priv->uncore.funcs.mmio_writew  = hsw_write16;
 720                         dev_priv->uncore.funcs.mmio_writel  = hsw_write32;
 721                         dev_priv->uncore.funcs.mmio_writeq  = hsw_write64;
 722                 } else {
 723                         dev_priv->uncore.funcs.mmio_writeb  = gen6_write8;
 724                         dev_priv->uncore.funcs.mmio_writew  = gen6_write16;
 725                         dev_priv->uncore.funcs.mmio_writel  = gen6_write32;
 726                         dev_priv->uncore.funcs.mmio_writeq  = gen6_write64;
 727                 }
 728
 729                 if (IS_VALLEYVIEW(dev)) {
 730                         dev_priv->uncore.funcs.mmio_readb  = vlv_read8;
 731                         dev_priv->uncore.funcs.mmio_readw  = vlv_read16;
 732                         dev_priv->uncore.funcs.mmio_readl  = vlv_read32;
 733                         dev_priv->uncore.funcs.mmio_readq  = vlv_read64;
 734                 } else {
 735                         dev_priv->uncore.funcs.mmio_readb  = gen6_read8;
 736                         dev_priv->uncore.funcs.mmio_readw  = gen6_read16;
 737                         dev_priv->uncore.funcs.mmio_readl  = gen6_read32;
 738                         dev_priv->uncore.funcs.mmio_readq  = gen6_read64;
 739                 }
 740                 break;
 741         case 5:
 742                 dev_priv->uncore.funcs.mmio_writeb  = gen5_write8;
 743                 dev_priv->uncore.funcs.mmio_writew  = gen5_write16;
 744                 dev_priv->uncore.funcs.mmio_writel  = gen5_write32;
 745                 dev_priv->uncore.funcs.mmio_writeq  = gen5_write64;
 746                 dev_priv->uncore.funcs.mmio_readb  = gen5_read8;
 747                 dev_priv->uncore.funcs.mmio_readw  = gen5_read16;
 748                 dev_priv->uncore.funcs.mmio_readl  = gen5_read32;
 749                 dev_priv->uncore.funcs.mmio_readq  = gen5_read64;
 750                 break;
 751         case 4:
 752         case 3:
 753         case 2:
 754                 dev_priv->uncore.funcs.mmio_writeb  = gen4_write8;
 755                 dev_priv->uncore.funcs.mmio_writew  = gen4_write16;
 756                 dev_priv->uncore.funcs.mmio_writel  = gen4_write32;
 757                 dev_priv->uncore.funcs.mmio_writeq  = gen4_write64;
 758                 dev_priv->uncore.funcs.mmio_readb  = gen4_read8;
 759                 dev_priv->uncore.funcs.mmio_readw  = gen4_read16;
 760                 dev_priv->uncore.funcs.mmio_readl  = gen4_read32;
 761                 dev_priv->uncore.funcs.mmio_readq  = gen4_read64;
 762                 break;
 763         }
 764 }
 765
 766 void intel_uncore_fini(struct drm_device *dev)
 767 {
 768         struct drm_i915_private *dev_priv = dev->dev_private;
 769
 770         flush_delayed_work(&dev_priv->uncore.force_wake_work);
 771
 772         /* Paranoia: make sure we have disabled everything before we exit. */
 773         intel_uncore_sanitize(dev);
 774 }
 775
 776 static const struct register_whitelist {
 777         uint64_t offset;
 778         uint32_t size;
 779         uint32_t gen_bitmask; /* support gens, 0x10 for 4, 0x30 for 4 and 5, etc. */
 780 } whitelist[] = {
 781         { RING_TIMESTAMP(RENDER_RING_BASE), 8, 0xF0 },
 782 };
 783
 784 int i915_reg_read_ioctl(struct drm_device *dev,
 785                         void *data, struct drm_file *file)
 786 {
 787         struct drm_i915_private *dev_priv = dev->dev_private;
 788         struct drm_i915_reg_read *reg = data;
 789         struct register_whitelist const *entry = whitelist;
 790         int i;
 791
 792         for (i = 0; i < ARRAY_SIZE(whitelist); i++, entry++) {
 793                 if (entry->offset == reg->offset &&
 794                     (1 << INTEL_INFO(dev)->gen & entry->gen_bitmask))
 795                         break;
 796         }
 797
 798         if (i == ARRAY_SIZE(whitelist))
 799                 return -EINVAL;
 800
 801         switch (entry->size) {
 802         case 8:
 803                 reg->val = I915_READ64(reg->offset);
 804                 break;
 805         case 4:
 806                 reg->val = I915_READ(reg->offset);
 807                 break;
 808         case 2:
 809                 reg->val = I915_READ16(reg->offset);
 810                 break;
 811         case 1:
 812                 reg->val = I915_READ8(reg->offset);
 813                 break;
 814         default:
 815                 WARN_ON(1);
 816                 return -EINVAL;
 817         }
 818
 819         return 0;
 820 }
 821
 822 int i915_get_reset_stats_ioctl(struct drm_device *dev,
 823                                void *data, struct drm_file *file)
 824 {
 825         struct drm_i915_private *dev_priv = dev->dev_private;
 826         struct drm_i915_reset_stats *args = data;
 827         struct i915_ctx_hang_stats *hs;
 828         int ret;
 829
 830         if (args->flags || args->pad)
 831                 return -EINVAL;
 832
 833         if (args->ctx_id == DEFAULT_CONTEXT_ID && !capable(CAP_SYS_ADMIN))
 834                 return -EPERM;
 835
 836         ret = mutex_lock_interruptible(&dev->struct_mutex);
 837         if (ret)
 838                 return ret;
 839
 840         hs = i915_gem_context_get_hang_stats(dev, file, args->ctx_id);
 841         if (IS_ERR(hs)) {
 842                 mutex_unlock(&dev->struct_mutex);
 843                 return PTR_ERR(hs);
 844         }
 845
 846         if (capable(CAP_SYS_ADMIN))
 847                 args->reset_count = i915_reset_count(&dev_priv->gpu_error);
 848         else
 849                 args->reset_count = 0;
 850
 851         args->batch_active = hs->batch_active;
 852         args->batch_pending = hs->batch_pending;
 853
 854         mutex_unlock(&dev->struct_mutex);
 855
 856         return 0;
 857 }
 858
 859 static int i965_reset_complete(struct drm_device *dev)
 860 {
 861         u8 gdrst;
 862         pci_read_config_byte(dev->pdev, I965_GDRST, &gdrst);
 863         return (gdrst & GRDOM_RESET_ENABLE) == 0;
 864 }
 865
 866 static int i965_do_reset(struct drm_device *dev)
 867 {
 868         int ret;
 869
 870         /*
 871          * Set the domains we want to reset (GRDOM/bits 2 and 3) as
 872          * well as the reset bit (GR/bit 0).  Setting the GR bit
 873          * triggers the reset; when done, the hardware will clear it.
 874          */
 875         pci_write_config_byte(dev->pdev, I965_GDRST,
 876                               GRDOM_RENDER | GRDOM_RESET_ENABLE);
 877         ret =  wait_for(i965_reset_complete(dev), 500);
 878         if (ret)
 879                 return ret;
 880
 881         /* We can't reset render&media without also resetting display ... */
 882         pci_write_config_byte(dev->pdev, I965_GDRST,
 883                               GRDOM_MEDIA | GRDOM_RESET_ENABLE);
 884
 885         ret =  wait_for(i965_reset_complete(dev), 500);
 886         if (ret)
 887                 return ret;
 888
 889         pci_write_config_byte(dev->pdev, I965_GDRST, 0);
 890
 891         return 0;
 892 }
 893
 894 static int ironlake_do_reset(struct drm_device *dev)
 895 {
 896         struct drm_i915_private *dev_priv = dev->dev_private;
 897         u32 gdrst;
 898         int ret;
 899
 900         gdrst = I915_READ(MCHBAR_MIRROR_BASE + ILK_GDSR);
 901         gdrst &= ~GRDOM_MASK;
 902         I915_WRITE(MCHBAR_MIRROR_BASE + ILK_GDSR,
 903                    gdrst | GRDOM_RENDER | GRDOM_RESET_ENABLE);
 904         ret = wait_for(I915_READ(MCHBAR_MIRROR_BASE + ILK_GDSR) & 0x1, 500);
 905         if (ret)
 906                 return ret;
 907
 908         /* We can't reset render&media without also resetting display ... */
 909         gdrst = I915_READ(MCHBAR_MIRROR_BASE + ILK_GDSR);
 910         gdrst &= ~GRDOM_MASK;
 911         I915_WRITE(MCHBAR_MIRROR_BASE + ILK_GDSR,
 912                    gdrst | GRDOM_MEDIA | GRDOM_RESET_ENABLE);
 913         return wait_for(I915_READ(MCHBAR_MIRROR_BASE + ILK_GDSR) & 0x1, 500);
 914 }
 915
 916 static int gen6_do_reset(struct drm_device *dev)
 917 {
 918         struct drm_i915_private *dev_priv = dev->dev_private;
 919         int     ret;
 920         unsigned long irqflags;
 921
 922         /* Hold uncore.lock across reset to prevent any register access
 923          * with forcewake not set correctly
 924          */
 925         spin_lock_irqsave(&dev_priv->uncore.lock, irqflags);
 926
 927         /* Reset the chip */
 928
 929         /* GEN6_GDRST is not in the gt power well, no need to check
 930          * for fifo space for the write or forcewake the chip for
 931          * the read
 932          */
 933         __raw_i915_write32(dev_priv, GEN6_GDRST, GEN6_GRDOM_FULL);
 934
 935         /* Spin waiting for the device to ack the reset request */
 936         ret = wait_for((__raw_i915_read32(dev_priv, GEN6_GDRST) & GEN6_GRDOM_FULL) == 0, 500);
 937
 938         intel_uncore_forcewake_reset(dev);
 939
 940         /* If reset with a user forcewake, try to restore, otherwise turn it off */
 941         if (dev_priv->uncore.forcewake_count)
 942                 dev_priv->uncore.funcs.force_wake_get(dev_priv, FORCEWAKE_ALL);
 943         else
 944                 dev_priv->uncore.funcs.force_wake_put(dev_priv, FORCEWAKE_ALL);
 945
 946         /* Restore fifo count */
 947         dev_priv->uncore.fifo_count = __raw_i915_read32(dev_priv, GTFIFOCTL) & GT_FIFO_FREE_ENTRIES_MASK;
 948
 949         spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags);
 950         return ret;
 951 }
 952
 953 int intel_gpu_reset(struct drm_device *dev)
 954 {
 955         switch (INTEL_INFO(dev)->gen) {
 956         case 8:
 957         case 7:
 958         case 6: return gen6_do_reset(dev);
 959         case 5: return ironlake_do_reset(dev);
 960         case 4: return i965_do_reset(dev);
 961         default: return -ENODEV;
 962         }
 963 }
 964
 965 void intel_uncore_check_errors(struct drm_device *dev)
 966 {
 967         struct drm_i915_private *dev_priv = dev->dev_private;
 968
 969         if (HAS_FPGA_DBG_UNCLAIMED(dev) &&
 970             (__raw_i915_read32(dev_priv, FPGA_DBG) & FPGA_DBG_RM_NOCLAIM)) {
 971                 DRM_ERROR("Unclaimed register before interrupt\n");
 972                 __raw_i915_write32(dev_priv, FPGA_DBG, FPGA_DBG_RM_NOCLAIM);
 973         }
 974 }