drivers/gpu/drm/i915/display/intel_fbc.c

   1 /*
   2  * Copyright © 2014 Intel Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
  21  * DEALINGS IN THE SOFTWARE.
  22  */
  23
  24 /**
  25  * DOC: Frame Buffer Compression (FBC)
  26  *
  27  * FBC tries to save memory bandwidth (and so power consumption) by
  28  * compressing the amount of memory used by the display. It is total
  29  * transparent to user space and completely handled in the kernel.
  30  *
  31  * The benefits of FBC are mostly visible with solid backgrounds and
  32  * variation-less patterns. It comes from keeping the memory footprint small
  33  * and having fewer memory pages opened and accessed for refreshing the display.
  34  *
  35  * i915 is responsible to reserve stolen memory for FBC and configure its
  36  * offset on proper registers. The hardware takes care of all
  37  * compress/decompress. However there are many known cases where we have to
  38  * forcibly disable it to allow proper screen updates.
  39  */
  40
  41 #include <linux/string_helpers.h>
  42
  43 #include <drm/drm_fourcc.h>
  44
  45 #include "i915_drv.h"
  46 #include "i915_utils.h"
  47 #include "i915_vgpu.h"
  48 #include "intel_cdclk.h"
  49 #include "intel_de.h"
  50 #include "intel_display_trace.h"
  51 #include "intel_display_types.h"
  52 #include "intel_fbc.h"
  53 #include "intel_frontbuffer.h"
  54
  55 #define for_each_fbc_id(__dev_priv, __fbc_id) \
  56         for ((__fbc_id) = INTEL_FBC_A; (__fbc_id) < I915_MAX_FBCS; (__fbc_id)++) \
  57                 for_each_if(INTEL_INFO(__dev_priv)->display.fbc_mask & BIT(__fbc_id))
  58
  59 #define for_each_intel_fbc(__dev_priv, __fbc, __fbc_id) \
  60         for_each_fbc_id((__dev_priv), (__fbc_id)) \
  61                 for_each_if((__fbc) = (__dev_priv)->fbc[(__fbc_id)])
  62
  63 struct intel_fbc_funcs {
  64         void (*activate)(struct intel_fbc *fbc);
  65         void (*deactivate)(struct intel_fbc *fbc);
  66         bool (*is_active)(struct intel_fbc *fbc);
  67         bool (*is_compressing)(struct intel_fbc *fbc);
  68         void (*nuke)(struct intel_fbc *fbc);
  69         void (*program_cfb)(struct intel_fbc *fbc);
  70         void (*set_false_color)(struct intel_fbc *fbc, bool enable);
  71 };
  72
  73 struct intel_fbc_state {
  74         struct intel_plane *plane;
  75         unsigned int cfb_stride;
  76         unsigned int cfb_size;
  77         unsigned int fence_y_offset;
  78         u16 override_cfb_stride;
  79         u16 interval;
  80         s8 fence_id;
  81 };
  82
  83 struct intel_fbc {
  84         struct drm_i915_private *i915;
  85         const struct intel_fbc_funcs *funcs;
  86
  87         /*
  88          * This is always the inner lock when overlapping with
  89          * struct_mutex and it's the outer lock when overlapping
  90          * with stolen_lock.
  91          */
  92         struct mutex lock;
  93         unsigned int busy_bits;
  94
  95         struct drm_mm_node compressed_fb;
  96         struct drm_mm_node compressed_llb;
  97
  98         enum intel_fbc_id id;
  99
 100         u8 limit;
 101
 102         bool false_color;
 103
 104         bool active;
 105         bool activated;
 106         bool flip_pending;
 107
 108         bool underrun_detected;
 109         struct work_struct underrun_work;
 110
 111         /*
 112          * This structure contains everything that's relevant to program the
 113          * hardware registers. When we want to figure out if we need to disable
 114          * and re-enable FBC for a new configuration we just check if there's
 115          * something different in the struct. The genx_fbc_activate functions
 116          * are supposed to read from it in order to program the registers.
 117          */
 118         struct intel_fbc_state state;
 119         const char *no_fbc_reason;
 120 };
 121
 122 /* plane stride in pixels */
 123 static unsigned int intel_fbc_plane_stride(const struct intel_plane_state *plane_state)
 124 {
 125         const struct drm_framebuffer *fb = plane_state->hw.fb;
 126         unsigned int stride;
 127
 128         stride = plane_state->view.color_plane[0].mapping_stride;
 129         if (!drm_rotation_90_or_270(plane_state->hw.rotation))
 130                 stride /= fb->format->cpp[0];
 131
 132         return stride;
 133 }
 134
 135 /* plane stride based cfb stride in bytes, assuming 1:1 compression limit */
 136 static unsigned int _intel_fbc_cfb_stride(const struct intel_plane_state *plane_state)
 137 {
 138         unsigned int cpp = 4; /* FBC always 4 bytes per pixel */
 139
 140         return intel_fbc_plane_stride(plane_state) * cpp;
 141 }
 142
 143 /* minimum acceptable cfb stride in bytes, assuming 1:1 compression limit */
 144 static unsigned int skl_fbc_min_cfb_stride(const struct intel_plane_state *plane_state)
 145 {
 146         struct drm_i915_private *i915 = to_i915(plane_state->uapi.plane->dev);
 147         unsigned int limit = 4; /* 1:4 compression limit is the worst case */
 148         unsigned int cpp = 4; /* FBC always 4 bytes per pixel */
 149         unsigned int width = drm_rect_width(&plane_state->uapi.src) >> 16;
 150         unsigned int height = 4; /* FBC segment is 4 lines */
 151         unsigned int stride;
 152
 153         /* minimum segment stride we can use */
 154         stride = width * cpp * height / limit;
 155
 156         /*
 157          * Wa_16011863758: icl+
 158          * Avoid some hardware segment address miscalculation.
 159          */
 160         if (DISPLAY_VER(i915) >= 11)
 161                 stride += 64;
 162
 163         /*
 164          * At least some of the platforms require each 4 line segment to
 165          * be 512 byte aligned. Just do it always for simplicity.
 166          */
 167         stride = ALIGN(stride, 512);
 168
 169         /* convert back to single line equivalent with 1:1 compression limit */
 170         return stride * limit / height;
 171 }
 172
 173 /* properly aligned cfb stride in bytes, assuming 1:1 compression limit */
 174 static unsigned int intel_fbc_cfb_stride(const struct intel_plane_state *plane_state)
 175 {
 176         struct drm_i915_private *i915 = to_i915(plane_state->uapi.plane->dev);
 177         unsigned int stride = _intel_fbc_cfb_stride(plane_state);
 178
 179         /*
 180          * At least some of the platforms require each 4 line segment to
 181          * be 512 byte aligned. Aligning each line to 512 bytes guarantees
 182          * that regardless of the compression limit we choose later.
 183          */
 184         if (DISPLAY_VER(i915) >= 9)
 185                 return max(ALIGN(stride, 512), skl_fbc_min_cfb_stride(plane_state));
 186         else
 187                 return stride;
 188 }
 189
 190 static unsigned int intel_fbc_cfb_size(const struct intel_plane_state *plane_state)
 191 {
 192         struct drm_i915_private *i915 = to_i915(plane_state->uapi.plane->dev);
 193         int lines = drm_rect_height(&plane_state->uapi.src) >> 16;
 194
 195         if (DISPLAY_VER(i915) == 7)
 196                 lines = min(lines, 2048);
 197         else if (DISPLAY_VER(i915) >= 8)
 198                 lines = min(lines, 2560);
 199
 200         return lines * intel_fbc_cfb_stride(plane_state);
 201 }
 202
 203 static u16 intel_fbc_override_cfb_stride(const struct intel_plane_state *plane_state)
 204 {
 205         struct drm_i915_private *i915 = to_i915(plane_state->uapi.plane->dev);
 206         unsigned int stride_aligned = intel_fbc_cfb_stride(plane_state);
 207         unsigned int stride = _intel_fbc_cfb_stride(plane_state);
 208         const struct drm_framebuffer *fb = plane_state->hw.fb;
 209
 210         /*
 211          * Override stride in 64 byte units per 4 line segment.
 212          *
 213          * Gen9 hw miscalculates cfb stride for linear as
 214          * PLANE_STRIDE*512 instead of PLANE_STRIDE*64, so
 215          * we always need to use the override there.
 216          */
 217         if (stride != stride_aligned ||
 218             (DISPLAY_VER(i915) == 9 && fb->modifier == DRM_FORMAT_MOD_LINEAR))
 219                 return stride_aligned * 4 / 64;
 220
 221         return 0;
 222 }
 223
 224 static u32 i8xx_fbc_ctl(struct intel_fbc *fbc)
 225 {
 226         const struct intel_fbc_state *fbc_state = &fbc->state;
 227         struct drm_i915_private *i915 = fbc->i915;
 228         unsigned int cfb_stride;
 229         u32 fbc_ctl;
 230
 231         cfb_stride = fbc_state->cfb_stride / fbc->limit;
 232
 233         /* FBC_CTL wants 32B or 64B units */
 234         if (DISPLAY_VER(i915) == 2)
 235                 cfb_stride = (cfb_stride / 32) - 1;
 236         else
 237                 cfb_stride = (cfb_stride / 64) - 1;
 238
 239         fbc_ctl = FBC_CTL_PERIODIC |
 240                 FBC_CTL_INTERVAL(fbc_state->interval) |
 241                 FBC_CTL_STRIDE(cfb_stride);
 242
 243         if (IS_I945GM(i915))
 244                 fbc_ctl |= FBC_CTL_C3_IDLE; /* 945 needs special SR handling */
 245
 246         if (fbc_state->fence_id >= 0)
 247                 fbc_ctl |= FBC_CTL_FENCENO(fbc_state->fence_id);
 248
 249         return fbc_ctl;
 250 }
 251
 252 static u32 i965_fbc_ctl2(struct intel_fbc *fbc)
 253 {
 254         const struct intel_fbc_state *fbc_state = &fbc->state;
 255         u32 fbc_ctl2;
 256
 257         fbc_ctl2 = FBC_CTL_FENCE_DBL | FBC_CTL_IDLE_IMM |
 258                 FBC_CTL_PLANE(fbc_state->plane->i9xx_plane);
 259
 260         if (fbc_state->fence_id >= 0)
 261                 fbc_ctl2 |= FBC_CTL_CPU_FENCE_EN;
 262
 263         return fbc_ctl2;
 264 }
 265
 266 static void i8xx_fbc_deactivate(struct intel_fbc *fbc)
 267 {
 268         struct drm_i915_private *i915 = fbc->i915;
 269         u32 fbc_ctl;
 270
 271         /* Disable compression */
 272         fbc_ctl = intel_de_read(i915, FBC_CONTROL);
 273         if ((fbc_ctl & FBC_CTL_EN) == 0)
 274                 return;
 275
 276         fbc_ctl &= ~FBC_CTL_EN;
 277         intel_de_write(i915, FBC_CONTROL, fbc_ctl);
 278
 279         /* Wait for compressing bit to clear */
 280         if (intel_de_wait_for_clear(i915, FBC_STATUS,
 281                                     FBC_STAT_COMPRESSING, 10)) {
 282                 drm_dbg_kms(&i915->drm, "FBC idle timed out\n");
 283                 return;
 284         }
 285 }
 286
 287 static void i8xx_fbc_activate(struct intel_fbc *fbc)
 288 {
 289         const struct intel_fbc_state *fbc_state = &fbc->state;
 290         struct drm_i915_private *i915 = fbc->i915;
 291         int i;
 292
 293         /* Clear old tags */
 294         for (i = 0; i < (FBC_LL_SIZE / 32) + 1; i++)
 295                 intel_de_write(i915, FBC_TAG(i), 0);
 296
 297         if (DISPLAY_VER(i915) == 4) {
 298                 intel_de_write(i915, FBC_CONTROL2,
 299                                i965_fbc_ctl2(fbc));
 300                 intel_de_write(i915, FBC_FENCE_OFF,
 301                                fbc_state->fence_y_offset);
 302         }
 303
 304         intel_de_write(i915, FBC_CONTROL,
 305                        FBC_CTL_EN | i8xx_fbc_ctl(fbc));
 306 }
 307
 308 static bool i8xx_fbc_is_active(struct intel_fbc *fbc)
 309 {
 310         return intel_de_read(fbc->i915, FBC_CONTROL) & FBC_CTL_EN;
 311 }
 312
 313 static bool i8xx_fbc_is_compressing(struct intel_fbc *fbc)
 314 {
 315         return intel_de_read(fbc->i915, FBC_STATUS) &
 316                 (FBC_STAT_COMPRESSING | FBC_STAT_COMPRESSED);
 317 }
 318
 319 static void i8xx_fbc_nuke(struct intel_fbc *fbc)
 320 {
 321         struct intel_fbc_state *fbc_state = &fbc->state;
 322         enum i9xx_plane_id i9xx_plane = fbc_state->plane->i9xx_plane;
 323         struct drm_i915_private *dev_priv = fbc->i915;
 324
 325         spin_lock_irq(&dev_priv->uncore.lock);
 326         intel_de_write_fw(dev_priv, DSPADDR(i9xx_plane),
 327                           intel_de_read_fw(dev_priv, DSPADDR(i9xx_plane)));
 328         spin_unlock_irq(&dev_priv->uncore.lock);
 329 }
 330
 331 static void i8xx_fbc_program_cfb(struct intel_fbc *fbc)
 332 {
 333         struct drm_i915_private *i915 = fbc->i915;
 334
 335         GEM_BUG_ON(range_overflows_end_t(u64, i915->dsm.start,
 336                                          fbc->compressed_fb.start, U32_MAX));
 337         GEM_BUG_ON(range_overflows_end_t(u64, i915->dsm.start,
 338                                          fbc->compressed_llb.start, U32_MAX));
 339
 340         intel_de_write(i915, FBC_CFB_BASE,
 341                        i915->dsm.start + fbc->compressed_fb.start);
 342         intel_de_write(i915, FBC_LL_BASE,
 343                        i915->dsm.start + fbc->compressed_llb.start);
 344 }
 345
 346 static const struct intel_fbc_funcs i8xx_fbc_funcs = {
 347         .activate = i8xx_fbc_activate,
 348         .deactivate = i8xx_fbc_deactivate,
 349         .is_active = i8xx_fbc_is_active,
 350         .is_compressing = i8xx_fbc_is_compressing,
 351         .nuke = i8xx_fbc_nuke,
 352         .program_cfb = i8xx_fbc_program_cfb,
 353 };
 354
 355 static void i965_fbc_nuke(struct intel_fbc *fbc)
 356 {
 357         struct intel_fbc_state *fbc_state = &fbc->state;
 358         enum i9xx_plane_id i9xx_plane = fbc_state->plane->i9xx_plane;
 359         struct drm_i915_private *dev_priv = fbc->i915;
 360
 361         spin_lock_irq(&dev_priv->uncore.lock);
 362         intel_de_write_fw(dev_priv, DSPSURF(i9xx_plane),
 363                           intel_de_read_fw(dev_priv, DSPSURF(i9xx_plane)));
 364         spin_unlock_irq(&dev_priv->uncore.lock);
 365 }
 366
 367 static const struct intel_fbc_funcs i965_fbc_funcs = {
 368         .activate = i8xx_fbc_activate,
 369         .deactivate = i8xx_fbc_deactivate,
 370         .is_active = i8xx_fbc_is_active,
 371         .is_compressing = i8xx_fbc_is_compressing,
 372         .nuke = i965_fbc_nuke,
 373         .program_cfb = i8xx_fbc_program_cfb,
 374 };
 375
 376 static u32 g4x_dpfc_ctl_limit(struct intel_fbc *fbc)
 377 {
 378         switch (fbc->limit) {
 379         default:
 380                 MISSING_CASE(fbc->limit);
 381                 fallthrough;
 382         case 1:
 383                 return DPFC_CTL_LIMIT_1X;
 384         case 2:
 385                 return DPFC_CTL_LIMIT_2X;
 386         case 4:
 387                 return DPFC_CTL_LIMIT_4X;
 388         }
 389 }
 390
 391 static u32 g4x_dpfc_ctl(struct intel_fbc *fbc)
 392 {
 393         const struct intel_fbc_state *fbc_state = &fbc->state;
 394         struct drm_i915_private *i915 = fbc->i915;
 395         u32 dpfc_ctl;
 396
 397         dpfc_ctl = g4x_dpfc_ctl_limit(fbc) |
 398                 DPFC_CTL_PLANE_G4X(fbc_state->plane->i9xx_plane);
 399
 400         if (IS_G4X(i915))
 401                 dpfc_ctl |= DPFC_CTL_SR_EN;
 402
 403         if (fbc_state->fence_id >= 0) {
 404                 dpfc_ctl |= DPFC_CTL_FENCE_EN_G4X;
 405
 406                 if (DISPLAY_VER(i915) < 6)
 407                         dpfc_ctl |= DPFC_CTL_FENCENO(fbc_state->fence_id);
 408         }
 409
 410         return dpfc_ctl;
 411 }
 412
 413 static void g4x_fbc_activate(struct intel_fbc *fbc)
 414 {
 415         const struct intel_fbc_state *fbc_state = &fbc->state;
 416         struct drm_i915_private *i915 = fbc->i915;
 417
 418         intel_de_write(i915, DPFC_FENCE_YOFF,
 419                        fbc_state->fence_y_offset);
 420
 421         intel_de_write(i915, DPFC_CONTROL,
 422                        DPFC_CTL_EN | g4x_dpfc_ctl(fbc));
 423 }
 424
 425 static void g4x_fbc_deactivate(struct intel_fbc *fbc)
 426 {
 427         struct drm_i915_private *i915 = fbc->i915;
 428         u32 dpfc_ctl;
 429
 430         /* Disable compression */
 431         dpfc_ctl = intel_de_read(i915, DPFC_CONTROL);
 432         if (dpfc_ctl & DPFC_CTL_EN) {
 433                 dpfc_ctl &= ~DPFC_CTL_EN;
 434                 intel_de_write(i915, DPFC_CONTROL, dpfc_ctl);
 435         }
 436 }
 437
 438 static bool g4x_fbc_is_active(struct intel_fbc *fbc)
 439 {
 440         return intel_de_read(fbc->i915, DPFC_CONTROL) & DPFC_CTL_EN;
 441 }
 442
 443 static bool g4x_fbc_is_compressing(struct intel_fbc *fbc)
 444 {
 445         return intel_de_read(fbc->i915, DPFC_STATUS) & DPFC_COMP_SEG_MASK;
 446 }
 447
 448 static void g4x_fbc_program_cfb(struct intel_fbc *fbc)
 449 {
 450         struct drm_i915_private *i915 = fbc->i915;
 451
 452         intel_de_write(i915, DPFC_CB_BASE, fbc->compressed_fb.start);
 453 }
 454
 455 static const struct intel_fbc_funcs g4x_fbc_funcs = {
 456         .activate = g4x_fbc_activate,
 457         .deactivate = g4x_fbc_deactivate,
 458         .is_active = g4x_fbc_is_active,
 459         .is_compressing = g4x_fbc_is_compressing,
 460         .nuke = i965_fbc_nuke,
 461         .program_cfb = g4x_fbc_program_cfb,
 462 };
 463
 464 static void ilk_fbc_activate(struct intel_fbc *fbc)
 465 {
 466         struct intel_fbc_state *fbc_state = &fbc->state;
 467         struct drm_i915_private *i915 = fbc->i915;
 468
 469         intel_de_write(i915, ILK_DPFC_FENCE_YOFF(fbc->id),
 470                        fbc_state->fence_y_offset);
 471
 472         intel_de_write(i915, ILK_DPFC_CONTROL(fbc->id),
 473                        DPFC_CTL_EN | g4x_dpfc_ctl(fbc));
 474 }
 475
 476 static void ilk_fbc_deactivate(struct intel_fbc *fbc)
 477 {
 478         struct drm_i915_private *i915 = fbc->i915;
 479         u32 dpfc_ctl;
 480
 481         /* Disable compression */
 482         dpfc_ctl = intel_de_read(i915, ILK_DPFC_CONTROL(fbc->id));
 483         if (dpfc_ctl & DPFC_CTL_EN) {
 484                 dpfc_ctl &= ~DPFC_CTL_EN;
 485                 intel_de_write(i915, ILK_DPFC_CONTROL(fbc->id), dpfc_ctl);
 486         }
 487 }
 488
 489 static bool ilk_fbc_is_active(struct intel_fbc *fbc)
 490 {
 491         return intel_de_read(fbc->i915, ILK_DPFC_CONTROL(fbc->id)) & DPFC_CTL_EN;
 492 }
 493
 494 static bool ilk_fbc_is_compressing(struct intel_fbc *fbc)
 495 {
 496         return intel_de_read(fbc->i915, ILK_DPFC_STATUS(fbc->id)) & DPFC_COMP_SEG_MASK;
 497 }
 498
 499 static void ilk_fbc_program_cfb(struct intel_fbc *fbc)
 500 {
 501         struct drm_i915_private *i915 = fbc->i915;
 502
 503         intel_de_write(i915, ILK_DPFC_CB_BASE(fbc->id), fbc->compressed_fb.start);
 504 }
 505
 506 static const struct intel_fbc_funcs ilk_fbc_funcs = {
 507         .activate = ilk_fbc_activate,
 508         .deactivate = ilk_fbc_deactivate,
 509         .is_active = ilk_fbc_is_active,
 510         .is_compressing = ilk_fbc_is_compressing,
 511         .nuke = i965_fbc_nuke,
 512         .program_cfb = ilk_fbc_program_cfb,
 513 };
 514
 515 static void snb_fbc_program_fence(struct intel_fbc *fbc)
 516 {
 517         const struct intel_fbc_state *fbc_state = &fbc->state;
 518         struct drm_i915_private *i915 = fbc->i915;
 519         u32 ctl = 0;
 520
 521         if (fbc_state->fence_id >= 0)
 522                 ctl = SNB_DPFC_FENCE_EN | SNB_DPFC_FENCENO(fbc_state->fence_id);
 523
 524         intel_de_write(i915, SNB_DPFC_CTL_SA, ctl);
 525         intel_de_write(i915, SNB_DPFC_CPU_FENCE_OFFSET, fbc_state->fence_y_offset);
 526 }
 527
 528 static void snb_fbc_activate(struct intel_fbc *fbc)
 529 {
 530         snb_fbc_program_fence(fbc);
 531
 532         ilk_fbc_activate(fbc);
 533 }
 534
 535 static void snb_fbc_nuke(struct intel_fbc *fbc)
 536 {
 537         struct drm_i915_private *i915 = fbc->i915;
 538
 539         intel_de_write(i915, MSG_FBC_REND_STATE(fbc->id), FBC_REND_NUKE);
 540         intel_de_posting_read(i915, MSG_FBC_REND_STATE(fbc->id));
 541 }
 542
 543 static const struct intel_fbc_funcs snb_fbc_funcs = {
 544         .activate = snb_fbc_activate,
 545         .deactivate = ilk_fbc_deactivate,
 546         .is_active = ilk_fbc_is_active,
 547         .is_compressing = ilk_fbc_is_compressing,
 548         .nuke = snb_fbc_nuke,
 549         .program_cfb = ilk_fbc_program_cfb,
 550 };
 551
 552 static void glk_fbc_program_cfb_stride(struct intel_fbc *fbc)
 553 {
 554         const struct intel_fbc_state *fbc_state = &fbc->state;
 555         struct drm_i915_private *i915 = fbc->i915;
 556         u32 val = 0;
 557
 558         if (fbc_state->override_cfb_stride)
 559                 val |= FBC_STRIDE_OVERRIDE |
 560                         FBC_STRIDE(fbc_state->override_cfb_stride / fbc->limit);
 561
 562         intel_de_write(i915, GLK_FBC_STRIDE(fbc->id), val);
 563 }
 564
 565 static void skl_fbc_program_cfb_stride(struct intel_fbc *fbc)
 566 {
 567         const struct intel_fbc_state *fbc_state = &fbc->state;
 568         struct drm_i915_private *i915 = fbc->i915;
 569         u32 val = 0;
 570
 571         /* Display WA #0529: skl, kbl, bxt. */
 572         if (fbc_state->override_cfb_stride)
 573                 val |= CHICKEN_FBC_STRIDE_OVERRIDE |
 574                         CHICKEN_FBC_STRIDE(fbc_state->override_cfb_stride / fbc->limit);
 575
 576         intel_de_rmw(i915, CHICKEN_MISC_4,
 577                      CHICKEN_FBC_STRIDE_OVERRIDE |
 578                      CHICKEN_FBC_STRIDE_MASK, val);
 579 }
 580
 581 static u32 ivb_dpfc_ctl(struct intel_fbc *fbc)
 582 {
 583         const struct intel_fbc_state *fbc_state = &fbc->state;
 584         struct drm_i915_private *i915 = fbc->i915;
 585         u32 dpfc_ctl;
 586
 587         dpfc_ctl = g4x_dpfc_ctl_limit(fbc);
 588
 589         if (IS_IVYBRIDGE(i915))
 590                 dpfc_ctl |= DPFC_CTL_PLANE_IVB(fbc_state->plane->i9xx_plane);
 591
 592         if (fbc_state->fence_id >= 0)
 593                 dpfc_ctl |= DPFC_CTL_FENCE_EN_IVB;
 594
 595         if (fbc->false_color)
 596                 dpfc_ctl |= DPFC_CTL_FALSE_COLOR;
 597
 598         return dpfc_ctl;
 599 }
 600
 601 static void ivb_fbc_activate(struct intel_fbc *fbc)
 602 {
 603         struct drm_i915_private *i915 = fbc->i915;
 604
 605         if (DISPLAY_VER(i915) >= 10)
 606                 glk_fbc_program_cfb_stride(fbc);
 607         else if (DISPLAY_VER(i915) == 9)
 608                 skl_fbc_program_cfb_stride(fbc);
 609
 610         if (to_gt(i915)->ggtt->num_fences)
 611                 snb_fbc_program_fence(fbc);
 612
 613         intel_de_write(i915, ILK_DPFC_CONTROL(fbc->id),
 614                        DPFC_CTL_EN | ivb_dpfc_ctl(fbc));
 615 }
 616
 617 static bool ivb_fbc_is_compressing(struct intel_fbc *fbc)
 618 {
 619         return intel_de_read(fbc->i915, ILK_DPFC_STATUS2(fbc->id)) & DPFC_COMP_SEG_MASK_IVB;
 620 }
 621
 622 static void ivb_fbc_set_false_color(struct intel_fbc *fbc,
 623                                     bool enable)
 624 {
 625         intel_de_rmw(fbc->i915, ILK_DPFC_CONTROL(fbc->id),
 626                      DPFC_CTL_FALSE_COLOR, enable ? DPFC_CTL_FALSE_COLOR : 0);
 627 }
 628
 629 static const struct intel_fbc_funcs ivb_fbc_funcs = {
 630         .activate = ivb_fbc_activate,
 631         .deactivate = ilk_fbc_deactivate,
 632         .is_active = ilk_fbc_is_active,
 633         .is_compressing = ivb_fbc_is_compressing,
 634         .nuke = snb_fbc_nuke,
 635         .program_cfb = ilk_fbc_program_cfb,
 636         .set_false_color = ivb_fbc_set_false_color,
 637 };
 638
 639 static bool intel_fbc_hw_is_active(struct intel_fbc *fbc)
 640 {
 641         return fbc->funcs->is_active(fbc);
 642 }
 643
 644 static void intel_fbc_hw_activate(struct intel_fbc *fbc)
 645 {
 646         trace_intel_fbc_activate(fbc->state.plane);
 647
 648         fbc->active = true;
 649         fbc->activated = true;
 650
 651         fbc->funcs->activate(fbc);
 652 }
 653
 654 static void intel_fbc_hw_deactivate(struct intel_fbc *fbc)
 655 {
 656         trace_intel_fbc_deactivate(fbc->state.plane);
 657
 658         fbc->active = false;
 659
 660         fbc->funcs->deactivate(fbc);
 661 }
 662
 663 static bool intel_fbc_is_compressing(struct intel_fbc *fbc)
 664 {
 665         return fbc->funcs->is_compressing(fbc);
 666 }
 667
 668 static void intel_fbc_nuke(struct intel_fbc *fbc)
 669 {
 670         struct drm_i915_private *i915 = fbc->i915;
 671
 672         drm_WARN_ON(&i915->drm, fbc->flip_pending);
 673
 674         trace_intel_fbc_nuke(fbc->state.plane);
 675
 676         fbc->funcs->nuke(fbc);
 677 }
 678
 679 static void intel_fbc_activate(struct intel_fbc *fbc)
 680 {
 681         intel_fbc_hw_activate(fbc);
 682         intel_fbc_nuke(fbc);
 683
 684         fbc->no_fbc_reason = NULL;
 685 }
 686
 687 static void intel_fbc_deactivate(struct intel_fbc *fbc, const char *reason)
 688 {
 689         struct drm_i915_private *i915 = fbc->i915;
 690
 691         drm_WARN_ON(&i915->drm, !mutex_is_locked(&fbc->lock));
 692
 693         if (fbc->active)
 694                 intel_fbc_hw_deactivate(fbc);
 695
 696         fbc->no_fbc_reason = reason;
 697 }
 698
 699 static u64 intel_fbc_cfb_base_max(struct drm_i915_private *i915)
 700 {
 701         if (DISPLAY_VER(i915) >= 5 || IS_G4X(i915))
 702                 return BIT_ULL(28);
 703         else
 704                 return BIT_ULL(32);
 705 }
 706
 707 static u64 intel_fbc_stolen_end(struct drm_i915_private *i915)
 708 {
 709         u64 end;
 710
 711         /* The FBC hardware for BDW/SKL doesn't have access to the stolen
 712          * reserved range size, so it always assumes the maximum (8mb) is used.
 713          * If we enable FBC using a CFB on that memory range we'll get FIFO
 714          * underruns, even if that range is not reserved by the BIOS. */
 715         if (IS_BROADWELL(i915) ||
 716             (DISPLAY_VER(i915) == 9 && !IS_BROXTON(i915)))
 717                 end = resource_size(&i915->dsm) - 8 * 1024 * 1024;
 718         else
 719                 end = U64_MAX;
 720
 721         return min(end, intel_fbc_cfb_base_max(i915));
 722 }
 723
 724 static int intel_fbc_min_limit(const struct intel_plane_state *plane_state)
 725 {
 726         return plane_state->hw.fb->format->cpp[0] == 2 ? 2 : 1;
 727 }
 728
 729 static int intel_fbc_max_limit(struct drm_i915_private *i915)
 730 {
 731         /* WaFbcOnly1to1Ratio:ctg */
 732         if (IS_G4X(i915))
 733                 return 1;
 734
 735         /*
 736          * FBC2 can only do 1:1, 1:2, 1:4, we limit
 737          * FBC1 to the same out of convenience.
 738          */
 739         return 4;
 740 }
 741
 742 static int find_compression_limit(struct intel_fbc *fbc,
 743                                   unsigned int size, int min_limit)
 744 {
 745         struct drm_i915_private *i915 = fbc->i915;
 746         u64 end = intel_fbc_stolen_end(i915);
 747         int ret, limit = min_limit;
 748
 749         size /= limit;
 750
 751         /* Try to over-allocate to reduce reallocations and fragmentation. */
 752         ret = i915_gem_stolen_insert_node_in_range(i915, &fbc->compressed_fb,
 753                                                    size <<= 1, 4096, 0, end);
 754         if (ret == 0)
 755                 return limit;
 756
 757         for (; limit <= intel_fbc_max_limit(i915); limit <<= 1) {
 758                 ret = i915_gem_stolen_insert_node_in_range(i915, &fbc->compressed_fb,
 759                                                            size >>= 1, 4096, 0, end);
 760                 if (ret == 0)
 761                         return limit;
 762         }
 763
 764         return 0;
 765 }
 766
 767 static int intel_fbc_alloc_cfb(struct intel_fbc *fbc,
 768                                unsigned int size, int min_limit)
 769 {
 770         struct drm_i915_private *i915 = fbc->i915;
 771         int ret;
 772
 773         drm_WARN_ON(&i915->drm,
 774                     drm_mm_node_allocated(&fbc->compressed_fb));
 775         drm_WARN_ON(&i915->drm,
 776                     drm_mm_node_allocated(&fbc->compressed_llb));
 777
 778         if (DISPLAY_VER(i915) < 5 && !IS_G4X(i915)) {
 779                 ret = i915_gem_stolen_insert_node(i915, &fbc->compressed_llb,
 780                                                   4096, 4096);
 781                 if (ret)
 782                         goto err;
 783         }
 784
 785         ret = find_compression_limit(fbc, size, min_limit);
 786         if (!ret)
 787                 goto err_llb;
 788         else if (ret > min_limit)
 789                 drm_info_once(&i915->drm,
 790                               "Reducing the compressed framebuffer size. This may lead to less power savings than a non-reduced-size. Try to increase stolen memory size if available in BIOS.\n");
 791
 792         fbc->limit = ret;
 793
 794         drm_dbg_kms(&i915->drm,
 795                     "reserved %llu bytes of contiguous stolen space for FBC, limit: %d\n",
 796                     fbc->compressed_fb.size, fbc->limit);
 797
 798         return 0;
 799
 800 err_llb:
 801         if (drm_mm_node_allocated(&fbc->compressed_llb))
 802                 i915_gem_stolen_remove_node(i915, &fbc->compressed_llb);
 803 err:
 804         if (drm_mm_initialized(&i915->mm.stolen))
 805                 drm_info_once(&i915->drm, "not enough stolen space for compressed buffer (need %d more bytes), disabling. Hint: you may be able to increase stolen memory size in the BIOS to avoid this.\n", size);
 806         return -ENOSPC;
 807 }
 808
 809 static void intel_fbc_program_cfb(struct intel_fbc *fbc)
 810 {
 811         fbc->funcs->program_cfb(fbc);
 812 }
 813
 814 static void intel_fbc_program_workarounds(struct intel_fbc *fbc)
 815 {
 816         /* Wa_22014263786:icl,jsl,tgl,dg1,rkl,adls,dg2,adlp */
 817         if (DISPLAY_VER(fbc->i915) >= 11)
 818                 intel_de_rmw(fbc->i915, ILK_DPFC_CHICKEN(fbc->id), 0,
 819                              DPFC_CHICKEN_FORCE_SLB_INVALIDATION);
 820 }
 821
 822 static void __intel_fbc_cleanup_cfb(struct intel_fbc *fbc)
 823 {
 824         struct drm_i915_private *i915 = fbc->i915;
 825
 826         if (WARN_ON(intel_fbc_hw_is_active(fbc)))
 827                 return;
 828
 829         if (drm_mm_node_allocated(&fbc->compressed_llb))
 830                 i915_gem_stolen_remove_node(i915, &fbc->compressed_llb);
 831         if (drm_mm_node_allocated(&fbc->compressed_fb))
 832                 i915_gem_stolen_remove_node(i915, &fbc->compressed_fb);
 833 }
 834
 835 void intel_fbc_cleanup(struct drm_i915_private *i915)
 836 {
 837         struct intel_fbc *fbc;
 838         enum intel_fbc_id fbc_id;
 839
 840         for_each_intel_fbc(i915, fbc, fbc_id) {
 841                 mutex_lock(&fbc->lock);
 842                 __intel_fbc_cleanup_cfb(fbc);
 843                 mutex_unlock(&fbc->lock);
 844
 845                 kfree(fbc);
 846         }
 847 }
 848
 849 static bool stride_is_valid(const struct intel_plane_state *plane_state)
 850 {
 851         struct drm_i915_private *i915 = to_i915(plane_state->uapi.plane->dev);
 852         const struct drm_framebuffer *fb = plane_state->hw.fb;
 853         unsigned int stride = intel_fbc_plane_stride(plane_state) *
 854                 fb->format->cpp[0];
 855
 856         /* This should have been caught earlier. */
 857         if (drm_WARN_ON_ONCE(&i915->drm, (stride & (64 - 1)) != 0))
 858                 return false;
 859
 860         /* Below are the additional FBC restrictions. */
 861         if (stride < 512)
 862                 return false;
 863
 864         if (DISPLAY_VER(i915) == 2 || DISPLAY_VER(i915) == 3)
 865                 return stride == 4096 || stride == 8192;
 866
 867         if (DISPLAY_VER(i915) == 4 && !IS_G4X(i915) && stride < 2048)
 868                 return false;
 869
 870         /* Display WA #1105: skl,bxt,kbl,cfl,glk */
 871         if ((DISPLAY_VER(i915) == 9 || IS_GEMINILAKE(i915)) &&
 872             fb->modifier == DRM_FORMAT_MOD_LINEAR && stride & 511)
 873                 return false;
 874
 875         if (stride > 16384)
 876                 return false;
 877
 878         return true;
 879 }
 880
 881 static bool pixel_format_is_valid(const struct intel_plane_state *plane_state)
 882 {
 883         struct drm_i915_private *i915 = to_i915(plane_state->uapi.plane->dev);
 884         const struct drm_framebuffer *fb = plane_state->hw.fb;
 885
 886         switch (fb->format->format) {
 887         case DRM_FORMAT_XRGB8888:
 888         case DRM_FORMAT_XBGR8888:
 889                 return true;
 890         case DRM_FORMAT_XRGB1555:
 891         case DRM_FORMAT_RGB565:
 892                 /* 16bpp not supported on gen2 */
 893                 if (DISPLAY_VER(i915) == 2)
 894                         return false;
 895                 /* WaFbcOnly1to1Ratio:ctg */
 896                 if (IS_G4X(i915))
 897                         return false;
 898                 return true;
 899         default:
 900                 return false;
 901         }
 902 }
 903
 904 static bool rotation_is_valid(const struct intel_plane_state *plane_state)
 905 {
 906         struct drm_i915_private *i915 = to_i915(plane_state->uapi.plane->dev);
 907         const struct drm_framebuffer *fb = plane_state->hw.fb;
 908         unsigned int rotation = plane_state->hw.rotation;
 909
 910         if (DISPLAY_VER(i915) >= 9 && fb->format->format == DRM_FORMAT_RGB565 &&
 911             drm_rotation_90_or_270(rotation))
 912                 return false;
 913         else if (DISPLAY_VER(i915) <= 4 && !IS_G4X(i915) &&
 914                  rotation != DRM_MODE_ROTATE_0)
 915                 return false;
 916
 917         return true;
 918 }
 919
 920 /*
 921  * For some reason, the hardware tracking starts looking at whatever we
 922  * programmed as the display plane base address register. It does not look at
 923  * the X and Y offset registers. That's why we include the src x/y offsets
 924  * instead of just looking at the plane size.
 925  */
 926 static bool intel_fbc_hw_tracking_covers_screen(const struct intel_plane_state *plane_state)
 927 {
 928         struct drm_i915_private *i915 = to_i915(plane_state->uapi.plane->dev);
 929         unsigned int effective_w, effective_h, max_w, max_h;
 930
 931         if (DISPLAY_VER(i915) >= 10) {
 932                 max_w = 5120;
 933                 max_h = 4096;
 934         } else if (DISPLAY_VER(i915) >= 8 || IS_HASWELL(i915)) {
 935                 max_w = 4096;
 936                 max_h = 4096;
 937         } else if (IS_G4X(i915) || DISPLAY_VER(i915) >= 5) {
 938                 max_w = 4096;
 939                 max_h = 2048;
 940         } else {
 941                 max_w = 2048;
 942                 max_h = 1536;
 943         }
 944
 945         effective_w = plane_state->view.color_plane[0].x +
 946                 (drm_rect_width(&plane_state->uapi.src) >> 16);
 947         effective_h = plane_state->view.color_plane[0].y +
 948                 (drm_rect_height(&plane_state->uapi.src) >> 16);
 949
 950         return effective_w <= max_w && effective_h <= max_h;
 951 }
 952
 953 static bool tiling_is_valid(const struct intel_plane_state *plane_state)
 954 {
 955         struct drm_i915_private *i915 = to_i915(plane_state->uapi.plane->dev);
 956         const struct drm_framebuffer *fb = plane_state->hw.fb;
 957
 958         switch (fb->modifier) {
 959         case DRM_FORMAT_MOD_LINEAR:
 960         case I915_FORMAT_MOD_Y_TILED:
 961         case I915_FORMAT_MOD_Yf_TILED:
 962                 return DISPLAY_VER(i915) >= 9;
 963         case I915_FORMAT_MOD_4_TILED:
 964         case I915_FORMAT_MOD_X_TILED:
 965                 return true;
 966         default:
 967                 return false;
 968         }
 969 }
 970
 971 static void intel_fbc_update_state(struct intel_atomic_state *state,
 972                                    struct intel_crtc *crtc,
 973                                    struct intel_plane *plane)
 974 {
 975         struct drm_i915_private *i915 = to_i915(state->base.dev);
 976         const struct intel_crtc_state *crtc_state =
 977                 intel_atomic_get_new_crtc_state(state, crtc);
 978         const struct intel_plane_state *plane_state =
 979                 intel_atomic_get_new_plane_state(state, plane);
 980         struct intel_fbc *fbc = plane->fbc;
 981         struct intel_fbc_state *fbc_state = &fbc->state;
 982
 983         WARN_ON(plane_state->no_fbc_reason);
 984         WARN_ON(fbc_state->plane && fbc_state->plane != plane);
 985
 986         fbc_state->plane = plane;
 987
 988         /* FBC1 compression interval: arbitrary choice of 1 second */
 989         fbc_state->interval = drm_mode_vrefresh(&crtc_state->hw.adjusted_mode);
 990
 991         fbc_state->fence_y_offset = intel_plane_fence_y_offset(plane_state);
 992
 993         drm_WARN_ON(&i915->drm, plane_state->flags & PLANE_HAS_FENCE &&
 994                     !plane_state->ggtt_vma->fence);
 995
 996         if (plane_state->flags & PLANE_HAS_FENCE &&
 997             plane_state->ggtt_vma->fence)
 998                 fbc_state->fence_id = plane_state->ggtt_vma->fence->id;
 999         else
1000                 fbc_state->fence_id = -1;
1001
1002         fbc_state->cfb_stride = intel_fbc_cfb_stride(plane_state);
1003         fbc_state->cfb_size = intel_fbc_cfb_size(plane_state);
1004         fbc_state->override_cfb_stride = intel_fbc_override_cfb_stride(plane_state);
1005 }
1006
1007 static bool intel_fbc_is_fence_ok(const struct intel_plane_state *plane_state)
1008 {
1009         struct drm_i915_private *i915 = to_i915(plane_state->uapi.plane->dev);
1010
1011         /* The use of a CPU fence is one of two ways to detect writes by the
1012          * CPU to the scanout and trigger updates to the FBC.
1013          *
1014          * The other method is by software tracking (see
1015          * intel_fbc_invalidate/flush()), it will manually notify FBC and nuke
1016          * the current compressed buffer and recompress it.
1017          *
1018          * Note that is possible for a tiled surface to be unmappable (and
1019          * so have no fence associated with it) due to aperture constraints
1020          * at the time of pinning.
1021          *
1022          * FIXME with 90/270 degree rotation we should use the fence on
1023          * the normal GTT view (the rotated view doesn't even have a
1024          * fence). Would need changes to the FBC fence Y offset as well.
1025          * For now this will effectively disable FBC with 90/270 degree
1026          * rotation.
1027          */
1028         return DISPLAY_VER(i915) >= 9 ||
1029                 (plane_state->flags & PLANE_HAS_FENCE &&
1030                  plane_state->ggtt_vma->fence);
1031 }
1032
1033 static bool intel_fbc_is_cfb_ok(const struct intel_plane_state *plane_state)
1034 {
1035         struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
1036         struct intel_fbc *fbc = plane->fbc;
1037
1038         return intel_fbc_min_limit(plane_state) <= fbc->limit &&
1039                 intel_fbc_cfb_size(plane_state) <= fbc->compressed_fb.size * fbc->limit;
1040 }
1041
1042 static bool intel_fbc_is_ok(const struct intel_plane_state *plane_state)
1043 {
1044         return !plane_state->no_fbc_reason &&
1045                 intel_fbc_is_fence_ok(plane_state) &&
1046                 intel_fbc_is_cfb_ok(plane_state);
1047 }
1048
1049 static int intel_fbc_check_plane(struct intel_atomic_state *state,
1050                                  struct intel_plane *plane)
1051 {
1052         struct drm_i915_private *i915 = to_i915(state->base.dev);
1053         struct intel_plane_state *plane_state =
1054                 intel_atomic_get_new_plane_state(state, plane);
1055         const struct drm_framebuffer *fb = plane_state->hw.fb;
1056         struct intel_crtc *crtc = to_intel_crtc(plane_state->hw.crtc);
1057         const struct intel_crtc_state *crtc_state;
1058         struct intel_fbc *fbc = plane->fbc;
1059
1060         if (!fbc)
1061                 return 0;
1062
1063         if (intel_vgpu_active(i915)) {
1064                 plane_state->no_fbc_reason = "VGPU active";
1065                 return 0;
1066         }
1067
1068         if (!i915->params.enable_fbc) {
1069                 plane_state->no_fbc_reason = "disabled per module param or by default";
1070                 return 0;
1071         }
1072
1073         if (!plane_state->uapi.visible) {
1074                 plane_state->no_fbc_reason = "plane not visible";
1075                 return 0;
1076         }
1077
1078         crtc_state = intel_atomic_get_new_crtc_state(state, crtc);
1079
1080         if (crtc_state->hw.adjusted_mode.flags & DRM_MODE_FLAG_INTERLACE) {
1081                 plane_state->no_fbc_reason = "interlaced mode not supported";
1082                 return 0;
1083         }
1084
1085         if (crtc_state->double_wide) {
1086                 plane_state->no_fbc_reason = "double wide pipe not supported";
1087                 return 0;
1088         }
1089
1090         /*
1091          * Display 12+ is not supporting FBC with PSR2.
1092          * Recommendation is to keep this combination disabled
1093          * Bspec: 50422 HSD: 14010260002
1094          */
1095         if (DISPLAY_VER(i915) >= 12 && crtc_state->has_psr2) {
1096                 plane_state->no_fbc_reason = "PSR2 enabled";
1097                 return 0;
1098         }
1099
1100         if (!pixel_format_is_valid(plane_state)) {
1101                 plane_state->no_fbc_reason = "pixel format not supported";
1102                 return 0;
1103         }
1104
1105         if (!tiling_is_valid(plane_state)) {
1106                 plane_state->no_fbc_reason = "tiling not supported";
1107                 return 0;
1108         }
1109
1110         if (!rotation_is_valid(plane_state)) {
1111                 plane_state->no_fbc_reason = "rotation not supported";
1112                 return 0;
1113         }
1114
1115         if (!stride_is_valid(plane_state)) {
1116                 plane_state->no_fbc_reason = "stride not supported";
1117                 return 0;
1118         }
1119
1120         if (plane_state->hw.pixel_blend_mode != DRM_MODE_BLEND_PIXEL_NONE &&
1121             fb->format->has_alpha) {
1122                 plane_state->no_fbc_reason = "per-pixel alpha not supported";
1123                 return 0;
1124         }
1125
1126         if (!intel_fbc_hw_tracking_covers_screen(plane_state)) {
1127                 plane_state->no_fbc_reason = "plane size too big";
1128                 return 0;
1129         }
1130
1131         /*
1132          * Work around a problem on GEN9+ HW, where enabling FBC on a plane
1133          * having a Y offset that isn't divisible by 4 causes FIFO underrun
1134          * and screen flicker.
1135          */
1136         if (DISPLAY_VER(i915) >= 9 &&
1137             plane_state->view.color_plane[0].y & 3) {
1138                 plane_state->no_fbc_reason = "plane start Y offset misaligned";
1139                 return 0;
1140         }
1141
1142         /* Wa_22010751166: icl, ehl, tgl, dg1, rkl */
1143         if (DISPLAY_VER(i915) >= 11 &&
1144             (plane_state->view.color_plane[0].y +
1145              (drm_rect_height(&plane_state->uapi.src) >> 16)) & 3) {
1146                 plane_state->no_fbc_reason = "plane end Y offset misaligned";
1147                 return 0;
1148         }
1149
1150         /* WaFbcExceedCdClockThreshold:hsw,bdw */
1151         if (IS_HASWELL(i915) || IS_BROADWELL(i915)) {
1152                 const struct intel_cdclk_state *cdclk_state;
1153
1154                 cdclk_state = intel_atomic_get_cdclk_state(state);
1155                 if (IS_ERR(cdclk_state))
1156                         return PTR_ERR(cdclk_state);
1157
1158                 if (crtc_state->pixel_rate >= cdclk_state->logical.cdclk * 95 / 100) {
1159                         plane_state->no_fbc_reason = "pixel rate too high";
1160                         return 0;
1161                 }
1162         }
1163
1164         plane_state->no_fbc_reason = NULL;
1165
1166         return 0;
1167 }
1168
1169
1170 static bool intel_fbc_can_flip_nuke(struct intel_atomic_state *state,
1171                                     struct intel_crtc *crtc,
1172                                     struct intel_plane *plane)
1173 {
1174         const struct intel_crtc_state *new_crtc_state =
1175                 intel_atomic_get_new_crtc_state(state, crtc);
1176         const struct intel_plane_state *old_plane_state =
1177                 intel_atomic_get_old_plane_state(state, plane);
1178         const struct intel_plane_state *new_plane_state =
1179                 intel_atomic_get_new_plane_state(state, plane);
1180         const struct drm_framebuffer *old_fb = old_plane_state->hw.fb;
1181         const struct drm_framebuffer *new_fb = new_plane_state->hw.fb;
1182
1183         if (drm_atomic_crtc_needs_modeset(&new_crtc_state->uapi))
1184                 return false;
1185
1186         if (!intel_fbc_is_ok(old_plane_state) ||
1187             !intel_fbc_is_ok(new_plane_state))
1188                 return false;
1189
1190         if (old_fb->format->format != new_fb->format->format)
1191                 return false;
1192
1193         if (old_fb->modifier != new_fb->modifier)
1194                 return false;
1195
1196         if (intel_fbc_plane_stride(old_plane_state) !=
1197             intel_fbc_plane_stride(new_plane_state))
1198                 return false;
1199
1200         if (intel_fbc_cfb_stride(old_plane_state) !=
1201             intel_fbc_cfb_stride(new_plane_state))
1202                 return false;
1203
1204         if (intel_fbc_cfb_size(old_plane_state) !=
1205             intel_fbc_cfb_size(new_plane_state))
1206                 return false;
1207
1208         if (intel_fbc_override_cfb_stride(old_plane_state) !=
1209             intel_fbc_override_cfb_stride(new_plane_state))
1210                 return false;
1211
1212         return true;
1213 }
1214
1215 static bool __intel_fbc_pre_update(struct intel_atomic_state *state,
1216                                    struct intel_crtc *crtc,
1217                                    struct intel_plane *plane)
1218 {
1219         struct drm_i915_private *i915 = to_i915(state->base.dev);
1220         struct intel_fbc *fbc = plane->fbc;
1221         bool need_vblank_wait = false;
1222
1223         fbc->flip_pending = true;
1224
1225         if (intel_fbc_can_flip_nuke(state, crtc, plane))
1226                 return need_vblank_wait;
1227
1228         intel_fbc_deactivate(fbc, "update pending");
1229
1230         /*
1231          * Display WA #1198: glk+
1232          * Need an extra vblank wait between FBC disable and most plane
1233          * updates. Bspec says this is only needed for plane disable, but
1234          * that is not true. Touching most plane registers will cause the
1235          * corruption to appear. Also SKL/derivatives do not seem to be
1236          * affected.
1237          *
1238          * TODO: could optimize this a bit by sampling the frame
1239          * counter when we disable FBC (if it was already done earlier)
1240          * and skipping the extra vblank wait before the plane update
1241          * if at least one frame has already passed.
1242          */
1243         if (fbc->activated && DISPLAY_VER(i915) >= 10)
1244                 need_vblank_wait = true;
1245         fbc->activated = false;
1246
1247         return need_vblank_wait;
1248 }
1249
1250 bool intel_fbc_pre_update(struct intel_atomic_state *state,
1251                           struct intel_crtc *crtc)
1252 {
1253         const struct intel_plane_state *plane_state;
1254         bool need_vblank_wait = false;
1255         struct intel_plane *plane;
1256         int i;
1257
1258         for_each_new_intel_plane_in_state(state, plane, plane_state, i) {
1259                 struct intel_fbc *fbc = plane->fbc;
1260
1261                 if (!fbc || plane->pipe != crtc->pipe)
1262                         continue;
1263
1264                 mutex_lock(&fbc->lock);
1265
1266                 if (fbc->state.plane == plane)
1267                         need_vblank_wait |= __intel_fbc_pre_update(state, crtc, plane);
1268
1269                 mutex_unlock(&fbc->lock);
1270         }
1271
1272         return need_vblank_wait;
1273 }
1274
1275 static void __intel_fbc_disable(struct intel_fbc *fbc)
1276 {
1277         struct drm_i915_private *i915 = fbc->i915;
1278         struct intel_plane *plane = fbc->state.plane;
1279
1280         drm_WARN_ON(&i915->drm, !mutex_is_locked(&fbc->lock));
1281         drm_WARN_ON(&i915->drm, fbc->active);
1282
1283         drm_dbg_kms(&i915->drm, "Disabling FBC on [PLANE:%d:%s]\n",
1284                     plane->base.base.id, plane->base.name);
1285
1286         __intel_fbc_cleanup_cfb(fbc);
1287
1288         fbc->state.plane = NULL;
1289         fbc->flip_pending = false;
1290         fbc->busy_bits = 0;
1291 }
1292
1293 static void __intel_fbc_post_update(struct intel_fbc *fbc)
1294 {
1295         struct drm_i915_private *i915 = fbc->i915;
1296
1297         drm_WARN_ON(&i915->drm, !mutex_is_locked(&fbc->lock));
1298
1299         if (!fbc->busy_bits)
1300                 intel_fbc_activate(fbc);
1301         else
1302                 intel_fbc_deactivate(fbc, "frontbuffer write");
1303 }
1304
1305 void intel_fbc_post_update(struct intel_atomic_state *state,
1306                            struct intel_crtc *crtc)
1307 {
1308         const struct intel_plane_state *plane_state;
1309         struct intel_plane *plane;
1310         int i;
1311
1312         for_each_new_intel_plane_in_state(state, plane, plane_state, i) {
1313                 struct intel_fbc *fbc = plane->fbc;
1314
1315                 if (!fbc || plane->pipe != crtc->pipe)
1316                         continue;
1317
1318                 mutex_lock(&fbc->lock);
1319
1320                 if (fbc->state.plane == plane) {
1321                         fbc->flip_pending = false;
1322                         __intel_fbc_post_update(fbc);
1323                 }
1324
1325                 mutex_unlock(&fbc->lock);
1326         }
1327 }
1328
1329 static unsigned int intel_fbc_get_frontbuffer_bit(struct intel_fbc *fbc)
1330 {
1331         if (fbc->state.plane)
1332                 return fbc->state.plane->frontbuffer_bit;
1333         else
1334                 return 0;
1335 }
1336
1337 static void __intel_fbc_invalidate(struct intel_fbc *fbc,
1338                                    unsigned int frontbuffer_bits,
1339                                    enum fb_op_origin origin)
1340 {
1341         if (origin == ORIGIN_FLIP || origin == ORIGIN_CURSOR_UPDATE)
1342                 return;
1343
1344         mutex_lock(&fbc->lock);
1345
1346         frontbuffer_bits &= intel_fbc_get_frontbuffer_bit(fbc);
1347         if (!frontbuffer_bits)
1348                 goto out;
1349
1350         fbc->busy_bits |= frontbuffer_bits;
1351         intel_fbc_deactivate(fbc, "frontbuffer write");
1352
1353 out:
1354         mutex_unlock(&fbc->lock);
1355 }
1356
1357 void intel_fbc_invalidate(struct drm_i915_private *i915,
1358                           unsigned int frontbuffer_bits,
1359                           enum fb_op_origin origin)
1360 {
1361         struct intel_fbc *fbc;
1362         enum intel_fbc_id fbc_id;
1363
1364         for_each_intel_fbc(i915, fbc, fbc_id)
1365                 __intel_fbc_invalidate(fbc, frontbuffer_bits, origin);
1366
1367 }
1368
1369 static void __intel_fbc_flush(struct intel_fbc *fbc,
1370                               unsigned int frontbuffer_bits,
1371                               enum fb_op_origin origin)
1372 {
1373         mutex_lock(&fbc->lock);
1374
1375         frontbuffer_bits &= intel_fbc_get_frontbuffer_bit(fbc);
1376         if (!frontbuffer_bits)
1377                 goto out;
1378
1379         fbc->busy_bits &= ~frontbuffer_bits;
1380
1381         if (origin == ORIGIN_FLIP || origin == ORIGIN_CURSOR_UPDATE)
1382                 goto out;
1383
1384         if (fbc->busy_bits || fbc->flip_pending)
1385                 goto out;
1386
1387         if (fbc->active)
1388                 intel_fbc_nuke(fbc);
1389         else
1390                 intel_fbc_activate(fbc);
1391
1392 out:
1393         mutex_unlock(&fbc->lock);
1394 }
1395
1396 void intel_fbc_flush(struct drm_i915_private *i915,
1397                      unsigned int frontbuffer_bits,
1398                      enum fb_op_origin origin)
1399 {
1400         struct intel_fbc *fbc;
1401         enum intel_fbc_id fbc_id;
1402
1403         for_each_intel_fbc(i915, fbc, fbc_id)
1404                 __intel_fbc_flush(fbc, frontbuffer_bits, origin);
1405 }
1406
1407 int intel_fbc_atomic_check(struct intel_atomic_state *state)
1408 {
1409         struct intel_plane_state *plane_state;
1410         struct intel_plane *plane;
1411         int i;
1412
1413         for_each_new_intel_plane_in_state(state, plane, plane_state, i) {
1414                 int ret;
1415
1416                 ret = intel_fbc_check_plane(state, plane);
1417                 if (ret)
1418                         return ret;
1419         }
1420
1421         return 0;
1422 }
1423
1424 static void __intel_fbc_enable(struct intel_atomic_state *state,
1425                                struct intel_crtc *crtc,
1426                                struct intel_plane *plane)
1427 {
1428         struct drm_i915_private *i915 = to_i915(state->base.dev);
1429         const struct intel_plane_state *plane_state =
1430                 intel_atomic_get_new_plane_state(state, plane);
1431         struct intel_fbc *fbc = plane->fbc;
1432
1433         if (fbc->state.plane) {
1434                 if (fbc->state.plane != plane)
1435                         return;
1436
1437                 if (intel_fbc_is_ok(plane_state)) {
1438                         intel_fbc_update_state(state, crtc, plane);
1439                         return;
1440                 }
1441
1442                 __intel_fbc_disable(fbc);
1443         }
1444
1445         drm_WARN_ON(&i915->drm, fbc->active);
1446
1447         fbc->no_fbc_reason = plane_state->no_fbc_reason;
1448         if (fbc->no_fbc_reason)
1449                 return;
1450
1451         if (!intel_fbc_is_fence_ok(plane_state)) {
1452                 fbc->no_fbc_reason = "framebuffer not fenced";
1453                 return;
1454         }
1455
1456         if (fbc->underrun_detected) {
1457                 fbc->no_fbc_reason = "FIFO underrun";
1458                 return;
1459         }
1460
1461         if (intel_fbc_alloc_cfb(fbc, intel_fbc_cfb_size(plane_state),
1462                                 intel_fbc_min_limit(plane_state))) {
1463                 fbc->no_fbc_reason = "not enough stolen memory";
1464                 return;
1465         }
1466
1467         drm_dbg_kms(&i915->drm, "Enabling FBC on [PLANE:%d:%s]\n",
1468                     plane->base.base.id, plane->base.name);
1469         fbc->no_fbc_reason = "FBC enabled but not active yet\n";
1470
1471         intel_fbc_update_state(state, crtc, plane);
1472
1473         intel_fbc_program_workarounds(fbc);
1474         intel_fbc_program_cfb(fbc);
1475 }
1476
1477 /**
1478  * intel_fbc_disable - disable FBC if it's associated with crtc
1479  * @crtc: the CRTC
1480  *
1481  * This function disables FBC if it's associated with the provided CRTC.
1482  */
1483 void intel_fbc_disable(struct intel_crtc *crtc)
1484 {
1485         struct drm_i915_private *i915 = to_i915(crtc->base.dev);
1486         struct intel_plane *plane;
1487
1488         for_each_intel_plane(&i915->drm, plane) {
1489                 struct intel_fbc *fbc = plane->fbc;
1490
1491                 if (!fbc || plane->pipe != crtc->pipe)
1492                         continue;
1493
1494                 mutex_lock(&fbc->lock);
1495                 if (fbc->state.plane == plane)
1496                         __intel_fbc_disable(fbc);
1497                 mutex_unlock(&fbc->lock);
1498         }
1499 }
1500
1501 void intel_fbc_update(struct intel_atomic_state *state,
1502                       struct intel_crtc *crtc)
1503 {
1504         const struct intel_crtc_state *crtc_state =
1505                 intel_atomic_get_new_crtc_state(state, crtc);
1506         const struct intel_plane_state *plane_state;
1507         struct intel_plane *plane;
1508         int i;
1509
1510         for_each_new_intel_plane_in_state(state, plane, plane_state, i) {
1511                 struct intel_fbc *fbc = plane->fbc;
1512
1513                 if (!fbc || plane->pipe != crtc->pipe)
1514                         continue;
1515
1516                 mutex_lock(&fbc->lock);
1517
1518                 if (crtc_state->update_pipe && plane_state->no_fbc_reason) {
1519                         if (fbc->state.plane == plane)
1520                                 __intel_fbc_disable(fbc);
1521                 } else {
1522                         __intel_fbc_enable(state, crtc, plane);
1523                 }
1524
1525                 mutex_unlock(&fbc->lock);
1526         }
1527 }
1528
1529 static void intel_fbc_underrun_work_fn(struct work_struct *work)
1530 {
1531         struct intel_fbc *fbc = container_of(work, typeof(*fbc), underrun_work);
1532         struct drm_i915_private *i915 = fbc->i915;
1533
1534         mutex_lock(&fbc->lock);
1535
1536         /* Maybe we were scheduled twice. */
1537         if (fbc->underrun_detected || !fbc->state.plane)
1538                 goto out;
1539
1540         drm_dbg_kms(&i915->drm, "Disabling FBC due to FIFO underrun.\n");
1541         fbc->underrun_detected = true;
1542
1543         intel_fbc_deactivate(fbc, "FIFO underrun");
1544         if (!fbc->flip_pending)
1545                 intel_crtc_wait_for_next_vblank(intel_crtc_for_pipe(i915, fbc->state.plane->pipe));
1546         __intel_fbc_disable(fbc);
1547 out:
1548         mutex_unlock(&fbc->lock);
1549 }
1550
1551 static void __intel_fbc_reset_underrun(struct intel_fbc *fbc)
1552 {
1553         struct drm_i915_private *i915 = fbc->i915;
1554
1555         cancel_work_sync(&fbc->underrun_work);
1556
1557         mutex_lock(&fbc->lock);
1558
1559         if (fbc->underrun_detected) {
1560                 drm_dbg_kms(&i915->drm,
1561                             "Re-allowing FBC after fifo underrun\n");
1562                 fbc->no_fbc_reason = "FIFO underrun cleared";
1563         }
1564
1565         fbc->underrun_detected = false;
1566         mutex_unlock(&fbc->lock);
1567 }
1568
1569 /*
1570  * intel_fbc_reset_underrun - reset FBC fifo underrun status.
1571  * @i915: the i915 device
1572  *
1573  * See intel_fbc_handle_fifo_underrun_irq(). For automated testing we
1574  * want to re-enable FBC after an underrun to increase test coverage.
1575  */
1576 void intel_fbc_reset_underrun(struct drm_i915_private *i915)
1577 {
1578         struct intel_fbc *fbc;
1579         enum intel_fbc_id fbc_id;
1580
1581         for_each_intel_fbc(i915, fbc, fbc_id)
1582                 __intel_fbc_reset_underrun(fbc);
1583 }
1584
1585 static void __intel_fbc_handle_fifo_underrun_irq(struct intel_fbc *fbc)
1586 {
1587         /*
1588          * There's no guarantee that underrun_detected won't be set to true
1589          * right after this check and before the work is scheduled, but that's
1590          * not a problem since we'll check it again under the work function
1591          * while FBC is locked. This check here is just to prevent us from
1592          * unnecessarily scheduling the work, and it relies on the fact that we
1593          * never switch underrun_detect back to false after it's true.
1594          */
1595         if (READ_ONCE(fbc->underrun_detected))
1596                 return;
1597
1598         schedule_work(&fbc->underrun_work);
1599 }
1600
1601 /**
1602  * intel_fbc_handle_fifo_underrun_irq - disable FBC when we get a FIFO underrun
1603  * @i915: i915 device
1604  *
1605  * Without FBC, most underruns are harmless and don't really cause too many
1606  * problems, except for an annoying message on dmesg. With FBC, underruns can
1607  * become black screens or even worse, especially when paired with bad
1608  * watermarks. So in order for us to be on the safe side, completely disable FBC
1609  * in case we ever detect a FIFO underrun on any pipe. An underrun on any pipe
1610  * already suggests that watermarks may be bad, so try to be as safe as
1611  * possible.
1612  *
1613  * This function is called from the IRQ handler.
1614  */
1615 void intel_fbc_handle_fifo_underrun_irq(struct drm_i915_private *i915)
1616 {
1617         struct intel_fbc *fbc;
1618         enum intel_fbc_id fbc_id;
1619
1620         for_each_intel_fbc(i915, fbc, fbc_id)
1621                 __intel_fbc_handle_fifo_underrun_irq(fbc);
1622 }
1623
1624 /*
1625  * The DDX driver changes its behavior depending on the value it reads from
1626  * i915.enable_fbc, so sanitize it by translating the default value into either
1627  * 0 or 1 in order to allow it to know what's going on.
1628  *
1629  * Notice that this is done at driver initialization and we still allow user
1630  * space to change the value during runtime without sanitizing it again. IGT
1631  * relies on being able to change i915.enable_fbc at runtime.
1632  */
1633 static int intel_sanitize_fbc_option(struct drm_i915_private *i915)
1634 {
1635         if (i915->params.enable_fbc >= 0)
1636                 return !!i915->params.enable_fbc;
1637
1638         if (!HAS_FBC(i915))
1639                 return 0;
1640
1641         if (IS_BROADWELL(i915) || DISPLAY_VER(i915) >= 9)
1642                 return 1;
1643
1644         return 0;
1645 }
1646
1647 static bool need_fbc_vtd_wa(struct drm_i915_private *i915)
1648 {
1649         /* WaFbcTurnOffFbcWhenHyperVisorIsUsed:skl,bxt */
1650         if (i915_vtd_active(i915) &&
1651             (IS_SKYLAKE(i915) || IS_BROXTON(i915))) {
1652                 drm_info(&i915->drm,
1653                          "Disabling framebuffer compression (FBC) to prevent screen flicker with VT-d enabled\n");
1654                 return true;
1655         }
1656
1657         return false;
1658 }
1659
1660 void intel_fbc_add_plane(struct intel_fbc *fbc, struct intel_plane *plane)
1661 {
1662         plane->fbc = fbc;
1663 }
1664
1665 static struct intel_fbc *intel_fbc_create(struct drm_i915_private *i915,
1666                                           enum intel_fbc_id fbc_id)
1667 {
1668         struct intel_fbc *fbc;
1669
1670         fbc = kzalloc(sizeof(*fbc), GFP_KERNEL);
1671         if (!fbc)
1672                 return NULL;
1673
1674         fbc->id = fbc_id;
1675         fbc->i915 = i915;
1676         INIT_WORK(&fbc->underrun_work, intel_fbc_underrun_work_fn);
1677         mutex_init(&fbc->lock);
1678
1679         if (DISPLAY_VER(i915) >= 7)
1680                 fbc->funcs = &ivb_fbc_funcs;
1681         else if (DISPLAY_VER(i915) == 6)
1682                 fbc->funcs = &snb_fbc_funcs;
1683         else if (DISPLAY_VER(i915) == 5)
1684                 fbc->funcs = &ilk_fbc_funcs;
1685         else if (IS_G4X(i915))
1686                 fbc->funcs = &g4x_fbc_funcs;
1687         else if (DISPLAY_VER(i915) == 4)
1688                 fbc->funcs = &i965_fbc_funcs;
1689         else
1690                 fbc->funcs = &i8xx_fbc_funcs;
1691
1692         return fbc;
1693 }
1694
1695 /**
1696  * intel_fbc_init - Initialize FBC
1697  * @i915: the i915 device
1698  *
1699  * This function might be called during PM init process.
1700  */
1701 void intel_fbc_init(struct drm_i915_private *i915)
1702 {
1703         enum intel_fbc_id fbc_id;
1704
1705         if (!drm_mm_initialized(&i915->mm.stolen))
1706                 mkwrite_device_info(i915)->display.fbc_mask = 0;
1707
1708         if (need_fbc_vtd_wa(i915))
1709                 mkwrite_device_info(i915)->display.fbc_mask = 0;
1710
1711         i915->params.enable_fbc = intel_sanitize_fbc_option(i915);
1712         drm_dbg_kms(&i915->drm, "Sanitized enable_fbc value: %d\n",
1713                     i915->params.enable_fbc);
1714
1715         for_each_fbc_id(i915, fbc_id)
1716                 i915->fbc[fbc_id] = intel_fbc_create(i915, fbc_id);
1717 }
1718
1719 /**
1720  * intel_fbc_sanitize - Sanitize FBC
1721  * @i915: the i915 device
1722  *
1723  * Make sure FBC is initially disabled since we have no
1724  * idea eg. into which parts of stolen it might be scribbling
1725  * into.
1726  */
1727 void intel_fbc_sanitize(struct drm_i915_private *i915)
1728 {
1729         struct intel_fbc *fbc;
1730         enum intel_fbc_id fbc_id;
1731
1732         for_each_intel_fbc(i915, fbc, fbc_id) {
1733                 if (intel_fbc_hw_is_active(fbc))
1734                         intel_fbc_hw_deactivate(fbc);
1735         }
1736 }
1737
1738 static int intel_fbc_debugfs_status_show(struct seq_file *m, void *unused)
1739 {
1740         struct intel_fbc *fbc = m->private;
1741         struct drm_i915_private *i915 = fbc->i915;
1742         struct intel_plane *plane;
1743         intel_wakeref_t wakeref;
1744
1745         drm_modeset_lock_all(&i915->drm);
1746
1747         wakeref = intel_runtime_pm_get(&i915->runtime_pm);
1748         mutex_lock(&fbc->lock);
1749
1750         if (fbc->active) {
1751                 seq_puts(m, "FBC enabled\n");
1752                 seq_printf(m, "Compressing: %s\n",
1753                            str_yes_no(intel_fbc_is_compressing(fbc)));
1754         } else {
1755                 seq_printf(m, "FBC disabled: %s\n", fbc->no_fbc_reason);
1756         }
1757
1758         for_each_intel_plane(&i915->drm, plane) {
1759                 const struct intel_plane_state *plane_state =
1760                         to_intel_plane_state(plane->base.state);
1761
1762                 if (plane->fbc != fbc)
1763                         continue;
1764
1765                 seq_printf(m, "%c [PLANE:%d:%s]: %s\n",
1766                            fbc->state.plane == plane ? '*' : ' ',
1767                            plane->base.base.id, plane->base.name,
1768                            plane_state->no_fbc_reason ?: "FBC possible");
1769         }
1770
1771         mutex_unlock(&fbc->lock);
1772         intel_runtime_pm_put(&i915->runtime_pm, wakeref);
1773
1774         drm_modeset_unlock_all(&i915->drm);
1775
1776         return 0;
1777 }
1778
1779 DEFINE_SHOW_ATTRIBUTE(intel_fbc_debugfs_status);
1780
1781 static int intel_fbc_debugfs_false_color_get(void *data, u64 *val)
1782 {
1783         struct intel_fbc *fbc = data;
1784
1785         *val = fbc->false_color;
1786
1787         return 0;
1788 }
1789
1790 static int intel_fbc_debugfs_false_color_set(void *data, u64 val)
1791 {
1792         struct intel_fbc *fbc = data;
1793
1794         mutex_lock(&fbc->lock);
1795
1796         fbc->false_color = val;
1797
1798         if (fbc->active)
1799                 fbc->funcs->set_false_color(fbc, fbc->false_color);
1800
1801         mutex_unlock(&fbc->lock);
1802
1803         return 0;
1804 }
1805
1806 DEFINE_SIMPLE_ATTRIBUTE(intel_fbc_debugfs_false_color_fops,
1807                         intel_fbc_debugfs_false_color_get,
1808                         intel_fbc_debugfs_false_color_set,
1809                         "%llu\n");
1810
1811 static void intel_fbc_debugfs_add(struct intel_fbc *fbc,
1812                                   struct dentry *parent)
1813 {
1814         debugfs_create_file("i915_fbc_status", 0444, parent,
1815                             fbc, &intel_fbc_debugfs_status_fops);
1816
1817         if (fbc->funcs->set_false_color)
1818                 debugfs_create_file("i915_fbc_false_color", 0644, parent,
1819                                     fbc, &intel_fbc_debugfs_false_color_fops);
1820 }
1821
1822 void intel_fbc_crtc_debugfs_add(struct intel_crtc *crtc)
1823 {
1824         struct intel_plane *plane = to_intel_plane(crtc->base.primary);
1825
1826         if (plane->fbc)
1827                 intel_fbc_debugfs_add(plane->fbc, crtc->base.debugfs_entry);
1828 }
1829
1830 /* FIXME: remove this once igt is on board with per-crtc stuff */
1831 void intel_fbc_debugfs_register(struct drm_i915_private *i915)
1832 {
1833         struct drm_minor *minor = i915->drm.primary;
1834         struct intel_fbc *fbc;
1835
1836         fbc = i915->fbc[INTEL_FBC_A];
1837         if (fbc)
1838                 intel_fbc_debugfs_add(fbc, minor->debugfs_root);
1839 }