drivers/gpu/drm/i915/display/intel_fbc.c

   1 /*
   2  * Copyright © 2014 Intel Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
  21  * DEALINGS IN THE SOFTWARE.
  22  */
  23
  24 /**
  25  * DOC: Frame Buffer Compression (FBC)
  26  *
  27  * FBC tries to save memory bandwidth (and so power consumption) by
  28  * compressing the amount of memory used by the display. It is total
  29  * transparent to user space and completely handled in the kernel.
  30  *
  31  * The benefits of FBC are mostly visible with solid backgrounds and
  32  * variation-less patterns. It comes from keeping the memory footprint small
  33  * and having fewer memory pages opened and accessed for refreshing the display.
  34  *
  35  * i915 is responsible to reserve stolen memory for FBC and configure its
  36  * offset on proper registers. The hardware takes care of all
  37  * compress/decompress. However there are many known cases where we have to
  38  * forcibly disable it to allow proper screen updates.
  39  */
  40
  41 #include <linux/debugfs.h>
  42 #include <linux/string_helpers.h>
  43
  44 #include <drm/drm_blend.h>
  45 #include <drm/drm_fourcc.h>
  46
  47 #include "gem/i915_gem_stolen.h"
  48 #include "gt/intel_gt_types.h"
  49 #include "i915_drv.h"
  50 #include "i915_reg.h"
  51 #include "i915_utils.h"
  52 #include "i915_vgpu.h"
  53 #include "i915_vma.h"
  54 #include "i9xx_plane_regs.h"
  55 #include "intel_cdclk.h"
  56 #include "intel_de.h"
  57 #include "intel_display_device.h"
  58 #include "intel_display_trace.h"
  59 #include "intel_display_types.h"
  60 #include "intel_display_wa.h"
  61 #include "intel_fbc.h"
  62 #include "intel_fbc_regs.h"
  63 #include "intel_frontbuffer.h"
  64
  65 #define for_each_fbc_id(__display, __fbc_id) \
  66         for ((__fbc_id) = INTEL_FBC_A; (__fbc_id) < I915_MAX_FBCS; (__fbc_id)++) \
  67                 for_each_if(DISPLAY_RUNTIME_INFO(__display)->fbc_mask & BIT(__fbc_id))
  68
  69 #define for_each_intel_fbc(__display, __fbc, __fbc_id) \
  70         for_each_fbc_id((__display), (__fbc_id)) \
  71                 for_each_if((__fbc) = (__display)->fbc[(__fbc_id)])
  72
  73 struct intel_fbc_funcs {
  74         void (*activate)(struct intel_fbc *fbc);
  75         void (*deactivate)(struct intel_fbc *fbc);
  76         bool (*is_active)(struct intel_fbc *fbc);
  77         bool (*is_compressing)(struct intel_fbc *fbc);
  78         void (*nuke)(struct intel_fbc *fbc);
  79         void (*program_cfb)(struct intel_fbc *fbc);
  80         void (*set_false_color)(struct intel_fbc *fbc, bool enable);
  81 };
  82
  83 struct intel_fbc_state {
  84         struct intel_plane *plane;
  85         unsigned int cfb_stride;
  86         unsigned int cfb_size;
  87         unsigned int fence_y_offset;
  88         u16 override_cfb_stride;
  89         u16 interval;
  90         s8 fence_id;
  91 };
  92
  93 struct intel_fbc {
  94         struct intel_display *display;
  95         const struct intel_fbc_funcs *funcs;
  96
  97         /*
  98          * This is always the inner lock when overlapping with
  99          * struct_mutex and it's the outer lock when overlapping
 100          * with stolen_lock.
 101          */
 102         struct mutex lock;
 103         unsigned int busy_bits;
 104
 105         struct i915_stolen_fb compressed_fb, compressed_llb;
 106
 107         enum intel_fbc_id id;
 108
 109         u8 limit;
 110
 111         bool false_color;
 112
 113         bool active;
 114         bool activated;
 115         bool flip_pending;
 116
 117         bool underrun_detected;
 118         struct work_struct underrun_work;
 119
 120         /*
 121          * This structure contains everything that's relevant to program the
 122          * hardware registers. When we want to figure out if we need to disable
 123          * and re-enable FBC for a new configuration we just check if there's
 124          * something different in the struct. The genx_fbc_activate functions
 125          * are supposed to read from it in order to program the registers.
 126          */
 127         struct intel_fbc_state state;
 128         const char *no_fbc_reason;
 129 };
 130
 131 /* plane stride in pixels */
 132 static unsigned int intel_fbc_plane_stride(const struct intel_plane_state *plane_state)
 133 {
 134         const struct drm_framebuffer *fb = plane_state->hw.fb;
 135         unsigned int stride;
 136
 137         stride = plane_state->view.color_plane[0].mapping_stride;
 138         if (!drm_rotation_90_or_270(plane_state->hw.rotation))
 139                 stride /= fb->format->cpp[0];
 140
 141         return stride;
 142 }
 143
 144 static unsigned int intel_fbc_cfb_cpp(void)
 145 {
 146         return 4; /* FBC always 4 bytes per pixel */
 147 }
 148
 149 /* plane stride based cfb stride in bytes, assuming 1:1 compression limit */
 150 static unsigned int intel_fbc_plane_cfb_stride(const struct intel_plane_state *plane_state)
 151 {
 152         unsigned int cpp = intel_fbc_cfb_cpp();
 153
 154         return intel_fbc_plane_stride(plane_state) * cpp;
 155 }
 156
 157 /* minimum acceptable cfb stride in bytes, assuming 1:1 compression limit */
 158 static unsigned int skl_fbc_min_cfb_stride(struct intel_display *display,
 159                                            unsigned int cpp, unsigned int width)
 160 {
 161         unsigned int limit = 4; /* 1:4 compression limit is the worst case */
 162         unsigned int height = 4; /* FBC segment is 4 lines */
 163         unsigned int stride;
 164
 165         /* minimum segment stride we can use */
 166         stride = width * cpp * height / limit;
 167
 168         /*
 169          * Wa_16011863758: icl+
 170          * Avoid some hardware segment address miscalculation.
 171          */
 172         if (DISPLAY_VER(display) >= 11)
 173                 stride += 64;
 174
 175         /*
 176          * At least some of the platforms require each 4 line segment to
 177          * be 512 byte aligned. Just do it always for simplicity.
 178          */
 179         stride = ALIGN(stride, 512);
 180
 181         /* convert back to single line equivalent with 1:1 compression limit */
 182         return stride * limit / height;
 183 }
 184
 185 /* properly aligned cfb stride in bytes, assuming 1:1 compression limit */
 186 static unsigned int _intel_fbc_cfb_stride(struct intel_display *display,
 187                                           unsigned int cpp, unsigned int width,
 188                                           unsigned int stride)
 189 {
 190         /*
 191          * At least some of the platforms require each 4 line segment to
 192          * be 512 byte aligned. Aligning each line to 512 bytes guarantees
 193          * that regardless of the compression limit we choose later.
 194          */
 195         if (DISPLAY_VER(display) >= 9)
 196                 return max(ALIGN(stride, 512), skl_fbc_min_cfb_stride(display, cpp, width));
 197         else
 198                 return stride;
 199 }
 200
 201 static unsigned int intel_fbc_cfb_stride(const struct intel_plane_state *plane_state)
 202 {
 203         struct intel_display *display = to_intel_display(plane_state->uapi.plane->dev);
 204         unsigned int stride = intel_fbc_plane_cfb_stride(plane_state);
 205         unsigned int width = drm_rect_width(&plane_state->uapi.src) >> 16;
 206         unsigned int cpp = intel_fbc_cfb_cpp();
 207
 208         return _intel_fbc_cfb_stride(display, cpp, width, stride);
 209 }
 210
 211 /*
 212  * Maximum height the hardware will compress, on HSW+
 213  * additional lines (up to the actual plane height) will
 214  * remain uncompressed.
 215  */
 216 static unsigned int intel_fbc_max_cfb_height(struct intel_display *display)
 217 {
 218         struct drm_i915_private *i915 = to_i915(display->drm);
 219
 220         if (DISPLAY_VER(display) >= 8)
 221                 return 2560;
 222         else if (DISPLAY_VER(display) >= 5 || IS_G4X(i915))
 223                 return 2048;
 224         else
 225                 return 1536;
 226 }
 227
 228 static unsigned int _intel_fbc_cfb_size(struct intel_display *display,
 229                                         unsigned int height, unsigned int stride)
 230 {
 231         return min(height, intel_fbc_max_cfb_height(display)) * stride;
 232 }
 233
 234 static unsigned int intel_fbc_cfb_size(const struct intel_plane_state *plane_state)
 235 {
 236         struct intel_display *display = to_intel_display(plane_state->uapi.plane->dev);
 237         unsigned int height = drm_rect_height(&plane_state->uapi.src) >> 16;
 238
 239         return _intel_fbc_cfb_size(display, height, intel_fbc_cfb_stride(plane_state));
 240 }
 241
 242 static u16 intel_fbc_override_cfb_stride(const struct intel_plane_state *plane_state)
 243 {
 244         struct intel_display *display = to_intel_display(plane_state->uapi.plane->dev);
 245         unsigned int stride_aligned = intel_fbc_cfb_stride(plane_state);
 246         unsigned int stride = intel_fbc_plane_cfb_stride(plane_state);
 247         const struct drm_framebuffer *fb = plane_state->hw.fb;
 248
 249         /*
 250          * Override stride in 64 byte units per 4 line segment.
 251          *
 252          * Gen9 hw miscalculates cfb stride for linear as
 253          * PLANE_STRIDE*512 instead of PLANE_STRIDE*64, so
 254          * we always need to use the override there.
 255          */
 256         if (stride != stride_aligned ||
 257             (DISPLAY_VER(display) == 9 && fb->modifier == DRM_FORMAT_MOD_LINEAR))
 258                 return stride_aligned * 4 / 64;
 259
 260         return 0;
 261 }
 262
 263 static bool intel_fbc_has_fences(struct intel_display *display)
 264 {
 265         struct drm_i915_private __maybe_unused *i915 = to_i915(display->drm);
 266
 267         return intel_gt_support_legacy_fencing(to_gt(i915));
 268 }
 269
 270 static u32 i8xx_fbc_ctl(struct intel_fbc *fbc)
 271 {
 272         const struct intel_fbc_state *fbc_state = &fbc->state;
 273         struct intel_display *display = fbc->display;
 274         struct drm_i915_private *i915 = to_i915(display->drm);
 275         unsigned int cfb_stride;
 276         u32 fbc_ctl;
 277
 278         cfb_stride = fbc_state->cfb_stride / fbc->limit;
 279
 280         /* FBC_CTL wants 32B or 64B units */
 281         if (DISPLAY_VER(display) == 2)
 282                 cfb_stride = (cfb_stride / 32) - 1;
 283         else
 284                 cfb_stride = (cfb_stride / 64) - 1;
 285
 286         fbc_ctl = FBC_CTL_PERIODIC |
 287                 FBC_CTL_INTERVAL(fbc_state->interval) |
 288                 FBC_CTL_STRIDE(cfb_stride);
 289
 290         if (IS_I945GM(i915))
 291                 fbc_ctl |= FBC_CTL_C3_IDLE; /* 945 needs special SR handling */
 292
 293         if (fbc_state->fence_id >= 0)
 294                 fbc_ctl |= FBC_CTL_FENCENO(fbc_state->fence_id);
 295
 296         return fbc_ctl;
 297 }
 298
 299 static u32 i965_fbc_ctl2(struct intel_fbc *fbc)
 300 {
 301         const struct intel_fbc_state *fbc_state = &fbc->state;
 302         u32 fbc_ctl2;
 303
 304         fbc_ctl2 = FBC_CTL_FENCE_DBL | FBC_CTL_IDLE_IMM |
 305                 FBC_CTL_PLANE(fbc_state->plane->i9xx_plane);
 306
 307         if (fbc_state->fence_id >= 0)
 308                 fbc_ctl2 |= FBC_CTL_CPU_FENCE_EN;
 309
 310         return fbc_ctl2;
 311 }
 312
 313 static void i8xx_fbc_deactivate(struct intel_fbc *fbc)
 314 {
 315         struct intel_display *display = fbc->display;
 316         u32 fbc_ctl;
 317
 318         /* Disable compression */
 319         fbc_ctl = intel_de_read(display, FBC_CONTROL);
 320         if ((fbc_ctl & FBC_CTL_EN) == 0)
 321                 return;
 322
 323         fbc_ctl &= ~FBC_CTL_EN;
 324         intel_de_write(display, FBC_CONTROL, fbc_ctl);
 325
 326         /* Wait for compressing bit to clear */
 327         if (intel_de_wait_for_clear(display, FBC_STATUS,
 328                                     FBC_STAT_COMPRESSING, 10)) {
 329                 drm_dbg_kms(display->drm, "FBC idle timed out\n");
 330                 return;
 331         }
 332 }
 333
 334 static void i8xx_fbc_activate(struct intel_fbc *fbc)
 335 {
 336         const struct intel_fbc_state *fbc_state = &fbc->state;
 337         struct intel_display *display = fbc->display;
 338         int i;
 339
 340         /* Clear old tags */
 341         for (i = 0; i < (FBC_LL_SIZE / 32) + 1; i++)
 342                 intel_de_write(display, FBC_TAG(i), 0);
 343
 344         if (DISPLAY_VER(display) == 4) {
 345                 intel_de_write(display, FBC_CONTROL2,
 346                                i965_fbc_ctl2(fbc));
 347                 intel_de_write(display, FBC_FENCE_OFF,
 348                                fbc_state->fence_y_offset);
 349         }
 350
 351         intel_de_write(display, FBC_CONTROL,
 352                        FBC_CTL_EN | i8xx_fbc_ctl(fbc));
 353 }
 354
 355 static bool i8xx_fbc_is_active(struct intel_fbc *fbc)
 356 {
 357         return intel_de_read(fbc->display, FBC_CONTROL) & FBC_CTL_EN;
 358 }
 359
 360 static bool i8xx_fbc_is_compressing(struct intel_fbc *fbc)
 361 {
 362         return intel_de_read(fbc->display, FBC_STATUS) &
 363                 (FBC_STAT_COMPRESSING | FBC_STAT_COMPRESSED);
 364 }
 365
 366 static void i8xx_fbc_nuke(struct intel_fbc *fbc)
 367 {
 368         struct intel_fbc_state *fbc_state = &fbc->state;
 369         enum i9xx_plane_id i9xx_plane = fbc_state->plane->i9xx_plane;
 370         struct drm_i915_private *dev_priv = to_i915(fbc->display->drm);
 371
 372         intel_de_write_fw(dev_priv, DSPADDR(dev_priv, i9xx_plane),
 373                           intel_de_read_fw(dev_priv, DSPADDR(dev_priv, i9xx_plane)));
 374 }
 375
 376 static void i8xx_fbc_program_cfb(struct intel_fbc *fbc)
 377 {
 378         struct intel_display *display = fbc->display;
 379         struct drm_i915_private *i915 = to_i915(display->drm);
 380
 381         drm_WARN_ON(display->drm,
 382                     range_overflows_end_t(u64, i915_gem_stolen_area_address(i915),
 383                                           i915_gem_stolen_node_offset(&fbc->compressed_fb),
 384                                           U32_MAX));
 385         drm_WARN_ON(display->drm,
 386                     range_overflows_end_t(u64, i915_gem_stolen_area_address(i915),
 387                                           i915_gem_stolen_node_offset(&fbc->compressed_llb),
 388                                           U32_MAX));
 389         intel_de_write(i915, FBC_CFB_BASE,
 390                        i915_gem_stolen_node_address(i915, &fbc->compressed_fb));
 391         intel_de_write(i915, FBC_LL_BASE,
 392                        i915_gem_stolen_node_address(i915, &fbc->compressed_llb));
 393 }
 394
 395 static const struct intel_fbc_funcs i8xx_fbc_funcs = {
 396         .activate = i8xx_fbc_activate,
 397         .deactivate = i8xx_fbc_deactivate,
 398         .is_active = i8xx_fbc_is_active,
 399         .is_compressing = i8xx_fbc_is_compressing,
 400         .nuke = i8xx_fbc_nuke,
 401         .program_cfb = i8xx_fbc_program_cfb,
 402 };
 403
 404 static void i965_fbc_nuke(struct intel_fbc *fbc)
 405 {
 406         struct intel_fbc_state *fbc_state = &fbc->state;
 407         enum i9xx_plane_id i9xx_plane = fbc_state->plane->i9xx_plane;
 408         struct drm_i915_private *dev_priv = to_i915(fbc->display->drm);
 409
 410         intel_de_write_fw(dev_priv, DSPSURF(dev_priv, i9xx_plane),
 411                           intel_de_read_fw(dev_priv, DSPSURF(dev_priv, i9xx_plane)));
 412 }
 413
 414 static const struct intel_fbc_funcs i965_fbc_funcs = {
 415         .activate = i8xx_fbc_activate,
 416         .deactivate = i8xx_fbc_deactivate,
 417         .is_active = i8xx_fbc_is_active,
 418         .is_compressing = i8xx_fbc_is_compressing,
 419         .nuke = i965_fbc_nuke,
 420         .program_cfb = i8xx_fbc_program_cfb,
 421 };
 422
 423 static u32 g4x_dpfc_ctl_limit(struct intel_fbc *fbc)
 424 {
 425         switch (fbc->limit) {
 426         default:
 427                 MISSING_CASE(fbc->limit);
 428                 fallthrough;
 429         case 1:
 430                 return DPFC_CTL_LIMIT_1X;
 431         case 2:
 432                 return DPFC_CTL_LIMIT_2X;
 433         case 4:
 434                 return DPFC_CTL_LIMIT_4X;
 435         }
 436 }
 437
 438 static u32 g4x_dpfc_ctl(struct intel_fbc *fbc)
 439 {
 440         const struct intel_fbc_state *fbc_state = &fbc->state;
 441         struct intel_display *display = fbc->display;
 442         struct drm_i915_private *i915 = to_i915(display->drm);
 443         u32 dpfc_ctl;
 444
 445         dpfc_ctl = g4x_dpfc_ctl_limit(fbc) |
 446                 DPFC_CTL_PLANE_G4X(fbc_state->plane->i9xx_plane);
 447
 448         if (IS_G4X(i915))
 449                 dpfc_ctl |= DPFC_CTL_SR_EN;
 450
 451         if (fbc_state->fence_id >= 0) {
 452                 dpfc_ctl |= DPFC_CTL_FENCE_EN_G4X;
 453
 454                 if (DISPLAY_VER(display) < 6)
 455                         dpfc_ctl |= DPFC_CTL_FENCENO(fbc_state->fence_id);
 456         }
 457
 458         return dpfc_ctl;
 459 }
 460
 461 static void g4x_fbc_activate(struct intel_fbc *fbc)
 462 {
 463         const struct intel_fbc_state *fbc_state = &fbc->state;
 464         struct intel_display *display = fbc->display;
 465
 466         intel_de_write(display, DPFC_FENCE_YOFF,
 467                        fbc_state->fence_y_offset);
 468
 469         intel_de_write(display, DPFC_CONTROL,
 470                        DPFC_CTL_EN | g4x_dpfc_ctl(fbc));
 471 }
 472
 473 static void g4x_fbc_deactivate(struct intel_fbc *fbc)
 474 {
 475         struct intel_display *display = fbc->display;
 476         u32 dpfc_ctl;
 477
 478         /* Disable compression */
 479         dpfc_ctl = intel_de_read(display, DPFC_CONTROL);
 480         if (dpfc_ctl & DPFC_CTL_EN) {
 481                 dpfc_ctl &= ~DPFC_CTL_EN;
 482                 intel_de_write(display, DPFC_CONTROL, dpfc_ctl);
 483         }
 484 }
 485
 486 static bool g4x_fbc_is_active(struct intel_fbc *fbc)
 487 {
 488         return intel_de_read(fbc->display, DPFC_CONTROL) & DPFC_CTL_EN;
 489 }
 490
 491 static bool g4x_fbc_is_compressing(struct intel_fbc *fbc)
 492 {
 493         return intel_de_read(fbc->display, DPFC_STATUS) & DPFC_COMP_SEG_MASK;
 494 }
 495
 496 static void g4x_fbc_program_cfb(struct intel_fbc *fbc)
 497 {
 498         struct intel_display *display = fbc->display;
 499
 500         intel_de_write(display, DPFC_CB_BASE,
 501                        i915_gem_stolen_node_offset(&fbc->compressed_fb));
 502 }
 503
 504 static const struct intel_fbc_funcs g4x_fbc_funcs = {
 505         .activate = g4x_fbc_activate,
 506         .deactivate = g4x_fbc_deactivate,
 507         .is_active = g4x_fbc_is_active,
 508         .is_compressing = g4x_fbc_is_compressing,
 509         .nuke = i965_fbc_nuke,
 510         .program_cfb = g4x_fbc_program_cfb,
 511 };
 512
 513 static void ilk_fbc_activate(struct intel_fbc *fbc)
 514 {
 515         struct intel_fbc_state *fbc_state = &fbc->state;
 516         struct intel_display *display = fbc->display;
 517
 518         intel_de_write(display, ILK_DPFC_FENCE_YOFF(fbc->id),
 519                        fbc_state->fence_y_offset);
 520
 521         intel_de_write(display, ILK_DPFC_CONTROL(fbc->id),
 522                        DPFC_CTL_EN | g4x_dpfc_ctl(fbc));
 523 }
 524
 525 static void ilk_fbc_deactivate(struct intel_fbc *fbc)
 526 {
 527         struct intel_display *display = fbc->display;
 528         u32 dpfc_ctl;
 529
 530         /* Disable compression */
 531         dpfc_ctl = intel_de_read(display, ILK_DPFC_CONTROL(fbc->id));
 532         if (dpfc_ctl & DPFC_CTL_EN) {
 533                 dpfc_ctl &= ~DPFC_CTL_EN;
 534                 intel_de_write(display, ILK_DPFC_CONTROL(fbc->id), dpfc_ctl);
 535         }
 536 }
 537
 538 static bool ilk_fbc_is_active(struct intel_fbc *fbc)
 539 {
 540         return intel_de_read(fbc->display, ILK_DPFC_CONTROL(fbc->id)) & DPFC_CTL_EN;
 541 }
 542
 543 static bool ilk_fbc_is_compressing(struct intel_fbc *fbc)
 544 {
 545         return intel_de_read(fbc->display, ILK_DPFC_STATUS(fbc->id)) & DPFC_COMP_SEG_MASK;
 546 }
 547
 548 static void ilk_fbc_program_cfb(struct intel_fbc *fbc)
 549 {
 550         struct intel_display *display = fbc->display;
 551
 552         intel_de_write(display, ILK_DPFC_CB_BASE(fbc->id),
 553                        i915_gem_stolen_node_offset(&fbc->compressed_fb));
 554 }
 555
 556 static const struct intel_fbc_funcs ilk_fbc_funcs = {
 557         .activate = ilk_fbc_activate,
 558         .deactivate = ilk_fbc_deactivate,
 559         .is_active = ilk_fbc_is_active,
 560         .is_compressing = ilk_fbc_is_compressing,
 561         .nuke = i965_fbc_nuke,
 562         .program_cfb = ilk_fbc_program_cfb,
 563 };
 564
 565 static void snb_fbc_program_fence(struct intel_fbc *fbc)
 566 {
 567         const struct intel_fbc_state *fbc_state = &fbc->state;
 568         struct intel_display *display = fbc->display;
 569         u32 ctl = 0;
 570
 571         if (fbc_state->fence_id >= 0)
 572                 ctl = SNB_DPFC_FENCE_EN | SNB_DPFC_FENCENO(fbc_state->fence_id);
 573
 574         intel_de_write(display, SNB_DPFC_CTL_SA, ctl);
 575         intel_de_write(display, SNB_DPFC_CPU_FENCE_OFFSET, fbc_state->fence_y_offset);
 576 }
 577
 578 static void snb_fbc_activate(struct intel_fbc *fbc)
 579 {
 580         snb_fbc_program_fence(fbc);
 581
 582         ilk_fbc_activate(fbc);
 583 }
 584
 585 static void snb_fbc_nuke(struct intel_fbc *fbc)
 586 {
 587         struct intel_display *display = fbc->display;
 588
 589         intel_de_write(display, MSG_FBC_REND_STATE(fbc->id), FBC_REND_NUKE);
 590         intel_de_posting_read(display, MSG_FBC_REND_STATE(fbc->id));
 591 }
 592
 593 static const struct intel_fbc_funcs snb_fbc_funcs = {
 594         .activate = snb_fbc_activate,
 595         .deactivate = ilk_fbc_deactivate,
 596         .is_active = ilk_fbc_is_active,
 597         .is_compressing = ilk_fbc_is_compressing,
 598         .nuke = snb_fbc_nuke,
 599         .program_cfb = ilk_fbc_program_cfb,
 600 };
 601
 602 static void glk_fbc_program_cfb_stride(struct intel_fbc *fbc)
 603 {
 604         const struct intel_fbc_state *fbc_state = &fbc->state;
 605         struct intel_display *display = fbc->display;
 606         u32 val = 0;
 607
 608         if (fbc_state->override_cfb_stride)
 609                 val |= FBC_STRIDE_OVERRIDE |
 610                         FBC_STRIDE(fbc_state->override_cfb_stride / fbc->limit);
 611
 612         intel_de_write(display, GLK_FBC_STRIDE(fbc->id), val);
 613 }
 614
 615 static void skl_fbc_program_cfb_stride(struct intel_fbc *fbc)
 616 {
 617         const struct intel_fbc_state *fbc_state = &fbc->state;
 618         struct intel_display *display = fbc->display;
 619         u32 val = 0;
 620
 621         /* Display WA #0529: skl, kbl, bxt. */
 622         if (fbc_state->override_cfb_stride)
 623                 val |= CHICKEN_FBC_STRIDE_OVERRIDE |
 624                         CHICKEN_FBC_STRIDE(fbc_state->override_cfb_stride / fbc->limit);
 625
 626         intel_de_rmw(display, CHICKEN_MISC_4,
 627                      CHICKEN_FBC_STRIDE_OVERRIDE |
 628                      CHICKEN_FBC_STRIDE_MASK, val);
 629 }
 630
 631 static u32 ivb_dpfc_ctl(struct intel_fbc *fbc)
 632 {
 633         const struct intel_fbc_state *fbc_state = &fbc->state;
 634         struct intel_display *display = fbc->display;
 635         struct drm_i915_private *i915 = to_i915(display->drm);
 636         u32 dpfc_ctl;
 637
 638         dpfc_ctl = g4x_dpfc_ctl_limit(fbc);
 639
 640         if (IS_IVYBRIDGE(i915))
 641                 dpfc_ctl |= DPFC_CTL_PLANE_IVB(fbc_state->plane->i9xx_plane);
 642
 643         if (DISPLAY_VER(display) >= 20)
 644                 dpfc_ctl |= DPFC_CTL_PLANE_BINDING(fbc_state->plane->id);
 645
 646         if (fbc_state->fence_id >= 0)
 647                 dpfc_ctl |= DPFC_CTL_FENCE_EN_IVB;
 648
 649         if (fbc->false_color)
 650                 dpfc_ctl |= DPFC_CTL_FALSE_COLOR;
 651
 652         return dpfc_ctl;
 653 }
 654
 655 static void ivb_fbc_activate(struct intel_fbc *fbc)
 656 {
 657         struct intel_display *display = fbc->display;
 658         u32 dpfc_ctl;
 659
 660         if (DISPLAY_VER(display) >= 10)
 661                 glk_fbc_program_cfb_stride(fbc);
 662         else if (DISPLAY_VER(display) == 9)
 663                 skl_fbc_program_cfb_stride(fbc);
 664
 665         if (intel_fbc_has_fences(display))
 666                 snb_fbc_program_fence(fbc);
 667
 668         /* wa_14019417088 Alternative WA*/
 669         dpfc_ctl = ivb_dpfc_ctl(fbc);
 670         if (DISPLAY_VER(display) >= 20)
 671                 intel_de_write(display, ILK_DPFC_CONTROL(fbc->id), dpfc_ctl);
 672
 673         intel_de_write(display, ILK_DPFC_CONTROL(fbc->id),
 674                        DPFC_CTL_EN | dpfc_ctl);
 675 }
 676
 677 static bool ivb_fbc_is_compressing(struct intel_fbc *fbc)
 678 {
 679         return intel_de_read(fbc->display, ILK_DPFC_STATUS2(fbc->id)) & DPFC_COMP_SEG_MASK_IVB;
 680 }
 681
 682 static void ivb_fbc_set_false_color(struct intel_fbc *fbc,
 683                                     bool enable)
 684 {
 685         intel_de_rmw(fbc->display, ILK_DPFC_CONTROL(fbc->id),
 686                      DPFC_CTL_FALSE_COLOR, enable ? DPFC_CTL_FALSE_COLOR : 0);
 687 }
 688
 689 static const struct intel_fbc_funcs ivb_fbc_funcs = {
 690         .activate = ivb_fbc_activate,
 691         .deactivate = ilk_fbc_deactivate,
 692         .is_active = ilk_fbc_is_active,
 693         .is_compressing = ivb_fbc_is_compressing,
 694         .nuke = snb_fbc_nuke,
 695         .program_cfb = ilk_fbc_program_cfb,
 696         .set_false_color = ivb_fbc_set_false_color,
 697 };
 698
 699 static bool intel_fbc_hw_is_active(struct intel_fbc *fbc)
 700 {
 701         return fbc->funcs->is_active(fbc);
 702 }
 703
 704 static void intel_fbc_hw_activate(struct intel_fbc *fbc)
 705 {
 706         trace_intel_fbc_activate(fbc->state.plane);
 707
 708         fbc->active = true;
 709         fbc->activated = true;
 710
 711         fbc->funcs->activate(fbc);
 712 }
 713
 714 static void intel_fbc_hw_deactivate(struct intel_fbc *fbc)
 715 {
 716         trace_intel_fbc_deactivate(fbc->state.plane);
 717
 718         fbc->active = false;
 719
 720         fbc->funcs->deactivate(fbc);
 721 }
 722
 723 static bool intel_fbc_is_compressing(struct intel_fbc *fbc)
 724 {
 725         return fbc->funcs->is_compressing(fbc);
 726 }
 727
 728 static void intel_fbc_nuke(struct intel_fbc *fbc)
 729 {
 730         struct intel_display *display = fbc->display;
 731
 732         lockdep_assert_held(&fbc->lock);
 733         drm_WARN_ON(display->drm, fbc->flip_pending);
 734
 735         trace_intel_fbc_nuke(fbc->state.plane);
 736
 737         fbc->funcs->nuke(fbc);
 738 }
 739
 740 static void intel_fbc_activate(struct intel_fbc *fbc)
 741 {
 742         lockdep_assert_held(&fbc->lock);
 743
 744         intel_fbc_hw_activate(fbc);
 745         intel_fbc_nuke(fbc);
 746
 747         fbc->no_fbc_reason = NULL;
 748 }
 749
 750 static void intel_fbc_deactivate(struct intel_fbc *fbc, const char *reason)
 751 {
 752         lockdep_assert_held(&fbc->lock);
 753
 754         if (fbc->active)
 755                 intel_fbc_hw_deactivate(fbc);
 756
 757         fbc->no_fbc_reason = reason;
 758 }
 759
 760 static u64 intel_fbc_cfb_base_max(struct intel_display *display)
 761 {
 762         struct drm_i915_private *i915 = to_i915(display->drm);
 763
 764         if (DISPLAY_VER(display) >= 5 || IS_G4X(i915))
 765                 return BIT_ULL(28);
 766         else
 767                 return BIT_ULL(32);
 768 }
 769
 770 static u64 intel_fbc_stolen_end(struct intel_display *display)
 771 {
 772         struct drm_i915_private __maybe_unused *i915 = to_i915(display->drm);
 773         u64 end;
 774
 775         /* The FBC hardware for BDW/SKL doesn't have access to the stolen
 776          * reserved range size, so it always assumes the maximum (8mb) is used.
 777          * If we enable FBC using a CFB on that memory range we'll get FIFO
 778          * underruns, even if that range is not reserved by the BIOS. */
 779         if (IS_BROADWELL(i915) ||
 780             (DISPLAY_VER(display) == 9 && !IS_BROXTON(i915)))
 781                 end = i915_gem_stolen_area_size(i915) - 8 * 1024 * 1024;
 782         else
 783                 end = U64_MAX;
 784
 785         return min(end, intel_fbc_cfb_base_max(display));
 786 }
 787
 788 static int intel_fbc_min_limit(const struct intel_plane_state *plane_state)
 789 {
 790         return plane_state->hw.fb->format->cpp[0] == 2 ? 2 : 1;
 791 }
 792
 793 static int intel_fbc_max_limit(struct intel_display *display)
 794 {
 795         struct drm_i915_private *i915 = to_i915(display->drm);
 796
 797         /* WaFbcOnly1to1Ratio:ctg */
 798         if (IS_G4X(i915))
 799                 return 1;
 800
 801         /*
 802          * FBC2 can only do 1:1, 1:2, 1:4, we limit
 803          * FBC1 to the same out of convenience.
 804          */
 805         return 4;
 806 }
 807
 808 static int find_compression_limit(struct intel_fbc *fbc,
 809                                   unsigned int size, int min_limit)
 810 {
 811         struct intel_display *display = fbc->display;
 812         struct drm_i915_private *i915 = to_i915(display->drm);
 813         u64 end = intel_fbc_stolen_end(display);
 814         int ret, limit = min_limit;
 815
 816         size /= limit;
 817
 818         /* Try to over-allocate to reduce reallocations and fragmentation. */
 819         ret = i915_gem_stolen_insert_node_in_range(i915, &fbc->compressed_fb,
 820                                                    size <<= 1, 4096, 0, end);
 821         if (ret == 0)
 822                 return limit;
 823
 824         for (; limit <= intel_fbc_max_limit(display); limit <<= 1) {
 825                 ret = i915_gem_stolen_insert_node_in_range(i915, &fbc->compressed_fb,
 826                                                            size >>= 1, 4096, 0, end);
 827                 if (ret == 0)
 828                         return limit;
 829         }
 830
 831         return 0;
 832 }
 833
 834 static int intel_fbc_alloc_cfb(struct intel_fbc *fbc,
 835                                unsigned int size, int min_limit)
 836 {
 837         struct intel_display *display = fbc->display;
 838         struct drm_i915_private *i915 = to_i915(display->drm);
 839         int ret;
 840
 841         drm_WARN_ON(display->drm,
 842                     i915_gem_stolen_node_allocated(&fbc->compressed_fb));
 843         drm_WARN_ON(display->drm,
 844                     i915_gem_stolen_node_allocated(&fbc->compressed_llb));
 845
 846         if (DISPLAY_VER(display) < 5 && !IS_G4X(i915)) {
 847                 ret = i915_gem_stolen_insert_node(i915, &fbc->compressed_llb,
 848                                                   4096, 4096);
 849                 if (ret)
 850                         goto err;
 851         }
 852
 853         ret = find_compression_limit(fbc, size, min_limit);
 854         if (!ret)
 855                 goto err_llb;
 856         else if (ret > min_limit)
 857                 drm_info_once(display->drm,
 858                               "Reducing the compressed framebuffer size. This may lead to less power savings than a non-reduced-size. Try to increase stolen memory size if available in BIOS.\n");
 859
 860         fbc->limit = ret;
 861
 862         drm_dbg_kms(display->drm,
 863                     "reserved %llu bytes of contiguous stolen space for FBC, limit: %d\n",
 864                     i915_gem_stolen_node_size(&fbc->compressed_fb), fbc->limit);
 865         return 0;
 866
 867 err_llb:
 868         if (i915_gem_stolen_node_allocated(&fbc->compressed_llb))
 869                 i915_gem_stolen_remove_node(i915, &fbc->compressed_llb);
 870 err:
 871         if (i915_gem_stolen_initialized(i915))
 872                 drm_info_once(display->drm,
 873                               "not enough stolen space for compressed buffer (need %d more bytes), disabling. Hint: you may be able to increase stolen memory size in the BIOS to avoid this.\n", size);
 874         return -ENOSPC;
 875 }
 876
 877 static void intel_fbc_program_cfb(struct intel_fbc *fbc)
 878 {
 879         fbc->funcs->program_cfb(fbc);
 880 }
 881
 882 static void intel_fbc_program_workarounds(struct intel_fbc *fbc)
 883 {
 884         struct intel_display *display = fbc->display;
 885         struct drm_i915_private *i915 = to_i915(display->drm);
 886
 887         if (IS_SKYLAKE(i915) || IS_BROXTON(i915)) {
 888                 /*
 889                  * WaFbcHighMemBwCorruptionAvoidance:skl,bxt
 890                  * Display WA #0883: skl,bxt
 891                  */
 892                 intel_de_rmw(display, ILK_DPFC_CHICKEN(fbc->id),
 893                              0, DPFC_DISABLE_DUMMY0);
 894         }
 895
 896         if (IS_SKYLAKE(i915) || IS_KABYLAKE(i915) ||
 897             IS_COFFEELAKE(i915) || IS_COMETLAKE(i915)) {
 898                 /*
 899                  * WaFbcNukeOnHostModify:skl,kbl,cfl
 900                  * Display WA #0873: skl,kbl,cfl
 901                  */
 902                 intel_de_rmw(display, ILK_DPFC_CHICKEN(fbc->id),
 903                              0, DPFC_NUKE_ON_ANY_MODIFICATION);
 904         }
 905
 906         /* Wa_1409120013:icl,jsl,tgl,dg1 */
 907         if (IS_DISPLAY_VER(display, 11, 12))
 908                 intel_de_rmw(display, ILK_DPFC_CHICKEN(fbc->id),
 909                              0, DPFC_CHICKEN_COMP_DUMMY_PIXEL);
 910
 911         /* Wa_22014263786:icl,jsl,tgl,dg1,rkl,adls,adlp,mtl */
 912         if (DISPLAY_VER(display) >= 11 && !IS_DG2(i915))
 913                 intel_de_rmw(display, ILK_DPFC_CHICKEN(fbc->id),
 914                              0, DPFC_CHICKEN_FORCE_SLB_INVALIDATION);
 915 }
 916
 917 static void __intel_fbc_cleanup_cfb(struct intel_fbc *fbc)
 918 {
 919         struct intel_display *display = fbc->display;
 920         struct drm_i915_private *i915 = to_i915(display->drm);
 921
 922         if (WARN_ON(intel_fbc_hw_is_active(fbc)))
 923                 return;
 924
 925         if (i915_gem_stolen_node_allocated(&fbc->compressed_llb))
 926                 i915_gem_stolen_remove_node(i915, &fbc->compressed_llb);
 927         if (i915_gem_stolen_node_allocated(&fbc->compressed_fb))
 928                 i915_gem_stolen_remove_node(i915, &fbc->compressed_fb);
 929 }
 930
 931 void intel_fbc_cleanup(struct intel_display *display)
 932 {
 933         struct intel_fbc *fbc;
 934         enum intel_fbc_id fbc_id;
 935
 936         for_each_intel_fbc(display, fbc, fbc_id) {
 937                 mutex_lock(&fbc->lock);
 938                 __intel_fbc_cleanup_cfb(fbc);
 939                 mutex_unlock(&fbc->lock);
 940
 941                 kfree(fbc);
 942         }
 943 }
 944
 945 static bool i8xx_fbc_stride_is_valid(const struct intel_plane_state *plane_state)
 946 {
 947         const struct drm_framebuffer *fb = plane_state->hw.fb;
 948         unsigned int stride = intel_fbc_plane_stride(plane_state) *
 949                 fb->format->cpp[0];
 950
 951         return stride == 4096 || stride == 8192;
 952 }
 953
 954 static bool i965_fbc_stride_is_valid(const struct intel_plane_state *plane_state)
 955 {
 956         const struct drm_framebuffer *fb = plane_state->hw.fb;
 957         unsigned int stride = intel_fbc_plane_stride(plane_state) *
 958                 fb->format->cpp[0];
 959
 960         return stride >= 2048 && stride <= 16384;
 961 }
 962
 963 static bool g4x_fbc_stride_is_valid(const struct intel_plane_state *plane_state)
 964 {
 965         return true;
 966 }
 967
 968 static bool skl_fbc_stride_is_valid(const struct intel_plane_state *plane_state)
 969 {
 970         const struct drm_framebuffer *fb = plane_state->hw.fb;
 971         unsigned int stride = intel_fbc_plane_stride(plane_state) *
 972                 fb->format->cpp[0];
 973
 974         /* Display WA #1105: skl,bxt,kbl,cfl,glk */
 975         if (fb->modifier == DRM_FORMAT_MOD_LINEAR && stride & 511)
 976                 return false;
 977
 978         return true;
 979 }
 980
 981 static bool icl_fbc_stride_is_valid(const struct intel_plane_state *plane_state)
 982 {
 983         return true;
 984 }
 985
 986 static bool stride_is_valid(const struct intel_plane_state *plane_state)
 987 {
 988         struct intel_display *display = to_intel_display(plane_state->uapi.plane->dev);
 989         struct drm_i915_private *i915 = to_i915(display->drm);
 990
 991         if (DISPLAY_VER(display) >= 11)
 992                 return icl_fbc_stride_is_valid(plane_state);
 993         else if (DISPLAY_VER(display) >= 9)
 994                 return skl_fbc_stride_is_valid(plane_state);
 995         else if (DISPLAY_VER(display) >= 5 || IS_G4X(i915))
 996                 return g4x_fbc_stride_is_valid(plane_state);
 997         else if (DISPLAY_VER(display) == 4)
 998                 return i965_fbc_stride_is_valid(plane_state);
 999         else
1000                 return i8xx_fbc_stride_is_valid(plane_state);
1001 }
1002
1003 static bool i8xx_fbc_pixel_format_is_valid(const struct intel_plane_state *plane_state)
1004 {
1005         struct intel_display *display = to_intel_display(plane_state->uapi.plane->dev);
1006         const struct drm_framebuffer *fb = plane_state->hw.fb;
1007
1008         switch (fb->format->format) {
1009         case DRM_FORMAT_XRGB8888:
1010         case DRM_FORMAT_XBGR8888:
1011                 return true;
1012         case DRM_FORMAT_XRGB1555:
1013         case DRM_FORMAT_RGB565:
1014                 /* 16bpp not supported on gen2 */
1015                 if (DISPLAY_VER(display) == 2)
1016                         return false;
1017                 return true;
1018         default:
1019                 return false;
1020         }
1021 }
1022
1023 static bool g4x_fbc_pixel_format_is_valid(const struct intel_plane_state *plane_state)
1024 {
1025         struct intel_display *display = to_intel_display(plane_state->uapi.plane->dev);
1026         struct drm_i915_private *i915 = to_i915(display->drm);
1027         const struct drm_framebuffer *fb = plane_state->hw.fb;
1028
1029         switch (fb->format->format) {
1030         case DRM_FORMAT_XRGB8888:
1031         case DRM_FORMAT_XBGR8888:
1032                 return true;
1033         case DRM_FORMAT_RGB565:
1034                 /* WaFbcOnly1to1Ratio:ctg */
1035                 if (IS_G4X(i915))
1036                         return false;
1037                 return true;
1038         default:
1039                 return false;
1040         }
1041 }
1042
1043 static bool lnl_fbc_pixel_format_is_valid(const struct intel_plane_state *plane_state)
1044 {
1045         const struct drm_framebuffer *fb = plane_state->hw.fb;
1046
1047         switch (fb->format->format) {
1048         case DRM_FORMAT_XRGB8888:
1049         case DRM_FORMAT_XBGR8888:
1050         case DRM_FORMAT_ARGB8888:
1051         case DRM_FORMAT_ABGR8888:
1052         case DRM_FORMAT_RGB565:
1053                 return true;
1054         default:
1055                 return false;
1056         }
1057 }
1058
1059 static bool pixel_format_is_valid(const struct intel_plane_state *plane_state)
1060 {
1061         struct intel_display *display = to_intel_display(plane_state->uapi.plane->dev);
1062         struct drm_i915_private *i915 = to_i915(display->drm);
1063
1064         if (DISPLAY_VER(display) >= 20)
1065                 return lnl_fbc_pixel_format_is_valid(plane_state);
1066         else if (DISPLAY_VER(display) >= 5 || IS_G4X(i915))
1067                 return g4x_fbc_pixel_format_is_valid(plane_state);
1068         else
1069                 return i8xx_fbc_pixel_format_is_valid(plane_state);
1070 }
1071
1072 static bool i8xx_fbc_rotation_is_valid(const struct intel_plane_state *plane_state)
1073 {
1074         return plane_state->hw.rotation == DRM_MODE_ROTATE_0;
1075 }
1076
1077 static bool g4x_fbc_rotation_is_valid(const struct intel_plane_state *plane_state)
1078 {
1079         return true;
1080 }
1081
1082 static bool skl_fbc_rotation_is_valid(const struct intel_plane_state *plane_state)
1083 {
1084         const struct drm_framebuffer *fb = plane_state->hw.fb;
1085         unsigned int rotation = plane_state->hw.rotation;
1086
1087         if (fb->format->format == DRM_FORMAT_RGB565 &&
1088             drm_rotation_90_or_270(rotation))
1089                 return false;
1090
1091         return true;
1092 }
1093
1094 static bool rotation_is_valid(const struct intel_plane_state *plane_state)
1095 {
1096         struct intel_display *display = to_intel_display(plane_state->uapi.plane->dev);
1097         struct drm_i915_private *i915 = to_i915(display->drm);
1098
1099         if (DISPLAY_VER(display) >= 9)
1100                 return skl_fbc_rotation_is_valid(plane_state);
1101         else if (DISPLAY_VER(display) >= 5 || IS_G4X(i915))
1102                 return g4x_fbc_rotation_is_valid(plane_state);
1103         else
1104                 return i8xx_fbc_rotation_is_valid(plane_state);
1105 }
1106
1107 static void intel_fbc_max_surface_size(struct intel_display *display,
1108                                        unsigned int *w, unsigned int *h)
1109 {
1110         struct drm_i915_private *i915 = to_i915(display->drm);
1111
1112         if (DISPLAY_VER(display) >= 11) {
1113                 *w = 8192;
1114                 *h = 4096;
1115         } else if (DISPLAY_VER(display) >= 10) {
1116                 *w = 5120;
1117                 *h = 4096;
1118         } else if (DISPLAY_VER(display) >= 7) {
1119                 *w = 4096;
1120                 *h = 4096;
1121         } else if (DISPLAY_VER(display) >= 5 || IS_G4X(i915)) {
1122                 *w = 4096;
1123                 *h = 2048;
1124         } else {
1125                 *w = 2048;
1126                 *h = 1536;
1127         }
1128 }
1129
1130 /*
1131  * For some reason, the hardware tracking starts looking at whatever we
1132  * programmed as the display plane base address register. It does not look at
1133  * the X and Y offset registers. That's why we include the src x/y offsets
1134  * instead of just looking at the plane size.
1135  */
1136 static bool intel_fbc_surface_size_ok(const struct intel_plane_state *plane_state)
1137 {
1138         struct intel_display *display = to_intel_display(plane_state->uapi.plane->dev);
1139         unsigned int effective_w, effective_h, max_w, max_h;
1140
1141         intel_fbc_max_surface_size(display, &max_w, &max_h);
1142
1143         effective_w = plane_state->view.color_plane[0].x +
1144                 (drm_rect_width(&plane_state->uapi.src) >> 16);
1145         effective_h = plane_state->view.color_plane[0].y +
1146                 (drm_rect_height(&plane_state->uapi.src) >> 16);
1147
1148         return effective_w <= max_w && effective_h <= max_h;
1149 }
1150
1151 static void intel_fbc_max_plane_size(struct intel_display *display,
1152                                      unsigned int *w, unsigned int *h)
1153 {
1154         struct drm_i915_private *i915 = to_i915(display->drm);
1155
1156         if (DISPLAY_VER(display) >= 10) {
1157                 *w = 5120;
1158                 *h = 4096;
1159         } else if (DISPLAY_VER(display) >= 8 || IS_HASWELL(i915)) {
1160                 *w = 4096;
1161                 *h = 4096;
1162         } else if (DISPLAY_VER(display) >= 5 || IS_G4X(i915)) {
1163                 *w = 4096;
1164                 *h = 2048;
1165         } else {
1166                 *w = 2048;
1167                 *h = 1536;
1168         }
1169 }
1170
1171 static bool intel_fbc_plane_size_valid(const struct intel_plane_state *plane_state)
1172 {
1173         struct intel_display *display = to_intel_display(plane_state->uapi.plane->dev);
1174         unsigned int w, h, max_w, max_h;
1175
1176         intel_fbc_max_plane_size(display, &max_w, &max_h);
1177
1178         w = drm_rect_width(&plane_state->uapi.src) >> 16;
1179         h = drm_rect_height(&plane_state->uapi.src) >> 16;
1180
1181         return w <= max_w && h <= max_h;
1182 }
1183
1184 static bool i8xx_fbc_tiling_valid(const struct intel_plane_state *plane_state)
1185 {
1186         const struct drm_framebuffer *fb = plane_state->hw.fb;
1187
1188         return fb->modifier == I915_FORMAT_MOD_X_TILED;
1189 }
1190
1191 static bool skl_fbc_tiling_valid(const struct intel_plane_state *plane_state)
1192 {
1193         return true;
1194 }
1195
1196 static bool tiling_is_valid(const struct intel_plane_state *plane_state)
1197 {
1198         struct intel_display *display = to_intel_display(plane_state->uapi.plane->dev);
1199
1200         if (DISPLAY_VER(display) >= 9)
1201                 return skl_fbc_tiling_valid(plane_state);
1202         else
1203                 return i8xx_fbc_tiling_valid(plane_state);
1204 }
1205
1206 static void intel_fbc_update_state(struct intel_atomic_state *state,
1207                                    struct intel_crtc *crtc,
1208                                    struct intel_plane *plane)
1209 {
1210         struct intel_display *display = to_intel_display(state->base.dev);
1211         const struct intel_crtc_state *crtc_state =
1212                 intel_atomic_get_new_crtc_state(state, crtc);
1213         const struct intel_plane_state *plane_state =
1214                 intel_atomic_get_new_plane_state(state, plane);
1215         struct intel_fbc *fbc = plane->fbc;
1216         struct intel_fbc_state *fbc_state = &fbc->state;
1217
1218         WARN_ON(plane_state->no_fbc_reason);
1219         WARN_ON(fbc_state->plane && fbc_state->plane != plane);
1220
1221         fbc_state->plane = plane;
1222
1223         /* FBC1 compression interval: arbitrary choice of 1 second */
1224         fbc_state->interval = drm_mode_vrefresh(&crtc_state->hw.adjusted_mode);
1225
1226         fbc_state->fence_y_offset = intel_plane_fence_y_offset(plane_state);
1227
1228         drm_WARN_ON(display->drm, plane_state->flags & PLANE_HAS_FENCE &&
1229                     !intel_fbc_has_fences(display));
1230
1231         if (plane_state->flags & PLANE_HAS_FENCE)
1232                 fbc_state->fence_id =  i915_vma_fence_id(plane_state->ggtt_vma);
1233         else
1234                 fbc_state->fence_id = -1;
1235
1236         fbc_state->cfb_stride = intel_fbc_cfb_stride(plane_state);
1237         fbc_state->cfb_size = intel_fbc_cfb_size(plane_state);
1238         fbc_state->override_cfb_stride = intel_fbc_override_cfb_stride(plane_state);
1239 }
1240
1241 static bool intel_fbc_is_fence_ok(const struct intel_plane_state *plane_state)
1242 {
1243         struct intel_display *display = to_intel_display(plane_state->uapi.plane->dev);
1244
1245         /*
1246          * The use of a CPU fence is one of two ways to detect writes by the
1247          * CPU to the scanout and trigger updates to the FBC.
1248          *
1249          * The other method is by software tracking (see
1250          * intel_fbc_invalidate/flush()), it will manually notify FBC and nuke
1251          * the current compressed buffer and recompress it.
1252          *
1253          * Note that is possible for a tiled surface to be unmappable (and
1254          * so have no fence associated with it) due to aperture constraints
1255          * at the time of pinning.
1256          */
1257         return DISPLAY_VER(display) >= 9 ||
1258                 (plane_state->flags & PLANE_HAS_FENCE &&
1259                  i915_vma_fence_id(plane_state->ggtt_vma) != -1);
1260 }
1261
1262 static bool intel_fbc_is_cfb_ok(const struct intel_plane_state *plane_state)
1263 {
1264         struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
1265         struct intel_fbc *fbc = plane->fbc;
1266
1267         return intel_fbc_min_limit(plane_state) <= fbc->limit &&
1268                 intel_fbc_cfb_size(plane_state) <= fbc->limit *
1269                         i915_gem_stolen_node_size(&fbc->compressed_fb);
1270 }
1271
1272 static bool intel_fbc_is_ok(const struct intel_plane_state *plane_state)
1273 {
1274         return !plane_state->no_fbc_reason &&
1275                 intel_fbc_is_fence_ok(plane_state) &&
1276                 intel_fbc_is_cfb_ok(plane_state);
1277 }
1278
1279 static int intel_fbc_check_plane(struct intel_atomic_state *state,
1280                                  struct intel_plane *plane)
1281 {
1282         struct intel_display *display = to_intel_display(state->base.dev);
1283         struct drm_i915_private *i915 = to_i915(display->drm);
1284         struct intel_plane_state *plane_state =
1285                 intel_atomic_get_new_plane_state(state, plane);
1286         const struct drm_framebuffer *fb = plane_state->hw.fb;
1287         struct intel_crtc *crtc = to_intel_crtc(plane_state->hw.crtc);
1288         const struct intel_crtc_state *crtc_state;
1289         struct intel_fbc *fbc = plane->fbc;
1290
1291         if (!fbc)
1292                 return 0;
1293
1294         if (!i915_gem_stolen_initialized(i915)) {
1295                 plane_state->no_fbc_reason = "stolen memory not initialised";
1296                 return 0;
1297         }
1298
1299         if (intel_vgpu_active(i915)) {
1300                 plane_state->no_fbc_reason = "VGPU active";
1301                 return 0;
1302         }
1303
1304         if (!display->params.enable_fbc) {
1305                 plane_state->no_fbc_reason = "disabled per module param or by default";
1306                 return 0;
1307         }
1308
1309         if (!plane_state->uapi.visible) {
1310                 plane_state->no_fbc_reason = "plane not visible";
1311                 return 0;
1312         }
1313
1314         if (intel_display_needs_wa_16023588340(i915)) {
1315                 plane_state->no_fbc_reason = "Wa_16023588340";
1316                 return 0;
1317         }
1318
1319         /* WaFbcTurnOffFbcWhenHyperVisorIsUsed:skl,bxt */
1320         if (i915_vtd_active(i915) && (IS_SKYLAKE(i915) || IS_BROXTON(i915))) {
1321                 plane_state->no_fbc_reason = "VT-d enabled";
1322                 return 0;
1323         }
1324
1325         crtc_state = intel_atomic_get_new_crtc_state(state, crtc);
1326
1327         if (crtc_state->hw.adjusted_mode.flags & DRM_MODE_FLAG_INTERLACE) {
1328                 plane_state->no_fbc_reason = "interlaced mode not supported";
1329                 return 0;
1330         }
1331
1332         if (crtc_state->double_wide) {
1333                 plane_state->no_fbc_reason = "double wide pipe not supported";
1334                 return 0;
1335         }
1336
1337         /*
1338          * Display 12+ is not supporting FBC with PSR2.
1339          * Recommendation is to keep this combination disabled
1340          * Bspec: 50422 HSD: 14010260002
1341          */
1342         if (IS_DISPLAY_VER(display, 12, 14) && crtc_state->has_sel_update &&
1343             !crtc_state->has_panel_replay) {
1344                 plane_state->no_fbc_reason = "PSR2 enabled";
1345                 return 0;
1346         }
1347
1348         /* Wa_14016291713 */
1349         if ((IS_DISPLAY_VER(display, 12, 13) ||
1350              IS_DISPLAY_VERx100_STEP(i915, 1400, STEP_A0, STEP_C0)) &&
1351             crtc_state->has_psr && !crtc_state->has_panel_replay) {
1352                 plane_state->no_fbc_reason = "PSR1 enabled (Wa_14016291713)";
1353                 return 0;
1354         }
1355
1356         if (!pixel_format_is_valid(plane_state)) {
1357                 plane_state->no_fbc_reason = "pixel format not supported";
1358                 return 0;
1359         }
1360
1361         if (!tiling_is_valid(plane_state)) {
1362                 plane_state->no_fbc_reason = "tiling not supported";
1363                 return 0;
1364         }
1365
1366         if (!rotation_is_valid(plane_state)) {
1367                 plane_state->no_fbc_reason = "rotation not supported";
1368                 return 0;
1369         }
1370
1371         if (!stride_is_valid(plane_state)) {
1372                 plane_state->no_fbc_reason = "stride not supported";
1373                 return 0;
1374         }
1375
1376         if (DISPLAY_VER(display) < 20 &&
1377             plane_state->hw.pixel_blend_mode != DRM_MODE_BLEND_PIXEL_NONE &&
1378             fb->format->has_alpha) {
1379                 plane_state->no_fbc_reason = "per-pixel alpha not supported";
1380                 return 0;
1381         }
1382
1383         if (!intel_fbc_plane_size_valid(plane_state)) {
1384                 plane_state->no_fbc_reason = "plane size too big";
1385                 return 0;
1386         }
1387
1388         if (!intel_fbc_surface_size_ok(plane_state)) {
1389                 plane_state->no_fbc_reason = "surface size too big";
1390                 return 0;
1391         }
1392
1393         /*
1394          * Work around a problem on GEN9+ HW, where enabling FBC on a plane
1395          * having a Y offset that isn't divisible by 4 causes FIFO underrun
1396          * and screen flicker.
1397          */
1398         if (DISPLAY_VER(display) >= 9 &&
1399             plane_state->view.color_plane[0].y & 3) {
1400                 plane_state->no_fbc_reason = "plane start Y offset misaligned";
1401                 return 0;
1402         }
1403
1404         /* Wa_22010751166: icl, ehl, tgl, dg1, rkl */
1405         if (DISPLAY_VER(display) >= 11 &&
1406             (plane_state->view.color_plane[0].y +
1407              (drm_rect_height(&plane_state->uapi.src) >> 16)) & 3) {
1408                 plane_state->no_fbc_reason = "plane end Y offset misaligned";
1409                 return 0;
1410         }
1411
1412         /* WaFbcExceedCdClockThreshold:hsw,bdw */
1413         if (IS_HASWELL(i915) || IS_BROADWELL(i915)) {
1414                 const struct intel_cdclk_state *cdclk_state;
1415
1416                 cdclk_state = intel_atomic_get_cdclk_state(state);
1417                 if (IS_ERR(cdclk_state))
1418                         return PTR_ERR(cdclk_state);
1419
1420                 if (crtc_state->pixel_rate >= cdclk_state->logical.cdclk * 95 / 100) {
1421                         plane_state->no_fbc_reason = "pixel rate too high";
1422                         return 0;
1423                 }
1424         }
1425
1426         plane_state->no_fbc_reason = NULL;
1427
1428         return 0;
1429 }
1430
1431
1432 static bool intel_fbc_can_flip_nuke(struct intel_atomic_state *state,
1433                                     struct intel_crtc *crtc,
1434                                     struct intel_plane *plane)
1435 {
1436         const struct intel_crtc_state *new_crtc_state =
1437                 intel_atomic_get_new_crtc_state(state, crtc);
1438         const struct intel_plane_state *old_plane_state =
1439                 intel_atomic_get_old_plane_state(state, plane);
1440         const struct intel_plane_state *new_plane_state =
1441                 intel_atomic_get_new_plane_state(state, plane);
1442         const struct drm_framebuffer *old_fb = old_plane_state->hw.fb;
1443         const struct drm_framebuffer *new_fb = new_plane_state->hw.fb;
1444
1445         if (intel_crtc_needs_modeset(new_crtc_state))
1446                 return false;
1447
1448         if (!intel_fbc_is_ok(old_plane_state) ||
1449             !intel_fbc_is_ok(new_plane_state))
1450                 return false;
1451
1452         if (old_fb->format->format != new_fb->format->format)
1453                 return false;
1454
1455         if (old_fb->modifier != new_fb->modifier)
1456                 return false;
1457
1458         if (intel_fbc_plane_stride(old_plane_state) !=
1459             intel_fbc_plane_stride(new_plane_state))
1460                 return false;
1461
1462         if (intel_fbc_cfb_stride(old_plane_state) !=
1463             intel_fbc_cfb_stride(new_plane_state))
1464                 return false;
1465
1466         if (intel_fbc_cfb_size(old_plane_state) !=
1467             intel_fbc_cfb_size(new_plane_state))
1468                 return false;
1469
1470         if (intel_fbc_override_cfb_stride(old_plane_state) !=
1471             intel_fbc_override_cfb_stride(new_plane_state))
1472                 return false;
1473
1474         return true;
1475 }
1476
1477 static bool __intel_fbc_pre_update(struct intel_atomic_state *state,
1478                                    struct intel_crtc *crtc,
1479                                    struct intel_plane *plane)
1480 {
1481         struct intel_display *display = to_intel_display(state->base.dev);
1482         struct intel_fbc *fbc = plane->fbc;
1483         bool need_vblank_wait = false;
1484
1485         lockdep_assert_held(&fbc->lock);
1486
1487         fbc->flip_pending = true;
1488
1489         if (intel_fbc_can_flip_nuke(state, crtc, plane))
1490                 return need_vblank_wait;
1491
1492         intel_fbc_deactivate(fbc, "update pending");
1493
1494         /*
1495          * Display WA #1198: glk+
1496          * Need an extra vblank wait between FBC disable and most plane
1497          * updates. Bspec says this is only needed for plane disable, but
1498          * that is not true. Touching most plane registers will cause the
1499          * corruption to appear. Also SKL/derivatives do not seem to be
1500          * affected.
1501          *
1502          * TODO: could optimize this a bit by sampling the frame
1503          * counter when we disable FBC (if it was already done earlier)
1504          * and skipping the extra vblank wait before the plane update
1505          * if at least one frame has already passed.
1506          */
1507         if (fbc->activated && DISPLAY_VER(display) >= 10)
1508                 need_vblank_wait = true;
1509         fbc->activated = false;
1510
1511         return need_vblank_wait;
1512 }
1513
1514 bool intel_fbc_pre_update(struct intel_atomic_state *state,
1515                           struct intel_crtc *crtc)
1516 {
1517         const struct intel_plane_state __maybe_unused *plane_state;
1518         bool need_vblank_wait = false;
1519         struct intel_plane *plane;
1520         int i;
1521
1522         for_each_new_intel_plane_in_state(state, plane, plane_state, i) {
1523                 struct intel_fbc *fbc = plane->fbc;
1524
1525                 if (!fbc || plane->pipe != crtc->pipe)
1526                         continue;
1527
1528                 mutex_lock(&fbc->lock);
1529
1530                 if (fbc->state.plane == plane)
1531                         need_vblank_wait |= __intel_fbc_pre_update(state, crtc, plane);
1532
1533                 mutex_unlock(&fbc->lock);
1534         }
1535
1536         return need_vblank_wait;
1537 }
1538
1539 static void __intel_fbc_disable(struct intel_fbc *fbc)
1540 {
1541         struct intel_display *display = fbc->display;
1542         struct intel_plane *plane = fbc->state.plane;
1543
1544         lockdep_assert_held(&fbc->lock);
1545         drm_WARN_ON(display->drm, fbc->active);
1546
1547         drm_dbg_kms(display->drm, "Disabling FBC on [PLANE:%d:%s]\n",
1548                     plane->base.base.id, plane->base.name);
1549
1550         __intel_fbc_cleanup_cfb(fbc);
1551
1552         fbc->state.plane = NULL;
1553         fbc->flip_pending = false;
1554         fbc->busy_bits = 0;
1555 }
1556
1557 static void __intel_fbc_post_update(struct intel_fbc *fbc)
1558 {
1559         lockdep_assert_held(&fbc->lock);
1560
1561         fbc->flip_pending = false;
1562         fbc->busy_bits = 0;
1563
1564         intel_fbc_activate(fbc);
1565 }
1566
1567 void intel_fbc_post_update(struct intel_atomic_state *state,
1568                            struct intel_crtc *crtc)
1569 {
1570         const struct intel_plane_state __maybe_unused *plane_state;
1571         struct intel_plane *plane;
1572         int i;
1573
1574         for_each_new_intel_plane_in_state(state, plane, plane_state, i) {
1575                 struct intel_fbc *fbc = plane->fbc;
1576
1577                 if (!fbc || plane->pipe != crtc->pipe)
1578                         continue;
1579
1580                 mutex_lock(&fbc->lock);
1581
1582                 if (fbc->state.plane == plane)
1583                         __intel_fbc_post_update(fbc);
1584
1585                 mutex_unlock(&fbc->lock);
1586         }
1587 }
1588
1589 static unsigned int intel_fbc_get_frontbuffer_bit(struct intel_fbc *fbc)
1590 {
1591         if (fbc->state.plane)
1592                 return fbc->state.plane->frontbuffer_bit;
1593         else
1594                 return 0;
1595 }
1596
1597 static void __intel_fbc_invalidate(struct intel_fbc *fbc,
1598                                    unsigned int frontbuffer_bits,
1599                                    enum fb_op_origin origin)
1600 {
1601         if (origin == ORIGIN_FLIP || origin == ORIGIN_CURSOR_UPDATE)
1602                 return;
1603
1604         mutex_lock(&fbc->lock);
1605
1606         frontbuffer_bits &= intel_fbc_get_frontbuffer_bit(fbc);
1607         if (!frontbuffer_bits)
1608                 goto out;
1609
1610         fbc->busy_bits |= frontbuffer_bits;
1611         intel_fbc_deactivate(fbc, "frontbuffer write");
1612
1613 out:
1614         mutex_unlock(&fbc->lock);
1615 }
1616
1617 void intel_fbc_invalidate(struct drm_i915_private *i915,
1618                           unsigned int frontbuffer_bits,
1619                           enum fb_op_origin origin)
1620 {
1621         struct intel_fbc *fbc;
1622         enum intel_fbc_id fbc_id;
1623
1624         for_each_intel_fbc(&i915->display, fbc, fbc_id)
1625                 __intel_fbc_invalidate(fbc, frontbuffer_bits, origin);
1626
1627 }
1628
1629 static void __intel_fbc_flush(struct intel_fbc *fbc,
1630                               unsigned int frontbuffer_bits,
1631                               enum fb_op_origin origin)
1632 {
1633         mutex_lock(&fbc->lock);
1634
1635         frontbuffer_bits &= intel_fbc_get_frontbuffer_bit(fbc);
1636         if (!frontbuffer_bits)
1637                 goto out;
1638
1639         fbc->busy_bits &= ~frontbuffer_bits;
1640
1641         if (origin == ORIGIN_FLIP || origin == ORIGIN_CURSOR_UPDATE)
1642                 goto out;
1643
1644         if (fbc->busy_bits || fbc->flip_pending)
1645                 goto out;
1646
1647         if (fbc->active)
1648                 intel_fbc_nuke(fbc);
1649         else
1650                 intel_fbc_activate(fbc);
1651
1652 out:
1653         mutex_unlock(&fbc->lock);
1654 }
1655
1656 void intel_fbc_flush(struct drm_i915_private *i915,
1657                      unsigned int frontbuffer_bits,
1658                      enum fb_op_origin origin)
1659 {
1660         struct intel_fbc *fbc;
1661         enum intel_fbc_id fbc_id;
1662
1663         for_each_intel_fbc(&i915->display, fbc, fbc_id)
1664                 __intel_fbc_flush(fbc, frontbuffer_bits, origin);
1665 }
1666
1667 int intel_fbc_atomic_check(struct intel_atomic_state *state)
1668 {
1669         struct intel_plane_state __maybe_unused *plane_state;
1670         struct intel_plane *plane;
1671         int i;
1672
1673         for_each_new_intel_plane_in_state(state, plane, plane_state, i) {
1674                 int ret;
1675
1676                 ret = intel_fbc_check_plane(state, plane);
1677                 if (ret)
1678                         return ret;
1679         }
1680
1681         return 0;
1682 }
1683
1684 static void __intel_fbc_enable(struct intel_atomic_state *state,
1685                                struct intel_crtc *crtc,
1686                                struct intel_plane *plane)
1687 {
1688         struct intel_display *display = to_intel_display(state->base.dev);
1689         const struct intel_plane_state *plane_state =
1690                 intel_atomic_get_new_plane_state(state, plane);
1691         struct intel_fbc *fbc = plane->fbc;
1692
1693         lockdep_assert_held(&fbc->lock);
1694
1695         if (fbc->state.plane) {
1696                 if (fbc->state.plane != plane)
1697                         return;
1698
1699                 if (intel_fbc_is_ok(plane_state)) {
1700                         intel_fbc_update_state(state, crtc, plane);
1701                         return;
1702                 }
1703
1704                 __intel_fbc_disable(fbc);
1705         }
1706
1707         drm_WARN_ON(display->drm, fbc->active);
1708
1709         fbc->no_fbc_reason = plane_state->no_fbc_reason;
1710         if (fbc->no_fbc_reason)
1711                 return;
1712
1713         if (!intel_fbc_is_fence_ok(plane_state)) {
1714                 fbc->no_fbc_reason = "framebuffer not fenced";
1715                 return;
1716         }
1717
1718         if (fbc->underrun_detected) {
1719                 fbc->no_fbc_reason = "FIFO underrun";
1720                 return;
1721         }
1722
1723         if (intel_fbc_alloc_cfb(fbc, intel_fbc_cfb_size(plane_state),
1724                                 intel_fbc_min_limit(plane_state))) {
1725                 fbc->no_fbc_reason = "not enough stolen memory";
1726                 return;
1727         }
1728
1729         drm_dbg_kms(display->drm, "Enabling FBC on [PLANE:%d:%s]\n",
1730                     plane->base.base.id, plane->base.name);
1731         fbc->no_fbc_reason = "FBC enabled but not active yet\n";
1732
1733         intel_fbc_update_state(state, crtc, plane);
1734
1735         intel_fbc_program_workarounds(fbc);
1736         intel_fbc_program_cfb(fbc);
1737 }
1738
1739 /**
1740  * intel_fbc_disable - disable FBC if it's associated with crtc
1741  * @crtc: the CRTC
1742  *
1743  * This function disables FBC if it's associated with the provided CRTC.
1744  */
1745 void intel_fbc_disable(struct intel_crtc *crtc)
1746 {
1747         struct intel_display *display = to_intel_display(crtc->base.dev);
1748         struct intel_plane *plane;
1749
1750         for_each_intel_plane(display->drm, plane) {
1751                 struct intel_fbc *fbc = plane->fbc;
1752
1753                 if (!fbc || plane->pipe != crtc->pipe)
1754                         continue;
1755
1756                 mutex_lock(&fbc->lock);
1757                 if (fbc->state.plane == plane)
1758                         __intel_fbc_disable(fbc);
1759                 mutex_unlock(&fbc->lock);
1760         }
1761 }
1762
1763 void intel_fbc_update(struct intel_atomic_state *state,
1764                       struct intel_crtc *crtc)
1765 {
1766         const struct intel_crtc_state *crtc_state =
1767                 intel_atomic_get_new_crtc_state(state, crtc);
1768         const struct intel_plane_state *plane_state;
1769         struct intel_plane *plane;
1770         int i;
1771
1772         for_each_new_intel_plane_in_state(state, plane, plane_state, i) {
1773                 struct intel_fbc *fbc = plane->fbc;
1774
1775                 if (!fbc || plane->pipe != crtc->pipe)
1776                         continue;
1777
1778                 mutex_lock(&fbc->lock);
1779
1780                 if (intel_crtc_needs_fastset(crtc_state) &&
1781                     plane_state->no_fbc_reason) {
1782                         if (fbc->state.plane == plane)
1783                                 __intel_fbc_disable(fbc);
1784                 } else {
1785                         __intel_fbc_enable(state, crtc, plane);
1786                 }
1787
1788                 mutex_unlock(&fbc->lock);
1789         }
1790 }
1791
1792 static void intel_fbc_underrun_work_fn(struct work_struct *work)
1793 {
1794         struct intel_fbc *fbc = container_of(work, typeof(*fbc), underrun_work);
1795         struct intel_display *display = fbc->display;
1796
1797         mutex_lock(&fbc->lock);
1798
1799         /* Maybe we were scheduled twice. */
1800         if (fbc->underrun_detected || !fbc->state.plane)
1801                 goto out;
1802
1803         drm_dbg_kms(display->drm, "Disabling FBC due to FIFO underrun.\n");
1804         fbc->underrun_detected = true;
1805
1806         intel_fbc_deactivate(fbc, "FIFO underrun");
1807         if (!fbc->flip_pending)
1808                 intel_crtc_wait_for_next_vblank(intel_crtc_for_pipe(display, fbc->state.plane->pipe));
1809         __intel_fbc_disable(fbc);
1810 out:
1811         mutex_unlock(&fbc->lock);
1812 }
1813
1814 static void __intel_fbc_reset_underrun(struct intel_fbc *fbc)
1815 {
1816         struct intel_display *display = fbc->display;
1817
1818         cancel_work_sync(&fbc->underrun_work);
1819
1820         mutex_lock(&fbc->lock);
1821
1822         if (fbc->underrun_detected) {
1823                 drm_dbg_kms(display->drm,
1824                             "Re-allowing FBC after fifo underrun\n");
1825                 fbc->no_fbc_reason = "FIFO underrun cleared";
1826         }
1827
1828         fbc->underrun_detected = false;
1829         mutex_unlock(&fbc->lock);
1830 }
1831
1832 /*
1833  * intel_fbc_reset_underrun - reset FBC fifo underrun status.
1834  * @display: display
1835  *
1836  * See intel_fbc_handle_fifo_underrun_irq(). For automated testing we
1837  * want to re-enable FBC after an underrun to increase test coverage.
1838  */
1839 void intel_fbc_reset_underrun(struct intel_display *display)
1840 {
1841         struct intel_fbc *fbc;
1842         enum intel_fbc_id fbc_id;
1843
1844         for_each_intel_fbc(display, fbc, fbc_id)
1845                 __intel_fbc_reset_underrun(fbc);
1846 }
1847
1848 static void __intel_fbc_handle_fifo_underrun_irq(struct intel_fbc *fbc)
1849 {
1850         struct drm_i915_private *i915 = to_i915(fbc->display->drm);
1851
1852         /*
1853          * There's no guarantee that underrun_detected won't be set to true
1854          * right after this check and before the work is scheduled, but that's
1855          * not a problem since we'll check it again under the work function
1856          * while FBC is locked. This check here is just to prevent us from
1857          * unnecessarily scheduling the work, and it relies on the fact that we
1858          * never switch underrun_detect back to false after it's true.
1859          */
1860         if (READ_ONCE(fbc->underrun_detected))
1861                 return;
1862
1863         queue_work(i915->unordered_wq, &fbc->underrun_work);
1864 }
1865
1866 /**
1867  * intel_fbc_handle_fifo_underrun_irq - disable FBC when we get a FIFO underrun
1868  * @display: display
1869  *
1870  * Without FBC, most underruns are harmless and don't really cause too many
1871  * problems, except for an annoying message on dmesg. With FBC, underruns can
1872  * become black screens or even worse, especially when paired with bad
1873  * watermarks. So in order for us to be on the safe side, completely disable FBC
1874  * in case we ever detect a FIFO underrun on any pipe. An underrun on any pipe
1875  * already suggests that watermarks may be bad, so try to be as safe as
1876  * possible.
1877  *
1878  * This function is called from the IRQ handler.
1879  */
1880 void intel_fbc_handle_fifo_underrun_irq(struct intel_display *display)
1881 {
1882         struct intel_fbc *fbc;
1883         enum intel_fbc_id fbc_id;
1884
1885         for_each_intel_fbc(display, fbc, fbc_id)
1886                 __intel_fbc_handle_fifo_underrun_irq(fbc);
1887 }
1888
1889 /*
1890  * The DDX driver changes its behavior depending on the value it reads from
1891  * i915.enable_fbc, so sanitize it by translating the default value into either
1892  * 0 or 1 in order to allow it to know what's going on.
1893  *
1894  * Notice that this is done at driver initialization and we still allow user
1895  * space to change the value during runtime without sanitizing it again. IGT
1896  * relies on being able to change i915.enable_fbc at runtime.
1897  */
1898 static int intel_sanitize_fbc_option(struct intel_display *display)
1899 {
1900         struct drm_i915_private *i915 = to_i915(display->drm);
1901
1902         if (display->params.enable_fbc >= 0)
1903                 return !!display->params.enable_fbc;
1904
1905         if (!HAS_FBC(display))
1906                 return 0;
1907
1908         if (IS_BROADWELL(i915) || DISPLAY_VER(display) >= 9)
1909                 return 1;
1910
1911         return 0;
1912 }
1913
1914 void intel_fbc_add_plane(struct intel_fbc *fbc, struct intel_plane *plane)
1915 {
1916         plane->fbc = fbc;
1917 }
1918
1919 static struct intel_fbc *intel_fbc_create(struct intel_display *display,
1920                                           enum intel_fbc_id fbc_id)
1921 {
1922         struct drm_i915_private *i915 = to_i915(display->drm);
1923         struct intel_fbc *fbc;
1924
1925         fbc = kzalloc(sizeof(*fbc), GFP_KERNEL);
1926         if (!fbc)
1927                 return NULL;
1928
1929         fbc->id = fbc_id;
1930         fbc->display = display;
1931         INIT_WORK(&fbc->underrun_work, intel_fbc_underrun_work_fn);
1932         mutex_init(&fbc->lock);
1933
1934         if (DISPLAY_VER(display) >= 7)
1935                 fbc->funcs = &ivb_fbc_funcs;
1936         else if (DISPLAY_VER(display) == 6)
1937                 fbc->funcs = &snb_fbc_funcs;
1938         else if (DISPLAY_VER(display) == 5)
1939                 fbc->funcs = &ilk_fbc_funcs;
1940         else if (IS_G4X(i915))
1941                 fbc->funcs = &g4x_fbc_funcs;
1942         else if (DISPLAY_VER(display) == 4)
1943                 fbc->funcs = &i965_fbc_funcs;
1944         else
1945                 fbc->funcs = &i8xx_fbc_funcs;
1946
1947         return fbc;
1948 }
1949
1950 /**
1951  * intel_fbc_init - Initialize FBC
1952  * @display: display
1953  *
1954  * This function might be called during PM init process.
1955  */
1956 void intel_fbc_init(struct intel_display *display)
1957 {
1958         enum intel_fbc_id fbc_id;
1959
1960         display->params.enable_fbc = intel_sanitize_fbc_option(display);
1961         drm_dbg_kms(display->drm, "Sanitized enable_fbc value: %d\n",
1962                     display->params.enable_fbc);
1963
1964         for_each_fbc_id(display, fbc_id)
1965                 display->fbc[fbc_id] = intel_fbc_create(display, fbc_id);
1966 }
1967
1968 /**
1969  * intel_fbc_sanitize - Sanitize FBC
1970  * @display: display
1971  *
1972  * Make sure FBC is initially disabled since we have no
1973  * idea eg. into which parts of stolen it might be scribbling
1974  * into.
1975  */
1976 void intel_fbc_sanitize(struct intel_display *display)
1977 {
1978         struct intel_fbc *fbc;
1979         enum intel_fbc_id fbc_id;
1980
1981         for_each_intel_fbc(display, fbc, fbc_id) {
1982                 if (intel_fbc_hw_is_active(fbc))
1983                         intel_fbc_hw_deactivate(fbc);
1984         }
1985 }
1986
1987 static int intel_fbc_debugfs_status_show(struct seq_file *m, void *unused)
1988 {
1989         struct intel_fbc *fbc = m->private;
1990         struct intel_display *display = fbc->display;
1991         struct drm_i915_private *i915 = to_i915(display->drm);
1992         struct intel_plane *plane;
1993         intel_wakeref_t wakeref;
1994
1995         drm_modeset_lock_all(display->drm);
1996
1997         wakeref = intel_runtime_pm_get(&i915->runtime_pm);
1998         mutex_lock(&fbc->lock);
1999
2000         if (fbc->active) {
2001                 seq_puts(m, "FBC enabled\n");
2002                 seq_printf(m, "Compressing: %s\n",
2003                            str_yes_no(intel_fbc_is_compressing(fbc)));
2004         } else {
2005                 seq_printf(m, "FBC disabled: %s\n", fbc->no_fbc_reason);
2006         }
2007
2008         for_each_intel_plane(display->drm, plane) {
2009                 const struct intel_plane_state *plane_state =
2010                         to_intel_plane_state(plane->base.state);
2011
2012                 if (plane->fbc != fbc)
2013                         continue;
2014
2015                 seq_printf(m, "%c [PLANE:%d:%s]: %s\n",
2016                            fbc->state.plane == plane ? '*' : ' ',
2017                            plane->base.base.id, plane->base.name,
2018                            plane_state->no_fbc_reason ?: "FBC possible");
2019         }
2020
2021         mutex_unlock(&fbc->lock);
2022         intel_runtime_pm_put(&i915->runtime_pm, wakeref);
2023
2024         drm_modeset_unlock_all(display->drm);
2025
2026         return 0;
2027 }
2028
2029 DEFINE_SHOW_ATTRIBUTE(intel_fbc_debugfs_status);
2030
2031 static int intel_fbc_debugfs_false_color_get(void *data, u64 *val)
2032 {
2033         struct intel_fbc *fbc = data;
2034
2035         *val = fbc->false_color;
2036
2037         return 0;
2038 }
2039
2040 static int intel_fbc_debugfs_false_color_set(void *data, u64 val)
2041 {
2042         struct intel_fbc *fbc = data;
2043
2044         mutex_lock(&fbc->lock);
2045
2046         fbc->false_color = val;
2047
2048         if (fbc->active)
2049                 fbc->funcs->set_false_color(fbc, fbc->false_color);
2050
2051         mutex_unlock(&fbc->lock);
2052
2053         return 0;
2054 }
2055
2056 DEFINE_DEBUGFS_ATTRIBUTE(intel_fbc_debugfs_false_color_fops,
2057                          intel_fbc_debugfs_false_color_get,
2058                          intel_fbc_debugfs_false_color_set,
2059                          "%llu\n");
2060
2061 static void intel_fbc_debugfs_add(struct intel_fbc *fbc,
2062                                   struct dentry *parent)
2063 {
2064         debugfs_create_file("i915_fbc_status", 0444, parent,
2065                             fbc, &intel_fbc_debugfs_status_fops);
2066
2067         if (fbc->funcs->set_false_color)
2068                 debugfs_create_file_unsafe("i915_fbc_false_color", 0644, parent,
2069                                            fbc, &intel_fbc_debugfs_false_color_fops);
2070 }
2071
2072 void intel_fbc_crtc_debugfs_add(struct intel_crtc *crtc)
2073 {
2074         struct intel_plane *plane = to_intel_plane(crtc->base.primary);
2075
2076         if (plane->fbc)
2077                 intel_fbc_debugfs_add(plane->fbc, crtc->base.debugfs_entry);
2078 }
2079
2080 /* FIXME: remove this once igt is on board with per-crtc stuff */
2081 void intel_fbc_debugfs_register(struct intel_display *display)
2082 {
2083         struct drm_minor *minor = display->drm->primary;
2084         struct intel_fbc *fbc;
2085
2086         fbc = display->fbc[INTEL_FBC_A];
2087         if (fbc)
2088                 intel_fbc_debugfs_add(fbc, minor->debugfs_root);
2089 }