drivers/gpu/drm/i915/display/intel_fbc.c

   1 /*
   2  * Copyright © 2014 Intel Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
  21  * DEALINGS IN THE SOFTWARE.
  22  */
  23
  24 /**
  25  * DOC: Frame Buffer Compression (FBC)
  26  *
  27  * FBC tries to save memory bandwidth (and so power consumption) by
  28  * compressing the amount of memory used by the display. It is total
  29  * transparent to user space and completely handled in the kernel.
  30  *
  31  * The benefits of FBC are mostly visible with solid backgrounds and
  32  * variation-less patterns. It comes from keeping the memory footprint small
  33  * and having fewer memory pages opened and accessed for refreshing the display.
  34  *
  35  * i915 is responsible to reserve stolen memory for FBC and configure its
  36  * offset on proper registers. The hardware takes care of all
  37  * compress/decompress. However there are many known cases where we have to
  38  * forcibly disable it to allow proper screen updates.
  39  */
  40
  41 #include <linux/string_helpers.h>
  42
  43 #include <drm/drm_blend.h>
  44 #include <drm/drm_fourcc.h>
  45
  46 #include "i915_drv.h"
  47 #include "i915_reg.h"
  48 #include "i915_utils.h"
  49 #include "i915_vgpu.h"
  50 #include "i915_vma.h"
  51 #include "intel_cdclk.h"
  52 #include "intel_de.h"
  53 #include "intel_display_device.h"
  54 #include "intel_display_trace.h"
  55 #include "intel_display_types.h"
  56 #include "intel_fbc.h"
  57 #include "intel_fbc_regs.h"
  58 #include "intel_frontbuffer.h"
  59
  60 #define for_each_fbc_id(__dev_priv, __fbc_id) \
  61         for ((__fbc_id) = INTEL_FBC_A; (__fbc_id) < I915_MAX_FBCS; (__fbc_id)++) \
  62                 for_each_if(DISPLAY_RUNTIME_INFO(__dev_priv)->fbc_mask & BIT(__fbc_id))
  63
  64 #define for_each_intel_fbc(__dev_priv, __fbc, __fbc_id) \
  65         for_each_fbc_id((__dev_priv), (__fbc_id)) \
  66                 for_each_if((__fbc) = (__dev_priv)->display.fbc[(__fbc_id)])
  67
  68 struct intel_fbc_funcs {
  69         void (*activate)(struct intel_fbc *fbc);
  70         void (*deactivate)(struct intel_fbc *fbc);
  71         bool (*is_active)(struct intel_fbc *fbc);
  72         bool (*is_compressing)(struct intel_fbc *fbc);
  73         void (*nuke)(struct intel_fbc *fbc);
  74         void (*program_cfb)(struct intel_fbc *fbc);
  75         void (*set_false_color)(struct intel_fbc *fbc, bool enable);
  76 };
  77
  78 struct intel_fbc_state {
  79         struct intel_plane *plane;
  80         unsigned int cfb_stride;
  81         unsigned int cfb_size;
  82         unsigned int fence_y_offset;
  83         u16 override_cfb_stride;
  84         u16 interval;
  85         s8 fence_id;
  86 };
  87
  88 struct intel_fbc {
  89         struct drm_i915_private *i915;
  90         const struct intel_fbc_funcs *funcs;
  91
  92         /*
  93          * This is always the inner lock when overlapping with
  94          * struct_mutex and it's the outer lock when overlapping
  95          * with stolen_lock.
  96          */
  97         struct mutex lock;
  98         unsigned int busy_bits;
  99
 100         struct i915_stolen_fb compressed_fb, compressed_llb;
 101
 102         enum intel_fbc_id id;
 103
 104         u8 limit;
 105
 106         bool false_color;
 107
 108         bool active;
 109         bool activated;
 110         bool flip_pending;
 111
 112         bool underrun_detected;
 113         struct work_struct underrun_work;
 114
 115         /*
 116          * This structure contains everything that's relevant to program the
 117          * hardware registers. When we want to figure out if we need to disable
 118          * and re-enable FBC for a new configuration we just check if there's
 119          * something different in the struct. The genx_fbc_activate functions
 120          * are supposed to read from it in order to program the registers.
 121          */
 122         struct intel_fbc_state state;
 123         const char *no_fbc_reason;
 124 };
 125
 126 /* plane stride in pixels */
 127 static unsigned int intel_fbc_plane_stride(const struct intel_plane_state *plane_state)
 128 {
 129         const struct drm_framebuffer *fb = plane_state->hw.fb;
 130         unsigned int stride;
 131
 132         stride = plane_state->view.color_plane[0].mapping_stride;
 133         if (!drm_rotation_90_or_270(plane_state->hw.rotation))
 134                 stride /= fb->format->cpp[0];
 135
 136         return stride;
 137 }
 138
 139 /* plane stride based cfb stride in bytes, assuming 1:1 compression limit */
 140 static unsigned int _intel_fbc_cfb_stride(const struct intel_plane_state *plane_state)
 141 {
 142         unsigned int cpp = 4; /* FBC always 4 bytes per pixel */
 143
 144         return intel_fbc_plane_stride(plane_state) * cpp;
 145 }
 146
 147 /* minimum acceptable cfb stride in bytes, assuming 1:1 compression limit */
 148 static unsigned int skl_fbc_min_cfb_stride(const struct intel_plane_state *plane_state)
 149 {
 150         struct drm_i915_private *i915 = to_i915(plane_state->uapi.plane->dev);
 151         unsigned int limit = 4; /* 1:4 compression limit is the worst case */
 152         unsigned int cpp = 4; /* FBC always 4 bytes per pixel */
 153         unsigned int width = drm_rect_width(&plane_state->uapi.src) >> 16;
 154         unsigned int height = 4; /* FBC segment is 4 lines */
 155         unsigned int stride;
 156
 157         /* minimum segment stride we can use */
 158         stride = width * cpp * height / limit;
 159
 160         /*
 161          * Wa_16011863758: icl+
 162          * Avoid some hardware segment address miscalculation.
 163          */
 164         if (DISPLAY_VER(i915) >= 11)
 165                 stride += 64;
 166
 167         /*
 168          * At least some of the platforms require each 4 line segment to
 169          * be 512 byte aligned. Just do it always for simplicity.
 170          */
 171         stride = ALIGN(stride, 512);
 172
 173         /* convert back to single line equivalent with 1:1 compression limit */
 174         return stride * limit / height;
 175 }
 176
 177 /* properly aligned cfb stride in bytes, assuming 1:1 compression limit */
 178 static unsigned int intel_fbc_cfb_stride(const struct intel_plane_state *plane_state)
 179 {
 180         struct drm_i915_private *i915 = to_i915(plane_state->uapi.plane->dev);
 181         unsigned int stride = _intel_fbc_cfb_stride(plane_state);
 182
 183         /*
 184          * At least some of the platforms require each 4 line segment to
 185          * be 512 byte aligned. Aligning each line to 512 bytes guarantees
 186          * that regardless of the compression limit we choose later.
 187          */
 188         if (DISPLAY_VER(i915) >= 9)
 189                 return max(ALIGN(stride, 512), skl_fbc_min_cfb_stride(plane_state));
 190         else
 191                 return stride;
 192 }
 193
 194 static unsigned int intel_fbc_cfb_size(const struct intel_plane_state *plane_state)
 195 {
 196         struct drm_i915_private *i915 = to_i915(plane_state->uapi.plane->dev);
 197         int lines = drm_rect_height(&plane_state->uapi.src) >> 16;
 198
 199         if (DISPLAY_VER(i915) == 7)
 200                 lines = min(lines, 2048);
 201         else if (DISPLAY_VER(i915) >= 8)
 202                 lines = min(lines, 2560);
 203
 204         return lines * intel_fbc_cfb_stride(plane_state);
 205 }
 206
 207 static u16 intel_fbc_override_cfb_stride(const struct intel_plane_state *plane_state)
 208 {
 209         struct drm_i915_private *i915 = to_i915(plane_state->uapi.plane->dev);
 210         unsigned int stride_aligned = intel_fbc_cfb_stride(plane_state);
 211         unsigned int stride = _intel_fbc_cfb_stride(plane_state);
 212         const struct drm_framebuffer *fb = plane_state->hw.fb;
 213
 214         /*
 215          * Override stride in 64 byte units per 4 line segment.
 216          *
 217          * Gen9 hw miscalculates cfb stride for linear as
 218          * PLANE_STRIDE*512 instead of PLANE_STRIDE*64, so
 219          * we always need to use the override there.
 220          */
 221         if (stride != stride_aligned ||
 222             (DISPLAY_VER(i915) == 9 && fb->modifier == DRM_FORMAT_MOD_LINEAR))
 223                 return stride_aligned * 4 / 64;
 224
 225         return 0;
 226 }
 227
 228 static u32 i8xx_fbc_ctl(struct intel_fbc *fbc)
 229 {
 230         const struct intel_fbc_state *fbc_state = &fbc->state;
 231         struct drm_i915_private *i915 = fbc->i915;
 232         unsigned int cfb_stride;
 233         u32 fbc_ctl;
 234
 235         cfb_stride = fbc_state->cfb_stride / fbc->limit;
 236
 237         /* FBC_CTL wants 32B or 64B units */
 238         if (DISPLAY_VER(i915) == 2)
 239                 cfb_stride = (cfb_stride / 32) - 1;
 240         else
 241                 cfb_stride = (cfb_stride / 64) - 1;
 242
 243         fbc_ctl = FBC_CTL_PERIODIC |
 244                 FBC_CTL_INTERVAL(fbc_state->interval) |
 245                 FBC_CTL_STRIDE(cfb_stride);
 246
 247         if (IS_I945GM(i915))
 248                 fbc_ctl |= FBC_CTL_C3_IDLE; /* 945 needs special SR handling */
 249
 250         if (fbc_state->fence_id >= 0)
 251                 fbc_ctl |= FBC_CTL_FENCENO(fbc_state->fence_id);
 252
 253         return fbc_ctl;
 254 }
 255
 256 static u32 i965_fbc_ctl2(struct intel_fbc *fbc)
 257 {
 258         const struct intel_fbc_state *fbc_state = &fbc->state;
 259         u32 fbc_ctl2;
 260
 261         fbc_ctl2 = FBC_CTL_FENCE_DBL | FBC_CTL_IDLE_IMM |
 262                 FBC_CTL_PLANE(fbc_state->plane->i9xx_plane);
 263
 264         if (fbc_state->fence_id >= 0)
 265                 fbc_ctl2 |= FBC_CTL_CPU_FENCE_EN;
 266
 267         return fbc_ctl2;
 268 }
 269
 270 static void i8xx_fbc_deactivate(struct intel_fbc *fbc)
 271 {
 272         struct drm_i915_private *i915 = fbc->i915;
 273         u32 fbc_ctl;
 274
 275         /* Disable compression */
 276         fbc_ctl = intel_de_read(i915, FBC_CONTROL);
 277         if ((fbc_ctl & FBC_CTL_EN) == 0)
 278                 return;
 279
 280         fbc_ctl &= ~FBC_CTL_EN;
 281         intel_de_write(i915, FBC_CONTROL, fbc_ctl);
 282
 283         /* Wait for compressing bit to clear */
 284         if (intel_de_wait_for_clear(i915, FBC_STATUS,
 285                                     FBC_STAT_COMPRESSING, 10)) {
 286                 drm_dbg_kms(&i915->drm, "FBC idle timed out\n");
 287                 return;
 288         }
 289 }
 290
 291 static void i8xx_fbc_activate(struct intel_fbc *fbc)
 292 {
 293         const struct intel_fbc_state *fbc_state = &fbc->state;
 294         struct drm_i915_private *i915 = fbc->i915;
 295         int i;
 296
 297         /* Clear old tags */
 298         for (i = 0; i < (FBC_LL_SIZE / 32) + 1; i++)
 299                 intel_de_write(i915, FBC_TAG(i), 0);
 300
 301         if (DISPLAY_VER(i915) == 4) {
 302                 intel_de_write(i915, FBC_CONTROL2,
 303                                i965_fbc_ctl2(fbc));
 304                 intel_de_write(i915, FBC_FENCE_OFF,
 305                                fbc_state->fence_y_offset);
 306         }
 307
 308         intel_de_write(i915, FBC_CONTROL,
 309                        FBC_CTL_EN | i8xx_fbc_ctl(fbc));
 310 }
 311
 312 static bool i8xx_fbc_is_active(struct intel_fbc *fbc)
 313 {
 314         return intel_de_read(fbc->i915, FBC_CONTROL) & FBC_CTL_EN;
 315 }
 316
 317 static bool i8xx_fbc_is_compressing(struct intel_fbc *fbc)
 318 {
 319         return intel_de_read(fbc->i915, FBC_STATUS) &
 320                 (FBC_STAT_COMPRESSING | FBC_STAT_COMPRESSED);
 321 }
 322
 323 static void i8xx_fbc_nuke(struct intel_fbc *fbc)
 324 {
 325         struct intel_fbc_state *fbc_state = &fbc->state;
 326         enum i9xx_plane_id i9xx_plane = fbc_state->plane->i9xx_plane;
 327         struct drm_i915_private *dev_priv = fbc->i915;
 328
 329         intel_de_write_fw(dev_priv, DSPADDR(i9xx_plane),
 330                           intel_de_read_fw(dev_priv, DSPADDR(i9xx_plane)));
 331 }
 332
 333 static void i8xx_fbc_program_cfb(struct intel_fbc *fbc)
 334 {
 335         struct drm_i915_private *i915 = fbc->i915;
 336
 337         drm_WARN_ON(&i915->drm,
 338                     range_overflows_end_t(u64, i915_gem_stolen_area_address(i915),
 339                                           i915_gem_stolen_node_offset(&fbc->compressed_fb),
 340                                           U32_MAX));
 341         drm_WARN_ON(&i915->drm,
 342                     range_overflows_end_t(u64, i915_gem_stolen_area_address(i915),
 343                                           i915_gem_stolen_node_offset(&fbc->compressed_llb),
 344                                           U32_MAX));
 345         intel_de_write(i915, FBC_CFB_BASE,
 346                        i915_gem_stolen_node_address(i915, &fbc->compressed_fb));
 347         intel_de_write(i915, FBC_LL_BASE,
 348                        i915_gem_stolen_node_address(i915, &fbc->compressed_llb));
 349 }
 350
 351 static const struct intel_fbc_funcs i8xx_fbc_funcs = {
 352         .activate = i8xx_fbc_activate,
 353         .deactivate = i8xx_fbc_deactivate,
 354         .is_active = i8xx_fbc_is_active,
 355         .is_compressing = i8xx_fbc_is_compressing,
 356         .nuke = i8xx_fbc_nuke,
 357         .program_cfb = i8xx_fbc_program_cfb,
 358 };
 359
 360 static void i965_fbc_nuke(struct intel_fbc *fbc)
 361 {
 362         struct intel_fbc_state *fbc_state = &fbc->state;
 363         enum i9xx_plane_id i9xx_plane = fbc_state->plane->i9xx_plane;
 364         struct drm_i915_private *dev_priv = fbc->i915;
 365
 366         intel_de_write_fw(dev_priv, DSPSURF(i9xx_plane),
 367                           intel_de_read_fw(dev_priv, DSPSURF(i9xx_plane)));
 368 }
 369
 370 static const struct intel_fbc_funcs i965_fbc_funcs = {
 371         .activate = i8xx_fbc_activate,
 372         .deactivate = i8xx_fbc_deactivate,
 373         .is_active = i8xx_fbc_is_active,
 374         .is_compressing = i8xx_fbc_is_compressing,
 375         .nuke = i965_fbc_nuke,
 376         .program_cfb = i8xx_fbc_program_cfb,
 377 };
 378
 379 static u32 g4x_dpfc_ctl_limit(struct intel_fbc *fbc)
 380 {
 381         switch (fbc->limit) {
 382         default:
 383                 MISSING_CASE(fbc->limit);
 384                 fallthrough;
 385         case 1:
 386                 return DPFC_CTL_LIMIT_1X;
 387         case 2:
 388                 return DPFC_CTL_LIMIT_2X;
 389         case 4:
 390                 return DPFC_CTL_LIMIT_4X;
 391         }
 392 }
 393
 394 static u32 g4x_dpfc_ctl(struct intel_fbc *fbc)
 395 {
 396         const struct intel_fbc_state *fbc_state = &fbc->state;
 397         struct drm_i915_private *i915 = fbc->i915;
 398         u32 dpfc_ctl;
 399
 400         dpfc_ctl = g4x_dpfc_ctl_limit(fbc) |
 401                 DPFC_CTL_PLANE_G4X(fbc_state->plane->i9xx_plane);
 402
 403         if (IS_G4X(i915))
 404                 dpfc_ctl |= DPFC_CTL_SR_EN;
 405
 406         if (fbc_state->fence_id >= 0) {
 407                 dpfc_ctl |= DPFC_CTL_FENCE_EN_G4X;
 408
 409                 if (DISPLAY_VER(i915) < 6)
 410                         dpfc_ctl |= DPFC_CTL_FENCENO(fbc_state->fence_id);
 411         }
 412
 413         return dpfc_ctl;
 414 }
 415
 416 static void g4x_fbc_activate(struct intel_fbc *fbc)
 417 {
 418         const struct intel_fbc_state *fbc_state = &fbc->state;
 419         struct drm_i915_private *i915 = fbc->i915;
 420
 421         intel_de_write(i915, DPFC_FENCE_YOFF,
 422                        fbc_state->fence_y_offset);
 423
 424         intel_de_write(i915, DPFC_CONTROL,
 425                        DPFC_CTL_EN | g4x_dpfc_ctl(fbc));
 426 }
 427
 428 static void g4x_fbc_deactivate(struct intel_fbc *fbc)
 429 {
 430         struct drm_i915_private *i915 = fbc->i915;
 431         u32 dpfc_ctl;
 432
 433         /* Disable compression */
 434         dpfc_ctl = intel_de_read(i915, DPFC_CONTROL);
 435         if (dpfc_ctl & DPFC_CTL_EN) {
 436                 dpfc_ctl &= ~DPFC_CTL_EN;
 437                 intel_de_write(i915, DPFC_CONTROL, dpfc_ctl);
 438         }
 439 }
 440
 441 static bool g4x_fbc_is_active(struct intel_fbc *fbc)
 442 {
 443         return intel_de_read(fbc->i915, DPFC_CONTROL) & DPFC_CTL_EN;
 444 }
 445
 446 static bool g4x_fbc_is_compressing(struct intel_fbc *fbc)
 447 {
 448         return intel_de_read(fbc->i915, DPFC_STATUS) & DPFC_COMP_SEG_MASK;
 449 }
 450
 451 static void g4x_fbc_program_cfb(struct intel_fbc *fbc)
 452 {
 453         struct drm_i915_private *i915 = fbc->i915;
 454
 455         intel_de_write(i915, DPFC_CB_BASE,
 456                        i915_gem_stolen_node_offset(&fbc->compressed_fb));
 457 }
 458
 459 static const struct intel_fbc_funcs g4x_fbc_funcs = {
 460         .activate = g4x_fbc_activate,
 461         .deactivate = g4x_fbc_deactivate,
 462         .is_active = g4x_fbc_is_active,
 463         .is_compressing = g4x_fbc_is_compressing,
 464         .nuke = i965_fbc_nuke,
 465         .program_cfb = g4x_fbc_program_cfb,
 466 };
 467
 468 static void ilk_fbc_activate(struct intel_fbc *fbc)
 469 {
 470         struct intel_fbc_state *fbc_state = &fbc->state;
 471         struct drm_i915_private *i915 = fbc->i915;
 472
 473         intel_de_write(i915, ILK_DPFC_FENCE_YOFF(fbc->id),
 474                        fbc_state->fence_y_offset);
 475
 476         intel_de_write(i915, ILK_DPFC_CONTROL(fbc->id),
 477                        DPFC_CTL_EN | g4x_dpfc_ctl(fbc));
 478 }
 479
 480 static void ilk_fbc_deactivate(struct intel_fbc *fbc)
 481 {
 482         struct drm_i915_private *i915 = fbc->i915;
 483         u32 dpfc_ctl;
 484
 485         /* Disable compression */
 486         dpfc_ctl = intel_de_read(i915, ILK_DPFC_CONTROL(fbc->id));
 487         if (dpfc_ctl & DPFC_CTL_EN) {
 488                 dpfc_ctl &= ~DPFC_CTL_EN;
 489                 intel_de_write(i915, ILK_DPFC_CONTROL(fbc->id), dpfc_ctl);
 490         }
 491 }
 492
 493 static bool ilk_fbc_is_active(struct intel_fbc *fbc)
 494 {
 495         return intel_de_read(fbc->i915, ILK_DPFC_CONTROL(fbc->id)) & DPFC_CTL_EN;
 496 }
 497
 498 static bool ilk_fbc_is_compressing(struct intel_fbc *fbc)
 499 {
 500         return intel_de_read(fbc->i915, ILK_DPFC_STATUS(fbc->id)) & DPFC_COMP_SEG_MASK;
 501 }
 502
 503 static void ilk_fbc_program_cfb(struct intel_fbc *fbc)
 504 {
 505         struct drm_i915_private *i915 = fbc->i915;
 506
 507         intel_de_write(i915, ILK_DPFC_CB_BASE(fbc->id),
 508                        i915_gem_stolen_node_offset(&fbc->compressed_fb));
 509 }
 510
 511 static const struct intel_fbc_funcs ilk_fbc_funcs = {
 512         .activate = ilk_fbc_activate,
 513         .deactivate = ilk_fbc_deactivate,
 514         .is_active = ilk_fbc_is_active,
 515         .is_compressing = ilk_fbc_is_compressing,
 516         .nuke = i965_fbc_nuke,
 517         .program_cfb = ilk_fbc_program_cfb,
 518 };
 519
 520 static void snb_fbc_program_fence(struct intel_fbc *fbc)
 521 {
 522         const struct intel_fbc_state *fbc_state = &fbc->state;
 523         struct drm_i915_private *i915 = fbc->i915;
 524         u32 ctl = 0;
 525
 526         if (fbc_state->fence_id >= 0)
 527                 ctl = SNB_DPFC_FENCE_EN | SNB_DPFC_FENCENO(fbc_state->fence_id);
 528
 529         intel_de_write(i915, SNB_DPFC_CTL_SA, ctl);
 530         intel_de_write(i915, SNB_DPFC_CPU_FENCE_OFFSET, fbc_state->fence_y_offset);
 531 }
 532
 533 static void snb_fbc_activate(struct intel_fbc *fbc)
 534 {
 535         snb_fbc_program_fence(fbc);
 536
 537         ilk_fbc_activate(fbc);
 538 }
 539
 540 static void snb_fbc_nuke(struct intel_fbc *fbc)
 541 {
 542         struct drm_i915_private *i915 = fbc->i915;
 543
 544         intel_de_write(i915, MSG_FBC_REND_STATE(fbc->id), FBC_REND_NUKE);
 545         intel_de_posting_read(i915, MSG_FBC_REND_STATE(fbc->id));
 546 }
 547
 548 static const struct intel_fbc_funcs snb_fbc_funcs = {
 549         .activate = snb_fbc_activate,
 550         .deactivate = ilk_fbc_deactivate,
 551         .is_active = ilk_fbc_is_active,
 552         .is_compressing = ilk_fbc_is_compressing,
 553         .nuke = snb_fbc_nuke,
 554         .program_cfb = ilk_fbc_program_cfb,
 555 };
 556
 557 static void glk_fbc_program_cfb_stride(struct intel_fbc *fbc)
 558 {
 559         const struct intel_fbc_state *fbc_state = &fbc->state;
 560         struct drm_i915_private *i915 = fbc->i915;
 561         u32 val = 0;
 562
 563         if (fbc_state->override_cfb_stride)
 564                 val |= FBC_STRIDE_OVERRIDE |
 565                         FBC_STRIDE(fbc_state->override_cfb_stride / fbc->limit);
 566
 567         intel_de_write(i915, GLK_FBC_STRIDE(fbc->id), val);
 568 }
 569
 570 static void skl_fbc_program_cfb_stride(struct intel_fbc *fbc)
 571 {
 572         const struct intel_fbc_state *fbc_state = &fbc->state;
 573         struct drm_i915_private *i915 = fbc->i915;
 574         u32 val = 0;
 575
 576         /* Display WA #0529: skl, kbl, bxt. */
 577         if (fbc_state->override_cfb_stride)
 578                 val |= CHICKEN_FBC_STRIDE_OVERRIDE |
 579                         CHICKEN_FBC_STRIDE(fbc_state->override_cfb_stride / fbc->limit);
 580
 581         intel_de_rmw(i915, CHICKEN_MISC_4,
 582                      CHICKEN_FBC_STRIDE_OVERRIDE |
 583                      CHICKEN_FBC_STRIDE_MASK, val);
 584 }
 585
 586 static u32 ivb_dpfc_ctl(struct intel_fbc *fbc)
 587 {
 588         const struct intel_fbc_state *fbc_state = &fbc->state;
 589         struct drm_i915_private *i915 = fbc->i915;
 590         u32 dpfc_ctl;
 591
 592         dpfc_ctl = g4x_dpfc_ctl_limit(fbc);
 593
 594         if (IS_IVYBRIDGE(i915))
 595                 dpfc_ctl |= DPFC_CTL_PLANE_IVB(fbc_state->plane->i9xx_plane);
 596
 597         if (DISPLAY_VER(i915) >= 20)
 598                 dpfc_ctl |= DPFC_CTL_PLANE_BINDING(fbc_state->plane->id);
 599
 600         if (fbc_state->fence_id >= 0)
 601                 dpfc_ctl |= DPFC_CTL_FENCE_EN_IVB;
 602
 603         if (fbc->false_color)
 604                 dpfc_ctl |= DPFC_CTL_FALSE_COLOR;
 605
 606         return dpfc_ctl;
 607 }
 608
 609 static void ivb_fbc_activate(struct intel_fbc *fbc)
 610 {
 611         struct drm_i915_private *i915 = fbc->i915;
 612         u32 dpfc_ctl;
 613
 614         if (DISPLAY_VER(i915) >= 10)
 615                 glk_fbc_program_cfb_stride(fbc);
 616         else if (DISPLAY_VER(i915) == 9)
 617                 skl_fbc_program_cfb_stride(fbc);
 618
 619         if (intel_gt_support_legacy_fencing(to_gt(i915)))
 620                 snb_fbc_program_fence(fbc);
 621
 622         /* wa_14019417088 Alternative WA*/
 623         dpfc_ctl = ivb_dpfc_ctl(fbc);
 624         if (DISPLAY_VER(i915) >= 20)
 625                 intel_de_write(i915, ILK_DPFC_CONTROL(fbc->id), dpfc_ctl);
 626
 627         intel_de_write(i915, ILK_DPFC_CONTROL(fbc->id),
 628                        DPFC_CTL_EN | dpfc_ctl);
 629 }
 630
 631 static bool ivb_fbc_is_compressing(struct intel_fbc *fbc)
 632 {
 633         return intel_de_read(fbc->i915, ILK_DPFC_STATUS2(fbc->id)) & DPFC_COMP_SEG_MASK_IVB;
 634 }
 635
 636 static void ivb_fbc_set_false_color(struct intel_fbc *fbc,
 637                                     bool enable)
 638 {
 639         intel_de_rmw(fbc->i915, ILK_DPFC_CONTROL(fbc->id),
 640                      DPFC_CTL_FALSE_COLOR, enable ? DPFC_CTL_FALSE_COLOR : 0);
 641 }
 642
 643 static const struct intel_fbc_funcs ivb_fbc_funcs = {
 644         .activate = ivb_fbc_activate,
 645         .deactivate = ilk_fbc_deactivate,
 646         .is_active = ilk_fbc_is_active,
 647         .is_compressing = ivb_fbc_is_compressing,
 648         .nuke = snb_fbc_nuke,
 649         .program_cfb = ilk_fbc_program_cfb,
 650         .set_false_color = ivb_fbc_set_false_color,
 651 };
 652
 653 static bool intel_fbc_hw_is_active(struct intel_fbc *fbc)
 654 {
 655         return fbc->funcs->is_active(fbc);
 656 }
 657
 658 static void intel_fbc_hw_activate(struct intel_fbc *fbc)
 659 {
 660         trace_intel_fbc_activate(fbc->state.plane);
 661
 662         fbc->active = true;
 663         fbc->activated = true;
 664
 665         fbc->funcs->activate(fbc);
 666 }
 667
 668 static void intel_fbc_hw_deactivate(struct intel_fbc *fbc)
 669 {
 670         trace_intel_fbc_deactivate(fbc->state.plane);
 671
 672         fbc->active = false;
 673
 674         fbc->funcs->deactivate(fbc);
 675 }
 676
 677 static bool intel_fbc_is_compressing(struct intel_fbc *fbc)
 678 {
 679         return fbc->funcs->is_compressing(fbc);
 680 }
 681
 682 static void intel_fbc_nuke(struct intel_fbc *fbc)
 683 {
 684         struct drm_i915_private *i915 = fbc->i915;
 685
 686         lockdep_assert_held(&fbc->lock);
 687         drm_WARN_ON(&i915->drm, fbc->flip_pending);
 688
 689         trace_intel_fbc_nuke(fbc->state.plane);
 690
 691         fbc->funcs->nuke(fbc);
 692 }
 693
 694 static void intel_fbc_activate(struct intel_fbc *fbc)
 695 {
 696         lockdep_assert_held(&fbc->lock);
 697
 698         intel_fbc_hw_activate(fbc);
 699         intel_fbc_nuke(fbc);
 700
 701         fbc->no_fbc_reason = NULL;
 702 }
 703
 704 static void intel_fbc_deactivate(struct intel_fbc *fbc, const char *reason)
 705 {
 706         lockdep_assert_held(&fbc->lock);
 707
 708         if (fbc->active)
 709                 intel_fbc_hw_deactivate(fbc);
 710
 711         fbc->no_fbc_reason = reason;
 712 }
 713
 714 static u64 intel_fbc_cfb_base_max(struct drm_i915_private *i915)
 715 {
 716         if (DISPLAY_VER(i915) >= 5 || IS_G4X(i915))
 717                 return BIT_ULL(28);
 718         else
 719                 return BIT_ULL(32);
 720 }
 721
 722 static u64 intel_fbc_stolen_end(struct drm_i915_private *i915)
 723 {
 724         u64 end;
 725
 726         /* The FBC hardware for BDW/SKL doesn't have access to the stolen
 727          * reserved range size, so it always assumes the maximum (8mb) is used.
 728          * If we enable FBC using a CFB on that memory range we'll get FIFO
 729          * underruns, even if that range is not reserved by the BIOS. */
 730         if (IS_BROADWELL(i915) ||
 731             (DISPLAY_VER(i915) == 9 && !IS_BROXTON(i915)))
 732                 end = i915_gem_stolen_area_size(i915) - 8 * 1024 * 1024;
 733         else
 734                 end = U64_MAX;
 735
 736         return min(end, intel_fbc_cfb_base_max(i915));
 737 }
 738
 739 static int intel_fbc_min_limit(const struct intel_plane_state *plane_state)
 740 {
 741         return plane_state->hw.fb->format->cpp[0] == 2 ? 2 : 1;
 742 }
 743
 744 static int intel_fbc_max_limit(struct drm_i915_private *i915)
 745 {
 746         /* WaFbcOnly1to1Ratio:ctg */
 747         if (IS_G4X(i915))
 748                 return 1;
 749
 750         /*
 751          * FBC2 can only do 1:1, 1:2, 1:4, we limit
 752          * FBC1 to the same out of convenience.
 753          */
 754         return 4;
 755 }
 756
 757 static int find_compression_limit(struct intel_fbc *fbc,
 758                                   unsigned int size, int min_limit)
 759 {
 760         struct drm_i915_private *i915 = fbc->i915;
 761         u64 end = intel_fbc_stolen_end(i915);
 762         int ret, limit = min_limit;
 763
 764         size /= limit;
 765
 766         /* Try to over-allocate to reduce reallocations and fragmentation. */
 767         ret = i915_gem_stolen_insert_node_in_range(i915, &fbc->compressed_fb,
 768                                                    size <<= 1, 4096, 0, end);
 769         if (ret == 0)
 770                 return limit;
 771
 772         for (; limit <= intel_fbc_max_limit(i915); limit <<= 1) {
 773                 ret = i915_gem_stolen_insert_node_in_range(i915, &fbc->compressed_fb,
 774                                                            size >>= 1, 4096, 0, end);
 775                 if (ret == 0)
 776                         return limit;
 777         }
 778
 779         return 0;
 780 }
 781
 782 static int intel_fbc_alloc_cfb(struct intel_fbc *fbc,
 783                                unsigned int size, int min_limit)
 784 {
 785         struct drm_i915_private *i915 = fbc->i915;
 786         int ret;
 787
 788         drm_WARN_ON(&i915->drm,
 789                     i915_gem_stolen_node_allocated(&fbc->compressed_fb));
 790         drm_WARN_ON(&i915->drm,
 791                     i915_gem_stolen_node_allocated(&fbc->compressed_llb));
 792
 793         if (DISPLAY_VER(i915) < 5 && !IS_G4X(i915)) {
 794                 ret = i915_gem_stolen_insert_node(i915, &fbc->compressed_llb,
 795                                                   4096, 4096);
 796                 if (ret)
 797                         goto err;
 798         }
 799
 800         ret = find_compression_limit(fbc, size, min_limit);
 801         if (!ret)
 802                 goto err_llb;
 803         else if (ret > min_limit)
 804                 drm_info_once(&i915->drm,
 805                               "Reducing the compressed framebuffer size. This may lead to less power savings than a non-reduced-size. Try to increase stolen memory size if available in BIOS.\n");
 806
 807         fbc->limit = ret;
 808
 809         drm_dbg_kms(&i915->drm,
 810                     "reserved %llu bytes of contiguous stolen space for FBC, limit: %d\n",
 811                     i915_gem_stolen_node_size(&fbc->compressed_fb), fbc->limit);
 812         return 0;
 813
 814 err_llb:
 815         if (i915_gem_stolen_node_allocated(&fbc->compressed_llb))
 816                 i915_gem_stolen_remove_node(i915, &fbc->compressed_llb);
 817 err:
 818         if (i915_gem_stolen_initialized(i915))
 819                 drm_info_once(&i915->drm, "not enough stolen space for compressed buffer (need %d more bytes), disabling. Hint: you may be able to increase stolen memory size in the BIOS to avoid this.\n", size);
 820         return -ENOSPC;
 821 }
 822
 823 static void intel_fbc_program_cfb(struct intel_fbc *fbc)
 824 {
 825         fbc->funcs->program_cfb(fbc);
 826 }
 827
 828 static void intel_fbc_program_workarounds(struct intel_fbc *fbc)
 829 {
 830         struct drm_i915_private *i915 = fbc->i915;
 831
 832         if (IS_SKYLAKE(i915) || IS_BROXTON(i915)) {
 833                 /*
 834                  * WaFbcHighMemBwCorruptionAvoidance:skl,bxt
 835                  * Display WA #0883: skl,bxt
 836                  */
 837                 intel_de_rmw(i915, ILK_DPFC_CHICKEN(fbc->id),
 838                              0, DPFC_DISABLE_DUMMY0);
 839         }
 840
 841         if (IS_SKYLAKE(i915) || IS_KABYLAKE(i915) ||
 842             IS_COFFEELAKE(i915) || IS_COMETLAKE(i915)) {
 843                 /*
 844                  * WaFbcNukeOnHostModify:skl,kbl,cfl
 845                  * Display WA #0873: skl,kbl,cfl
 846                  */
 847                 intel_de_rmw(i915, ILK_DPFC_CHICKEN(fbc->id),
 848                              0, DPFC_NUKE_ON_ANY_MODIFICATION);
 849         }
 850
 851         /* Wa_1409120013:icl,jsl,tgl,dg1 */
 852         if (IS_DISPLAY_VER(i915, 11, 12))
 853                 intel_de_rmw(i915, ILK_DPFC_CHICKEN(fbc->id),
 854                              0, DPFC_CHICKEN_COMP_DUMMY_PIXEL);
 855
 856         /* Wa_22014263786:icl,jsl,tgl,dg1,rkl,adls,adlp,mtl */
 857         if (DISPLAY_VER(i915) >= 11 && !IS_DG2(i915))
 858                 intel_de_rmw(i915, ILK_DPFC_CHICKEN(fbc->id),
 859                              0, DPFC_CHICKEN_FORCE_SLB_INVALIDATION);
 860 }
 861
 862 static void __intel_fbc_cleanup_cfb(struct intel_fbc *fbc)
 863 {
 864         struct drm_i915_private *i915 = fbc->i915;
 865
 866         if (WARN_ON(intel_fbc_hw_is_active(fbc)))
 867                 return;
 868
 869         if (i915_gem_stolen_node_allocated(&fbc->compressed_llb))
 870                 i915_gem_stolen_remove_node(i915, &fbc->compressed_llb);
 871         if (i915_gem_stolen_node_allocated(&fbc->compressed_fb))
 872                 i915_gem_stolen_remove_node(i915, &fbc->compressed_fb);
 873 }
 874
 875 void intel_fbc_cleanup(struct drm_i915_private *i915)
 876 {
 877         struct intel_fbc *fbc;
 878         enum intel_fbc_id fbc_id;
 879
 880         for_each_intel_fbc(i915, fbc, fbc_id) {
 881                 mutex_lock(&fbc->lock);
 882                 __intel_fbc_cleanup_cfb(fbc);
 883                 mutex_unlock(&fbc->lock);
 884
 885                 kfree(fbc);
 886         }
 887 }
 888
 889 static bool i8xx_fbc_stride_is_valid(const struct intel_plane_state *plane_state)
 890 {
 891         const struct drm_framebuffer *fb = plane_state->hw.fb;
 892         unsigned int stride = intel_fbc_plane_stride(plane_state) *
 893                 fb->format->cpp[0];
 894
 895         return stride == 4096 || stride == 8192;
 896 }
 897
 898 static bool i965_fbc_stride_is_valid(const struct intel_plane_state *plane_state)
 899 {
 900         const struct drm_framebuffer *fb = plane_state->hw.fb;
 901         unsigned int stride = intel_fbc_plane_stride(plane_state) *
 902                 fb->format->cpp[0];
 903
 904         return stride >= 2048 && stride <= 16384;
 905 }
 906
 907 static bool g4x_fbc_stride_is_valid(const struct intel_plane_state *plane_state)
 908 {
 909         return true;
 910 }
 911
 912 static bool skl_fbc_stride_is_valid(const struct intel_plane_state *plane_state)
 913 {
 914         const struct drm_framebuffer *fb = plane_state->hw.fb;
 915         unsigned int stride = intel_fbc_plane_stride(plane_state) *
 916                 fb->format->cpp[0];
 917
 918         /* Display WA #1105: skl,bxt,kbl,cfl,glk */
 919         if (fb->modifier == DRM_FORMAT_MOD_LINEAR && stride & 511)
 920                 return false;
 921
 922         return true;
 923 }
 924
 925 static bool icl_fbc_stride_is_valid(const struct intel_plane_state *plane_state)
 926 {
 927         return true;
 928 }
 929
 930 static bool stride_is_valid(const struct intel_plane_state *plane_state)
 931 {
 932         struct drm_i915_private *i915 = to_i915(plane_state->uapi.plane->dev);
 933
 934         if (DISPLAY_VER(i915) >= 11)
 935                 return icl_fbc_stride_is_valid(plane_state);
 936         else if (DISPLAY_VER(i915) >= 9)
 937                 return skl_fbc_stride_is_valid(plane_state);
 938         else if (DISPLAY_VER(i915) >= 5 || IS_G4X(i915))
 939                 return g4x_fbc_stride_is_valid(plane_state);
 940         else if (DISPLAY_VER(i915) == 4)
 941                 return i965_fbc_stride_is_valid(plane_state);
 942         else
 943                 return i8xx_fbc_stride_is_valid(plane_state);
 944 }
 945
 946 static bool i8xx_fbc_pixel_format_is_valid(const struct intel_plane_state *plane_state)
 947 {
 948         struct drm_i915_private *i915 = to_i915(plane_state->uapi.plane->dev);
 949         const struct drm_framebuffer *fb = plane_state->hw.fb;
 950
 951         switch (fb->format->format) {
 952         case DRM_FORMAT_XRGB8888:
 953         case DRM_FORMAT_XBGR8888:
 954                 return true;
 955         case DRM_FORMAT_XRGB1555:
 956         case DRM_FORMAT_RGB565:
 957                 /* 16bpp not supported on gen2 */
 958                 if (DISPLAY_VER(i915) == 2)
 959                         return false;
 960                 return true;
 961         default:
 962                 return false;
 963         }
 964 }
 965
 966 static bool g4x_fbc_pixel_format_is_valid(const struct intel_plane_state *plane_state)
 967 {
 968         struct drm_i915_private *i915 = to_i915(plane_state->uapi.plane->dev);
 969         const struct drm_framebuffer *fb = plane_state->hw.fb;
 970
 971         switch (fb->format->format) {
 972         case DRM_FORMAT_XRGB8888:
 973         case DRM_FORMAT_XBGR8888:
 974                 return true;
 975         case DRM_FORMAT_RGB565:
 976                 /* WaFbcOnly1to1Ratio:ctg */
 977                 if (IS_G4X(i915))
 978                         return false;
 979                 return true;
 980         default:
 981                 return false;
 982         }
 983 }
 984
 985 static bool lnl_fbc_pixel_format_is_valid(const struct intel_plane_state *plane_state)
 986 {
 987         const struct drm_framebuffer *fb = plane_state->hw.fb;
 988
 989         switch (fb->format->format) {
 990         case DRM_FORMAT_XRGB8888:
 991         case DRM_FORMAT_XBGR8888:
 992         case DRM_FORMAT_ARGB8888:
 993         case DRM_FORMAT_ABGR8888:
 994         case DRM_FORMAT_RGB565:
 995                 return true;
 996         default:
 997                 return false;
 998         }
 999 }
1000
1001 static bool pixel_format_is_valid(const struct intel_plane_state *plane_state)
1002 {
1003         struct drm_i915_private *i915 = to_i915(plane_state->uapi.plane->dev);
1004
1005         if (DISPLAY_VER(i915) >= 20)
1006                 return lnl_fbc_pixel_format_is_valid(plane_state);
1007         else if (DISPLAY_VER(i915) >= 5 || IS_G4X(i915))
1008                 return g4x_fbc_pixel_format_is_valid(plane_state);
1009         else
1010                 return i8xx_fbc_pixel_format_is_valid(plane_state);
1011 }
1012
1013 static bool i8xx_fbc_rotation_is_valid(const struct intel_plane_state *plane_state)
1014 {
1015         return plane_state->hw.rotation == DRM_MODE_ROTATE_0;
1016 }
1017
1018 static bool g4x_fbc_rotation_is_valid(const struct intel_plane_state *plane_state)
1019 {
1020         return true;
1021 }
1022
1023 static bool skl_fbc_rotation_is_valid(const struct intel_plane_state *plane_state)
1024 {
1025         const struct drm_framebuffer *fb = plane_state->hw.fb;
1026         unsigned int rotation = plane_state->hw.rotation;
1027
1028         if (fb->format->format == DRM_FORMAT_RGB565 &&
1029             drm_rotation_90_or_270(rotation))
1030                 return false;
1031
1032         return true;
1033 }
1034
1035 static bool rotation_is_valid(const struct intel_plane_state *plane_state)
1036 {
1037         struct drm_i915_private *i915 = to_i915(plane_state->uapi.plane->dev);
1038
1039         if (DISPLAY_VER(i915) >= 9)
1040                 return skl_fbc_rotation_is_valid(plane_state);
1041         else if (DISPLAY_VER(i915) >= 5 || IS_G4X(i915))
1042                 return g4x_fbc_rotation_is_valid(plane_state);
1043         else
1044                 return i8xx_fbc_rotation_is_valid(plane_state);
1045 }
1046
1047 /*
1048  * For some reason, the hardware tracking starts looking at whatever we
1049  * programmed as the display plane base address register. It does not look at
1050  * the X and Y offset registers. That's why we include the src x/y offsets
1051  * instead of just looking at the plane size.
1052  */
1053 static bool intel_fbc_hw_tracking_covers_screen(const struct intel_plane_state *plane_state)
1054 {
1055         struct drm_i915_private *i915 = to_i915(plane_state->uapi.plane->dev);
1056         unsigned int effective_w, effective_h, max_w, max_h;
1057
1058         if (DISPLAY_VER(i915) >= 11) {
1059                 max_w = 8192;
1060                 max_h = 4096;
1061         } else if (DISPLAY_VER(i915) >= 10) {
1062                 max_w = 5120;
1063                 max_h = 4096;
1064         } else if (DISPLAY_VER(i915) >= 7) {
1065                 max_w = 4096;
1066                 max_h = 4096;
1067         } else if (IS_G4X(i915) || DISPLAY_VER(i915) >= 5) {
1068                 max_w = 4096;
1069                 max_h = 2048;
1070         } else {
1071                 max_w = 2048;
1072                 max_h = 1536;
1073         }
1074
1075         effective_w = plane_state->view.color_plane[0].x +
1076                 (drm_rect_width(&plane_state->uapi.src) >> 16);
1077         effective_h = plane_state->view.color_plane[0].y +
1078                 (drm_rect_height(&plane_state->uapi.src) >> 16);
1079
1080         return effective_w <= max_w && effective_h <= max_h;
1081 }
1082
1083 static bool intel_fbc_plane_size_valid(const struct intel_plane_state *plane_state)
1084 {
1085         struct drm_i915_private *i915 = to_i915(plane_state->uapi.plane->dev);
1086         unsigned int w, h, max_w, max_h;
1087
1088         if (DISPLAY_VER(i915) >= 10) {
1089                 max_w = 5120;
1090                 max_h = 4096;
1091         } else if (DISPLAY_VER(i915) >= 8 || IS_HASWELL(i915)) {
1092                 max_w = 4096;
1093                 max_h = 4096;
1094         } else if (IS_G4X(i915) || DISPLAY_VER(i915) >= 5) {
1095                 max_w = 4096;
1096                 max_h = 2048;
1097         } else {
1098                 max_w = 2048;
1099                 max_h = 1536;
1100         }
1101
1102         w = drm_rect_width(&plane_state->uapi.src) >> 16;
1103         h = drm_rect_height(&plane_state->uapi.src) >> 16;
1104
1105         return w <= max_w && h <= max_h;
1106 }
1107
1108 static bool i8xx_fbc_tiling_valid(const struct intel_plane_state *plane_state)
1109 {
1110         const struct drm_framebuffer *fb = plane_state->hw.fb;
1111
1112         return fb->modifier == I915_FORMAT_MOD_X_TILED;
1113 }
1114
1115 static bool skl_fbc_tiling_valid(const struct intel_plane_state *plane_state)
1116 {
1117         return true;
1118 }
1119
1120 static bool tiling_is_valid(const struct intel_plane_state *plane_state)
1121 {
1122         struct drm_i915_private *i915 = to_i915(plane_state->uapi.plane->dev);
1123
1124         if (DISPLAY_VER(i915) >= 9)
1125                 return skl_fbc_tiling_valid(plane_state);
1126         else
1127                 return i8xx_fbc_tiling_valid(plane_state);
1128 }
1129
1130 static void intel_fbc_update_state(struct intel_atomic_state *state,
1131                                    struct intel_crtc *crtc,
1132                                    struct intel_plane *plane)
1133 {
1134         struct drm_i915_private *i915 = to_i915(state->base.dev);
1135         const struct intel_crtc_state *crtc_state =
1136                 intel_atomic_get_new_crtc_state(state, crtc);
1137         const struct intel_plane_state *plane_state =
1138                 intel_atomic_get_new_plane_state(state, plane);
1139         struct intel_fbc *fbc = plane->fbc;
1140         struct intel_fbc_state *fbc_state = &fbc->state;
1141
1142         WARN_ON(plane_state->no_fbc_reason);
1143         WARN_ON(fbc_state->plane && fbc_state->plane != plane);
1144
1145         fbc_state->plane = plane;
1146
1147         /* FBC1 compression interval: arbitrary choice of 1 second */
1148         fbc_state->interval = drm_mode_vrefresh(&crtc_state->hw.adjusted_mode);
1149
1150         fbc_state->fence_y_offset = intel_plane_fence_y_offset(plane_state);
1151
1152         drm_WARN_ON(&i915->drm, plane_state->flags & PLANE_HAS_FENCE &&
1153                     !intel_gt_support_legacy_fencing(to_gt(i915)));
1154
1155         if (plane_state->flags & PLANE_HAS_FENCE)
1156                 fbc_state->fence_id =  i915_vma_fence_id(plane_state->ggtt_vma);
1157         else
1158                 fbc_state->fence_id = -1;
1159
1160         fbc_state->cfb_stride = intel_fbc_cfb_stride(plane_state);
1161         fbc_state->cfb_size = intel_fbc_cfb_size(plane_state);
1162         fbc_state->override_cfb_stride = intel_fbc_override_cfb_stride(plane_state);
1163 }
1164
1165 static bool intel_fbc_is_fence_ok(const struct intel_plane_state *plane_state)
1166 {
1167         struct drm_i915_private *i915 = to_i915(plane_state->uapi.plane->dev);
1168
1169         /*
1170          * The use of a CPU fence is one of two ways to detect writes by the
1171          * CPU to the scanout and trigger updates to the FBC.
1172          *
1173          * The other method is by software tracking (see
1174          * intel_fbc_invalidate/flush()), it will manually notify FBC and nuke
1175          * the current compressed buffer and recompress it.
1176          *
1177          * Note that is possible for a tiled surface to be unmappable (and
1178          * so have no fence associated with it) due to aperture constraints
1179          * at the time of pinning.
1180          */
1181         return DISPLAY_VER(i915) >= 9 ||
1182                 (plane_state->flags & PLANE_HAS_FENCE &&
1183                  i915_vma_fence_id(plane_state->ggtt_vma) != -1);
1184 }
1185
1186 static bool intel_fbc_is_cfb_ok(const struct intel_plane_state *plane_state)
1187 {
1188         struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
1189         struct intel_fbc *fbc = plane->fbc;
1190
1191         return intel_fbc_min_limit(plane_state) <= fbc->limit &&
1192                 intel_fbc_cfb_size(plane_state) <= fbc->limit *
1193                         i915_gem_stolen_node_size(&fbc->compressed_fb);
1194 }
1195
1196 static bool intel_fbc_is_ok(const struct intel_plane_state *plane_state)
1197 {
1198         return !plane_state->no_fbc_reason &&
1199                 intel_fbc_is_fence_ok(plane_state) &&
1200                 intel_fbc_is_cfb_ok(plane_state);
1201 }
1202
1203 static int intel_fbc_check_plane(struct intel_atomic_state *state,
1204                                  struct intel_plane *plane)
1205 {
1206         struct drm_i915_private *i915 = to_i915(state->base.dev);
1207         struct intel_plane_state *plane_state =
1208                 intel_atomic_get_new_plane_state(state, plane);
1209         const struct drm_framebuffer *fb = plane_state->hw.fb;
1210         struct intel_crtc *crtc = to_intel_crtc(plane_state->hw.crtc);
1211         const struct intel_crtc_state *crtc_state;
1212         struct intel_fbc *fbc = plane->fbc;
1213
1214         if (!fbc)
1215                 return 0;
1216
1217         if (!i915_gem_stolen_initialized(i915)) {
1218                 plane_state->no_fbc_reason = "stolen memory not initialised";
1219                 return 0;
1220         }
1221
1222         if (intel_vgpu_active(i915)) {
1223                 plane_state->no_fbc_reason = "VGPU active";
1224                 return 0;
1225         }
1226
1227         if (!i915->display.params.enable_fbc) {
1228                 plane_state->no_fbc_reason = "disabled per module param or by default";
1229                 return 0;
1230         }
1231
1232         if (!plane_state->uapi.visible) {
1233                 plane_state->no_fbc_reason = "plane not visible";
1234                 return 0;
1235         }
1236
1237         crtc_state = intel_atomic_get_new_crtc_state(state, crtc);
1238
1239         if (crtc_state->hw.adjusted_mode.flags & DRM_MODE_FLAG_INTERLACE) {
1240                 plane_state->no_fbc_reason = "interlaced mode not supported";
1241                 return 0;
1242         }
1243
1244         if (crtc_state->double_wide) {
1245                 plane_state->no_fbc_reason = "double wide pipe not supported";
1246                 return 0;
1247         }
1248
1249         /*
1250          * Display 12+ is not supporting FBC with PSR2.
1251          * Recommendation is to keep this combination disabled
1252          * Bspec: 50422 HSD: 14010260002
1253          */
1254         if (IS_DISPLAY_VER(i915, 12, 14) && crtc_state->has_psr2) {
1255                 plane_state->no_fbc_reason = "PSR2 enabled";
1256                 return 0;
1257         }
1258
1259         /* Wa_14016291713 */
1260         if ((IS_DISPLAY_VER(i915, 12, 13) ||
1261              IS_DISPLAY_IP_STEP(i915, IP_VER(14, 0), STEP_A0, STEP_C0)) &&
1262             crtc_state->has_psr) {
1263                 plane_state->no_fbc_reason = "PSR1 enabled (Wa_14016291713)";
1264                 return 0;
1265         }
1266
1267         if (!pixel_format_is_valid(plane_state)) {
1268                 plane_state->no_fbc_reason = "pixel format not supported";
1269                 return 0;
1270         }
1271
1272         if (!tiling_is_valid(plane_state)) {
1273                 plane_state->no_fbc_reason = "tiling not supported";
1274                 return 0;
1275         }
1276
1277         if (!rotation_is_valid(plane_state)) {
1278                 plane_state->no_fbc_reason = "rotation not supported";
1279                 return 0;
1280         }
1281
1282         if (!stride_is_valid(plane_state)) {
1283                 plane_state->no_fbc_reason = "stride not supported";
1284                 return 0;
1285         }
1286
1287         if (DISPLAY_VER(i915) < 20 &&
1288             plane_state->hw.pixel_blend_mode != DRM_MODE_BLEND_PIXEL_NONE &&
1289             fb->format->has_alpha) {
1290                 plane_state->no_fbc_reason = "per-pixel alpha not supported";
1291                 return 0;
1292         }
1293
1294         if (!intel_fbc_plane_size_valid(plane_state)) {
1295                 plane_state->no_fbc_reason = "plane size too big";
1296                 return 0;
1297         }
1298
1299         if (!intel_fbc_hw_tracking_covers_screen(plane_state)) {
1300                 plane_state->no_fbc_reason = "surface size too big";
1301                 return 0;
1302         }
1303
1304         /*
1305          * Work around a problem on GEN9+ HW, where enabling FBC on a plane
1306          * having a Y offset that isn't divisible by 4 causes FIFO underrun
1307          * and screen flicker.
1308          */
1309         if (DISPLAY_VER(i915) >= 9 &&
1310             plane_state->view.color_plane[0].y & 3) {
1311                 plane_state->no_fbc_reason = "plane start Y offset misaligned";
1312                 return 0;
1313         }
1314
1315         /* Wa_22010751166: icl, ehl, tgl, dg1, rkl */
1316         if (DISPLAY_VER(i915) >= 11 &&
1317             (plane_state->view.color_plane[0].y +
1318              (drm_rect_height(&plane_state->uapi.src) >> 16)) & 3) {
1319                 plane_state->no_fbc_reason = "plane end Y offset misaligned";
1320                 return 0;
1321         }
1322
1323         /* WaFbcExceedCdClockThreshold:hsw,bdw */
1324         if (IS_HASWELL(i915) || IS_BROADWELL(i915)) {
1325                 const struct intel_cdclk_state *cdclk_state;
1326
1327                 cdclk_state = intel_atomic_get_cdclk_state(state);
1328                 if (IS_ERR(cdclk_state))
1329                         return PTR_ERR(cdclk_state);
1330
1331                 if (crtc_state->pixel_rate >= cdclk_state->logical.cdclk * 95 / 100) {
1332                         plane_state->no_fbc_reason = "pixel rate too high";
1333                         return 0;
1334                 }
1335         }
1336
1337         plane_state->no_fbc_reason = NULL;
1338
1339         return 0;
1340 }
1341
1342
1343 static bool intel_fbc_can_flip_nuke(struct intel_atomic_state *state,
1344                                     struct intel_crtc *crtc,
1345                                     struct intel_plane *plane)
1346 {
1347         const struct intel_crtc_state *new_crtc_state =
1348                 intel_atomic_get_new_crtc_state(state, crtc);
1349         const struct intel_plane_state *old_plane_state =
1350                 intel_atomic_get_old_plane_state(state, plane);
1351         const struct intel_plane_state *new_plane_state =
1352                 intel_atomic_get_new_plane_state(state, plane);
1353         const struct drm_framebuffer *old_fb = old_plane_state->hw.fb;
1354         const struct drm_framebuffer *new_fb = new_plane_state->hw.fb;
1355
1356         if (intel_crtc_needs_modeset(new_crtc_state))
1357                 return false;
1358
1359         if (!intel_fbc_is_ok(old_plane_state) ||
1360             !intel_fbc_is_ok(new_plane_state))
1361                 return false;
1362
1363         if (old_fb->format->format != new_fb->format->format)
1364                 return false;
1365
1366         if (old_fb->modifier != new_fb->modifier)
1367                 return false;
1368
1369         if (intel_fbc_plane_stride(old_plane_state) !=
1370             intel_fbc_plane_stride(new_plane_state))
1371                 return false;
1372
1373         if (intel_fbc_cfb_stride(old_plane_state) !=
1374             intel_fbc_cfb_stride(new_plane_state))
1375                 return false;
1376
1377         if (intel_fbc_cfb_size(old_plane_state) !=
1378             intel_fbc_cfb_size(new_plane_state))
1379                 return false;
1380
1381         if (intel_fbc_override_cfb_stride(old_plane_state) !=
1382             intel_fbc_override_cfb_stride(new_plane_state))
1383                 return false;
1384
1385         return true;
1386 }
1387
1388 static bool __intel_fbc_pre_update(struct intel_atomic_state *state,
1389                                    struct intel_crtc *crtc,
1390                                    struct intel_plane *plane)
1391 {
1392         struct drm_i915_private *i915 = to_i915(state->base.dev);
1393         struct intel_fbc *fbc = plane->fbc;
1394         bool need_vblank_wait = false;
1395
1396         lockdep_assert_held(&fbc->lock);
1397
1398         fbc->flip_pending = true;
1399
1400         if (intel_fbc_can_flip_nuke(state, crtc, plane))
1401                 return need_vblank_wait;
1402
1403         intel_fbc_deactivate(fbc, "update pending");
1404
1405         /*
1406          * Display WA #1198: glk+
1407          * Need an extra vblank wait between FBC disable and most plane
1408          * updates. Bspec says this is only needed for plane disable, but
1409          * that is not true. Touching most plane registers will cause the
1410          * corruption to appear. Also SKL/derivatives do not seem to be
1411          * affected.
1412          *
1413          * TODO: could optimize this a bit by sampling the frame
1414          * counter when we disable FBC (if it was already done earlier)
1415          * and skipping the extra vblank wait before the plane update
1416          * if at least one frame has already passed.
1417          */
1418         if (fbc->activated && DISPLAY_VER(i915) >= 10)
1419                 need_vblank_wait = true;
1420         fbc->activated = false;
1421
1422         return need_vblank_wait;
1423 }
1424
1425 bool intel_fbc_pre_update(struct intel_atomic_state *state,
1426                           struct intel_crtc *crtc)
1427 {
1428         const struct intel_plane_state __maybe_unused *plane_state;
1429         bool need_vblank_wait = false;
1430         struct intel_plane *plane;
1431         int i;
1432
1433         for_each_new_intel_plane_in_state(state, plane, plane_state, i) {
1434                 struct intel_fbc *fbc = plane->fbc;
1435
1436                 if (!fbc || plane->pipe != crtc->pipe)
1437                         continue;
1438
1439                 mutex_lock(&fbc->lock);
1440
1441                 if (fbc->state.plane == plane)
1442                         need_vblank_wait |= __intel_fbc_pre_update(state, crtc, plane);
1443
1444                 mutex_unlock(&fbc->lock);
1445         }
1446
1447         return need_vblank_wait;
1448 }
1449
1450 static void __intel_fbc_disable(struct intel_fbc *fbc)
1451 {
1452         struct drm_i915_private *i915 = fbc->i915;
1453         struct intel_plane *plane = fbc->state.plane;
1454
1455         lockdep_assert_held(&fbc->lock);
1456         drm_WARN_ON(&i915->drm, fbc->active);
1457
1458         drm_dbg_kms(&i915->drm, "Disabling FBC on [PLANE:%d:%s]\n",
1459                     plane->base.base.id, plane->base.name);
1460
1461         __intel_fbc_cleanup_cfb(fbc);
1462
1463         fbc->state.plane = NULL;
1464         fbc->flip_pending = false;
1465         fbc->busy_bits = 0;
1466 }
1467
1468 static void __intel_fbc_post_update(struct intel_fbc *fbc)
1469 {
1470         lockdep_assert_held(&fbc->lock);
1471
1472         fbc->flip_pending = false;
1473         fbc->busy_bits = 0;
1474
1475         intel_fbc_activate(fbc);
1476 }
1477
1478 void intel_fbc_post_update(struct intel_atomic_state *state,
1479                            struct intel_crtc *crtc)
1480 {
1481         const struct intel_plane_state __maybe_unused *plane_state;
1482         struct intel_plane *plane;
1483         int i;
1484
1485         for_each_new_intel_plane_in_state(state, plane, plane_state, i) {
1486                 struct intel_fbc *fbc = plane->fbc;
1487
1488                 if (!fbc || plane->pipe != crtc->pipe)
1489                         continue;
1490
1491                 mutex_lock(&fbc->lock);
1492
1493                 if (fbc->state.plane == plane)
1494                         __intel_fbc_post_update(fbc);
1495
1496                 mutex_unlock(&fbc->lock);
1497         }
1498 }
1499
1500 static unsigned int intel_fbc_get_frontbuffer_bit(struct intel_fbc *fbc)
1501 {
1502         if (fbc->state.plane)
1503                 return fbc->state.plane->frontbuffer_bit;
1504         else
1505                 return 0;
1506 }
1507
1508 static void __intel_fbc_invalidate(struct intel_fbc *fbc,
1509                                    unsigned int frontbuffer_bits,
1510                                    enum fb_op_origin origin)
1511 {
1512         if (origin == ORIGIN_FLIP || origin == ORIGIN_CURSOR_UPDATE)
1513                 return;
1514
1515         mutex_lock(&fbc->lock);
1516
1517         frontbuffer_bits &= intel_fbc_get_frontbuffer_bit(fbc);
1518         if (!frontbuffer_bits)
1519                 goto out;
1520
1521         fbc->busy_bits |= frontbuffer_bits;
1522         intel_fbc_deactivate(fbc, "frontbuffer write");
1523
1524 out:
1525         mutex_unlock(&fbc->lock);
1526 }
1527
1528 void intel_fbc_invalidate(struct drm_i915_private *i915,
1529                           unsigned int frontbuffer_bits,
1530                           enum fb_op_origin origin)
1531 {
1532         struct intel_fbc *fbc;
1533         enum intel_fbc_id fbc_id;
1534
1535         for_each_intel_fbc(i915, fbc, fbc_id)
1536                 __intel_fbc_invalidate(fbc, frontbuffer_bits, origin);
1537
1538 }
1539
1540 static void __intel_fbc_flush(struct intel_fbc *fbc,
1541                               unsigned int frontbuffer_bits,
1542                               enum fb_op_origin origin)
1543 {
1544         mutex_lock(&fbc->lock);
1545
1546         frontbuffer_bits &= intel_fbc_get_frontbuffer_bit(fbc);
1547         if (!frontbuffer_bits)
1548                 goto out;
1549
1550         fbc->busy_bits &= ~frontbuffer_bits;
1551
1552         if (origin == ORIGIN_FLIP || origin == ORIGIN_CURSOR_UPDATE)
1553                 goto out;
1554
1555         if (fbc->busy_bits || fbc->flip_pending)
1556                 goto out;
1557
1558         if (fbc->active)
1559                 intel_fbc_nuke(fbc);
1560         else
1561                 intel_fbc_activate(fbc);
1562
1563 out:
1564         mutex_unlock(&fbc->lock);
1565 }
1566
1567 void intel_fbc_flush(struct drm_i915_private *i915,
1568                      unsigned int frontbuffer_bits,
1569                      enum fb_op_origin origin)
1570 {
1571         struct intel_fbc *fbc;
1572         enum intel_fbc_id fbc_id;
1573
1574         for_each_intel_fbc(i915, fbc, fbc_id)
1575                 __intel_fbc_flush(fbc, frontbuffer_bits, origin);
1576 }
1577
1578 int intel_fbc_atomic_check(struct intel_atomic_state *state)
1579 {
1580         struct intel_plane_state __maybe_unused *plane_state;
1581         struct intel_plane *plane;
1582         int i;
1583
1584         for_each_new_intel_plane_in_state(state, plane, plane_state, i) {
1585                 int ret;
1586
1587                 ret = intel_fbc_check_plane(state, plane);
1588                 if (ret)
1589                         return ret;
1590         }
1591
1592         return 0;
1593 }
1594
1595 static void __intel_fbc_enable(struct intel_atomic_state *state,
1596                                struct intel_crtc *crtc,
1597                                struct intel_plane *plane)
1598 {
1599         struct drm_i915_private *i915 = to_i915(state->base.dev);
1600         const struct intel_plane_state *plane_state =
1601                 intel_atomic_get_new_plane_state(state, plane);
1602         struct intel_fbc *fbc = plane->fbc;
1603
1604         lockdep_assert_held(&fbc->lock);
1605
1606         if (fbc->state.plane) {
1607                 if (fbc->state.plane != plane)
1608                         return;
1609
1610                 if (intel_fbc_is_ok(plane_state)) {
1611                         intel_fbc_update_state(state, crtc, plane);
1612                         return;
1613                 }
1614
1615                 __intel_fbc_disable(fbc);
1616         }
1617
1618         drm_WARN_ON(&i915->drm, fbc->active);
1619
1620         fbc->no_fbc_reason = plane_state->no_fbc_reason;
1621         if (fbc->no_fbc_reason)
1622                 return;
1623
1624         if (!intel_fbc_is_fence_ok(plane_state)) {
1625                 fbc->no_fbc_reason = "framebuffer not fenced";
1626                 return;
1627         }
1628
1629         if (fbc->underrun_detected) {
1630                 fbc->no_fbc_reason = "FIFO underrun";
1631                 return;
1632         }
1633
1634         if (intel_fbc_alloc_cfb(fbc, intel_fbc_cfb_size(plane_state),
1635                                 intel_fbc_min_limit(plane_state))) {
1636                 fbc->no_fbc_reason = "not enough stolen memory";
1637                 return;
1638         }
1639
1640         drm_dbg_kms(&i915->drm, "Enabling FBC on [PLANE:%d:%s]\n",
1641                     plane->base.base.id, plane->base.name);
1642         fbc->no_fbc_reason = "FBC enabled but not active yet\n";
1643
1644         intel_fbc_update_state(state, crtc, plane);
1645
1646         intel_fbc_program_workarounds(fbc);
1647         intel_fbc_program_cfb(fbc);
1648 }
1649
1650 /**
1651  * intel_fbc_disable - disable FBC if it's associated with crtc
1652  * @crtc: the CRTC
1653  *
1654  * This function disables FBC if it's associated with the provided CRTC.
1655  */
1656 void intel_fbc_disable(struct intel_crtc *crtc)
1657 {
1658         struct drm_i915_private *i915 = to_i915(crtc->base.dev);
1659         struct intel_plane *plane;
1660
1661         for_each_intel_plane(&i915->drm, plane) {
1662                 struct intel_fbc *fbc = plane->fbc;
1663
1664                 if (!fbc || plane->pipe != crtc->pipe)
1665                         continue;
1666
1667                 mutex_lock(&fbc->lock);
1668                 if (fbc->state.plane == plane)
1669                         __intel_fbc_disable(fbc);
1670                 mutex_unlock(&fbc->lock);
1671         }
1672 }
1673
1674 void intel_fbc_update(struct intel_atomic_state *state,
1675                       struct intel_crtc *crtc)
1676 {
1677         const struct intel_crtc_state *crtc_state =
1678                 intel_atomic_get_new_crtc_state(state, crtc);
1679         const struct intel_plane_state *plane_state;
1680         struct intel_plane *plane;
1681         int i;
1682
1683         for_each_new_intel_plane_in_state(state, plane, plane_state, i) {
1684                 struct intel_fbc *fbc = plane->fbc;
1685
1686                 if (!fbc || plane->pipe != crtc->pipe)
1687                         continue;
1688
1689                 mutex_lock(&fbc->lock);
1690
1691                 if (intel_crtc_needs_fastset(crtc_state) &&
1692                     plane_state->no_fbc_reason) {
1693                         if (fbc->state.plane == plane)
1694                                 __intel_fbc_disable(fbc);
1695                 } else {
1696                         __intel_fbc_enable(state, crtc, plane);
1697                 }
1698
1699                 mutex_unlock(&fbc->lock);
1700         }
1701 }
1702
1703 static void intel_fbc_underrun_work_fn(struct work_struct *work)
1704 {
1705         struct intel_fbc *fbc = container_of(work, typeof(*fbc), underrun_work);
1706         struct drm_i915_private *i915 = fbc->i915;
1707
1708         mutex_lock(&fbc->lock);
1709
1710         /* Maybe we were scheduled twice. */
1711         if (fbc->underrun_detected || !fbc->state.plane)
1712                 goto out;
1713
1714         drm_dbg_kms(&i915->drm, "Disabling FBC due to FIFO underrun.\n");
1715         fbc->underrun_detected = true;
1716
1717         intel_fbc_deactivate(fbc, "FIFO underrun");
1718         if (!fbc->flip_pending)
1719                 intel_crtc_wait_for_next_vblank(intel_crtc_for_pipe(i915, fbc->state.plane->pipe));
1720         __intel_fbc_disable(fbc);
1721 out:
1722         mutex_unlock(&fbc->lock);
1723 }
1724
1725 static void __intel_fbc_reset_underrun(struct intel_fbc *fbc)
1726 {
1727         struct drm_i915_private *i915 = fbc->i915;
1728
1729         cancel_work_sync(&fbc->underrun_work);
1730
1731         mutex_lock(&fbc->lock);
1732
1733         if (fbc->underrun_detected) {
1734                 drm_dbg_kms(&i915->drm,
1735                             "Re-allowing FBC after fifo underrun\n");
1736                 fbc->no_fbc_reason = "FIFO underrun cleared";
1737         }
1738
1739         fbc->underrun_detected = false;
1740         mutex_unlock(&fbc->lock);
1741 }
1742
1743 /*
1744  * intel_fbc_reset_underrun - reset FBC fifo underrun status.
1745  * @i915: the i915 device
1746  *
1747  * See intel_fbc_handle_fifo_underrun_irq(). For automated testing we
1748  * want to re-enable FBC after an underrun to increase test coverage.
1749  */
1750 void intel_fbc_reset_underrun(struct drm_i915_private *i915)
1751 {
1752         struct intel_fbc *fbc;
1753         enum intel_fbc_id fbc_id;
1754
1755         for_each_intel_fbc(i915, fbc, fbc_id)
1756                 __intel_fbc_reset_underrun(fbc);
1757 }
1758
1759 static void __intel_fbc_handle_fifo_underrun_irq(struct intel_fbc *fbc)
1760 {
1761         /*
1762          * There's no guarantee that underrun_detected won't be set to true
1763          * right after this check and before the work is scheduled, but that's
1764          * not a problem since we'll check it again under the work function
1765          * while FBC is locked. This check here is just to prevent us from
1766          * unnecessarily scheduling the work, and it relies on the fact that we
1767          * never switch underrun_detect back to false after it's true.
1768          */
1769         if (READ_ONCE(fbc->underrun_detected))
1770                 return;
1771
1772         queue_work(fbc->i915->unordered_wq, &fbc->underrun_work);
1773 }
1774
1775 /**
1776  * intel_fbc_handle_fifo_underrun_irq - disable FBC when we get a FIFO underrun
1777  * @i915: i915 device
1778  *
1779  * Without FBC, most underruns are harmless and don't really cause too many
1780  * problems, except for an annoying message on dmesg. With FBC, underruns can
1781  * become black screens or even worse, especially when paired with bad
1782  * watermarks. So in order for us to be on the safe side, completely disable FBC
1783  * in case we ever detect a FIFO underrun on any pipe. An underrun on any pipe
1784  * already suggests that watermarks may be bad, so try to be as safe as
1785  * possible.
1786  *
1787  * This function is called from the IRQ handler.
1788  */
1789 void intel_fbc_handle_fifo_underrun_irq(struct drm_i915_private *i915)
1790 {
1791         struct intel_fbc *fbc;
1792         enum intel_fbc_id fbc_id;
1793
1794         for_each_intel_fbc(i915, fbc, fbc_id)
1795                 __intel_fbc_handle_fifo_underrun_irq(fbc);
1796 }
1797
1798 /*
1799  * The DDX driver changes its behavior depending on the value it reads from
1800  * i915.enable_fbc, so sanitize it by translating the default value into either
1801  * 0 or 1 in order to allow it to know what's going on.
1802  *
1803  * Notice that this is done at driver initialization and we still allow user
1804  * space to change the value during runtime without sanitizing it again. IGT
1805  * relies on being able to change i915.enable_fbc at runtime.
1806  */
1807 static int intel_sanitize_fbc_option(struct drm_i915_private *i915)
1808 {
1809         if (i915->display.params.enable_fbc >= 0)
1810                 return !!i915->display.params.enable_fbc;
1811
1812         if (!HAS_FBC(i915))
1813                 return 0;
1814
1815         if (IS_BROADWELL(i915) || DISPLAY_VER(i915) >= 9)
1816                 return 1;
1817
1818         return 0;
1819 }
1820
1821 static bool need_fbc_vtd_wa(struct drm_i915_private *i915)
1822 {
1823         /* WaFbcTurnOffFbcWhenHyperVisorIsUsed:skl,bxt */
1824         if (i915_vtd_active(i915) &&
1825             (IS_SKYLAKE(i915) || IS_BROXTON(i915))) {
1826                 drm_info(&i915->drm,
1827                          "Disabling framebuffer compression (FBC) to prevent screen flicker with VT-d enabled\n");
1828                 return true;
1829         }
1830
1831         return false;
1832 }
1833
1834 void intel_fbc_add_plane(struct intel_fbc *fbc, struct intel_plane *plane)
1835 {
1836         plane->fbc = fbc;
1837 }
1838
1839 static struct intel_fbc *intel_fbc_create(struct drm_i915_private *i915,
1840                                           enum intel_fbc_id fbc_id)
1841 {
1842         struct intel_fbc *fbc;
1843
1844         fbc = kzalloc(sizeof(*fbc), GFP_KERNEL);
1845         if (!fbc)
1846                 return NULL;
1847
1848         fbc->id = fbc_id;
1849         fbc->i915 = i915;
1850         INIT_WORK(&fbc->underrun_work, intel_fbc_underrun_work_fn);
1851         mutex_init(&fbc->lock);
1852
1853         if (DISPLAY_VER(i915) >= 7)
1854                 fbc->funcs = &ivb_fbc_funcs;
1855         else if (DISPLAY_VER(i915) == 6)
1856                 fbc->funcs = &snb_fbc_funcs;
1857         else if (DISPLAY_VER(i915) == 5)
1858                 fbc->funcs = &ilk_fbc_funcs;
1859         else if (IS_G4X(i915))
1860                 fbc->funcs = &g4x_fbc_funcs;
1861         else if (DISPLAY_VER(i915) == 4)
1862                 fbc->funcs = &i965_fbc_funcs;
1863         else
1864                 fbc->funcs = &i8xx_fbc_funcs;
1865
1866         return fbc;
1867 }
1868
1869 /**
1870  * intel_fbc_init - Initialize FBC
1871  * @i915: the i915 device
1872  *
1873  * This function might be called during PM init process.
1874  */
1875 void intel_fbc_init(struct drm_i915_private *i915)
1876 {
1877         enum intel_fbc_id fbc_id;
1878
1879         if (need_fbc_vtd_wa(i915))
1880                 DISPLAY_RUNTIME_INFO(i915)->fbc_mask = 0;
1881
1882         i915->display.params.enable_fbc = intel_sanitize_fbc_option(i915);
1883         drm_dbg_kms(&i915->drm, "Sanitized enable_fbc value: %d\n",
1884                     i915->display.params.enable_fbc);
1885
1886         for_each_fbc_id(i915, fbc_id)
1887                 i915->display.fbc[fbc_id] = intel_fbc_create(i915, fbc_id);
1888 }
1889
1890 /**
1891  * intel_fbc_sanitize - Sanitize FBC
1892  * @i915: the i915 device
1893  *
1894  * Make sure FBC is initially disabled since we have no
1895  * idea eg. into which parts of stolen it might be scribbling
1896  * into.
1897  */
1898 void intel_fbc_sanitize(struct drm_i915_private *i915)
1899 {
1900         struct intel_fbc *fbc;
1901         enum intel_fbc_id fbc_id;
1902
1903         for_each_intel_fbc(i915, fbc, fbc_id) {
1904                 if (intel_fbc_hw_is_active(fbc))
1905                         intel_fbc_hw_deactivate(fbc);
1906         }
1907 }
1908
1909 static int intel_fbc_debugfs_status_show(struct seq_file *m, void *unused)
1910 {
1911         struct intel_fbc *fbc = m->private;
1912         struct drm_i915_private *i915 = fbc->i915;
1913         struct intel_plane *plane;
1914         intel_wakeref_t wakeref;
1915
1916         drm_modeset_lock_all(&i915->drm);
1917
1918         wakeref = intel_runtime_pm_get(&i915->runtime_pm);
1919         mutex_lock(&fbc->lock);
1920
1921         if (fbc->active) {
1922                 seq_puts(m, "FBC enabled\n");
1923                 seq_printf(m, "Compressing: %s\n",
1924                            str_yes_no(intel_fbc_is_compressing(fbc)));
1925         } else {
1926                 seq_printf(m, "FBC disabled: %s\n", fbc->no_fbc_reason);
1927         }
1928
1929         for_each_intel_plane(&i915->drm, plane) {
1930                 const struct intel_plane_state *plane_state =
1931                         to_intel_plane_state(plane->base.state);
1932
1933                 if (plane->fbc != fbc)
1934                         continue;
1935
1936                 seq_printf(m, "%c [PLANE:%d:%s]: %s\n",
1937                            fbc->state.plane == plane ? '*' : ' ',
1938                            plane->base.base.id, plane->base.name,
1939                            plane_state->no_fbc_reason ?: "FBC possible");
1940         }
1941
1942         mutex_unlock(&fbc->lock);
1943         intel_runtime_pm_put(&i915->runtime_pm, wakeref);
1944
1945         drm_modeset_unlock_all(&i915->drm);
1946
1947         return 0;
1948 }
1949
1950 DEFINE_SHOW_ATTRIBUTE(intel_fbc_debugfs_status);
1951
1952 static int intel_fbc_debugfs_false_color_get(void *data, u64 *val)
1953 {
1954         struct intel_fbc *fbc = data;
1955
1956         *val = fbc->false_color;
1957
1958         return 0;
1959 }
1960
1961 static int intel_fbc_debugfs_false_color_set(void *data, u64 val)
1962 {
1963         struct intel_fbc *fbc = data;
1964
1965         mutex_lock(&fbc->lock);
1966
1967         fbc->false_color = val;
1968
1969         if (fbc->active)
1970                 fbc->funcs->set_false_color(fbc, fbc->false_color);
1971
1972         mutex_unlock(&fbc->lock);
1973
1974         return 0;
1975 }
1976
1977 DEFINE_DEBUGFS_ATTRIBUTE(intel_fbc_debugfs_false_color_fops,
1978                          intel_fbc_debugfs_false_color_get,
1979                          intel_fbc_debugfs_false_color_set,
1980                          "%llu\n");
1981
1982 static void intel_fbc_debugfs_add(struct intel_fbc *fbc,
1983                                   struct dentry *parent)
1984 {
1985         debugfs_create_file("i915_fbc_status", 0444, parent,
1986                             fbc, &intel_fbc_debugfs_status_fops);
1987
1988         if (fbc->funcs->set_false_color)
1989                 debugfs_create_file_unsafe("i915_fbc_false_color", 0644, parent,
1990                                            fbc, &intel_fbc_debugfs_false_color_fops);
1991 }
1992
1993 void intel_fbc_crtc_debugfs_add(struct intel_crtc *crtc)
1994 {
1995         struct intel_plane *plane = to_intel_plane(crtc->base.primary);
1996
1997         if (plane->fbc)
1998                 intel_fbc_debugfs_add(plane->fbc, crtc->base.debugfs_entry);
1999 }
2000
2001 /* FIXME: remove this once igt is on board with per-crtc stuff */
2002 void intel_fbc_debugfs_register(struct drm_i915_private *i915)
2003 {
2004         struct drm_minor *minor = i915->drm.primary;
2005         struct intel_fbc *fbc;
2006
2007         fbc = i915->display.fbc[INTEL_FBC_A];
2008         if (fbc)
2009                 intel_fbc_debugfs_add(fbc, minor->debugfs_root);
2010 }