drivers/gpu/drm/i915/display/intel_crtc.c

   1 // SPDX-License-Identifier: MIT
   2 /*
   3  * Copyright © 2020 Intel Corporation
   4  */
   5 #include <linux/kernel.h>
   6 #include <linux/pm_qos.h>
   7 #include <linux/slab.h>
   8
   9 #include <drm/drm_atomic_helper.h>
  10 #include <drm/drm_fourcc.h>
  11 #include <drm/drm_plane.h>
  12 #include <drm/drm_vblank.h>
  13 #include <drm/drm_vblank_work.h>
  14
  15 #include "i915_drv.h"
  16 #include "i915_vgpu.h"
  17 #include "i9xx_plane.h"
  18 #include "icl_dsi.h"
  19 #include "intel_atomic.h"
  20 #include "intel_atomic_plane.h"
  21 #include "intel_color.h"
  22 #include "intel_crtc.h"
  23 #include "intel_cursor.h"
  24 #include "intel_display_debugfs.h"
  25 #include "intel_display_irq.h"
  26 #include "intel_display_trace.h"
  27 #include "intel_display_types.h"
  28 #include "intel_drrs.h"
  29 #include "intel_dsi.h"
  30 #include "intel_fifo_underrun.h"
  31 #include "intel_pipe_crc.h"
  32 #include "intel_psr.h"
  33 #include "intel_sprite.h"
  34 #include "intel_vblank.h"
  35 #include "intel_vrr.h"
  36 #include "skl_universal_plane.h"
  37
  38 static void assert_vblank_disabled(struct drm_crtc *crtc)
  39 {
  40         struct intel_display *display = to_intel_display(crtc->dev);
  41
  42         if (INTEL_DISPLAY_STATE_WARN(display, drm_crtc_vblank_get(crtc) == 0,
  43                                      "[CRTC:%d:%s] vblank assertion failure (expected off, current on)\n",
  44                                      crtc->base.id, crtc->name))
  45                 drm_crtc_vblank_put(crtc);
  46 }
  47
  48 struct intel_crtc *intel_first_crtc(struct drm_i915_private *i915)
  49 {
  50         return to_intel_crtc(drm_crtc_from_index(&i915->drm, 0));
  51 }
  52
  53 struct intel_crtc *intel_crtc_for_pipe(struct intel_display *display,
  54                                        enum pipe pipe)
  55 {
  56         struct intel_crtc *crtc;
  57
  58         for_each_intel_crtc(display->drm, crtc) {
  59                 if (crtc->pipe == pipe)
  60                         return crtc;
  61         }
  62
  63         return NULL;
  64 }
  65
  66 void intel_crtc_wait_for_next_vblank(struct intel_crtc *crtc)
  67 {
  68         drm_crtc_wait_one_vblank(&crtc->base);
  69 }
  70
  71 void intel_wait_for_vblank_if_active(struct drm_i915_private *i915,
  72                                      enum pipe pipe)
  73 {
  74         struct intel_display *display = &i915->display;
  75         struct intel_crtc *crtc = intel_crtc_for_pipe(display, pipe);
  76
  77         if (crtc->active)
  78                 intel_crtc_wait_for_next_vblank(crtc);
  79 }
  80
  81 u32 intel_crtc_get_vblank_counter(struct intel_crtc *crtc)
  82 {
  83         struct drm_vblank_crtc *vblank = drm_crtc_vblank_crtc(&crtc->base);
  84
  85         if (!crtc->active)
  86                 return 0;
  87
  88         if (!vblank->max_vblank_count)
  89                 return (u32)drm_crtc_accurate_vblank_count(&crtc->base);
  90
  91         return crtc->base.funcs->get_vblank_counter(&crtc->base);
  92 }
  93
  94 u32 intel_crtc_max_vblank_count(const struct intel_crtc_state *crtc_state)
  95 {
  96         struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
  97
  98         /*
  99          * From Gen 11, In case of dsi cmd mode, frame counter wouldnt
 100          * have updated at the beginning of TE, if we want to use
 101          * the hw counter, then we would find it updated in only
 102          * the next TE, hence switching to sw counter.
 103          */
 104         if (crtc_state->mode_flags & (I915_MODE_FLAG_DSI_USE_TE0 |
 105                                       I915_MODE_FLAG_DSI_USE_TE1))
 106                 return 0;
 107
 108         /*
 109          * On i965gm the hardware frame counter reads
 110          * zero when the TV encoder is enabled :(
 111          */
 112         if (IS_I965GM(dev_priv) &&
 113             (crtc_state->output_types & BIT(INTEL_OUTPUT_TVOUT)))
 114                 return 0;
 115
 116         if (DISPLAY_VER(dev_priv) >= 5 || IS_G4X(dev_priv))
 117                 return 0xffffffff; /* full 32 bit counter */
 118         else if (DISPLAY_VER(dev_priv) >= 3)
 119                 return 0xffffff; /* only 24 bits of frame count */
 120         else
 121                 return 0; /* Gen2 doesn't have a hardware frame counter */
 122 }
 123
 124 void intel_crtc_vblank_on(const struct intel_crtc_state *crtc_state)
 125 {
 126         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
 127
 128         crtc->block_dc_for_vblank = intel_psr_needs_block_dc_vblank(crtc_state);
 129
 130         assert_vblank_disabled(&crtc->base);
 131         drm_crtc_set_max_vblank_count(&crtc->base,
 132                                       intel_crtc_max_vblank_count(crtc_state));
 133         drm_crtc_vblank_on(&crtc->base);
 134
 135         /*
 136          * Should really happen exactly when we enable the pipe
 137          * but we want the frame counters in the trace, and that
 138          * requires vblank support on some platforms/outputs.
 139          */
 140         trace_intel_pipe_enable(crtc);
 141 }
 142
 143 void intel_crtc_vblank_off(const struct intel_crtc_state *crtc_state)
 144 {
 145         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
 146         struct intel_display *display = to_intel_display(crtc);
 147
 148         /*
 149          * Should really happen exactly when we disable the pipe
 150          * but we want the frame counters in the trace, and that
 151          * requires vblank support on some platforms/outputs.
 152          */
 153         trace_intel_pipe_disable(crtc);
 154
 155         drm_crtc_vblank_off(&crtc->base);
 156         assert_vblank_disabled(&crtc->base);
 157
 158         crtc->block_dc_for_vblank = false;
 159
 160         flush_work(&display->irq.vblank_dc_work);
 161 }
 162
 163 struct intel_crtc_state *intel_crtc_state_alloc(struct intel_crtc *crtc)
 164 {
 165         struct intel_crtc_state *crtc_state;
 166
 167         crtc_state = kmalloc(sizeof(*crtc_state), GFP_KERNEL);
 168
 169         if (crtc_state)
 170                 intel_crtc_state_reset(crtc_state, crtc);
 171
 172         return crtc_state;
 173 }
 174
 175 void intel_crtc_state_reset(struct intel_crtc_state *crtc_state,
 176                             struct intel_crtc *crtc)
 177 {
 178         memset(crtc_state, 0, sizeof(*crtc_state));
 179
 180         __drm_atomic_helper_crtc_state_reset(&crtc_state->uapi, &crtc->base);
 181
 182         crtc_state->cpu_transcoder = INVALID_TRANSCODER;
 183         crtc_state->master_transcoder = INVALID_TRANSCODER;
 184         crtc_state->hsw_workaround_pipe = INVALID_PIPE;
 185         crtc_state->scaler_state.scaler_id = -1;
 186         crtc_state->mst_master_transcoder = INVALID_TRANSCODER;
 187         crtc_state->max_link_bpp_x16 = INT_MAX;
 188 }
 189
 190 static struct intel_crtc *intel_crtc_alloc(void)
 191 {
 192         struct intel_crtc_state *crtc_state;
 193         struct intel_crtc *crtc;
 194
 195         crtc = kzalloc(sizeof(*crtc), GFP_KERNEL);
 196         if (!crtc)
 197                 return ERR_PTR(-ENOMEM);
 198
 199         crtc_state = intel_crtc_state_alloc(crtc);
 200         if (!crtc_state) {
 201                 kfree(crtc);
 202                 return ERR_PTR(-ENOMEM);
 203         }
 204
 205         crtc->base.state = &crtc_state->uapi;
 206         crtc->config = crtc_state;
 207
 208         return crtc;
 209 }
 210
 211 static void intel_crtc_free(struct intel_crtc *crtc)
 212 {
 213         intel_crtc_destroy_state(&crtc->base, crtc->base.state);
 214         kfree(crtc);
 215 }
 216
 217 static void intel_crtc_destroy(struct drm_crtc *_crtc)
 218 {
 219         struct intel_crtc *crtc = to_intel_crtc(_crtc);
 220
 221         cpu_latency_qos_remove_request(&crtc->vblank_pm_qos);
 222
 223         drm_crtc_cleanup(&crtc->base);
 224         kfree(crtc);
 225 }
 226
 227 static int intel_crtc_late_register(struct drm_crtc *crtc)
 228 {
 229         intel_crtc_debugfs_add(to_intel_crtc(crtc));
 230         return 0;
 231 }
 232
 233 #define INTEL_CRTC_FUNCS \
 234         .set_config = drm_atomic_helper_set_config, \
 235         .destroy = intel_crtc_destroy, \
 236         .page_flip = drm_atomic_helper_page_flip, \
 237         .atomic_duplicate_state = intel_crtc_duplicate_state, \
 238         .atomic_destroy_state = intel_crtc_destroy_state, \
 239         .set_crc_source = intel_crtc_set_crc_source, \
 240         .verify_crc_source = intel_crtc_verify_crc_source, \
 241         .get_crc_sources = intel_crtc_get_crc_sources, \
 242         .late_register = intel_crtc_late_register
 243
 244 static const struct drm_crtc_funcs bdw_crtc_funcs = {
 245         INTEL_CRTC_FUNCS,
 246
 247         .get_vblank_counter = g4x_get_vblank_counter,
 248         .enable_vblank = bdw_enable_vblank,
 249         .disable_vblank = bdw_disable_vblank,
 250         .get_vblank_timestamp = intel_crtc_get_vblank_timestamp,
 251 };
 252
 253 static const struct drm_crtc_funcs ilk_crtc_funcs = {
 254         INTEL_CRTC_FUNCS,
 255
 256         .get_vblank_counter = g4x_get_vblank_counter,
 257         .enable_vblank = ilk_enable_vblank,
 258         .disable_vblank = ilk_disable_vblank,
 259         .get_vblank_timestamp = intel_crtc_get_vblank_timestamp,
 260 };
 261
 262 static const struct drm_crtc_funcs g4x_crtc_funcs = {
 263         INTEL_CRTC_FUNCS,
 264
 265         .get_vblank_counter = g4x_get_vblank_counter,
 266         .enable_vblank = i965_enable_vblank,
 267         .disable_vblank = i965_disable_vblank,
 268         .get_vblank_timestamp = intel_crtc_get_vblank_timestamp,
 269 };
 270
 271 static const struct drm_crtc_funcs i965_crtc_funcs = {
 272         INTEL_CRTC_FUNCS,
 273
 274         .get_vblank_counter = i915_get_vblank_counter,
 275         .enable_vblank = i965_enable_vblank,
 276         .disable_vblank = i965_disable_vblank,
 277         .get_vblank_timestamp = intel_crtc_get_vblank_timestamp,
 278 };
 279
 280 static const struct drm_crtc_funcs i915gm_crtc_funcs = {
 281         INTEL_CRTC_FUNCS,
 282
 283         .get_vblank_counter = i915_get_vblank_counter,
 284         .enable_vblank = i915gm_enable_vblank,
 285         .disable_vblank = i915gm_disable_vblank,
 286         .get_vblank_timestamp = intel_crtc_get_vblank_timestamp,
 287 };
 288
 289 static const struct drm_crtc_funcs i915_crtc_funcs = {
 290         INTEL_CRTC_FUNCS,
 291
 292         .get_vblank_counter = i915_get_vblank_counter,
 293         .enable_vblank = i8xx_enable_vblank,
 294         .disable_vblank = i8xx_disable_vblank,
 295         .get_vblank_timestamp = intel_crtc_get_vblank_timestamp,
 296 };
 297
 298 static const struct drm_crtc_funcs i8xx_crtc_funcs = {
 299         INTEL_CRTC_FUNCS,
 300
 301         /* no hw vblank counter */
 302         .enable_vblank = i8xx_enable_vblank,
 303         .disable_vblank = i8xx_disable_vblank,
 304         .get_vblank_timestamp = intel_crtc_get_vblank_timestamp,
 305 };
 306
 307 int intel_crtc_init(struct drm_i915_private *dev_priv, enum pipe pipe)
 308 {
 309         struct intel_plane *primary, *cursor;
 310         const struct drm_crtc_funcs *funcs;
 311         struct intel_crtc *crtc;
 312         int sprite, ret;
 313
 314         crtc = intel_crtc_alloc();
 315         if (IS_ERR(crtc))
 316                 return PTR_ERR(crtc);
 317
 318         crtc->pipe = pipe;
 319         crtc->num_scalers = DISPLAY_RUNTIME_INFO(dev_priv)->num_scalers[pipe];
 320
 321         if (DISPLAY_VER(dev_priv) >= 9)
 322                 primary = skl_universal_plane_create(dev_priv, pipe, PLANE_1);
 323         else
 324                 primary = intel_primary_plane_create(dev_priv, pipe);
 325         if (IS_ERR(primary)) {
 326                 ret = PTR_ERR(primary);
 327                 goto fail;
 328         }
 329         crtc->plane_ids_mask |= BIT(primary->id);
 330
 331         intel_init_fifo_underrun_reporting(dev_priv, crtc, false);
 332
 333         for_each_sprite(dev_priv, pipe, sprite) {
 334                 struct intel_plane *plane;
 335
 336                 if (DISPLAY_VER(dev_priv) >= 9)
 337                         plane = skl_universal_plane_create(dev_priv, pipe, PLANE_2 + sprite);
 338                 else
 339                         plane = intel_sprite_plane_create(dev_priv, pipe, sprite);
 340                 if (IS_ERR(plane)) {
 341                         ret = PTR_ERR(plane);
 342                         goto fail;
 343                 }
 344                 crtc->plane_ids_mask |= BIT(plane->id);
 345         }
 346
 347         cursor = intel_cursor_plane_create(dev_priv, pipe);
 348         if (IS_ERR(cursor)) {
 349                 ret = PTR_ERR(cursor);
 350                 goto fail;
 351         }
 352         crtc->plane_ids_mask |= BIT(cursor->id);
 353
 354         if (HAS_GMCH(dev_priv)) {
 355                 if (IS_CHERRYVIEW(dev_priv) ||
 356                     IS_VALLEYVIEW(dev_priv) || IS_G4X(dev_priv))
 357                         funcs = &g4x_crtc_funcs;
 358                 else if (DISPLAY_VER(dev_priv) == 4)
 359                         funcs = &i965_crtc_funcs;
 360                 else if (IS_I945GM(dev_priv) || IS_I915GM(dev_priv))
 361                         funcs = &i915gm_crtc_funcs;
 362                 else if (DISPLAY_VER(dev_priv) == 3)
 363                         funcs = &i915_crtc_funcs;
 364                 else
 365                         funcs = &i8xx_crtc_funcs;
 366         } else {
 367                 if (DISPLAY_VER(dev_priv) >= 8)
 368                         funcs = &bdw_crtc_funcs;
 369                 else
 370                         funcs = &ilk_crtc_funcs;
 371         }
 372
 373         ret = drm_crtc_init_with_planes(&dev_priv->drm, &crtc->base,
 374                                         &primary->base, &cursor->base,
 375                                         funcs, "pipe %c", pipe_name(pipe));
 376         if (ret)
 377                 goto fail;
 378
 379         if (DISPLAY_VER(dev_priv) >= 11)
 380                 drm_crtc_create_scaling_filter_property(&crtc->base,
 381                                                 BIT(DRM_SCALING_FILTER_DEFAULT) |
 382                                                 BIT(DRM_SCALING_FILTER_NEAREST_NEIGHBOR));
 383
 384         intel_color_crtc_init(crtc);
 385         intel_drrs_crtc_init(crtc);
 386         intel_crtc_crc_init(crtc);
 387
 388         cpu_latency_qos_add_request(&crtc->vblank_pm_qos, PM_QOS_DEFAULT_VALUE);
 389
 390         drm_WARN_ON(&dev_priv->drm, drm_crtc_index(&crtc->base) != crtc->pipe);
 391
 392         return 0;
 393
 394 fail:
 395         intel_crtc_free(crtc);
 396
 397         return ret;
 398 }
 399
 400 int intel_crtc_get_pipe_from_crtc_id_ioctl(struct drm_device *dev, void *data,
 401                                            struct drm_file *file)
 402 {
 403         struct drm_i915_get_pipe_from_crtc_id *pipe_from_crtc_id = data;
 404         struct drm_crtc *drm_crtc;
 405         struct intel_crtc *crtc;
 406
 407         drm_crtc = drm_crtc_find(dev, file, pipe_from_crtc_id->crtc_id);
 408         if (!drm_crtc)
 409                 return -ENOENT;
 410
 411         crtc = to_intel_crtc(drm_crtc);
 412         pipe_from_crtc_id->pipe = crtc->pipe;
 413
 414         return 0;
 415 }
 416
 417 static bool intel_crtc_needs_vblank_work(const struct intel_crtc_state *crtc_state)
 418 {
 419         return crtc_state->hw.active &&
 420                 !crtc_state->preload_luts &&
 421                 !intel_crtc_needs_modeset(crtc_state) &&
 422                 intel_crtc_needs_color_update(crtc_state) &&
 423                 !intel_color_uses_dsb(crtc_state) &&
 424                 !crtc_state->use_dsb;
 425 }
 426
 427 static void intel_crtc_vblank_work(struct kthread_work *base)
 428 {
 429         struct drm_vblank_work *work = to_drm_vblank_work(base);
 430         struct intel_crtc_state *crtc_state =
 431                 container_of(work, typeof(*crtc_state), vblank_work);
 432         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
 433
 434         trace_intel_crtc_vblank_work_start(crtc);
 435
 436         intel_color_load_luts(crtc_state);
 437
 438         if (crtc_state->uapi.event) {
 439                 spin_lock_irq(&crtc->base.dev->event_lock);
 440                 drm_crtc_send_vblank_event(&crtc->base, crtc_state->uapi.event);
 441                 spin_unlock_irq(&crtc->base.dev->event_lock);
 442                 crtc_state->uapi.event = NULL;
 443         }
 444
 445         trace_intel_crtc_vblank_work_end(crtc);
 446 }
 447
 448 static void intel_crtc_vblank_work_init(struct intel_crtc_state *crtc_state)
 449 {
 450         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
 451
 452         drm_vblank_work_init(&crtc_state->vblank_work, &crtc->base,
 453                              intel_crtc_vblank_work);
 454         /*
 455          * Interrupt latency is critical for getting the vblank
 456          * work executed as early as possible during the vblank.
 457          */
 458         cpu_latency_qos_update_request(&crtc->vblank_pm_qos, 0);
 459 }
 460
 461 void intel_wait_for_vblank_workers(struct intel_atomic_state *state)
 462 {
 463         struct intel_crtc_state *crtc_state;
 464         struct intel_crtc *crtc;
 465         int i;
 466
 467         for_each_new_intel_crtc_in_state(state, crtc, crtc_state, i) {
 468                 if (!intel_crtc_needs_vblank_work(crtc_state))
 469                         continue;
 470
 471                 drm_vblank_work_flush(&crtc_state->vblank_work);
 472                 cpu_latency_qos_update_request(&crtc->vblank_pm_qos,
 473                                                PM_QOS_DEFAULT_VALUE);
 474         }
 475 }
 476
 477 int intel_usecs_to_scanlines(const struct drm_display_mode *adjusted_mode,
 478                              int usecs)
 479 {
 480         /* paranoia */
 481         if (!adjusted_mode->crtc_htotal)
 482                 return 1;
 483
 484         return DIV_ROUND_UP_ULL(mul_u32_u32(usecs, adjusted_mode->crtc_clock),
 485                                 1000 * adjusted_mode->crtc_htotal);
 486 }
 487
 488 int intel_scanlines_to_usecs(const struct drm_display_mode *adjusted_mode,
 489                              int scanlines)
 490 {
 491         /* paranoia */
 492         if (!adjusted_mode->crtc_clock)
 493                 return 1;
 494
 495         return DIV_ROUND_UP_ULL(mul_u32_u32(scanlines, adjusted_mode->crtc_htotal * 1000),
 496                                 adjusted_mode->crtc_clock);
 497 }
 498
 499 /**
 500  * intel_pipe_update_start() - start update of a set of display registers
 501  * @state: the atomic state
 502  * @crtc: the crtc
 503  *
 504  * Mark the start of an update to pipe registers that should be updated
 505  * atomically regarding vblank. If the next vblank will happens within
 506  * the next 100 us, this function waits until the vblank passes.
 507  *
 508  * After a successful call to this function, interrupts will be disabled
 509  * until a subsequent call to intel_pipe_update_end(). That is done to
 510  * avoid random delays.
 511  */
 512 void intel_pipe_update_start(struct intel_atomic_state *state,
 513                              struct intel_crtc *crtc)
 514 {
 515         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
 516         const struct intel_crtc_state *old_crtc_state =
 517                 intel_atomic_get_old_crtc_state(state, crtc);
 518         struct intel_crtc_state *new_crtc_state =
 519                 intel_atomic_get_new_crtc_state(state, crtc);
 520         struct intel_vblank_evade_ctx evade;
 521         int scanline;
 522
 523         intel_psr_lock(new_crtc_state);
 524
 525         if (new_crtc_state->do_async_flip) {
 526                 intel_crtc_prepare_vblank_event(new_crtc_state,
 527                                                 &crtc->flip_done_event);
 528                 return;
 529         }
 530
 531         if (intel_crtc_needs_vblank_work(new_crtc_state))
 532                 intel_crtc_vblank_work_init(new_crtc_state);
 533
 534         if (state->base.legacy_cursor_update) {
 535                 struct intel_plane *plane;
 536                 struct intel_plane_state *old_plane_state, *new_plane_state;
 537                 int i;
 538
 539                 for_each_oldnew_intel_plane_in_state(state, plane, old_plane_state,
 540                                                      new_plane_state, i) {
 541                         if (old_plane_state->uapi.crtc == &crtc->base)
 542                                 intel_plane_init_cursor_vblank_work(old_plane_state,
 543                                                                     new_plane_state);
 544                 }
 545         }
 546
 547         intel_vblank_evade_init(old_crtc_state, new_crtc_state, &evade);
 548
 549         if (drm_WARN_ON(&dev_priv->drm, drm_crtc_vblank_get(&crtc->base)))
 550                 goto irq_disable;
 551
 552         /*
 553          * Wait for psr to idle out after enabling the VBL interrupts
 554          * VBL interrupts will start the PSR exit and prevent a PSR
 555          * re-entry as well.
 556          */
 557         intel_psr_wait_for_idle_locked(new_crtc_state);
 558
 559         local_irq_disable();
 560
 561         crtc->debug.min_vbl = evade.min;
 562         crtc->debug.max_vbl = evade.max;
 563         trace_intel_pipe_update_start(crtc);
 564
 565         scanline = intel_vblank_evade(&evade);
 566
 567         drm_crtc_vblank_put(&crtc->base);
 568
 569         crtc->debug.scanline_start = scanline;
 570         crtc->debug.start_vbl_time = ktime_get();
 571         crtc->debug.start_vbl_count = intel_crtc_get_vblank_counter(crtc);
 572
 573         trace_intel_pipe_update_vblank_evaded(crtc);
 574         return;
 575
 576 irq_disable:
 577         local_irq_disable();
 578 }
 579
 580 #if IS_ENABLED(CONFIG_DRM_I915_DEBUG_VBLANK_EVADE)
 581 static void dbg_vblank_evade(struct intel_crtc *crtc, ktime_t end)
 582 {
 583         u64 delta = ktime_to_ns(ktime_sub(end, crtc->debug.start_vbl_time));
 584         unsigned int h;
 585
 586         h = ilog2(delta >> 9);
 587         if (h >= ARRAY_SIZE(crtc->debug.vbl.times))
 588                 h = ARRAY_SIZE(crtc->debug.vbl.times) - 1;
 589         crtc->debug.vbl.times[h]++;
 590
 591         crtc->debug.vbl.sum += delta;
 592         if (!crtc->debug.vbl.min || delta < crtc->debug.vbl.min)
 593                 crtc->debug.vbl.min = delta;
 594         if (delta > crtc->debug.vbl.max)
 595                 crtc->debug.vbl.max = delta;
 596
 597         if (delta > 1000 * VBLANK_EVASION_TIME_US) {
 598                 drm_dbg_kms(crtc->base.dev,
 599                             "Atomic update on pipe (%c) took %lld us, max time under evasion is %u us\n",
 600                             pipe_name(crtc->pipe),
 601                             div_u64(delta, 1000),
 602                             VBLANK_EVASION_TIME_US);
 603                 crtc->debug.vbl.over++;
 604         }
 605 }
 606 #else
 607 static void dbg_vblank_evade(struct intel_crtc *crtc, ktime_t end) {}
 608 #endif
 609
 610 void intel_crtc_arm_vblank_event(struct intel_crtc_state *crtc_state)
 611 {
 612         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
 613         unsigned long irqflags;
 614
 615         if (!crtc_state->uapi.event)
 616                 return;
 617
 618         drm_WARN_ON(crtc->base.dev, drm_crtc_vblank_get(&crtc->base) != 0);
 619
 620         spin_lock_irqsave(&crtc->base.dev->event_lock, irqflags);
 621         drm_crtc_arm_vblank_event(&crtc->base, crtc_state->uapi.event);
 622         spin_unlock_irqrestore(&crtc->base.dev->event_lock, irqflags);
 623
 624         crtc_state->uapi.event = NULL;
 625 }
 626
 627 void intel_crtc_prepare_vblank_event(struct intel_crtc_state *crtc_state,
 628                                      struct drm_pending_vblank_event **event)
 629 {
 630         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
 631         unsigned long irqflags;
 632
 633         spin_lock_irqsave(&crtc->base.dev->event_lock, irqflags);
 634         *event = crtc_state->uapi.event;
 635         spin_unlock_irqrestore(&crtc->base.dev->event_lock, irqflags);
 636
 637         crtc_state->uapi.event = NULL;
 638 }
 639
 640 /**
 641  * intel_pipe_update_end() - end update of a set of display registers
 642  * @state: the atomic state
 643  * @crtc: the crtc
 644  *
 645  * Mark the end of an update started with intel_pipe_update_start(). This
 646  * re-enables interrupts and verifies the update was actually completed
 647  * before a vblank.
 648  */
 649 void intel_pipe_update_end(struct intel_atomic_state *state,
 650                            struct intel_crtc *crtc)
 651 {
 652         struct intel_crtc_state *new_crtc_state =
 653                 intel_atomic_get_new_crtc_state(state, crtc);
 654         enum pipe pipe = crtc->pipe;
 655         int scanline_end = intel_get_crtc_scanline(crtc);
 656         u32 end_vbl_count = intel_crtc_get_vblank_counter(crtc);
 657         ktime_t end_vbl_time = ktime_get();
 658         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
 659
 660         if (new_crtc_state->do_async_flip)
 661                 goto out;
 662
 663         trace_intel_pipe_update_end(crtc, end_vbl_count, scanline_end);
 664
 665         /*
 666          * Incase of mipi dsi command mode, we need to set frame update
 667          * request for every commit.
 668          */
 669         if (DISPLAY_VER(dev_priv) >= 11 &&
 670             intel_crtc_has_type(new_crtc_state, INTEL_OUTPUT_DSI))
 671                 icl_dsi_frame_update(new_crtc_state);
 672
 673         /* We're still in the vblank-evade critical section, this can't race.
 674          * Would be slightly nice to just grab the vblank count and arm the
 675          * event outside of the critical section - the spinlock might spin for a
 676          * while ... */
 677         if (intel_crtc_needs_vblank_work(new_crtc_state)) {
 678                 drm_vblank_work_schedule(&new_crtc_state->vblank_work,
 679                                          drm_crtc_accurate_vblank_count(&crtc->base) + 1,
 680                                          false);
 681         } else {
 682                 intel_crtc_arm_vblank_event(new_crtc_state);
 683         }
 684
 685         if (state->base.legacy_cursor_update) {
 686                 struct intel_plane *plane;
 687                 struct intel_plane_state *old_plane_state;
 688                 int i;
 689
 690                 for_each_old_intel_plane_in_state(state, plane, old_plane_state, i) {
 691                         if (old_plane_state->uapi.crtc == &crtc->base &&
 692                             old_plane_state->unpin_work.vblank) {
 693                                 drm_vblank_work_schedule(&old_plane_state->unpin_work,
 694                                                          drm_crtc_accurate_vblank_count(&crtc->base) + 1,
 695                                                          false);
 696
 697                                 /* Remove plane from atomic state, cleanup/free is done from vblank worker. */
 698                                 memset(&state->base.planes[i], 0, sizeof(state->base.planes[i]));
 699                         }
 700                 }
 701         }
 702
 703         /*
 704          * Send VRR Push to terminate Vblank. If we are already in vblank
 705          * this has to be done _after_ sampling the frame counter, as
 706          * otherwise the push would immediately terminate the vblank and
 707          * the sampled frame counter would correspond to the next frame
 708          * instead of the current frame.
 709          *
 710          * There is a tiny race here (iff vblank evasion failed us) where
 711          * we might sample the frame counter just before vmax vblank start
 712          * but the push would be sent just after it. That would cause the
 713          * push to affect the next frame instead of the current frame,
 714          * which would cause the next frame to terminate already at vmin
 715          * vblank start instead of vmax vblank start.
 716          */
 717         intel_vrr_send_push(new_crtc_state);
 718
 719         local_irq_enable();
 720
 721         if (intel_vgpu_active(dev_priv))
 722                 goto out;
 723
 724         if (crtc->debug.start_vbl_count &&
 725             crtc->debug.start_vbl_count != end_vbl_count) {
 726                 drm_err(&dev_priv->drm,
 727                         "Atomic update failure on pipe %c (start=%u end=%u) time %lld us, min %d, max %d, scanline start %d, end %d\n",
 728                         pipe_name(pipe), crtc->debug.start_vbl_count,
 729                         end_vbl_count,
 730                         ktime_us_delta(end_vbl_time,
 731                                        crtc->debug.start_vbl_time),
 732                         crtc->debug.min_vbl, crtc->debug.max_vbl,
 733                         crtc->debug.scanline_start, scanline_end);
 734         }
 735
 736         dbg_vblank_evade(crtc, end_vbl_time);
 737
 738 out:
 739         intel_psr_unlock(new_crtc_state);
 740 }