drivers/gpu/drm/i915/display/intel_crtc.c

   1 // SPDX-License-Identifier: MIT
   2 /*
   3  * Copyright © 2020 Intel Corporation
   4  */
   5 #include <linux/kernel.h>
   6 #include <linux/pm_qos.h>
   7 #include <linux/slab.h>
   8
   9 #include <drm/drm_atomic_helper.h>
  10 #include <drm/drm_fourcc.h>
  11 #include <drm/drm_plane.h>
  12 #include <drm/drm_vblank.h>
  13 #include <drm/drm_vblank_work.h>
  14
  15 #include "i915_vgpu.h"
  16 #include "i9xx_plane.h"
  17 #include "icl_dsi.h"
  18 #include "intel_atomic.h"
  19 #include "intel_atomic_plane.h"
  20 #include "intel_color.h"
  21 #include "intel_crtc.h"
  22 #include "intel_cursor.h"
  23 #include "intel_display_debugfs.h"
  24 #include "intel_display_irq.h"
  25 #include "intel_display_trace.h"
  26 #include "intel_display_types.h"
  27 #include "intel_drrs.h"
  28 #include "intel_dsi.h"
  29 #include "intel_fifo_underrun.h"
  30 #include "intel_pipe_crc.h"
  31 #include "intel_psr.h"
  32 #include "intel_sprite.h"
  33 #include "intel_vblank.h"
  34 #include "intel_vrr.h"
  35 #include "skl_universal_plane.h"
  36
  37 static void assert_vblank_disabled(struct drm_crtc *crtc)
  38 {
  39         struct intel_display *display = to_intel_display(crtc->dev);
  40
  41         if (INTEL_DISPLAY_STATE_WARN(display, drm_crtc_vblank_get(crtc) == 0,
  42                                      "[CRTC:%d:%s] vblank assertion failure (expected off, current on)\n",
  43                                      crtc->base.id, crtc->name))
  44                 drm_crtc_vblank_put(crtc);
  45 }
  46
  47 struct intel_crtc *intel_first_crtc(struct drm_i915_private *i915)
  48 {
  49         return to_intel_crtc(drm_crtc_from_index(&i915->drm, 0));
  50 }
  51
  52 struct intel_crtc *intel_crtc_for_pipe(struct intel_display *display,
  53                                        enum pipe pipe)
  54 {
  55         struct intel_crtc *crtc;
  56
  57         for_each_intel_crtc(display->drm, crtc) {
  58                 if (crtc->pipe == pipe)
  59                         return crtc;
  60         }
  61
  62         return NULL;
  63 }
  64
  65 void intel_crtc_wait_for_next_vblank(struct intel_crtc *crtc)
  66 {
  67         drm_crtc_wait_one_vblank(&crtc->base);
  68 }
  69
  70 void intel_wait_for_vblank_if_active(struct drm_i915_private *i915,
  71                                      enum pipe pipe)
  72 {
  73         struct intel_display *display = &i915->display;
  74         struct intel_crtc *crtc = intel_crtc_for_pipe(display, pipe);
  75
  76         if (crtc->active)
  77                 intel_crtc_wait_for_next_vblank(crtc);
  78 }
  79
  80 u32 intel_crtc_get_vblank_counter(struct intel_crtc *crtc)
  81 {
  82         struct drm_vblank_crtc *vblank = drm_crtc_vblank_crtc(&crtc->base);
  83
  84         if (!crtc->active)
  85                 return 0;
  86
  87         if (!vblank->max_vblank_count)
  88                 return (u32)drm_crtc_accurate_vblank_count(&crtc->base);
  89
  90         return crtc->base.funcs->get_vblank_counter(&crtc->base);
  91 }
  92
  93 u32 intel_crtc_max_vblank_count(const struct intel_crtc_state *crtc_state)
  94 {
  95         struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
  96
  97         /*
  98          * From Gen 11, In case of dsi cmd mode, frame counter wouldnt
  99          * have updated at the beginning of TE, if we want to use
 100          * the hw counter, then we would find it updated in only
 101          * the next TE, hence switching to sw counter.
 102          */
 103         if (crtc_state->mode_flags & (I915_MODE_FLAG_DSI_USE_TE0 |
 104                                       I915_MODE_FLAG_DSI_USE_TE1))
 105                 return 0;
 106
 107         /*
 108          * On i965gm the hardware frame counter reads
 109          * zero when the TV encoder is enabled :(
 110          */
 111         if (IS_I965GM(dev_priv) &&
 112             (crtc_state->output_types & BIT(INTEL_OUTPUT_TVOUT)))
 113                 return 0;
 114
 115         if (DISPLAY_VER(dev_priv) >= 5 || IS_G4X(dev_priv))
 116                 return 0xffffffff; /* full 32 bit counter */
 117         else if (DISPLAY_VER(dev_priv) >= 3)
 118                 return 0xffffff; /* only 24 bits of frame count */
 119         else
 120                 return 0; /* Gen2 doesn't have a hardware frame counter */
 121 }
 122
 123 void intel_crtc_vblank_on(const struct intel_crtc_state *crtc_state)
 124 {
 125         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
 126
 127         crtc->block_dc_for_vblank = intel_psr_needs_block_dc_vblank(crtc_state);
 128
 129         assert_vblank_disabled(&crtc->base);
 130         drm_crtc_set_max_vblank_count(&crtc->base,
 131                                       intel_crtc_max_vblank_count(crtc_state));
 132         drm_crtc_vblank_on(&crtc->base);
 133
 134         /*
 135          * Should really happen exactly when we enable the pipe
 136          * but we want the frame counters in the trace, and that
 137          * requires vblank support on some platforms/outputs.
 138          */
 139         trace_intel_pipe_enable(crtc);
 140 }
 141
 142 void intel_crtc_vblank_off(const struct intel_crtc_state *crtc_state)
 143 {
 144         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
 145         struct intel_display *display = to_intel_display(crtc);
 146
 147         /*
 148          * Should really happen exactly when we disable the pipe
 149          * but we want the frame counters in the trace, and that
 150          * requires vblank support on some platforms/outputs.
 151          */
 152         trace_intel_pipe_disable(crtc);
 153
 154         drm_crtc_vblank_off(&crtc->base);
 155         assert_vblank_disabled(&crtc->base);
 156
 157         crtc->block_dc_for_vblank = false;
 158
 159         flush_work(&display->irq.vblank_dc_work);
 160 }
 161
 162 struct intel_crtc_state *intel_crtc_state_alloc(struct intel_crtc *crtc)
 163 {
 164         struct intel_crtc_state *crtc_state;
 165
 166         crtc_state = kmalloc(sizeof(*crtc_state), GFP_KERNEL);
 167
 168         if (crtc_state)
 169                 intel_crtc_state_reset(crtc_state, crtc);
 170
 171         return crtc_state;
 172 }
 173
 174 void intel_crtc_state_reset(struct intel_crtc_state *crtc_state,
 175                             struct intel_crtc *crtc)
 176 {
 177         memset(crtc_state, 0, sizeof(*crtc_state));
 178
 179         __drm_atomic_helper_crtc_state_reset(&crtc_state->uapi, &crtc->base);
 180
 181         crtc_state->cpu_transcoder = INVALID_TRANSCODER;
 182         crtc_state->master_transcoder = INVALID_TRANSCODER;
 183         crtc_state->hsw_workaround_pipe = INVALID_PIPE;
 184         crtc_state->scaler_state.scaler_id = -1;
 185         crtc_state->mst_master_transcoder = INVALID_TRANSCODER;
 186         crtc_state->max_link_bpp_x16 = INT_MAX;
 187 }
 188
 189 static struct intel_crtc *intel_crtc_alloc(void)
 190 {
 191         struct intel_crtc_state *crtc_state;
 192         struct intel_crtc *crtc;
 193
 194         crtc = kzalloc(sizeof(*crtc), GFP_KERNEL);
 195         if (!crtc)
 196                 return ERR_PTR(-ENOMEM);
 197
 198         crtc_state = intel_crtc_state_alloc(crtc);
 199         if (!crtc_state) {
 200                 kfree(crtc);
 201                 return ERR_PTR(-ENOMEM);
 202         }
 203
 204         crtc->base.state = &crtc_state->uapi;
 205         crtc->config = crtc_state;
 206
 207         return crtc;
 208 }
 209
 210 static void intel_crtc_free(struct intel_crtc *crtc)
 211 {
 212         intel_crtc_destroy_state(&crtc->base, crtc->base.state);
 213         kfree(crtc);
 214 }
 215
 216 static void intel_crtc_destroy(struct drm_crtc *_crtc)
 217 {
 218         struct intel_crtc *crtc = to_intel_crtc(_crtc);
 219
 220         cpu_latency_qos_remove_request(&crtc->vblank_pm_qos);
 221
 222         drm_crtc_cleanup(&crtc->base);
 223         kfree(crtc);
 224 }
 225
 226 static int intel_crtc_late_register(struct drm_crtc *crtc)
 227 {
 228         intel_crtc_debugfs_add(to_intel_crtc(crtc));
 229         return 0;
 230 }
 231
 232 #define INTEL_CRTC_FUNCS \
 233         .set_config = drm_atomic_helper_set_config, \
 234         .destroy = intel_crtc_destroy, \
 235         .page_flip = drm_atomic_helper_page_flip, \
 236         .atomic_duplicate_state = intel_crtc_duplicate_state, \
 237         .atomic_destroy_state = intel_crtc_destroy_state, \
 238         .set_crc_source = intel_crtc_set_crc_source, \
 239         .verify_crc_source = intel_crtc_verify_crc_source, \
 240         .get_crc_sources = intel_crtc_get_crc_sources, \
 241         .late_register = intel_crtc_late_register
 242
 243 static const struct drm_crtc_funcs bdw_crtc_funcs = {
 244         INTEL_CRTC_FUNCS,
 245
 246         .get_vblank_counter = g4x_get_vblank_counter,
 247         .enable_vblank = bdw_enable_vblank,
 248         .disable_vblank = bdw_disable_vblank,
 249         .get_vblank_timestamp = intel_crtc_get_vblank_timestamp,
 250 };
 251
 252 static const struct drm_crtc_funcs ilk_crtc_funcs = {
 253         INTEL_CRTC_FUNCS,
 254
 255         .get_vblank_counter = g4x_get_vblank_counter,
 256         .enable_vblank = ilk_enable_vblank,
 257         .disable_vblank = ilk_disable_vblank,
 258         .get_vblank_timestamp = intel_crtc_get_vblank_timestamp,
 259 };
 260
 261 static const struct drm_crtc_funcs g4x_crtc_funcs = {
 262         INTEL_CRTC_FUNCS,
 263
 264         .get_vblank_counter = g4x_get_vblank_counter,
 265         .enable_vblank = i965_enable_vblank,
 266         .disable_vblank = i965_disable_vblank,
 267         .get_vblank_timestamp = intel_crtc_get_vblank_timestamp,
 268 };
 269
 270 static const struct drm_crtc_funcs i965_crtc_funcs = {
 271         INTEL_CRTC_FUNCS,
 272
 273         .get_vblank_counter = i915_get_vblank_counter,
 274         .enable_vblank = i965_enable_vblank,
 275         .disable_vblank = i965_disable_vblank,
 276         .get_vblank_timestamp = intel_crtc_get_vblank_timestamp,
 277 };
 278
 279 static const struct drm_crtc_funcs i915gm_crtc_funcs = {
 280         INTEL_CRTC_FUNCS,
 281
 282         .get_vblank_counter = i915_get_vblank_counter,
 283         .enable_vblank = i915gm_enable_vblank,
 284         .disable_vblank = i915gm_disable_vblank,
 285         .get_vblank_timestamp = intel_crtc_get_vblank_timestamp,
 286 };
 287
 288 static const struct drm_crtc_funcs i915_crtc_funcs = {
 289         INTEL_CRTC_FUNCS,
 290
 291         .get_vblank_counter = i915_get_vblank_counter,
 292         .enable_vblank = i8xx_enable_vblank,
 293         .disable_vblank = i8xx_disable_vblank,
 294         .get_vblank_timestamp = intel_crtc_get_vblank_timestamp,
 295 };
 296
 297 static const struct drm_crtc_funcs i8xx_crtc_funcs = {
 298         INTEL_CRTC_FUNCS,
 299
 300         /* no hw vblank counter */
 301         .enable_vblank = i8xx_enable_vblank,
 302         .disable_vblank = i8xx_disable_vblank,
 303         .get_vblank_timestamp = intel_crtc_get_vblank_timestamp,
 304 };
 305
 306 int intel_crtc_init(struct drm_i915_private *dev_priv, enum pipe pipe)
 307 {
 308         struct intel_plane *primary, *cursor;
 309         const struct drm_crtc_funcs *funcs;
 310         struct intel_crtc *crtc;
 311         int sprite, ret;
 312
 313         crtc = intel_crtc_alloc();
 314         if (IS_ERR(crtc))
 315                 return PTR_ERR(crtc);
 316
 317         crtc->pipe = pipe;
 318         crtc->num_scalers = DISPLAY_RUNTIME_INFO(dev_priv)->num_scalers[pipe];
 319
 320         if (DISPLAY_VER(dev_priv) >= 9)
 321                 primary = skl_universal_plane_create(dev_priv, pipe, PLANE_1);
 322         else
 323                 primary = intel_primary_plane_create(dev_priv, pipe);
 324         if (IS_ERR(primary)) {
 325                 ret = PTR_ERR(primary);
 326                 goto fail;
 327         }
 328         crtc->plane_ids_mask |= BIT(primary->id);
 329
 330         intel_init_fifo_underrun_reporting(dev_priv, crtc, false);
 331
 332         for_each_sprite(dev_priv, pipe, sprite) {
 333                 struct intel_plane *plane;
 334
 335                 if (DISPLAY_VER(dev_priv) >= 9)
 336                         plane = skl_universal_plane_create(dev_priv, pipe, PLANE_2 + sprite);
 337                 else
 338                         plane = intel_sprite_plane_create(dev_priv, pipe, sprite);
 339                 if (IS_ERR(plane)) {
 340                         ret = PTR_ERR(plane);
 341                         goto fail;
 342                 }
 343                 crtc->plane_ids_mask |= BIT(plane->id);
 344         }
 345
 346         cursor = intel_cursor_plane_create(dev_priv, pipe);
 347         if (IS_ERR(cursor)) {
 348                 ret = PTR_ERR(cursor);
 349                 goto fail;
 350         }
 351         crtc->plane_ids_mask |= BIT(cursor->id);
 352
 353         if (HAS_GMCH(dev_priv)) {
 354                 if (IS_CHERRYVIEW(dev_priv) ||
 355                     IS_VALLEYVIEW(dev_priv) || IS_G4X(dev_priv))
 356                         funcs = &g4x_crtc_funcs;
 357                 else if (DISPLAY_VER(dev_priv) == 4)
 358                         funcs = &i965_crtc_funcs;
 359                 else if (IS_I945GM(dev_priv) || IS_I915GM(dev_priv))
 360                         funcs = &i915gm_crtc_funcs;
 361                 else if (DISPLAY_VER(dev_priv) == 3)
 362                         funcs = &i915_crtc_funcs;
 363                 else
 364                         funcs = &i8xx_crtc_funcs;
 365         } else {
 366                 if (DISPLAY_VER(dev_priv) >= 8)
 367                         funcs = &bdw_crtc_funcs;
 368                 else
 369                         funcs = &ilk_crtc_funcs;
 370         }
 371
 372         ret = drm_crtc_init_with_planes(&dev_priv->drm, &crtc->base,
 373                                         &primary->base, &cursor->base,
 374                                         funcs, "pipe %c", pipe_name(pipe));
 375         if (ret)
 376                 goto fail;
 377
 378         if (DISPLAY_VER(dev_priv) >= 11)
 379                 drm_crtc_create_scaling_filter_property(&crtc->base,
 380                                                 BIT(DRM_SCALING_FILTER_DEFAULT) |
 381                                                 BIT(DRM_SCALING_FILTER_NEAREST_NEIGHBOR));
 382
 383         intel_color_crtc_init(crtc);
 384         intel_drrs_crtc_init(crtc);
 385         intel_crtc_crc_init(crtc);
 386
 387         cpu_latency_qos_add_request(&crtc->vblank_pm_qos, PM_QOS_DEFAULT_VALUE);
 388
 389         drm_WARN_ON(&dev_priv->drm, drm_crtc_index(&crtc->base) != crtc->pipe);
 390
 391         return 0;
 392
 393 fail:
 394         intel_crtc_free(crtc);
 395
 396         return ret;
 397 }
 398
 399 int intel_crtc_get_pipe_from_crtc_id_ioctl(struct drm_device *dev, void *data,
 400                                            struct drm_file *file)
 401 {
 402         struct drm_i915_get_pipe_from_crtc_id *pipe_from_crtc_id = data;
 403         struct drm_crtc *drm_crtc;
 404         struct intel_crtc *crtc;
 405
 406         drm_crtc = drm_crtc_find(dev, file, pipe_from_crtc_id->crtc_id);
 407         if (!drm_crtc)
 408                 return -ENOENT;
 409
 410         crtc = to_intel_crtc(drm_crtc);
 411         pipe_from_crtc_id->pipe = crtc->pipe;
 412
 413         return 0;
 414 }
 415
 416 static bool intel_crtc_needs_vblank_work(const struct intel_crtc_state *crtc_state)
 417 {
 418         return crtc_state->hw.active &&
 419                 !crtc_state->preload_luts &&
 420                 !intel_crtc_needs_modeset(crtc_state) &&
 421                 intel_crtc_needs_color_update(crtc_state) &&
 422                 !intel_color_uses_dsb(crtc_state) &&
 423                 !crtc_state->use_dsb;
 424 }
 425
 426 static void intel_crtc_vblank_work(struct kthread_work *base)
 427 {
 428         struct drm_vblank_work *work = to_drm_vblank_work(base);
 429         struct intel_crtc_state *crtc_state =
 430                 container_of(work, typeof(*crtc_state), vblank_work);
 431         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
 432
 433         trace_intel_crtc_vblank_work_start(crtc);
 434
 435         intel_color_load_luts(crtc_state);
 436
 437         if (crtc_state->uapi.event) {
 438                 spin_lock_irq(&crtc->base.dev->event_lock);
 439                 drm_crtc_send_vblank_event(&crtc->base, crtc_state->uapi.event);
 440                 spin_unlock_irq(&crtc->base.dev->event_lock);
 441                 crtc_state->uapi.event = NULL;
 442         }
 443
 444         trace_intel_crtc_vblank_work_end(crtc);
 445 }
 446
 447 static void intel_crtc_vblank_work_init(struct intel_crtc_state *crtc_state)
 448 {
 449         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
 450
 451         drm_vblank_work_init(&crtc_state->vblank_work, &crtc->base,
 452                              intel_crtc_vblank_work);
 453         /*
 454          * Interrupt latency is critical for getting the vblank
 455          * work executed as early as possible during the vblank.
 456          */
 457         cpu_latency_qos_update_request(&crtc->vblank_pm_qos, 0);
 458 }
 459
 460 void intel_wait_for_vblank_workers(struct intel_atomic_state *state)
 461 {
 462         struct intel_crtc_state *crtc_state;
 463         struct intel_crtc *crtc;
 464         int i;
 465
 466         for_each_new_intel_crtc_in_state(state, crtc, crtc_state, i) {
 467                 if (!intel_crtc_needs_vblank_work(crtc_state))
 468                         continue;
 469
 470                 drm_vblank_work_flush(&crtc_state->vblank_work);
 471                 cpu_latency_qos_update_request(&crtc->vblank_pm_qos,
 472                                                PM_QOS_DEFAULT_VALUE);
 473         }
 474 }
 475
 476 int intel_usecs_to_scanlines(const struct drm_display_mode *adjusted_mode,
 477                              int usecs)
 478 {
 479         /* paranoia */
 480         if (!adjusted_mode->crtc_htotal)
 481                 return 1;
 482
 483         return DIV_ROUND_UP_ULL(mul_u32_u32(usecs, adjusted_mode->crtc_clock),
 484                                 1000 * adjusted_mode->crtc_htotal);
 485 }
 486
 487 int intel_scanlines_to_usecs(const struct drm_display_mode *adjusted_mode,
 488                              int scanlines)
 489 {
 490         /* paranoia */
 491         if (!adjusted_mode->crtc_clock)
 492                 return 1;
 493
 494         return DIV_ROUND_UP_ULL(mul_u32_u32(scanlines, adjusted_mode->crtc_htotal * 1000),
 495                                 adjusted_mode->crtc_clock);
 496 }
 497
 498 /**
 499  * intel_pipe_update_start() - start update of a set of display registers
 500  * @state: the atomic state
 501  * @crtc: the crtc
 502  *
 503  * Mark the start of an update to pipe registers that should be updated
 504  * atomically regarding vblank. If the next vblank will happens within
 505  * the next 100 us, this function waits until the vblank passes.
 506  *
 507  * After a successful call to this function, interrupts will be disabled
 508  * until a subsequent call to intel_pipe_update_end(). That is done to
 509  * avoid random delays.
 510  */
 511 void intel_pipe_update_start(struct intel_atomic_state *state,
 512                              struct intel_crtc *crtc)
 513 {
 514         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
 515         const struct intel_crtc_state *old_crtc_state =
 516                 intel_atomic_get_old_crtc_state(state, crtc);
 517         struct intel_crtc_state *new_crtc_state =
 518                 intel_atomic_get_new_crtc_state(state, crtc);
 519         struct intel_vblank_evade_ctx evade;
 520         int scanline;
 521
 522         intel_psr_lock(new_crtc_state);
 523
 524         if (new_crtc_state->do_async_flip) {
 525                 intel_crtc_prepare_vblank_event(new_crtc_state,
 526                                                 &crtc->flip_done_event);
 527                 return;
 528         }
 529
 530         if (intel_crtc_needs_vblank_work(new_crtc_state))
 531                 intel_crtc_vblank_work_init(new_crtc_state);
 532
 533         if (state->base.legacy_cursor_update) {
 534                 struct intel_plane *plane;
 535                 struct intel_plane_state *old_plane_state, *new_plane_state;
 536                 int i;
 537
 538                 for_each_oldnew_intel_plane_in_state(state, plane, old_plane_state,
 539                                                      new_plane_state, i) {
 540                         if (old_plane_state->uapi.crtc == &crtc->base)
 541                                 intel_plane_init_cursor_vblank_work(old_plane_state,
 542                                                                     new_plane_state);
 543                 }
 544         }
 545
 546         intel_vblank_evade_init(old_crtc_state, new_crtc_state, &evade);
 547
 548         if (drm_WARN_ON(&dev_priv->drm, drm_crtc_vblank_get(&crtc->base)))
 549                 goto irq_disable;
 550
 551         /*
 552          * Wait for psr to idle out after enabling the VBL interrupts
 553          * VBL interrupts will start the PSR exit and prevent a PSR
 554          * re-entry as well.
 555          */
 556         intel_psr_wait_for_idle_locked(new_crtc_state);
 557
 558         local_irq_disable();
 559
 560         crtc->debug.min_vbl = evade.min;
 561         crtc->debug.max_vbl = evade.max;
 562         trace_intel_pipe_update_start(crtc);
 563
 564         scanline = intel_vblank_evade(&evade);
 565
 566         drm_crtc_vblank_put(&crtc->base);
 567
 568         crtc->debug.scanline_start = scanline;
 569         crtc->debug.start_vbl_time = ktime_get();
 570         crtc->debug.start_vbl_count = intel_crtc_get_vblank_counter(crtc);
 571
 572         trace_intel_pipe_update_vblank_evaded(crtc);
 573         return;
 574
 575 irq_disable:
 576         local_irq_disable();
 577 }
 578
 579 #if IS_ENABLED(CONFIG_DRM_I915_DEBUG_VBLANK_EVADE)
 580 static void dbg_vblank_evade(struct intel_crtc *crtc, ktime_t end)
 581 {
 582         u64 delta = ktime_to_ns(ktime_sub(end, crtc->debug.start_vbl_time));
 583         unsigned int h;
 584
 585         h = ilog2(delta >> 9);
 586         if (h >= ARRAY_SIZE(crtc->debug.vbl.times))
 587                 h = ARRAY_SIZE(crtc->debug.vbl.times) - 1;
 588         crtc->debug.vbl.times[h]++;
 589
 590         crtc->debug.vbl.sum += delta;
 591         if (!crtc->debug.vbl.min || delta < crtc->debug.vbl.min)
 592                 crtc->debug.vbl.min = delta;
 593         if (delta > crtc->debug.vbl.max)
 594                 crtc->debug.vbl.max = delta;
 595
 596         if (delta > 1000 * VBLANK_EVASION_TIME_US) {
 597                 drm_dbg_kms(crtc->base.dev,
 598                             "Atomic update on pipe (%c) took %lld us, max time under evasion is %u us\n",
 599                             pipe_name(crtc->pipe),
 600                             div_u64(delta, 1000),
 601                             VBLANK_EVASION_TIME_US);
 602                 crtc->debug.vbl.over++;
 603         }
 604 }
 605 #else
 606 static void dbg_vblank_evade(struct intel_crtc *crtc, ktime_t end) {}
 607 #endif
 608
 609 void intel_crtc_arm_vblank_event(struct intel_crtc_state *crtc_state)
 610 {
 611         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
 612         unsigned long irqflags;
 613
 614         if (!crtc_state->uapi.event)
 615                 return;
 616
 617         drm_WARN_ON(crtc->base.dev, drm_crtc_vblank_get(&crtc->base) != 0);
 618
 619         spin_lock_irqsave(&crtc->base.dev->event_lock, irqflags);
 620         drm_crtc_arm_vblank_event(&crtc->base, crtc_state->uapi.event);
 621         spin_unlock_irqrestore(&crtc->base.dev->event_lock, irqflags);
 622
 623         crtc_state->uapi.event = NULL;
 624 }
 625
 626 void intel_crtc_prepare_vblank_event(struct intel_crtc_state *crtc_state,
 627                                      struct drm_pending_vblank_event **event)
 628 {
 629         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
 630         unsigned long irqflags;
 631
 632         spin_lock_irqsave(&crtc->base.dev->event_lock, irqflags);
 633         *event = crtc_state->uapi.event;
 634         spin_unlock_irqrestore(&crtc->base.dev->event_lock, irqflags);
 635
 636         crtc_state->uapi.event = NULL;
 637 }
 638
 639 /**
 640  * intel_pipe_update_end() - end update of a set of display registers
 641  * @state: the atomic state
 642  * @crtc: the crtc
 643  *
 644  * Mark the end of an update started with intel_pipe_update_start(). This
 645  * re-enables interrupts and verifies the update was actually completed
 646  * before a vblank.
 647  */
 648 void intel_pipe_update_end(struct intel_atomic_state *state,
 649                            struct intel_crtc *crtc)
 650 {
 651         struct intel_crtc_state *new_crtc_state =
 652                 intel_atomic_get_new_crtc_state(state, crtc);
 653         enum pipe pipe = crtc->pipe;
 654         int scanline_end = intel_get_crtc_scanline(crtc);
 655         u32 end_vbl_count = intel_crtc_get_vblank_counter(crtc);
 656         ktime_t end_vbl_time = ktime_get();
 657         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
 658
 659         if (new_crtc_state->do_async_flip)
 660                 goto out;
 661
 662         trace_intel_pipe_update_end(crtc, end_vbl_count, scanline_end);
 663
 664         /*
 665          * Incase of mipi dsi command mode, we need to set frame update
 666          * request for every commit.
 667          */
 668         if (DISPLAY_VER(dev_priv) >= 11 &&
 669             intel_crtc_has_type(new_crtc_state, INTEL_OUTPUT_DSI))
 670                 icl_dsi_frame_update(new_crtc_state);
 671
 672         /* We're still in the vblank-evade critical section, this can't race.
 673          * Would be slightly nice to just grab the vblank count and arm the
 674          * event outside of the critical section - the spinlock might spin for a
 675          * while ... */
 676         if (intel_crtc_needs_vblank_work(new_crtc_state)) {
 677                 drm_vblank_work_schedule(&new_crtc_state->vblank_work,
 678                                          drm_crtc_accurate_vblank_count(&crtc->base) + 1,
 679                                          false);
 680         } else {
 681                 intel_crtc_arm_vblank_event(new_crtc_state);
 682         }
 683
 684         if (state->base.legacy_cursor_update) {
 685                 struct intel_plane *plane;
 686                 struct intel_plane_state *old_plane_state;
 687                 int i;
 688
 689                 for_each_old_intel_plane_in_state(state, plane, old_plane_state, i) {
 690                         if (old_plane_state->uapi.crtc == &crtc->base &&
 691                             old_plane_state->unpin_work.vblank) {
 692                                 drm_vblank_work_schedule(&old_plane_state->unpin_work,
 693                                                          drm_crtc_accurate_vblank_count(&crtc->base) + 1,
 694                                                          false);
 695
 696                                 /* Remove plane from atomic state, cleanup/free is done from vblank worker. */
 697                                 memset(&state->base.planes[i], 0, sizeof(state->base.planes[i]));
 698                         }
 699                 }
 700         }
 701
 702         /*
 703          * Send VRR Push to terminate Vblank. If we are already in vblank
 704          * this has to be done _after_ sampling the frame counter, as
 705          * otherwise the push would immediately terminate the vblank and
 706          * the sampled frame counter would correspond to the next frame
 707          * instead of the current frame.
 708          *
 709          * There is a tiny race here (iff vblank evasion failed us) where
 710          * we might sample the frame counter just before vmax vblank start
 711          * but the push would be sent just after it. That would cause the
 712          * push to affect the next frame instead of the current frame,
 713          * which would cause the next frame to terminate already at vmin
 714          * vblank start instead of vmax vblank start.
 715          */
 716         intel_vrr_send_push(new_crtc_state);
 717
 718         local_irq_enable();
 719
 720         if (intel_vgpu_active(dev_priv))
 721                 goto out;
 722
 723         if (crtc->debug.start_vbl_count &&
 724             crtc->debug.start_vbl_count != end_vbl_count) {
 725                 drm_err(&dev_priv->drm,
 726                         "Atomic update failure on pipe %c (start=%u end=%u) time %lld us, min %d, max %d, scanline start %d, end %d\n",
 727                         pipe_name(pipe), crtc->debug.start_vbl_count,
 728                         end_vbl_count,
 729                         ktime_us_delta(end_vbl_time,
 730                                        crtc->debug.start_vbl_time),
 731                         crtc->debug.min_vbl, crtc->debug.max_vbl,
 732                         crtc->debug.scanline_start, scanline_end);
 733         }
 734
 735         dbg_vblank_evade(crtc, end_vbl_time);
 736
 737 out:
 738         intel_psr_unlock(new_crtc_state);
 739 }