drivers/gpu/drm/i915/display/intel_crtc.c

   1 // SPDX-License-Identifier: MIT
   2 /*
   3  * Copyright © 2020 Intel Corporation
   4  */
   5 #include <linux/kernel.h>
   6 #include <linux/pm_qos.h>
   7 #include <linux/slab.h>
   8
   9 #include <drm/drm_atomic_helper.h>
  10 #include <drm/drm_fourcc.h>
  11 #include <drm/drm_plane.h>
  12 #include <drm/drm_vblank_work.h>
  13
  14 #include "i915_vgpu.h"
  15 #include "i9xx_plane.h"
  16 #include "icl_dsi.h"
  17 #include "intel_atomic.h"
  18 #include "intel_atomic_plane.h"
  19 #include "intel_color.h"
  20 #include "intel_crtc.h"
  21 #include "intel_cursor.h"
  22 #include "intel_display_debugfs.h"
  23 #include "intel_display_irq.h"
  24 #include "intel_display_trace.h"
  25 #include "intel_display_types.h"
  26 #include "intel_drrs.h"
  27 #include "intel_dsb.h"
  28 #include "intel_dsi.h"
  29 #include "intel_fifo_underrun.h"
  30 #include "intel_pipe_crc.h"
  31 #include "intel_psr.h"
  32 #include "intel_sprite.h"
  33 #include "intel_vblank.h"
  34 #include "intel_vrr.h"
  35 #include "skl_universal_plane.h"
  36
  37 static void assert_vblank_disabled(struct drm_crtc *crtc)
  38 {
  39         struct drm_i915_private *i915 = to_i915(crtc->dev);
  40
  41         if (I915_STATE_WARN(i915, drm_crtc_vblank_get(crtc) == 0,
  42                             "[CRTC:%d:%s] vblank assertion failure (expected off, current on)\n",
  43                             crtc->base.id, crtc->name))
  44                 drm_crtc_vblank_put(crtc);
  45 }
  46
  47 struct intel_crtc *intel_first_crtc(struct drm_i915_private *i915)
  48 {
  49         return to_intel_crtc(drm_crtc_from_index(&i915->drm, 0));
  50 }
  51
  52 struct intel_crtc *intel_crtc_for_pipe(struct drm_i915_private *i915,
  53                                        enum pipe pipe)
  54 {
  55         struct intel_crtc *crtc;
  56
  57         for_each_intel_crtc(&i915->drm, crtc) {
  58                 if (crtc->pipe == pipe)
  59                         return crtc;
  60         }
  61
  62         return NULL;
  63 }
  64
  65 void intel_crtc_wait_for_next_vblank(struct intel_crtc *crtc)
  66 {
  67         drm_crtc_wait_one_vblank(&crtc->base);
  68 }
  69
  70 void intel_wait_for_vblank_if_active(struct drm_i915_private *i915,
  71                                      enum pipe pipe)
  72 {
  73         struct intel_crtc *crtc = intel_crtc_for_pipe(i915, pipe);
  74
  75         if (crtc->active)
  76                 intel_crtc_wait_for_next_vblank(crtc);
  77 }
  78
  79 u32 intel_crtc_get_vblank_counter(struct intel_crtc *crtc)
  80 {
  81         struct drm_device *dev = crtc->base.dev;
  82         struct drm_vblank_crtc *vblank = &dev->vblank[drm_crtc_index(&crtc->base)];
  83
  84         if (!crtc->active)
  85                 return 0;
  86
  87         if (!vblank->max_vblank_count)
  88                 return (u32)drm_crtc_accurate_vblank_count(&crtc->base);
  89
  90         return crtc->base.funcs->get_vblank_counter(&crtc->base);
  91 }
  92
  93 u32 intel_crtc_max_vblank_count(const struct intel_crtc_state *crtc_state)
  94 {
  95         struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
  96
  97         /*
  98          * From Gen 11, In case of dsi cmd mode, frame counter wouldnt
  99          * have updated at the beginning of TE, if we want to use
 100          * the hw counter, then we would find it updated in only
 101          * the next TE, hence switching to sw counter.
 102          */
 103         if (crtc_state->mode_flags & (I915_MODE_FLAG_DSI_USE_TE0 |
 104                                       I915_MODE_FLAG_DSI_USE_TE1))
 105                 return 0;
 106
 107         /*
 108          * On i965gm the hardware frame counter reads
 109          * zero when the TV encoder is enabled :(
 110          */
 111         if (IS_I965GM(dev_priv) &&
 112             (crtc_state->output_types & BIT(INTEL_OUTPUT_TVOUT)))
 113                 return 0;
 114
 115         if (DISPLAY_VER(dev_priv) >= 5 || IS_G4X(dev_priv))
 116                 return 0xffffffff; /* full 32 bit counter */
 117         else if (DISPLAY_VER(dev_priv) >= 3)
 118                 return 0xffffff; /* only 24 bits of frame count */
 119         else
 120                 return 0; /* Gen2 doesn't have a hardware frame counter */
 121 }
 122
 123 void intel_crtc_vblank_on(const struct intel_crtc_state *crtc_state)
 124 {
 125         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
 126
 127         assert_vblank_disabled(&crtc->base);
 128         drm_crtc_set_max_vblank_count(&crtc->base,
 129                                       intel_crtc_max_vblank_count(crtc_state));
 130         drm_crtc_vblank_on(&crtc->base);
 131
 132         /*
 133          * Should really happen exactly when we enable the pipe
 134          * but we want the frame counters in the trace, and that
 135          * requires vblank support on some platforms/outputs.
 136          */
 137         trace_intel_pipe_enable(crtc);
 138 }
 139
 140 void intel_crtc_vblank_off(const struct intel_crtc_state *crtc_state)
 141 {
 142         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
 143
 144         /*
 145          * Should really happen exactly when we disable the pipe
 146          * but we want the frame counters in the trace, and that
 147          * requires vblank support on some platforms/outputs.
 148          */
 149         trace_intel_pipe_disable(crtc);
 150
 151         drm_crtc_vblank_off(&crtc->base);
 152         assert_vblank_disabled(&crtc->base);
 153 }
 154
 155 struct intel_crtc_state *intel_crtc_state_alloc(struct intel_crtc *crtc)
 156 {
 157         struct intel_crtc_state *crtc_state;
 158
 159         crtc_state = kmalloc(sizeof(*crtc_state), GFP_KERNEL);
 160
 161         if (crtc_state)
 162                 intel_crtc_state_reset(crtc_state, crtc);
 163
 164         return crtc_state;
 165 }
 166
 167 void intel_crtc_state_reset(struct intel_crtc_state *crtc_state,
 168                             struct intel_crtc *crtc)
 169 {
 170         memset(crtc_state, 0, sizeof(*crtc_state));
 171
 172         __drm_atomic_helper_crtc_state_reset(&crtc_state->uapi, &crtc->base);
 173
 174         crtc_state->cpu_transcoder = INVALID_TRANSCODER;
 175         crtc_state->master_transcoder = INVALID_TRANSCODER;
 176         crtc_state->hsw_workaround_pipe = INVALID_PIPE;
 177         crtc_state->scaler_state.scaler_id = -1;
 178         crtc_state->mst_master_transcoder = INVALID_TRANSCODER;
 179         crtc_state->max_link_bpp_x16 = INT_MAX;
 180 }
 181
 182 static struct intel_crtc *intel_crtc_alloc(void)
 183 {
 184         struct intel_crtc_state *crtc_state;
 185         struct intel_crtc *crtc;
 186
 187         crtc = kzalloc(sizeof(*crtc), GFP_KERNEL);
 188         if (!crtc)
 189                 return ERR_PTR(-ENOMEM);
 190
 191         crtc_state = intel_crtc_state_alloc(crtc);
 192         if (!crtc_state) {
 193                 kfree(crtc);
 194                 return ERR_PTR(-ENOMEM);
 195         }
 196
 197         crtc->base.state = &crtc_state->uapi;
 198         crtc->config = crtc_state;
 199
 200         return crtc;
 201 }
 202
 203 static void intel_crtc_free(struct intel_crtc *crtc)
 204 {
 205         intel_crtc_destroy_state(&crtc->base, crtc->base.state);
 206         kfree(crtc);
 207 }
 208
 209 static void intel_crtc_destroy(struct drm_crtc *_crtc)
 210 {
 211         struct intel_crtc *crtc = to_intel_crtc(_crtc);
 212
 213         cpu_latency_qos_remove_request(&crtc->vblank_pm_qos);
 214
 215         drm_crtc_cleanup(&crtc->base);
 216         kfree(crtc);
 217 }
 218
 219 static int intel_crtc_late_register(struct drm_crtc *crtc)
 220 {
 221         intel_crtc_debugfs_add(to_intel_crtc(crtc));
 222         return 0;
 223 }
 224
 225 #define INTEL_CRTC_FUNCS \
 226         .set_config = drm_atomic_helper_set_config, \
 227         .destroy = intel_crtc_destroy, \
 228         .page_flip = drm_atomic_helper_page_flip, \
 229         .atomic_duplicate_state = intel_crtc_duplicate_state, \
 230         .atomic_destroy_state = intel_crtc_destroy_state, \
 231         .set_crc_source = intel_crtc_set_crc_source, \
 232         .verify_crc_source = intel_crtc_verify_crc_source, \
 233         .get_crc_sources = intel_crtc_get_crc_sources, \
 234         .late_register = intel_crtc_late_register
 235
 236 static const struct drm_crtc_funcs bdw_crtc_funcs = {
 237         INTEL_CRTC_FUNCS,
 238
 239         .get_vblank_counter = g4x_get_vblank_counter,
 240         .enable_vblank = bdw_enable_vblank,
 241         .disable_vblank = bdw_disable_vblank,
 242         .get_vblank_timestamp = intel_crtc_get_vblank_timestamp,
 243 };
 244
 245 static const struct drm_crtc_funcs ilk_crtc_funcs = {
 246         INTEL_CRTC_FUNCS,
 247
 248         .get_vblank_counter = g4x_get_vblank_counter,
 249         .enable_vblank = ilk_enable_vblank,
 250         .disable_vblank = ilk_disable_vblank,
 251         .get_vblank_timestamp = intel_crtc_get_vblank_timestamp,
 252 };
 253
 254 static const struct drm_crtc_funcs g4x_crtc_funcs = {
 255         INTEL_CRTC_FUNCS,
 256
 257         .get_vblank_counter = g4x_get_vblank_counter,
 258         .enable_vblank = i965_enable_vblank,
 259         .disable_vblank = i965_disable_vblank,
 260         .get_vblank_timestamp = intel_crtc_get_vblank_timestamp,
 261 };
 262
 263 static const struct drm_crtc_funcs i965_crtc_funcs = {
 264         INTEL_CRTC_FUNCS,
 265
 266         .get_vblank_counter = i915_get_vblank_counter,
 267         .enable_vblank = i965_enable_vblank,
 268         .disable_vblank = i965_disable_vblank,
 269         .get_vblank_timestamp = intel_crtc_get_vblank_timestamp,
 270 };
 271
 272 static const struct drm_crtc_funcs i915gm_crtc_funcs = {
 273         INTEL_CRTC_FUNCS,
 274
 275         .get_vblank_counter = i915_get_vblank_counter,
 276         .enable_vblank = i915gm_enable_vblank,
 277         .disable_vblank = i915gm_disable_vblank,
 278         .get_vblank_timestamp = intel_crtc_get_vblank_timestamp,
 279 };
 280
 281 static const struct drm_crtc_funcs i915_crtc_funcs = {
 282         INTEL_CRTC_FUNCS,
 283
 284         .get_vblank_counter = i915_get_vblank_counter,
 285         .enable_vblank = i8xx_enable_vblank,
 286         .disable_vblank = i8xx_disable_vblank,
 287         .get_vblank_timestamp = intel_crtc_get_vblank_timestamp,
 288 };
 289
 290 static const struct drm_crtc_funcs i8xx_crtc_funcs = {
 291         INTEL_CRTC_FUNCS,
 292
 293         /* no hw vblank counter */
 294         .enable_vblank = i8xx_enable_vblank,
 295         .disable_vblank = i8xx_disable_vblank,
 296         .get_vblank_timestamp = intel_crtc_get_vblank_timestamp,
 297 };
 298
 299 int intel_crtc_init(struct drm_i915_private *dev_priv, enum pipe pipe)
 300 {
 301         struct intel_plane *primary, *cursor;
 302         const struct drm_crtc_funcs *funcs;
 303         struct intel_crtc *crtc;
 304         int sprite, ret;
 305
 306         crtc = intel_crtc_alloc();
 307         if (IS_ERR(crtc))
 308                 return PTR_ERR(crtc);
 309
 310         crtc->pipe = pipe;
 311         crtc->num_scalers = DISPLAY_RUNTIME_INFO(dev_priv)->num_scalers[pipe];
 312
 313         if (DISPLAY_VER(dev_priv) >= 9)
 314                 primary = skl_universal_plane_create(dev_priv, pipe,
 315                                                      PLANE_PRIMARY);
 316         else
 317                 primary = intel_primary_plane_create(dev_priv, pipe);
 318         if (IS_ERR(primary)) {
 319                 ret = PTR_ERR(primary);
 320                 goto fail;
 321         }
 322         crtc->plane_ids_mask |= BIT(primary->id);
 323
 324         intel_init_fifo_underrun_reporting(dev_priv, crtc, false);
 325
 326         for_each_sprite(dev_priv, pipe, sprite) {
 327                 struct intel_plane *plane;
 328
 329                 if (DISPLAY_VER(dev_priv) >= 9)
 330                         plane = skl_universal_plane_create(dev_priv, pipe,
 331                                                            PLANE_SPRITE0 + sprite);
 332                 else
 333                         plane = intel_sprite_plane_create(dev_priv, pipe, sprite);
 334                 if (IS_ERR(plane)) {
 335                         ret = PTR_ERR(plane);
 336                         goto fail;
 337                 }
 338                 crtc->plane_ids_mask |= BIT(plane->id);
 339         }
 340
 341         cursor = intel_cursor_plane_create(dev_priv, pipe);
 342         if (IS_ERR(cursor)) {
 343                 ret = PTR_ERR(cursor);
 344                 goto fail;
 345         }
 346         crtc->plane_ids_mask |= BIT(cursor->id);
 347
 348         if (HAS_GMCH(dev_priv)) {
 349                 if (IS_CHERRYVIEW(dev_priv) ||
 350                     IS_VALLEYVIEW(dev_priv) || IS_G4X(dev_priv))
 351                         funcs = &g4x_crtc_funcs;
 352                 else if (DISPLAY_VER(dev_priv) == 4)
 353                         funcs = &i965_crtc_funcs;
 354                 else if (IS_I945GM(dev_priv) || IS_I915GM(dev_priv))
 355                         funcs = &i915gm_crtc_funcs;
 356                 else if (DISPLAY_VER(dev_priv) == 3)
 357                         funcs = &i915_crtc_funcs;
 358                 else
 359                         funcs = &i8xx_crtc_funcs;
 360         } else {
 361                 if (DISPLAY_VER(dev_priv) >= 8)
 362                         funcs = &bdw_crtc_funcs;
 363                 else
 364                         funcs = &ilk_crtc_funcs;
 365         }
 366
 367         ret = drm_crtc_init_with_planes(&dev_priv->drm, &crtc->base,
 368                                         &primary->base, &cursor->base,
 369                                         funcs, "pipe %c", pipe_name(pipe));
 370         if (ret)
 371                 goto fail;
 372
 373         if (DISPLAY_VER(dev_priv) >= 11)
 374                 drm_crtc_create_scaling_filter_property(&crtc->base,
 375                                                 BIT(DRM_SCALING_FILTER_DEFAULT) |
 376                                                 BIT(DRM_SCALING_FILTER_NEAREST_NEIGHBOR));
 377
 378         intel_color_crtc_init(crtc);
 379         intel_drrs_crtc_init(crtc);
 380         intel_crtc_crc_init(crtc);
 381
 382         cpu_latency_qos_add_request(&crtc->vblank_pm_qos, PM_QOS_DEFAULT_VALUE);
 383
 384         drm_WARN_ON(&dev_priv->drm, drm_crtc_index(&crtc->base) != crtc->pipe);
 385
 386         return 0;
 387
 388 fail:
 389         intel_crtc_free(crtc);
 390
 391         return ret;
 392 }
 393
 394 static bool intel_crtc_needs_vblank_work(const struct intel_crtc_state *crtc_state)
 395 {
 396         return crtc_state->hw.active &&
 397                 !intel_crtc_needs_modeset(crtc_state) &&
 398                 !crtc_state->preload_luts &&
 399                 intel_crtc_needs_color_update(crtc_state) &&
 400                 !intel_color_uses_dsb(crtc_state);
 401 }
 402
 403 static void intel_crtc_vblank_work(struct kthread_work *base)
 404 {
 405         struct drm_vblank_work *work = to_drm_vblank_work(base);
 406         struct intel_crtc_state *crtc_state =
 407                 container_of(work, typeof(*crtc_state), vblank_work);
 408         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
 409
 410         trace_intel_crtc_vblank_work_start(crtc);
 411
 412         intel_color_load_luts(crtc_state);
 413
 414         if (crtc_state->uapi.event) {
 415                 spin_lock_irq(&crtc->base.dev->event_lock);
 416                 drm_crtc_send_vblank_event(&crtc->base, crtc_state->uapi.event);
 417                 crtc_state->uapi.event = NULL;
 418                 spin_unlock_irq(&crtc->base.dev->event_lock);
 419         }
 420
 421         trace_intel_crtc_vblank_work_end(crtc);
 422 }
 423
 424 static void intel_crtc_vblank_work_init(struct intel_crtc_state *crtc_state)
 425 {
 426         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
 427
 428         drm_vblank_work_init(&crtc_state->vblank_work, &crtc->base,
 429                              intel_crtc_vblank_work);
 430         /*
 431          * Interrupt latency is critical for getting the vblank
 432          * work executed as early as possible during the vblank.
 433          */
 434         cpu_latency_qos_update_request(&crtc->vblank_pm_qos, 0);
 435 }
 436
 437 void intel_wait_for_vblank_workers(struct intel_atomic_state *state)
 438 {
 439         struct intel_crtc_state *crtc_state;
 440         struct intel_crtc *crtc;
 441         int i;
 442
 443         for_each_new_intel_crtc_in_state(state, crtc, crtc_state, i) {
 444                 if (!intel_crtc_needs_vblank_work(crtc_state))
 445                         continue;
 446
 447                 drm_vblank_work_flush(&crtc_state->vblank_work);
 448                 cpu_latency_qos_update_request(&crtc->vblank_pm_qos,
 449                                                PM_QOS_DEFAULT_VALUE);
 450         }
 451 }
 452
 453 int intel_usecs_to_scanlines(const struct drm_display_mode *adjusted_mode,
 454                              int usecs)
 455 {
 456         /* paranoia */
 457         if (!adjusted_mode->crtc_htotal)
 458                 return 1;
 459
 460         return DIV_ROUND_UP(usecs * adjusted_mode->crtc_clock,
 461                             1000 * adjusted_mode->crtc_htotal);
 462 }
 463
 464 static int intel_mode_vblank_start(const struct drm_display_mode *mode)
 465 {
 466         int vblank_start = mode->crtc_vblank_start;
 467
 468         if (mode->flags & DRM_MODE_FLAG_INTERLACE)
 469                 vblank_start = DIV_ROUND_UP(vblank_start, 2);
 470
 471         return vblank_start;
 472 }
 473
 474 static void intel_crtc_vblank_evade_scanlines(struct intel_atomic_state *state,
 475                                               struct intel_crtc *crtc,
 476                                               int *min, int *max, int *vblank_start)
 477 {
 478         const struct intel_crtc_state *old_crtc_state =
 479                 intel_atomic_get_old_crtc_state(state, crtc);
 480         const struct intel_crtc_state *new_crtc_state =
 481                 intel_atomic_get_new_crtc_state(state, crtc);
 482         const struct intel_crtc_state *crtc_state;
 483         const struct drm_display_mode *adjusted_mode;
 484
 485         /*
 486          * During fastsets/etc. the transcoder is still
 487          * running with the old timings at this point.
 488          *
 489          * TODO: maybe just use the active timings here?
 490          */
 491         if (intel_crtc_needs_modeset(new_crtc_state))
 492                 crtc_state = new_crtc_state;
 493         else
 494                 crtc_state = old_crtc_state;
 495
 496         adjusted_mode = &crtc_state->hw.adjusted_mode;
 497
 498         if (crtc->mode_flags & I915_MODE_FLAG_VRR) {
 499                 /* timing changes should happen with VRR disabled */
 500                 drm_WARN_ON(state->base.dev, intel_crtc_needs_modeset(new_crtc_state) ||
 501                             new_crtc_state->update_m_n || new_crtc_state->update_lrr);
 502
 503                 if (intel_vrr_is_push_sent(crtc_state))
 504                         *vblank_start = intel_vrr_vmin_vblank_start(crtc_state);
 505                 else
 506                         *vblank_start = intel_vrr_vmax_vblank_start(crtc_state);
 507         } else {
 508                 *vblank_start = intel_mode_vblank_start(adjusted_mode);
 509         }
 510
 511         /* FIXME needs to be calibrated sensibly */
 512         *min = *vblank_start - intel_usecs_to_scanlines(adjusted_mode,
 513                                                         VBLANK_EVASION_TIME_US);
 514         *max = *vblank_start - 1;
 515
 516         /*
 517          * M/N and TRANS_VTOTAL are double buffered on the transcoder's
 518          * undelayed vblank, so with seamless M/N and LRR we must evade
 519          * both vblanks.
 520          *
 521          * DSB execution waits for the transcoder's undelayed vblank,
 522          * hence we must kick off the commit before that.
 523          */
 524         if (new_crtc_state->dsb || new_crtc_state->update_m_n || new_crtc_state->update_lrr)
 525                 *min -= adjusted_mode->crtc_vblank_start - adjusted_mode->crtc_vdisplay;
 526 }
 527
 528 /**
 529  * intel_pipe_update_start() - start update of a set of display registers
 530  * @state: the atomic state
 531  * @crtc: the crtc
 532  *
 533  * Mark the start of an update to pipe registers that should be updated
 534  * atomically regarding vblank. If the next vblank will happens within
 535  * the next 100 us, this function waits until the vblank passes.
 536  *
 537  * After a successful call to this function, interrupts will be disabled
 538  * until a subsequent call to intel_pipe_update_end(). That is done to
 539  * avoid random delays.
 540  */
 541 void intel_pipe_update_start(struct intel_atomic_state *state,
 542                              struct intel_crtc *crtc)
 543 {
 544         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
 545         struct intel_crtc_state *new_crtc_state =
 546                 intel_atomic_get_new_crtc_state(state, crtc);
 547         long timeout = msecs_to_jiffies_timeout(1);
 548         int scanline, min, max, vblank_start;
 549         wait_queue_head_t *wq = drm_crtc_vblank_waitqueue(&crtc->base);
 550         bool need_vlv_dsi_wa = (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) &&
 551                 intel_crtc_has_type(new_crtc_state, INTEL_OUTPUT_DSI);
 552         DEFINE_WAIT(wait);
 553
 554         intel_psr_lock(new_crtc_state);
 555
 556         if (new_crtc_state->do_async_flip)
 557                 return;
 558
 559         if (intel_crtc_needs_vblank_work(new_crtc_state))
 560                 intel_crtc_vblank_work_init(new_crtc_state);
 561
 562         intel_crtc_vblank_evade_scanlines(state, crtc, &min, &max, &vblank_start);
 563         if (min <= 0 || max <= 0)
 564                 goto irq_disable;
 565
 566         if (drm_WARN_ON(&dev_priv->drm, drm_crtc_vblank_get(&crtc->base)))
 567                 goto irq_disable;
 568
 569         /*
 570          * Wait for psr to idle out after enabling the VBL interrupts
 571          * VBL interrupts will start the PSR exit and prevent a PSR
 572          * re-entry as well.
 573          */
 574         intel_psr_wait_for_idle_locked(new_crtc_state);
 575
 576         local_irq_disable();
 577
 578         crtc->debug.min_vbl = min;
 579         crtc->debug.max_vbl = max;
 580         trace_intel_pipe_update_start(crtc);
 581
 582         for (;;) {
 583                 /*
 584                  * prepare_to_wait() has a memory barrier, which guarantees
 585                  * other CPUs can see the task state update by the time we
 586                  * read the scanline.
 587                  */
 588                 prepare_to_wait(wq, &wait, TASK_UNINTERRUPTIBLE);
 589
 590                 scanline = intel_get_crtc_scanline(crtc);
 591                 if (scanline < min || scanline > max)
 592                         break;
 593
 594                 if (!timeout) {
 595                         drm_err(&dev_priv->drm,
 596                                 "Potential atomic update failure on pipe %c\n",
 597                                 pipe_name(crtc->pipe));
 598                         break;
 599                 }
 600
 601                 local_irq_enable();
 602
 603                 timeout = schedule_timeout(timeout);
 604
 605                 local_irq_disable();
 606         }
 607
 608         finish_wait(wq, &wait);
 609
 610         drm_crtc_vblank_put(&crtc->base);
 611
 612         /*
 613          * On VLV/CHV DSI the scanline counter would appear to
 614          * increment approx. 1/3 of a scanline before start of vblank.
 615          * The registers still get latched at start of vblank however.
 616          * This means we must not write any registers on the first
 617          * line of vblank (since not the whole line is actually in
 618          * vblank). And unfortunately we can't use the interrupt to
 619          * wait here since it will fire too soon. We could use the
 620          * frame start interrupt instead since it will fire after the
 621          * critical scanline, but that would require more changes
 622          * in the interrupt code. So for now we'll just do the nasty
 623          * thing and poll for the bad scanline to pass us by.
 624          *
 625          * FIXME figure out if BXT+ DSI suffers from this as well
 626          */
 627         while (need_vlv_dsi_wa && scanline == vblank_start)
 628                 scanline = intel_get_crtc_scanline(crtc);
 629
 630         crtc->debug.scanline_start = scanline;
 631         crtc->debug.start_vbl_time = ktime_get();
 632         crtc->debug.start_vbl_count = intel_crtc_get_vblank_counter(crtc);
 633
 634         trace_intel_pipe_update_vblank_evaded(crtc);
 635         return;
 636
 637 irq_disable:
 638         local_irq_disable();
 639 }
 640
 641 #if IS_ENABLED(CONFIG_DRM_I915_DEBUG_VBLANK_EVADE)
 642 static void dbg_vblank_evade(struct intel_crtc *crtc, ktime_t end)
 643 {
 644         u64 delta = ktime_to_ns(ktime_sub(end, crtc->debug.start_vbl_time));
 645         unsigned int h;
 646
 647         h = ilog2(delta >> 9);
 648         if (h >= ARRAY_SIZE(crtc->debug.vbl.times))
 649                 h = ARRAY_SIZE(crtc->debug.vbl.times) - 1;
 650         crtc->debug.vbl.times[h]++;
 651
 652         crtc->debug.vbl.sum += delta;
 653         if (!crtc->debug.vbl.min || delta < crtc->debug.vbl.min)
 654                 crtc->debug.vbl.min = delta;
 655         if (delta > crtc->debug.vbl.max)
 656                 crtc->debug.vbl.max = delta;
 657
 658         if (delta > 1000 * VBLANK_EVASION_TIME_US) {
 659                 drm_dbg_kms(crtc->base.dev,
 660                             "Atomic update on pipe (%c) took %lld us, max time under evasion is %u us\n",
 661                             pipe_name(crtc->pipe),
 662                             div_u64(delta, 1000),
 663                             VBLANK_EVASION_TIME_US);
 664                 crtc->debug.vbl.over++;
 665         }
 666 }
 667 #else
 668 static void dbg_vblank_evade(struct intel_crtc *crtc, ktime_t end) {}
 669 #endif
 670
 671 /**
 672  * intel_pipe_update_end() - end update of a set of display registers
 673  * @state: the atomic state
 674  * @crtc: the crtc
 675  *
 676  * Mark the end of an update started with intel_pipe_update_start(). This
 677  * re-enables interrupts and verifies the update was actually completed
 678  * before a vblank.
 679  */
 680 void intel_pipe_update_end(struct intel_atomic_state *state,
 681                            struct intel_crtc *crtc)
 682 {
 683         struct intel_crtc_state *new_crtc_state =
 684                 intel_atomic_get_new_crtc_state(state, crtc);
 685         enum pipe pipe = crtc->pipe;
 686         int scanline_end = intel_get_crtc_scanline(crtc);
 687         u32 end_vbl_count = intel_crtc_get_vblank_counter(crtc);
 688         ktime_t end_vbl_time = ktime_get();
 689         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
 690
 691         if (new_crtc_state->do_async_flip)
 692                 goto out;
 693
 694         trace_intel_pipe_update_end(crtc, end_vbl_count, scanline_end);
 695
 696         /*
 697          * Incase of mipi dsi command mode, we need to set frame update
 698          * request for every commit.
 699          */
 700         if (DISPLAY_VER(dev_priv) >= 11 &&
 701             intel_crtc_has_type(new_crtc_state, INTEL_OUTPUT_DSI))
 702                 icl_dsi_frame_update(new_crtc_state);
 703
 704         /* We're still in the vblank-evade critical section, this can't race.
 705          * Would be slightly nice to just grab the vblank count and arm the
 706          * event outside of the critical section - the spinlock might spin for a
 707          * while ... */
 708         if (intel_crtc_needs_vblank_work(new_crtc_state)) {
 709                 drm_vblank_work_schedule(&new_crtc_state->vblank_work,
 710                                          drm_crtc_accurate_vblank_count(&crtc->base) + 1,
 711                                          false);
 712         } else if (new_crtc_state->uapi.event) {
 713                 drm_WARN_ON(&dev_priv->drm,
 714                             drm_crtc_vblank_get(&crtc->base) != 0);
 715
 716                 spin_lock(&crtc->base.dev->event_lock);
 717                 drm_crtc_arm_vblank_event(&crtc->base,
 718                                           new_crtc_state->uapi.event);
 719                 spin_unlock(&crtc->base.dev->event_lock);
 720
 721                 new_crtc_state->uapi.event = NULL;
 722         }
 723
 724         /*
 725          * Send VRR Push to terminate Vblank. If we are already in vblank
 726          * this has to be done _after_ sampling the frame counter, as
 727          * otherwise the push would immediately terminate the vblank and
 728          * the sampled frame counter would correspond to the next frame
 729          * instead of the current frame.
 730          *
 731          * There is a tiny race here (iff vblank evasion failed us) where
 732          * we might sample the frame counter just before vmax vblank start
 733          * but the push would be sent just after it. That would cause the
 734          * push to affect the next frame instead of the current frame,
 735          * which would cause the next frame to terminate already at vmin
 736          * vblank start instead of vmax vblank start.
 737          */
 738         intel_vrr_send_push(new_crtc_state);
 739
 740         local_irq_enable();
 741
 742         if (intel_vgpu_active(dev_priv))
 743                 goto out;
 744
 745         if (crtc->debug.start_vbl_count &&
 746             crtc->debug.start_vbl_count != end_vbl_count) {
 747                 drm_err(&dev_priv->drm,
 748                         "Atomic update failure on pipe %c (start=%u end=%u) time %lld us, min %d, max %d, scanline start %d, end %d\n",
 749                         pipe_name(pipe), crtc->debug.start_vbl_count,
 750                         end_vbl_count,
 751                         ktime_us_delta(end_vbl_time,
 752                                        crtc->debug.start_vbl_time),
 753                         crtc->debug.min_vbl, crtc->debug.max_vbl,
 754                         crtc->debug.scanline_start, scanline_end);
 755         }
 756
 757         dbg_vblank_evade(crtc, end_vbl_time);
 758
 759 out:
 760         intel_psr_unlock(new_crtc_state);
 761 }