1 // SPDX-License-Identifier: MIT
3 * Copyright © 2020 Intel Corporation
5 #include <linux/kernel.h>
6 #include <linux/pm_qos.h>
7 #include <linux/slab.h>
9 #include <drm/drm_atomic_helper.h>
10 #include <drm/drm_fourcc.h>
11 #include <drm/drm_plane.h>
12 #include <drm/drm_vblank_work.h>
15 #include "i915_vgpu.h"
16 #include "i9xx_plane.h"
18 #include "intel_atomic.h"
19 #include "intel_atomic_plane.h"
20 #include "intel_color.h"
21 #include "intel_crtc.h"
22 #include "intel_cursor.h"
23 #include "intel_display_debugfs.h"
24 #include "intel_display_trace.h"
25 #include "intel_display_types.h"
26 #include "intel_drrs.h"
27 #include "intel_dsi.h"
28 #include "intel_fifo_underrun.h"
29 #include "intel_pipe_crc.h"
30 #include "intel_psr.h"
31 #include "intel_sprite.h"
32 #include "intel_vblank.h"
33 #include "intel_vrr.h"
34 #include "skl_universal_plane.h"
36 static void assert_vblank_disabled(struct drm_crtc *crtc)
38 if (I915_STATE_WARN_ON(drm_crtc_vblank_get(crtc) == 0))
39 drm_crtc_vblank_put(crtc);
42 struct intel_crtc *intel_first_crtc(struct drm_i915_private *i915)
44 return to_intel_crtc(drm_crtc_from_index(&i915->drm, 0));
47 struct intel_crtc *intel_crtc_for_pipe(struct drm_i915_private *i915,
50 struct intel_crtc *crtc;
52 for_each_intel_crtc(&i915->drm, crtc) {
53 if (crtc->pipe == pipe)
60 void intel_crtc_wait_for_next_vblank(struct intel_crtc *crtc)
62 drm_crtc_wait_one_vblank(&crtc->base);
65 void intel_wait_for_vblank_if_active(struct drm_i915_private *i915,
68 struct intel_crtc *crtc = intel_crtc_for_pipe(i915, pipe);
71 intel_crtc_wait_for_next_vblank(crtc);
74 u32 intel_crtc_get_vblank_counter(struct intel_crtc *crtc)
76 struct drm_device *dev = crtc->base.dev;
77 struct drm_vblank_crtc *vblank = &dev->vblank[drm_crtc_index(&crtc->base)];
82 if (!vblank->max_vblank_count)
83 return (u32)drm_crtc_accurate_vblank_count(&crtc->base);
85 return crtc->base.funcs->get_vblank_counter(&crtc->base);
88 u32 intel_crtc_max_vblank_count(const struct intel_crtc_state *crtc_state)
90 struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
93 * From Gen 11, In case of dsi cmd mode, frame counter wouldnt
94 * have updated at the beginning of TE, if we want to use
95 * the hw counter, then we would find it updated in only
96 * the next TE, hence switching to sw counter.
98 if (crtc_state->mode_flags & (I915_MODE_FLAG_DSI_USE_TE0 |
99 I915_MODE_FLAG_DSI_USE_TE1))
103 * On i965gm the hardware frame counter reads
104 * zero when the TV encoder is enabled :(
106 if (IS_I965GM(dev_priv) &&
107 (crtc_state->output_types & BIT(INTEL_OUTPUT_TVOUT)))
110 if (DISPLAY_VER(dev_priv) >= 5 || IS_G4X(dev_priv))
111 return 0xffffffff; /* full 32 bit counter */
112 else if (DISPLAY_VER(dev_priv) >= 3)
113 return 0xffffff; /* only 24 bits of frame count */
115 return 0; /* Gen2 doesn't have a hardware frame counter */
118 void intel_crtc_vblank_on(const struct intel_crtc_state *crtc_state)
120 struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
122 assert_vblank_disabled(&crtc->base);
123 drm_crtc_set_max_vblank_count(&crtc->base,
124 intel_crtc_max_vblank_count(crtc_state));
125 drm_crtc_vblank_on(&crtc->base);
128 * Should really happen exactly when we enable the pipe
129 * but we want the frame counters in the trace, and that
130 * requires vblank support on some platforms/outputs.
132 trace_intel_pipe_enable(crtc);
135 void intel_crtc_vblank_off(const struct intel_crtc_state *crtc_state)
137 struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
140 * Should really happen exactly when we disable the pipe
141 * but we want the frame counters in the trace, and that
142 * requires vblank support on some platforms/outputs.
144 trace_intel_pipe_disable(crtc);
146 drm_crtc_vblank_off(&crtc->base);
147 assert_vblank_disabled(&crtc->base);
150 struct intel_crtc_state *intel_crtc_state_alloc(struct intel_crtc *crtc)
152 struct intel_crtc_state *crtc_state;
154 crtc_state = kmalloc(sizeof(*crtc_state), GFP_KERNEL);
157 intel_crtc_state_reset(crtc_state, crtc);
162 void intel_crtc_state_reset(struct intel_crtc_state *crtc_state,
163 struct intel_crtc *crtc)
165 memset(crtc_state, 0, sizeof(*crtc_state));
167 __drm_atomic_helper_crtc_state_reset(&crtc_state->uapi, &crtc->base);
169 crtc_state->cpu_transcoder = INVALID_TRANSCODER;
170 crtc_state->master_transcoder = INVALID_TRANSCODER;
171 crtc_state->hsw_workaround_pipe = INVALID_PIPE;
172 crtc_state->scaler_state.scaler_id = -1;
173 crtc_state->mst_master_transcoder = INVALID_TRANSCODER;
176 static struct intel_crtc *intel_crtc_alloc(void)
178 struct intel_crtc_state *crtc_state;
179 struct intel_crtc *crtc;
181 crtc = kzalloc(sizeof(*crtc), GFP_KERNEL);
183 return ERR_PTR(-ENOMEM);
185 crtc_state = intel_crtc_state_alloc(crtc);
188 return ERR_PTR(-ENOMEM);
191 crtc->base.state = &crtc_state->uapi;
192 crtc->config = crtc_state;
197 static void intel_crtc_free(struct intel_crtc *crtc)
199 intel_crtc_destroy_state(&crtc->base, crtc->base.state);
203 static void intel_crtc_destroy(struct drm_crtc *_crtc)
205 struct intel_crtc *crtc = to_intel_crtc(_crtc);
207 cpu_latency_qos_remove_request(&crtc->vblank_pm_qos);
209 drm_crtc_cleanup(&crtc->base);
213 static int intel_crtc_late_register(struct drm_crtc *crtc)
215 intel_crtc_debugfs_add(to_intel_crtc(crtc));
219 #define INTEL_CRTC_FUNCS \
220 .set_config = drm_atomic_helper_set_config, \
221 .destroy = intel_crtc_destroy, \
222 .page_flip = drm_atomic_helper_page_flip, \
223 .atomic_duplicate_state = intel_crtc_duplicate_state, \
224 .atomic_destroy_state = intel_crtc_destroy_state, \
225 .set_crc_source = intel_crtc_set_crc_source, \
226 .verify_crc_source = intel_crtc_verify_crc_source, \
227 .get_crc_sources = intel_crtc_get_crc_sources, \
228 .late_register = intel_crtc_late_register
230 static const struct drm_crtc_funcs bdw_crtc_funcs = {
233 .get_vblank_counter = g4x_get_vblank_counter,
234 .enable_vblank = bdw_enable_vblank,
235 .disable_vblank = bdw_disable_vblank,
236 .get_vblank_timestamp = intel_crtc_get_vblank_timestamp,
239 static const struct drm_crtc_funcs ilk_crtc_funcs = {
242 .get_vblank_counter = g4x_get_vblank_counter,
243 .enable_vblank = ilk_enable_vblank,
244 .disable_vblank = ilk_disable_vblank,
245 .get_vblank_timestamp = intel_crtc_get_vblank_timestamp,
248 static const struct drm_crtc_funcs g4x_crtc_funcs = {
251 .get_vblank_counter = g4x_get_vblank_counter,
252 .enable_vblank = i965_enable_vblank,
253 .disable_vblank = i965_disable_vblank,
254 .get_vblank_timestamp = intel_crtc_get_vblank_timestamp,
257 static const struct drm_crtc_funcs i965_crtc_funcs = {
260 .get_vblank_counter = i915_get_vblank_counter,
261 .enable_vblank = i965_enable_vblank,
262 .disable_vblank = i965_disable_vblank,
263 .get_vblank_timestamp = intel_crtc_get_vblank_timestamp,
266 static const struct drm_crtc_funcs i915gm_crtc_funcs = {
269 .get_vblank_counter = i915_get_vblank_counter,
270 .enable_vblank = i915gm_enable_vblank,
271 .disable_vblank = i915gm_disable_vblank,
272 .get_vblank_timestamp = intel_crtc_get_vblank_timestamp,
275 static const struct drm_crtc_funcs i915_crtc_funcs = {
278 .get_vblank_counter = i915_get_vblank_counter,
279 .enable_vblank = i8xx_enable_vblank,
280 .disable_vblank = i8xx_disable_vblank,
281 .get_vblank_timestamp = intel_crtc_get_vblank_timestamp,
284 static const struct drm_crtc_funcs i8xx_crtc_funcs = {
287 /* no hw vblank counter */
288 .enable_vblank = i8xx_enable_vblank,
289 .disable_vblank = i8xx_disable_vblank,
290 .get_vblank_timestamp = intel_crtc_get_vblank_timestamp,
293 int intel_crtc_init(struct drm_i915_private *dev_priv, enum pipe pipe)
295 struct intel_plane *primary, *cursor;
296 const struct drm_crtc_funcs *funcs;
297 struct intel_crtc *crtc;
300 crtc = intel_crtc_alloc();
302 return PTR_ERR(crtc);
305 crtc->num_scalers = RUNTIME_INFO(dev_priv)->num_scalers[pipe];
307 if (DISPLAY_VER(dev_priv) >= 9)
308 primary = skl_universal_plane_create(dev_priv, pipe,
311 primary = intel_primary_plane_create(dev_priv, pipe);
312 if (IS_ERR(primary)) {
313 ret = PTR_ERR(primary);
316 crtc->plane_ids_mask |= BIT(primary->id);
318 intel_init_fifo_underrun_reporting(dev_priv, crtc, false);
320 for_each_sprite(dev_priv, pipe, sprite) {
321 struct intel_plane *plane;
323 if (DISPLAY_VER(dev_priv) >= 9)
324 plane = skl_universal_plane_create(dev_priv, pipe,
325 PLANE_SPRITE0 + sprite);
327 plane = intel_sprite_plane_create(dev_priv, pipe, sprite);
329 ret = PTR_ERR(plane);
332 crtc->plane_ids_mask |= BIT(plane->id);
335 cursor = intel_cursor_plane_create(dev_priv, pipe);
336 if (IS_ERR(cursor)) {
337 ret = PTR_ERR(cursor);
340 crtc->plane_ids_mask |= BIT(cursor->id);
342 if (HAS_GMCH(dev_priv)) {
343 if (IS_CHERRYVIEW(dev_priv) ||
344 IS_VALLEYVIEW(dev_priv) || IS_G4X(dev_priv))
345 funcs = &g4x_crtc_funcs;
346 else if (DISPLAY_VER(dev_priv) == 4)
347 funcs = &i965_crtc_funcs;
348 else if (IS_I945GM(dev_priv) || IS_I915GM(dev_priv))
349 funcs = &i915gm_crtc_funcs;
350 else if (DISPLAY_VER(dev_priv) == 3)
351 funcs = &i915_crtc_funcs;
353 funcs = &i8xx_crtc_funcs;
355 if (DISPLAY_VER(dev_priv) >= 8)
356 funcs = &bdw_crtc_funcs;
358 funcs = &ilk_crtc_funcs;
361 ret = drm_crtc_init_with_planes(&dev_priv->drm, &crtc->base,
362 &primary->base, &cursor->base,
363 funcs, "pipe %c", pipe_name(pipe));
367 if (DISPLAY_VER(dev_priv) >= 11)
368 drm_crtc_create_scaling_filter_property(&crtc->base,
369 BIT(DRM_SCALING_FILTER_DEFAULT) |
370 BIT(DRM_SCALING_FILTER_NEAREST_NEIGHBOR));
372 intel_color_crtc_init(crtc);
373 intel_drrs_crtc_init(crtc);
374 intel_crtc_crc_init(crtc);
376 cpu_latency_qos_add_request(&crtc->vblank_pm_qos, PM_QOS_DEFAULT_VALUE);
378 drm_WARN_ON(&dev_priv->drm, drm_crtc_index(&crtc->base) != crtc->pipe);
383 intel_crtc_free(crtc);
388 static bool intel_crtc_needs_vblank_work(const struct intel_crtc_state *crtc_state)
390 return crtc_state->hw.active &&
391 !intel_crtc_needs_modeset(crtc_state) &&
392 !crtc_state->preload_luts &&
393 intel_crtc_needs_color_update(crtc_state);
396 static void intel_crtc_vblank_work(struct kthread_work *base)
398 struct drm_vblank_work *work = to_drm_vblank_work(base);
399 struct intel_crtc_state *crtc_state =
400 container_of(work, typeof(*crtc_state), vblank_work);
401 struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
403 trace_intel_crtc_vblank_work_start(crtc);
405 intel_color_load_luts(crtc_state);
407 if (crtc_state->uapi.event) {
408 spin_lock_irq(&crtc->base.dev->event_lock);
409 drm_crtc_send_vblank_event(&crtc->base, crtc_state->uapi.event);
410 crtc_state->uapi.event = NULL;
411 spin_unlock_irq(&crtc->base.dev->event_lock);
414 trace_intel_crtc_vblank_work_end(crtc);
417 static void intel_crtc_vblank_work_init(struct intel_crtc_state *crtc_state)
419 struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
421 drm_vblank_work_init(&crtc_state->vblank_work, &crtc->base,
422 intel_crtc_vblank_work);
424 * Interrupt latency is critical for getting the vblank
425 * work executed as early as possible during the vblank.
427 cpu_latency_qos_update_request(&crtc->vblank_pm_qos, 0);
430 void intel_wait_for_vblank_workers(struct intel_atomic_state *state)
432 struct intel_crtc_state *crtc_state;
433 struct intel_crtc *crtc;
436 for_each_new_intel_crtc_in_state(state, crtc, crtc_state, i) {
437 if (!intel_crtc_needs_vblank_work(crtc_state))
440 drm_vblank_work_flush(&crtc_state->vblank_work);
441 cpu_latency_qos_update_request(&crtc->vblank_pm_qos,
442 PM_QOS_DEFAULT_VALUE);
446 int intel_usecs_to_scanlines(const struct drm_display_mode *adjusted_mode,
450 if (!adjusted_mode->crtc_htotal)
453 return DIV_ROUND_UP(usecs * adjusted_mode->crtc_clock,
454 1000 * adjusted_mode->crtc_htotal);
457 static int intel_mode_vblank_start(const struct drm_display_mode *mode)
459 int vblank_start = mode->crtc_vblank_start;
461 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
462 vblank_start = DIV_ROUND_UP(vblank_start, 2);
468 * intel_pipe_update_start() - start update of a set of display registers
469 * @new_crtc_state: the new crtc state
471 * Mark the start of an update to pipe registers that should be updated
472 * atomically regarding vblank. If the next vblank will happens within
473 * the next 100 us, this function waits until the vblank passes.
475 * After a successful call to this function, interrupts will be disabled
476 * until a subsequent call to intel_pipe_update_end(). That is done to
477 * avoid random delays.
479 void intel_pipe_update_start(struct intel_crtc_state *new_crtc_state)
481 struct intel_crtc *crtc = to_intel_crtc(new_crtc_state->uapi.crtc);
482 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
483 const struct drm_display_mode *adjusted_mode = &new_crtc_state->hw.adjusted_mode;
484 long timeout = msecs_to_jiffies_timeout(1);
485 int scanline, min, max, vblank_start;
486 wait_queue_head_t *wq = drm_crtc_vblank_waitqueue(&crtc->base);
487 bool need_vlv_dsi_wa = (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) &&
488 intel_crtc_has_type(new_crtc_state, INTEL_OUTPUT_DSI);
491 intel_psr_lock(new_crtc_state);
493 if (new_crtc_state->do_async_flip)
496 if (intel_crtc_needs_vblank_work(new_crtc_state))
497 intel_crtc_vblank_work_init(new_crtc_state);
499 if (new_crtc_state->vrr.enable) {
500 if (intel_vrr_is_push_sent(new_crtc_state))
501 vblank_start = intel_vrr_vmin_vblank_start(new_crtc_state);
503 vblank_start = intel_vrr_vmax_vblank_start(new_crtc_state);
505 vblank_start = intel_mode_vblank_start(adjusted_mode);
508 /* FIXME needs to be calibrated sensibly */
509 min = vblank_start - intel_usecs_to_scanlines(adjusted_mode,
510 VBLANK_EVASION_TIME_US);
511 max = vblank_start - 1;
513 if (min <= 0 || max <= 0)
516 if (drm_WARN_ON(&dev_priv->drm, drm_crtc_vblank_get(&crtc->base)))
520 * Wait for psr to idle out after enabling the VBL interrupts
521 * VBL interrupts will start the PSR exit and prevent a PSR
524 intel_psr_wait_for_idle_locked(new_crtc_state);
528 crtc->debug.min_vbl = min;
529 crtc->debug.max_vbl = max;
530 trace_intel_pipe_update_start(crtc);
534 * prepare_to_wait() has a memory barrier, which guarantees
535 * other CPUs can see the task state update by the time we
538 prepare_to_wait(wq, &wait, TASK_UNINTERRUPTIBLE);
540 scanline = intel_get_crtc_scanline(crtc);
541 if (scanline < min || scanline > max)
545 drm_err(&dev_priv->drm,
546 "Potential atomic update failure on pipe %c\n",
547 pipe_name(crtc->pipe));
553 timeout = schedule_timeout(timeout);
558 finish_wait(wq, &wait);
560 drm_crtc_vblank_put(&crtc->base);
563 * On VLV/CHV DSI the scanline counter would appear to
564 * increment approx. 1/3 of a scanline before start of vblank.
565 * The registers still get latched at start of vblank however.
566 * This means we must not write any registers on the first
567 * line of vblank (since not the whole line is actually in
568 * vblank). And unfortunately we can't use the interrupt to
569 * wait here since it will fire too soon. We could use the
570 * frame start interrupt instead since it will fire after the
571 * critical scanline, but that would require more changes
572 * in the interrupt code. So for now we'll just do the nasty
573 * thing and poll for the bad scanline to pass us by.
575 * FIXME figure out if BXT+ DSI suffers from this as well
577 while (need_vlv_dsi_wa && scanline == vblank_start)
578 scanline = intel_get_crtc_scanline(crtc);
580 crtc->debug.scanline_start = scanline;
581 crtc->debug.start_vbl_time = ktime_get();
582 crtc->debug.start_vbl_count = intel_crtc_get_vblank_counter(crtc);
584 trace_intel_pipe_update_vblank_evaded(crtc);
591 #if IS_ENABLED(CONFIG_DRM_I915_DEBUG_VBLANK_EVADE)
592 static void dbg_vblank_evade(struct intel_crtc *crtc, ktime_t end)
594 u64 delta = ktime_to_ns(ktime_sub(end, crtc->debug.start_vbl_time));
597 h = ilog2(delta >> 9);
598 if (h >= ARRAY_SIZE(crtc->debug.vbl.times))
599 h = ARRAY_SIZE(crtc->debug.vbl.times) - 1;
600 crtc->debug.vbl.times[h]++;
602 crtc->debug.vbl.sum += delta;
603 if (!crtc->debug.vbl.min || delta < crtc->debug.vbl.min)
604 crtc->debug.vbl.min = delta;
605 if (delta > crtc->debug.vbl.max)
606 crtc->debug.vbl.max = delta;
608 if (delta > 1000 * VBLANK_EVASION_TIME_US) {
609 drm_dbg_kms(crtc->base.dev,
610 "Atomic update on pipe (%c) took %lld us, max time under evasion is %u us\n",
611 pipe_name(crtc->pipe),
612 div_u64(delta, 1000),
613 VBLANK_EVASION_TIME_US);
614 crtc->debug.vbl.over++;
618 static void dbg_vblank_evade(struct intel_crtc *crtc, ktime_t end) {}
622 * intel_pipe_update_end() - end update of a set of display registers
623 * @new_crtc_state: the new crtc state
625 * Mark the end of an update started with intel_pipe_update_start(). This
626 * re-enables interrupts and verifies the update was actually completed
629 void intel_pipe_update_end(struct intel_crtc_state *new_crtc_state)
631 struct intel_crtc *crtc = to_intel_crtc(new_crtc_state->uapi.crtc);
632 enum pipe pipe = crtc->pipe;
633 int scanline_end = intel_get_crtc_scanline(crtc);
634 u32 end_vbl_count = intel_crtc_get_vblank_counter(crtc);
635 ktime_t end_vbl_time = ktime_get();
636 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
638 intel_psr_unlock(new_crtc_state);
640 if (new_crtc_state->do_async_flip)
643 trace_intel_pipe_update_end(crtc, end_vbl_count, scanline_end);
646 * Incase of mipi dsi command mode, we need to set frame update
647 * request for every commit.
649 if (DISPLAY_VER(dev_priv) >= 11 &&
650 intel_crtc_has_type(new_crtc_state, INTEL_OUTPUT_DSI))
651 icl_dsi_frame_update(new_crtc_state);
653 /* We're still in the vblank-evade critical section, this can't race.
654 * Would be slightly nice to just grab the vblank count and arm the
655 * event outside of the critical section - the spinlock might spin for a
657 if (intel_crtc_needs_vblank_work(new_crtc_state)) {
658 drm_vblank_work_schedule(&new_crtc_state->vblank_work,
659 drm_crtc_accurate_vblank_count(&crtc->base) + 1,
661 } else if (new_crtc_state->uapi.event) {
662 drm_WARN_ON(&dev_priv->drm,
663 drm_crtc_vblank_get(&crtc->base) != 0);
665 spin_lock(&crtc->base.dev->event_lock);
666 drm_crtc_arm_vblank_event(&crtc->base,
667 new_crtc_state->uapi.event);
668 spin_unlock(&crtc->base.dev->event_lock);
670 new_crtc_state->uapi.event = NULL;
674 * Send VRR Push to terminate Vblank. If we are already in vblank
675 * this has to be done _after_ sampling the frame counter, as
676 * otherwise the push would immediately terminate the vblank and
677 * the sampled frame counter would correspond to the next frame
678 * instead of the current frame.
680 * There is a tiny race here (iff vblank evasion failed us) where
681 * we might sample the frame counter just before vmax vblank start
682 * but the push would be sent just after it. That would cause the
683 * push to affect the next frame instead of the current frame,
684 * which would cause the next frame to terminate already at vmin
685 * vblank start instead of vmax vblank start.
687 intel_vrr_send_push(new_crtc_state);
690 * Seamless M/N update may need to update frame timings.
692 * FIXME Should be synchronized with the start of vblank somehow...
694 if (new_crtc_state->seamless_m_n && intel_crtc_needs_fastset(new_crtc_state))
695 intel_crtc_update_active_timings(new_crtc_state);
699 if (intel_vgpu_active(dev_priv))
702 if (crtc->debug.start_vbl_count &&
703 crtc->debug.start_vbl_count != end_vbl_count) {
704 drm_err(&dev_priv->drm,
705 "Atomic update failure on pipe %c (start=%u end=%u) time %lld us, min %d, max %d, scanline start %d, end %d\n",
706 pipe_name(pipe), crtc->debug.start_vbl_count,
708 ktime_us_delta(end_vbl_time,
709 crtc->debug.start_vbl_time),
710 crtc->debug.min_vbl, crtc->debug.max_vbl,
711 crtc->debug.scanline_start, scanline_end);
714 dbg_vblank_evade(crtc, end_vbl_time);