drivers/gpu/drm/i915/display/intel_display.c

   1 /*
   2  * Copyright © 2006-2007 Intel Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
  21  * DEALINGS IN THE SOFTWARE.
  22  *
  23  * Authors:
  24  *      Eric Anholt <[email protected]>
  25  */
  26
  27 #include <acpi/video.h>
  28 #include <linux/i2c.h>
  29 #include <linux/input.h>
  30 #include <linux/intel-iommu.h>
  31 #include <linux/kernel.h>
  32 #include <linux/module.h>
  33 #include <linux/dma-resv.h>
  34 #include <linux/slab.h>
  35
  36 #include <drm/drm_atomic.h>
  37 #include <drm/drm_atomic_helper.h>
  38 #include <drm/drm_atomic_uapi.h>
  39 #include <drm/drm_damage_helper.h>
  40 #include <drm/drm_dp_helper.h>
  41 #include <drm/drm_edid.h>
  42 #include <drm/drm_fourcc.h>
  43 #include <drm/drm_plane_helper.h>
  44 #include <drm/drm_probe_helper.h>
  45 #include <drm/drm_rect.h>
  46
  47 #include "display/intel_audio.h"
  48 #include "display/intel_crt.h"
  49 #include "display/intel_ddi.h"
  50 #include "display/intel_display_debugfs.h"
  51 #include "display/intel_dp.h"
  52 #include "display/intel_dp_mst.h"
  53 #include "display/intel_dpll.h"
  54 #include "display/intel_dpll_mgr.h"
  55 #include "display/intel_drrs.h"
  56 #include "display/intel_dsi.h"
  57 #include "display/intel_dvo.h"
  58 #include "display/intel_fb.h"
  59 #include "display/intel_gmbus.h"
  60 #include "display/intel_hdmi.h"
  61 #include "display/intel_lvds.h"
  62 #include "display/intel_sdvo.h"
  63 #include "display/intel_snps_phy.h"
  64 #include "display/intel_tv.h"
  65 #include "display/intel_vdsc.h"
  66 #include "display/intel_vrr.h"
  67
  68 #include "gem/i915_gem_lmem.h"
  69 #include "gem/i915_gem_object.h"
  70
  71 #include "gt/intel_rps.h"
  72 #include "gt/gen8_ppgtt.h"
  73
  74 #include "pxp/intel_pxp.h"
  75
  76 #include "g4x_dp.h"
  77 #include "g4x_hdmi.h"
  78 #include "i915_drv.h"
  79 #include "intel_acpi.h"
  80 #include "intel_atomic.h"
  81 #include "intel_atomic_plane.h"
  82 #include "intel_bw.h"
  83 #include "intel_cdclk.h"
  84 #include "intel_color.h"
  85 #include "intel_crtc.h"
  86 #include "intel_de.h"
  87 #include "intel_display_types.h"
  88 #include "intel_dmc.h"
  89 #include "intel_dp_link_training.h"
  90 #include "intel_dpt.h"
  91 #include "intel_fbc.h"
  92 #include "intel_fdi.h"
  93 #include "intel_fbdev.h"
  94 #include "intel_fifo_underrun.h"
  95 #include "intel_frontbuffer.h"
  96 #include "intel_hdcp.h"
  97 #include "intel_hotplug.h"
  98 #include "intel_overlay.h"
  99 #include "intel_panel.h"
 100 #include "intel_pipe_crc.h"
 101 #include "intel_pm.h"
 102 #include "intel_pps.h"
 103 #include "intel_psr.h"
 104 #include "intel_quirks.h"
 105 #include "intel_sideband.h"
 106 #include "intel_sprite.h"
 107 #include "intel_tc.h"
 108 #include "intel_vga.h"
 109 #include "i9xx_plane.h"
 110 #include "skl_scaler.h"
 111 #include "skl_universal_plane.h"
 112
 113 static void i9xx_crtc_clock_get(struct intel_crtc *crtc,
 114                                 struct intel_crtc_state *pipe_config);
 115 static void ilk_pch_clock_get(struct intel_crtc *crtc,
 116                               struct intel_crtc_state *pipe_config);
 117
 118 static void intel_set_transcoder_timings(const struct intel_crtc_state *crtc_state);
 119 static void intel_set_pipe_src_size(const struct intel_crtc_state *crtc_state);
 120 static void intel_cpu_transcoder_set_m_n(const struct intel_crtc_state *crtc_state,
 121                                          const struct intel_link_m_n *m_n,
 122                                          const struct intel_link_m_n *m2_n2);
 123 static void i9xx_set_pipeconf(const struct intel_crtc_state *crtc_state);
 124 static void ilk_set_pipeconf(const struct intel_crtc_state *crtc_state);
 125 static void hsw_set_transconf(const struct intel_crtc_state *crtc_state);
 126 static void bdw_set_pipemisc(const struct intel_crtc_state *crtc_state);
 127 static void ilk_pfit_enable(const struct intel_crtc_state *crtc_state);
 128 static void intel_modeset_setup_hw_state(struct drm_device *dev,
 129                                          struct drm_modeset_acquire_ctx *ctx);
 130
 131 /**
 132  * intel_update_watermarks - update FIFO watermark values based on current modes
 133  * @dev_priv: i915 device
 134  *
 135  * Calculate watermark values for the various WM regs based on current mode
 136  * and plane configuration.
 137  *
 138  * There are several cases to deal with here:
 139  *   - normal (i.e. non-self-refresh)
 140  *   - self-refresh (SR) mode
 141  *   - lines are large relative to FIFO size (buffer can hold up to 2)
 142  *   - lines are small relative to FIFO size (buffer can hold more than 2
 143  *     lines), so need to account for TLB latency
 144  *
 145  *   The normal calculation is:
 146  *     watermark = dotclock * bytes per pixel * latency
 147  *   where latency is platform & configuration dependent (we assume pessimal
 148  *   values here).
 149  *
 150  *   The SR calculation is:
 151  *     watermark = (trunc(latency/line time)+1) * surface width *
 152  *       bytes per pixel
 153  *   where
 154  *     line time = htotal / dotclock
 155  *     surface width = hdisplay for normal plane and 64 for cursor
 156  *   and latency is assumed to be high, as above.
 157  *
 158  * The final value programmed to the register should always be rounded up,
 159  * and include an extra 2 entries to account for clock crossings.
 160  *
 161  * We don't use the sprite, so we can ignore that.  And on Crestline we have
 162  * to set the non-SR watermarks to 8.
 163  */
 164 static void intel_update_watermarks(struct drm_i915_private *dev_priv)
 165 {
 166         if (dev_priv->wm_disp->update_wm)
 167                 dev_priv->wm_disp->update_wm(dev_priv);
 168 }
 169
 170 static int intel_compute_pipe_wm(struct intel_atomic_state *state,
 171                                  struct intel_crtc *crtc)
 172 {
 173         struct drm_i915_private *dev_priv = to_i915(state->base.dev);
 174         if (dev_priv->wm_disp->compute_pipe_wm)
 175                 return dev_priv->wm_disp->compute_pipe_wm(state, crtc);
 176         return 0;
 177 }
 178
 179 static int intel_compute_intermediate_wm(struct intel_atomic_state *state,
 180                                          struct intel_crtc *crtc)
 181 {
 182         struct drm_i915_private *dev_priv = to_i915(state->base.dev);
 183         if (!dev_priv->wm_disp->compute_intermediate_wm)
 184                 return 0;
 185         if (drm_WARN_ON(&dev_priv->drm,
 186                         !dev_priv->wm_disp->compute_pipe_wm))
 187                 return 0;
 188         return dev_priv->wm_disp->compute_intermediate_wm(state, crtc);
 189 }
 190
 191 static bool intel_initial_watermarks(struct intel_atomic_state *state,
 192                                      struct intel_crtc *crtc)
 193 {
 194         struct drm_i915_private *dev_priv = to_i915(state->base.dev);
 195         if (dev_priv->wm_disp->initial_watermarks) {
 196                 dev_priv->wm_disp->initial_watermarks(state, crtc);
 197                 return true;
 198         }
 199         return false;
 200 }
 201
 202 static void intel_atomic_update_watermarks(struct intel_atomic_state *state,
 203                                            struct intel_crtc *crtc)
 204 {
 205         struct drm_i915_private *dev_priv = to_i915(state->base.dev);
 206         if (dev_priv->wm_disp->atomic_update_watermarks)
 207                 dev_priv->wm_disp->atomic_update_watermarks(state, crtc);
 208 }
 209
 210 static void intel_optimize_watermarks(struct intel_atomic_state *state,
 211                                       struct intel_crtc *crtc)
 212 {
 213         struct drm_i915_private *dev_priv = to_i915(state->base.dev);
 214         if (dev_priv->wm_disp->optimize_watermarks)
 215                 dev_priv->wm_disp->optimize_watermarks(state, crtc);
 216 }
 217
 218 static int intel_compute_global_watermarks(struct intel_atomic_state *state)
 219 {
 220         struct drm_i915_private *dev_priv = to_i915(state->base.dev);
 221         if (dev_priv->wm_disp->compute_global_watermarks)
 222                 return dev_priv->wm_disp->compute_global_watermarks(state);
 223         return 0;
 224 }
 225
 226 /* returns HPLL frequency in kHz */
 227 int vlv_get_hpll_vco(struct drm_i915_private *dev_priv)
 228 {
 229         int hpll_freq, vco_freq[] = { 800, 1600, 2000, 2400 };
 230
 231         /* Obtain SKU information */
 232         hpll_freq = vlv_cck_read(dev_priv, CCK_FUSE_REG) &
 233                 CCK_FUSE_HPLL_FREQ_MASK;
 234
 235         return vco_freq[hpll_freq] * 1000;
 236 }
 237
 238 int vlv_get_cck_clock(struct drm_i915_private *dev_priv,
 239                       const char *name, u32 reg, int ref_freq)
 240 {
 241         u32 val;
 242         int divider;
 243
 244         val = vlv_cck_read(dev_priv, reg);
 245         divider = val & CCK_FREQUENCY_VALUES;
 246
 247         drm_WARN(&dev_priv->drm, (val & CCK_FREQUENCY_STATUS) !=
 248                  (divider << CCK_FREQUENCY_STATUS_SHIFT),
 249                  "%s change in progress\n", name);
 250
 251         return DIV_ROUND_CLOSEST(ref_freq << 1, divider + 1);
 252 }
 253
 254 int vlv_get_cck_clock_hpll(struct drm_i915_private *dev_priv,
 255                            const char *name, u32 reg)
 256 {
 257         int hpll;
 258
 259         vlv_cck_get(dev_priv);
 260
 261         if (dev_priv->hpll_freq == 0)
 262                 dev_priv->hpll_freq = vlv_get_hpll_vco(dev_priv);
 263
 264         hpll = vlv_get_cck_clock(dev_priv, name, reg, dev_priv->hpll_freq);
 265
 266         vlv_cck_put(dev_priv);
 267
 268         return hpll;
 269 }
 270
 271 static void intel_update_czclk(struct drm_i915_private *dev_priv)
 272 {
 273         if (!(IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)))
 274                 return;
 275
 276         dev_priv->czclk_freq = vlv_get_cck_clock_hpll(dev_priv, "czclk",
 277                                                       CCK_CZ_CLOCK_CONTROL);
 278
 279         drm_dbg(&dev_priv->drm, "CZ clock rate: %d kHz\n",
 280                 dev_priv->czclk_freq);
 281 }
 282
 283 static bool is_hdr_mode(const struct intel_crtc_state *crtc_state)
 284 {
 285         return (crtc_state->active_planes &
 286                 ~(icl_hdr_plane_mask() | BIT(PLANE_CURSOR))) == 0;
 287 }
 288
 289 /* WA Display #0827: Gen9:all */
 290 static void
 291 skl_wa_827(struct drm_i915_private *dev_priv, enum pipe pipe, bool enable)
 292 {
 293         if (enable)
 294                 intel_de_write(dev_priv, CLKGATE_DIS_PSL(pipe),
 295                                intel_de_read(dev_priv, CLKGATE_DIS_PSL(pipe)) | DUPS1_GATING_DIS | DUPS2_GATING_DIS);
 296         else
 297                 intel_de_write(dev_priv, CLKGATE_DIS_PSL(pipe),
 298                                intel_de_read(dev_priv, CLKGATE_DIS_PSL(pipe)) & ~(DUPS1_GATING_DIS | DUPS2_GATING_DIS));
 299 }
 300
 301 /* Wa_2006604312:icl,ehl */
 302 static void
 303 icl_wa_scalerclkgating(struct drm_i915_private *dev_priv, enum pipe pipe,
 304                        bool enable)
 305 {
 306         if (enable)
 307                 intel_de_write(dev_priv, CLKGATE_DIS_PSL(pipe),
 308                                intel_de_read(dev_priv, CLKGATE_DIS_PSL(pipe)) | DPFR_GATING_DIS);
 309         else
 310                 intel_de_write(dev_priv, CLKGATE_DIS_PSL(pipe),
 311                                intel_de_read(dev_priv, CLKGATE_DIS_PSL(pipe)) & ~DPFR_GATING_DIS);
 312 }
 313
 314 /* Wa_1604331009:icl,jsl,ehl */
 315 static void
 316 icl_wa_cursorclkgating(struct drm_i915_private *dev_priv, enum pipe pipe,
 317                        bool enable)
 318 {
 319         intel_de_rmw(dev_priv, CLKGATE_DIS_PSL(pipe), CURSOR_GATING_DIS,
 320                      enable ? CURSOR_GATING_DIS : 0);
 321 }
 322
 323 static bool
 324 is_trans_port_sync_slave(const struct intel_crtc_state *crtc_state)
 325 {
 326         return crtc_state->master_transcoder != INVALID_TRANSCODER;
 327 }
 328
 329 static bool
 330 is_trans_port_sync_master(const struct intel_crtc_state *crtc_state)
 331 {
 332         return crtc_state->sync_mode_slaves_mask != 0;
 333 }
 334
 335 bool
 336 is_trans_port_sync_mode(const struct intel_crtc_state *crtc_state)
 337 {
 338         return is_trans_port_sync_master(crtc_state) ||
 339                 is_trans_port_sync_slave(crtc_state);
 340 }
 341
 342 static bool pipe_scanline_is_moving(struct drm_i915_private *dev_priv,
 343                                     enum pipe pipe)
 344 {
 345         i915_reg_t reg = PIPEDSL(pipe);
 346         u32 line1, line2;
 347         u32 line_mask;
 348
 349         if (DISPLAY_VER(dev_priv) == 2)
 350                 line_mask = DSL_LINEMASK_GEN2;
 351         else
 352                 line_mask = DSL_LINEMASK_GEN3;
 353
 354         line1 = intel_de_read(dev_priv, reg) & line_mask;
 355         msleep(5);
 356         line2 = intel_de_read(dev_priv, reg) & line_mask;
 357
 358         return line1 != line2;
 359 }
 360
 361 static void wait_for_pipe_scanline_moving(struct intel_crtc *crtc, bool state)
 362 {
 363         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
 364         enum pipe pipe = crtc->pipe;
 365
 366         /* Wait for the display line to settle/start moving */
 367         if (wait_for(pipe_scanline_is_moving(dev_priv, pipe) == state, 100))
 368                 drm_err(&dev_priv->drm,
 369                         "pipe %c scanline %s wait timed out\n",
 370                         pipe_name(pipe), onoff(state));
 371 }
 372
 373 static void intel_wait_for_pipe_scanline_stopped(struct intel_crtc *crtc)
 374 {
 375         wait_for_pipe_scanline_moving(crtc, false);
 376 }
 377
 378 static void intel_wait_for_pipe_scanline_moving(struct intel_crtc *crtc)
 379 {
 380         wait_for_pipe_scanline_moving(crtc, true);
 381 }
 382
 383 static void
 384 intel_wait_for_pipe_off(const struct intel_crtc_state *old_crtc_state)
 385 {
 386         struct intel_crtc *crtc = to_intel_crtc(old_crtc_state->uapi.crtc);
 387         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
 388
 389         if (DISPLAY_VER(dev_priv) >= 4) {
 390                 enum transcoder cpu_transcoder = old_crtc_state->cpu_transcoder;
 391                 i915_reg_t reg = PIPECONF(cpu_transcoder);
 392
 393                 /* Wait for the Pipe State to go off */
 394                 if (intel_de_wait_for_clear(dev_priv, reg,
 395                                             I965_PIPECONF_ACTIVE, 100))
 396                         drm_WARN(&dev_priv->drm, 1,
 397                                  "pipe_off wait timed out\n");
 398         } else {
 399                 intel_wait_for_pipe_scanline_stopped(crtc);
 400         }
 401 }
 402
 403 void assert_transcoder(struct drm_i915_private *dev_priv,
 404                        enum transcoder cpu_transcoder, bool state)
 405 {
 406         bool cur_state;
 407         enum intel_display_power_domain power_domain;
 408         intel_wakeref_t wakeref;
 409
 410         /* we keep both pipes enabled on 830 */
 411         if (IS_I830(dev_priv))
 412                 state = true;
 413
 414         power_domain = POWER_DOMAIN_TRANSCODER(cpu_transcoder);
 415         wakeref = intel_display_power_get_if_enabled(dev_priv, power_domain);
 416         if (wakeref) {
 417                 u32 val = intel_de_read(dev_priv, PIPECONF(cpu_transcoder));
 418                 cur_state = !!(val & PIPECONF_ENABLE);
 419
 420                 intel_display_power_put(dev_priv, power_domain, wakeref);
 421         } else {
 422                 cur_state = false;
 423         }
 424
 425         I915_STATE_WARN(cur_state != state,
 426                         "transcoder %s assertion failure (expected %s, current %s)\n",
 427                         transcoder_name(cpu_transcoder),
 428                         onoff(state), onoff(cur_state));
 429 }
 430
 431 static void assert_plane(struct intel_plane *plane, bool state)
 432 {
 433         enum pipe pipe;
 434         bool cur_state;
 435
 436         cur_state = plane->get_hw_state(plane, &pipe);
 437
 438         I915_STATE_WARN(cur_state != state,
 439                         "%s assertion failure (expected %s, current %s)\n",
 440                         plane->base.name, onoff(state), onoff(cur_state));
 441 }
 442
 443 #define assert_plane_enabled(p) assert_plane(p, true)
 444 #define assert_plane_disabled(p) assert_plane(p, false)
 445
 446 static void assert_planes_disabled(struct intel_crtc *crtc)
 447 {
 448         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
 449         struct intel_plane *plane;
 450
 451         for_each_intel_plane_on_crtc(&dev_priv->drm, crtc, plane)
 452                 assert_plane_disabled(plane);
 453 }
 454
 455 void assert_pch_transcoder_disabled(struct drm_i915_private *dev_priv,
 456                                     enum pipe pipe)
 457 {
 458         u32 val;
 459         bool enabled;
 460
 461         val = intel_de_read(dev_priv, PCH_TRANSCONF(pipe));
 462         enabled = !!(val & TRANS_ENABLE);
 463         I915_STATE_WARN(enabled,
 464              "transcoder assertion failed, should be off on pipe %c but is still active\n",
 465              pipe_name(pipe));
 466 }
 467
 468 static void assert_pch_dp_disabled(struct drm_i915_private *dev_priv,
 469                                    enum pipe pipe, enum port port,
 470                                    i915_reg_t dp_reg)
 471 {
 472         enum pipe port_pipe;
 473         bool state;
 474
 475         state = g4x_dp_port_enabled(dev_priv, dp_reg, port, &port_pipe);
 476
 477         I915_STATE_WARN(state && port_pipe == pipe,
 478                         "PCH DP %c enabled on transcoder %c, should be disabled\n",
 479                         port_name(port), pipe_name(pipe));
 480
 481         I915_STATE_WARN(HAS_PCH_IBX(dev_priv) && !state && port_pipe == PIPE_B,
 482                         "IBX PCH DP %c still using transcoder B\n",
 483                         port_name(port));
 484 }
 485
 486 static void assert_pch_hdmi_disabled(struct drm_i915_private *dev_priv,
 487                                      enum pipe pipe, enum port port,
 488                                      i915_reg_t hdmi_reg)
 489 {
 490         enum pipe port_pipe;
 491         bool state;
 492
 493         state = intel_sdvo_port_enabled(dev_priv, hdmi_reg, &port_pipe);
 494
 495         I915_STATE_WARN(state && port_pipe == pipe,
 496                         "PCH HDMI %c enabled on transcoder %c, should be disabled\n",
 497                         port_name(port), pipe_name(pipe));
 498
 499         I915_STATE_WARN(HAS_PCH_IBX(dev_priv) && !state && port_pipe == PIPE_B,
 500                         "IBX PCH HDMI %c still using transcoder B\n",
 501                         port_name(port));
 502 }
 503
 504 static void assert_pch_ports_disabled(struct drm_i915_private *dev_priv,
 505                                       enum pipe pipe)
 506 {
 507         enum pipe port_pipe;
 508
 509         assert_pch_dp_disabled(dev_priv, pipe, PORT_B, PCH_DP_B);
 510         assert_pch_dp_disabled(dev_priv, pipe, PORT_C, PCH_DP_C);
 511         assert_pch_dp_disabled(dev_priv, pipe, PORT_D, PCH_DP_D);
 512
 513         I915_STATE_WARN(intel_crt_port_enabled(dev_priv, PCH_ADPA, &port_pipe) &&
 514                         port_pipe == pipe,
 515                         "PCH VGA enabled on transcoder %c, should be disabled\n",
 516                         pipe_name(pipe));
 517
 518         I915_STATE_WARN(intel_lvds_port_enabled(dev_priv, PCH_LVDS, &port_pipe) &&
 519                         port_pipe == pipe,
 520                         "PCH LVDS enabled on transcoder %c, should be disabled\n",
 521                         pipe_name(pipe));
 522
 523         /* PCH SDVOB multiplex with HDMIB */
 524         assert_pch_hdmi_disabled(dev_priv, pipe, PORT_B, PCH_HDMIB);
 525         assert_pch_hdmi_disabled(dev_priv, pipe, PORT_C, PCH_HDMIC);
 526         assert_pch_hdmi_disabled(dev_priv, pipe, PORT_D, PCH_HDMID);
 527 }
 528
 529 void vlv_wait_port_ready(struct drm_i915_private *dev_priv,
 530                          struct intel_digital_port *dig_port,
 531                          unsigned int expected_mask)
 532 {
 533         u32 port_mask;
 534         i915_reg_t dpll_reg;
 535
 536         switch (dig_port->base.port) {
 537         case PORT_B:
 538                 port_mask = DPLL_PORTB_READY_MASK;
 539                 dpll_reg = DPLL(0);
 540                 break;
 541         case PORT_C:
 542                 port_mask = DPLL_PORTC_READY_MASK;
 543                 dpll_reg = DPLL(0);
 544                 expected_mask <<= 4;
 545                 break;
 546         case PORT_D:
 547                 port_mask = DPLL_PORTD_READY_MASK;
 548                 dpll_reg = DPIO_PHY_STATUS;
 549                 break;
 550         default:
 551                 BUG();
 552         }
 553
 554         if (intel_de_wait_for_register(dev_priv, dpll_reg,
 555                                        port_mask, expected_mask, 1000))
 556                 drm_WARN(&dev_priv->drm, 1,
 557                          "timed out waiting for [ENCODER:%d:%s] port ready: got 0x%x, expected 0x%x\n",
 558                          dig_port->base.base.base.id, dig_port->base.base.name,
 559                          intel_de_read(dev_priv, dpll_reg) & port_mask,
 560                          expected_mask);
 561 }
 562
 563 static void ilk_enable_pch_transcoder(const struct intel_crtc_state *crtc_state)
 564 {
 565         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
 566         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
 567         enum pipe pipe = crtc->pipe;
 568         i915_reg_t reg;
 569         u32 val, pipeconf_val;
 570
 571         /* Make sure PCH DPLL is enabled */
 572         assert_shared_dpll_enabled(dev_priv, crtc_state->shared_dpll);
 573
 574         /* FDI must be feeding us bits for PCH ports */
 575         assert_fdi_tx_enabled(dev_priv, pipe);
 576         assert_fdi_rx_enabled(dev_priv, pipe);
 577
 578         if (HAS_PCH_CPT(dev_priv)) {
 579                 reg = TRANS_CHICKEN2(pipe);
 580                 val = intel_de_read(dev_priv, reg);
 581                 /*
 582                  * Workaround: Set the timing override bit
 583                  * before enabling the pch transcoder.
 584                  */
 585                 val |= TRANS_CHICKEN2_TIMING_OVERRIDE;
 586                 /* Configure frame start delay to match the CPU */
 587                 val &= ~TRANS_CHICKEN2_FRAME_START_DELAY_MASK;
 588                 val |= TRANS_CHICKEN2_FRAME_START_DELAY(dev_priv->framestart_delay - 1);
 589                 intel_de_write(dev_priv, reg, val);
 590         }
 591
 592         reg = PCH_TRANSCONF(pipe);
 593         val = intel_de_read(dev_priv, reg);
 594         pipeconf_val = intel_de_read(dev_priv, PIPECONF(pipe));
 595
 596         if (HAS_PCH_IBX(dev_priv)) {
 597                 /* Configure frame start delay to match the CPU */
 598                 val &= ~TRANS_FRAME_START_DELAY_MASK;
 599                 val |= TRANS_FRAME_START_DELAY(dev_priv->framestart_delay - 1);
 600
 601                 /*
 602                  * Make the BPC in transcoder be consistent with
 603                  * that in pipeconf reg. For HDMI we must use 8bpc
 604                  * here for both 8bpc and 12bpc.
 605                  */
 606                 val &= ~PIPECONF_BPC_MASK;
 607                 if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_HDMI))
 608                         val |= PIPECONF_8BPC;
 609                 else
 610                         val |= pipeconf_val & PIPECONF_BPC_MASK;
 611         }
 612
 613         val &= ~TRANS_INTERLACE_MASK;
 614         if ((pipeconf_val & PIPECONF_INTERLACE_MASK) == PIPECONF_INTERLACED_ILK) {
 615                 if (HAS_PCH_IBX(dev_priv) &&
 616                     intel_crtc_has_type(crtc_state, INTEL_OUTPUT_SDVO))
 617                         val |= TRANS_LEGACY_INTERLACED_ILK;
 618                 else
 619                         val |= TRANS_INTERLACED;
 620         } else {
 621                 val |= TRANS_PROGRESSIVE;
 622         }
 623
 624         intel_de_write(dev_priv, reg, val | TRANS_ENABLE);
 625         if (intel_de_wait_for_set(dev_priv, reg, TRANS_STATE_ENABLE, 100))
 626                 drm_err(&dev_priv->drm, "failed to enable transcoder %c\n",
 627                         pipe_name(pipe));
 628 }
 629
 630 static void lpt_enable_pch_transcoder(struct drm_i915_private *dev_priv,
 631                                       enum transcoder cpu_transcoder)
 632 {
 633         u32 val, pipeconf_val;
 634
 635         /* FDI must be feeding us bits for PCH ports */
 636         assert_fdi_tx_enabled(dev_priv, (enum pipe) cpu_transcoder);
 637         assert_fdi_rx_enabled(dev_priv, PIPE_A);
 638
 639         val = intel_de_read(dev_priv, TRANS_CHICKEN2(PIPE_A));
 640         /* Workaround: set timing override bit. */
 641         val |= TRANS_CHICKEN2_TIMING_OVERRIDE;
 642         /* Configure frame start delay to match the CPU */
 643         val &= ~TRANS_CHICKEN2_FRAME_START_DELAY_MASK;
 644         val |= TRANS_CHICKEN2_FRAME_START_DELAY(dev_priv->framestart_delay - 1);
 645         intel_de_write(dev_priv, TRANS_CHICKEN2(PIPE_A), val);
 646
 647         val = TRANS_ENABLE;
 648         pipeconf_val = intel_de_read(dev_priv, PIPECONF(cpu_transcoder));
 649
 650         if ((pipeconf_val & PIPECONF_INTERLACE_MASK_HSW) ==
 651             PIPECONF_INTERLACED_ILK)
 652                 val |= TRANS_INTERLACED;
 653         else
 654                 val |= TRANS_PROGRESSIVE;
 655
 656         intel_de_write(dev_priv, LPT_TRANSCONF, val);
 657         if (intel_de_wait_for_set(dev_priv, LPT_TRANSCONF,
 658                                   TRANS_STATE_ENABLE, 100))
 659                 drm_err(&dev_priv->drm, "Failed to enable PCH transcoder\n");
 660 }
 661
 662 static void ilk_disable_pch_transcoder(struct drm_i915_private *dev_priv,
 663                                        enum pipe pipe)
 664 {
 665         i915_reg_t reg;
 666         u32 val;
 667
 668         /* FDI relies on the transcoder */
 669         assert_fdi_tx_disabled(dev_priv, pipe);
 670         assert_fdi_rx_disabled(dev_priv, pipe);
 671
 672         /* Ports must be off as well */
 673         assert_pch_ports_disabled(dev_priv, pipe);
 674
 675         reg = PCH_TRANSCONF(pipe);
 676         val = intel_de_read(dev_priv, reg);
 677         val &= ~TRANS_ENABLE;
 678         intel_de_write(dev_priv, reg, val);
 679         /* wait for PCH transcoder off, transcoder state */
 680         if (intel_de_wait_for_clear(dev_priv, reg, TRANS_STATE_ENABLE, 50))
 681                 drm_err(&dev_priv->drm, "failed to disable transcoder %c\n",
 682                         pipe_name(pipe));
 683
 684         if (HAS_PCH_CPT(dev_priv)) {
 685                 /* Workaround: Clear the timing override chicken bit again. */
 686                 reg = TRANS_CHICKEN2(pipe);
 687                 val = intel_de_read(dev_priv, reg);
 688                 val &= ~TRANS_CHICKEN2_TIMING_OVERRIDE;
 689                 intel_de_write(dev_priv, reg, val);
 690         }
 691 }
 692
 693 void lpt_disable_pch_transcoder(struct drm_i915_private *dev_priv)
 694 {
 695         u32 val;
 696
 697         val = intel_de_read(dev_priv, LPT_TRANSCONF);
 698         val &= ~TRANS_ENABLE;
 699         intel_de_write(dev_priv, LPT_TRANSCONF, val);
 700         /* wait for PCH transcoder off, transcoder state */
 701         if (intel_de_wait_for_clear(dev_priv, LPT_TRANSCONF,
 702                                     TRANS_STATE_ENABLE, 50))
 703                 drm_err(&dev_priv->drm, "Failed to disable PCH transcoder\n");
 704
 705         /* Workaround: clear timing override bit. */
 706         val = intel_de_read(dev_priv, TRANS_CHICKEN2(PIPE_A));
 707         val &= ~TRANS_CHICKEN2_TIMING_OVERRIDE;
 708         intel_de_write(dev_priv, TRANS_CHICKEN2(PIPE_A), val);
 709 }
 710
 711 enum pipe intel_crtc_pch_transcoder(struct intel_crtc *crtc)
 712 {
 713         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
 714
 715         if (HAS_PCH_LPT(dev_priv))
 716                 return PIPE_A;
 717         else
 718                 return crtc->pipe;
 719 }
 720
 721 void intel_enable_transcoder(const struct intel_crtc_state *new_crtc_state)
 722 {
 723         struct intel_crtc *crtc = to_intel_crtc(new_crtc_state->uapi.crtc);
 724         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
 725         enum transcoder cpu_transcoder = new_crtc_state->cpu_transcoder;
 726         enum pipe pipe = crtc->pipe;
 727         i915_reg_t reg;
 728         u32 val;
 729
 730         drm_dbg_kms(&dev_priv->drm, "enabling pipe %c\n", pipe_name(pipe));
 731
 732         assert_planes_disabled(crtc);
 733
 734         /*
 735          * A pipe without a PLL won't actually be able to drive bits from
 736          * a plane.  On ILK+ the pipe PLLs are integrated, so we don't
 737          * need the check.
 738          */
 739         if (HAS_GMCH(dev_priv)) {
 740                 if (intel_crtc_has_type(new_crtc_state, INTEL_OUTPUT_DSI))
 741                         assert_dsi_pll_enabled(dev_priv);
 742                 else
 743                         assert_pll_enabled(dev_priv, pipe);
 744         } else {
 745                 if (new_crtc_state->has_pch_encoder) {
 746                         /* if driving the PCH, we need FDI enabled */
 747                         assert_fdi_rx_pll_enabled(dev_priv,
 748                                                   intel_crtc_pch_transcoder(crtc));
 749                         assert_fdi_tx_pll_enabled(dev_priv,
 750                                                   (enum pipe) cpu_transcoder);
 751                 }
 752                 /* FIXME: assert CPU port conditions for SNB+ */
 753         }
 754
 755         /* Wa_22012358565:adl-p */
 756         if (DISPLAY_VER(dev_priv) == 13)
 757                 intel_de_rmw(dev_priv, PIPE_ARB_CTL(pipe),
 758                              0, PIPE_ARB_USE_PROG_SLOTS);
 759
 760         reg = PIPECONF(cpu_transcoder);
 761         val = intel_de_read(dev_priv, reg);
 762         if (val & PIPECONF_ENABLE) {
 763                 /* we keep both pipes enabled on 830 */
 764                 drm_WARN_ON(&dev_priv->drm, !IS_I830(dev_priv));
 765                 return;
 766         }
 767
 768         intel_de_write(dev_priv, reg, val | PIPECONF_ENABLE);
 769         intel_de_posting_read(dev_priv, reg);
 770
 771         /*
 772          * Until the pipe starts PIPEDSL reads will return a stale value,
 773          * which causes an apparent vblank timestamp jump when PIPEDSL
 774          * resets to its proper value. That also messes up the frame count
 775          * when it's derived from the timestamps. So let's wait for the
 776          * pipe to start properly before we call drm_crtc_vblank_on()
 777          */
 778         if (intel_crtc_max_vblank_count(new_crtc_state) == 0)
 779                 intel_wait_for_pipe_scanline_moving(crtc);
 780 }
 781
 782 void intel_disable_transcoder(const struct intel_crtc_state *old_crtc_state)
 783 {
 784         struct intel_crtc *crtc = to_intel_crtc(old_crtc_state->uapi.crtc);
 785         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
 786         enum transcoder cpu_transcoder = old_crtc_state->cpu_transcoder;
 787         enum pipe pipe = crtc->pipe;
 788         i915_reg_t reg;
 789         u32 val;
 790
 791         drm_dbg_kms(&dev_priv->drm, "disabling pipe %c\n", pipe_name(pipe));
 792
 793         /*
 794          * Make sure planes won't keep trying to pump pixels to us,
 795          * or we might hang the display.
 796          */
 797         assert_planes_disabled(crtc);
 798
 799         reg = PIPECONF(cpu_transcoder);
 800         val = intel_de_read(dev_priv, reg);
 801         if ((val & PIPECONF_ENABLE) == 0)
 802                 return;
 803
 804         /*
 805          * Double wide has implications for planes
 806          * so best keep it disabled when not needed.
 807          */
 808         if (old_crtc_state->double_wide)
 809                 val &= ~PIPECONF_DOUBLE_WIDE;
 810
 811         /* Don't disable pipe or pipe PLLs if needed */
 812         if (!IS_I830(dev_priv))
 813                 val &= ~PIPECONF_ENABLE;
 814
 815         if (DISPLAY_VER(dev_priv) >= 12)
 816                 intel_de_rmw(dev_priv, CHICKEN_TRANS(cpu_transcoder),
 817                              FECSTALL_DIS_DPTSTREAM_DPTTG, 0);
 818
 819         intel_de_write(dev_priv, reg, val);
 820         if ((val & PIPECONF_ENABLE) == 0)
 821                 intel_wait_for_pipe_off(old_crtc_state);
 822 }
 823
 824 bool
 825 intel_format_info_is_yuv_semiplanar(const struct drm_format_info *info,
 826                                     u64 modifier)
 827 {
 828         return info->is_yuv &&
 829                info->num_planes == (is_ccs_modifier(modifier) ? 4 : 2);
 830 }
 831
 832 unsigned int intel_rotation_info_size(const struct intel_rotation_info *rot_info)
 833 {
 834         unsigned int size = 0;
 835         int i;
 836
 837         for (i = 0 ; i < ARRAY_SIZE(rot_info->plane); i++)
 838                 size += rot_info->plane[i].dst_stride * rot_info->plane[i].width;
 839
 840         return size;
 841 }
 842
 843 unsigned int intel_remapped_info_size(const struct intel_remapped_info *rem_info)
 844 {
 845         unsigned int size = 0;
 846         int i;
 847
 848         for (i = 0 ; i < ARRAY_SIZE(rem_info->plane); i++) {
 849                 if (rem_info->plane_alignment)
 850                         size = ALIGN(size, rem_info->plane_alignment);
 851                 size += rem_info->plane[i].dst_stride * rem_info->plane[i].height;
 852         }
 853
 854         return size;
 855 }
 856
 857 static bool intel_plane_uses_fence(const struct intel_plane_state *plane_state)
 858 {
 859         struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
 860         struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
 861
 862         return DISPLAY_VER(dev_priv) < 4 ||
 863                 (plane->has_fbc &&
 864                  plane_state->view.gtt.type == I915_GGTT_VIEW_NORMAL);
 865 }
 866
 867 static struct i915_vma *
 868 intel_pin_fb_obj_dpt(struct drm_framebuffer *fb,
 869                      const struct i915_ggtt_view *view,
 870                      bool uses_fence,
 871                      unsigned long *out_flags,
 872                      struct i915_address_space *vm)
 873 {
 874         struct drm_device *dev = fb->dev;
 875         struct drm_i915_private *dev_priv = to_i915(dev);
 876         struct drm_i915_gem_object *obj = intel_fb_obj(fb);
 877         struct i915_vma *vma;
 878         u32 alignment;
 879         int ret;
 880
 881         if (WARN_ON(!i915_gem_object_is_framebuffer(obj)))
 882                 return ERR_PTR(-EINVAL);
 883
 884         alignment = 4096 * 512;
 885
 886         atomic_inc(&dev_priv->gpu_error.pending_fb_pin);
 887
 888         ret = i915_gem_object_set_cache_level(obj, I915_CACHE_NONE);
 889         if (ret) {
 890                 vma = ERR_PTR(ret);
 891                 goto err;
 892         }
 893
 894         vma = i915_vma_instance(obj, vm, view);
 895         if (IS_ERR(vma))
 896                 goto err;
 897
 898         if (i915_vma_misplaced(vma, 0, alignment, 0)) {
 899                 ret = i915_vma_unbind(vma);
 900                 if (ret) {
 901                         vma = ERR_PTR(ret);
 902                         goto err;
 903                 }
 904         }
 905
 906         ret = i915_vma_pin(vma, 0, alignment, PIN_GLOBAL);
 907         if (ret) {
 908                 vma = ERR_PTR(ret);
 909                 goto err;
 910         }
 911
 912         vma->display_alignment = max_t(u64, vma->display_alignment, alignment);
 913
 914         i915_gem_object_flush_if_display(obj);
 915
 916         i915_vma_get(vma);
 917 err:
 918         atomic_dec(&dev_priv->gpu_error.pending_fb_pin);
 919
 920         return vma;
 921 }
 922
 923 struct i915_vma *
 924 intel_pin_and_fence_fb_obj(struct drm_framebuffer *fb,
 925                            bool phys_cursor,
 926                            const struct i915_ggtt_view *view,
 927                            bool uses_fence,
 928                            unsigned long *out_flags)
 929 {
 930         struct drm_device *dev = fb->dev;
 931         struct drm_i915_private *dev_priv = to_i915(dev);
 932         struct drm_i915_gem_object *obj = intel_fb_obj(fb);
 933         intel_wakeref_t wakeref;
 934         struct i915_gem_ww_ctx ww;
 935         struct i915_vma *vma;
 936         unsigned int pinctl;
 937         u32 alignment;
 938         int ret;
 939
 940         if (drm_WARN_ON(dev, !i915_gem_object_is_framebuffer(obj)))
 941                 return ERR_PTR(-EINVAL);
 942
 943         if (phys_cursor)
 944                 alignment = intel_cursor_alignment(dev_priv);
 945         else
 946                 alignment = intel_surf_alignment(fb, 0);
 947         if (drm_WARN_ON(dev, alignment && !is_power_of_2(alignment)))
 948                 return ERR_PTR(-EINVAL);
 949
 950         /* Note that the w/a also requires 64 PTE of padding following the
 951          * bo. We currently fill all unused PTE with the shadow page and so
 952          * we should always have valid PTE following the scanout preventing
 953          * the VT-d warning.
 954          */
 955         if (intel_scanout_needs_vtd_wa(dev_priv) && alignment < 256 * 1024)
 956                 alignment = 256 * 1024;
 957
 958         /*
 959          * Global gtt pte registers are special registers which actually forward
 960          * writes to a chunk of system memory. Which means that there is no risk
 961          * that the register values disappear as soon as we call
 962          * intel_runtime_pm_put(), so it is correct to wrap only the
 963          * pin/unpin/fence and not more.
 964          */
 965         wakeref = intel_runtime_pm_get(&dev_priv->runtime_pm);
 966
 967         atomic_inc(&dev_priv->gpu_error.pending_fb_pin);
 968
 969         /*
 970          * Valleyview is definitely limited to scanning out the first
 971          * 512MiB. Lets presume this behaviour was inherited from the
 972          * g4x display engine and that all earlier gen are similarly
 973          * limited. Testing suggests that it is a little more
 974          * complicated than this. For example, Cherryview appears quite
 975          * happy to scanout from anywhere within its global aperture.
 976          */
 977         pinctl = 0;
 978         if (HAS_GMCH(dev_priv))
 979                 pinctl |= PIN_MAPPABLE;
 980
 981         i915_gem_ww_ctx_init(&ww, true);
 982 retry:
 983         ret = i915_gem_object_lock(obj, &ww);
 984         if (!ret && phys_cursor)
 985                 ret = i915_gem_object_attach_phys(obj, alignment);
 986         else if (!ret && HAS_LMEM(dev_priv))
 987                 ret = i915_gem_object_migrate(obj, &ww, INTEL_REGION_LMEM);
 988         /* TODO: Do we need to sync when migration becomes async? */
 989         if (!ret)
 990                 ret = i915_gem_object_pin_pages(obj);
 991         if (ret)
 992                 goto err;
 993
 994         if (!ret) {
 995                 vma = i915_gem_object_pin_to_display_plane(obj, &ww, alignment,
 996                                                            view, pinctl);
 997                 if (IS_ERR(vma)) {
 998                         ret = PTR_ERR(vma);
 999                         goto err_unpin;
1000                 }
1001         }
1002
1003         if (uses_fence && i915_vma_is_map_and_fenceable(vma)) {
1004                 /*
1005                  * Install a fence for tiled scan-out. Pre-i965 always needs a
1006                  * fence, whereas 965+ only requires a fence if using
1007                  * framebuffer compression.  For simplicity, we always, when
1008                  * possible, install a fence as the cost is not that onerous.
1009                  *
1010                  * If we fail to fence the tiled scanout, then either the
1011                  * modeset will reject the change (which is highly unlikely as
1012                  * the affected systems, all but one, do not have unmappable
1013                  * space) or we will not be able to enable full powersaving
1014                  * techniques (also likely not to apply due to various limits
1015                  * FBC and the like impose on the size of the buffer, which
1016                  * presumably we violated anyway with this unmappable buffer).
1017                  * Anyway, it is presumably better to stumble onwards with
1018                  * something and try to run the system in a "less than optimal"
1019                  * mode that matches the user configuration.
1020                  */
1021                 ret = i915_vma_pin_fence(vma);
1022                 if (ret != 0 && DISPLAY_VER(dev_priv) < 4) {
1023                         i915_vma_unpin(vma);
1024                         goto err_unpin;
1025                 }
1026                 ret = 0;
1027
1028                 if (vma->fence)
1029                         *out_flags |= PLANE_HAS_FENCE;
1030         }
1031
1032         i915_vma_get(vma);
1033
1034 err_unpin:
1035         i915_gem_object_unpin_pages(obj);
1036 err:
1037         if (ret == -EDEADLK) {
1038                 ret = i915_gem_ww_ctx_backoff(&ww);
1039                 if (!ret)
1040                         goto retry;
1041         }
1042         i915_gem_ww_ctx_fini(&ww);
1043         if (ret)
1044                 vma = ERR_PTR(ret);
1045
1046         atomic_dec(&dev_priv->gpu_error.pending_fb_pin);
1047         intel_runtime_pm_put(&dev_priv->runtime_pm, wakeref);
1048         return vma;
1049 }
1050
1051 void intel_unpin_fb_vma(struct i915_vma *vma, unsigned long flags)
1052 {
1053         if (flags & PLANE_HAS_FENCE)
1054                 i915_vma_unpin_fence(vma);
1055         i915_vma_unpin(vma);
1056         i915_vma_put(vma);
1057 }
1058
1059 /*
1060  * Convert the x/y offsets into a linear offset.
1061  * Only valid with 0/180 degree rotation, which is fine since linear
1062  * offset is only used with linear buffers on pre-hsw and tiled buffers
1063  * with gen2/3, and 90/270 degree rotations isn't supported on any of them.
1064  */
1065 u32 intel_fb_xy_to_linear(int x, int y,
1066                           const struct intel_plane_state *state,
1067                           int color_plane)
1068 {
1069         const struct drm_framebuffer *fb = state->hw.fb;
1070         unsigned int cpp = fb->format->cpp[color_plane];
1071         unsigned int pitch = state->view.color_plane[color_plane].stride;
1072
1073         return y * pitch + x * cpp;
1074 }
1075
1076 /*
1077  * Add the x/y offsets derived from fb->offsets[] to the user
1078  * specified plane src x/y offsets. The resulting x/y offsets
1079  * specify the start of scanout from the beginning of the gtt mapping.
1080  */
1081 void intel_add_fb_offsets(int *x, int *y,
1082                           const struct intel_plane_state *state,
1083                           int color_plane)
1084
1085 {
1086         *x += state->view.color_plane[color_plane].x;
1087         *y += state->view.color_plane[color_plane].y;
1088 }
1089
1090 /*
1091  * From the Sky Lake PRM:
1092  * "The Color Control Surface (CCS) contains the compression status of
1093  *  the cache-line pairs. The compression state of the cache-line pair
1094  *  is specified by 2 bits in the CCS. Each CCS cache-line represents
1095  *  an area on the main surface of 16 x16 sets of 128 byte Y-tiled
1096  *  cache-line-pairs. CCS is always Y tiled."
1097  *
1098  * Since cache line pairs refers to horizontally adjacent cache lines,
1099  * each cache line in the CCS corresponds to an area of 32x16 cache
1100  * lines on the main surface. Since each pixel is 4 bytes, this gives
1101  * us a ratio of one byte in the CCS for each 8x16 pixels in the
1102  * main surface.
1103  */
1104 static const struct drm_format_info skl_ccs_formats[] = {
1105         { .format = DRM_FORMAT_XRGB8888, .depth = 24, .num_planes = 2,
1106           .cpp = { 4, 1, }, .hsub = 8, .vsub = 16, },
1107         { .format = DRM_FORMAT_XBGR8888, .depth = 24, .num_planes = 2,
1108           .cpp = { 4, 1, }, .hsub = 8, .vsub = 16, },
1109         { .format = DRM_FORMAT_ARGB8888, .depth = 32, .num_planes = 2,
1110           .cpp = { 4, 1, }, .hsub = 8, .vsub = 16, .has_alpha = true, },
1111         { .format = DRM_FORMAT_ABGR8888, .depth = 32, .num_planes = 2,
1112           .cpp = { 4, 1, }, .hsub = 8, .vsub = 16, .has_alpha = true, },
1113 };
1114
1115 /*
1116  * Gen-12 compression uses 4 bits of CCS data for each cache line pair in the
1117  * main surface. And each 64B CCS cache line represents an area of 4x1 Y-tiles
1118  * in the main surface. With 4 byte pixels and each Y-tile having dimensions of
1119  * 32x32 pixels, the ratio turns out to 1B in the CCS for every 2x32 pixels in
1120  * the main surface.
1121  */
1122 static const struct drm_format_info gen12_ccs_formats[] = {
1123         { .format = DRM_FORMAT_XRGB8888, .depth = 24, .num_planes = 2,
1124           .char_per_block = { 4, 1 }, .block_w = { 1, 2 }, .block_h = { 1, 1 },
1125           .hsub = 1, .vsub = 1, },
1126         { .format = DRM_FORMAT_XBGR8888, .depth = 24, .num_planes = 2,
1127           .char_per_block = { 4, 1 }, .block_w = { 1, 2 }, .block_h = { 1, 1 },
1128           .hsub = 1, .vsub = 1, },
1129         { .format = DRM_FORMAT_ARGB8888, .depth = 32, .num_planes = 2,
1130           .char_per_block = { 4, 1 }, .block_w = { 1, 2 }, .block_h = { 1, 1 },
1131           .hsub = 1, .vsub = 1, .has_alpha = true },
1132         { .format = DRM_FORMAT_ABGR8888, .depth = 32, .num_planes = 2,
1133           .char_per_block = { 4, 1 }, .block_w = { 1, 2 }, .block_h = { 1, 1 },
1134           .hsub = 1, .vsub = 1, .has_alpha = true },
1135         { .format = DRM_FORMAT_YUYV, .num_planes = 2,
1136           .char_per_block = { 2, 1 }, .block_w = { 1, 2 }, .block_h = { 1, 1 },
1137           .hsub = 2, .vsub = 1, .is_yuv = true },
1138         { .format = DRM_FORMAT_YVYU, .num_planes = 2,
1139           .char_per_block = { 2, 1 }, .block_w = { 1, 2 }, .block_h = { 1, 1 },
1140           .hsub = 2, .vsub = 1, .is_yuv = true },
1141         { .format = DRM_FORMAT_UYVY, .num_planes = 2,
1142           .char_per_block = { 2, 1 }, .block_w = { 1, 2 }, .block_h = { 1, 1 },
1143           .hsub = 2, .vsub = 1, .is_yuv = true },
1144         { .format = DRM_FORMAT_VYUY, .num_planes = 2,
1145           .char_per_block = { 2, 1 }, .block_w = { 1, 2 }, .block_h = { 1, 1 },
1146           .hsub = 2, .vsub = 1, .is_yuv = true },
1147         { .format = DRM_FORMAT_XYUV8888, .num_planes = 2,
1148           .char_per_block = { 4, 1 }, .block_w = { 1, 2 }, .block_h = { 1, 1 },
1149           .hsub = 1, .vsub = 1, .is_yuv = true },
1150         { .format = DRM_FORMAT_NV12, .num_planes = 4,
1151           .char_per_block = { 1, 2, 1, 1 }, .block_w = { 1, 1, 4, 4 }, .block_h = { 1, 1, 1, 1 },
1152           .hsub = 2, .vsub = 2, .is_yuv = true },
1153         { .format = DRM_FORMAT_P010, .num_planes = 4,
1154           .char_per_block = { 2, 4, 1, 1 }, .block_w = { 1, 1, 2, 2 }, .block_h = { 1, 1, 1, 1 },
1155           .hsub = 2, .vsub = 2, .is_yuv = true },
1156         { .format = DRM_FORMAT_P012, .num_planes = 4,
1157           .char_per_block = { 2, 4, 1, 1 }, .block_w = { 1, 1, 2, 2 }, .block_h = { 1, 1, 1, 1 },
1158           .hsub = 2, .vsub = 2, .is_yuv = true },
1159         { .format = DRM_FORMAT_P016, .num_planes = 4,
1160           .char_per_block = { 2, 4, 1, 1 }, .block_w = { 1, 1, 2, 2 }, .block_h = { 1, 1, 1, 1 },
1161           .hsub = 2, .vsub = 2, .is_yuv = true },
1162 };
1163
1164 /*
1165  * Same as gen12_ccs_formats[] above, but with additional surface used
1166  * to pass Clear Color information in plane 2 with 64 bits of data.
1167  */
1168 static const struct drm_format_info gen12_ccs_cc_formats[] = {
1169         { .format = DRM_FORMAT_XRGB8888, .depth = 24, .num_planes = 3,
1170           .char_per_block = { 4, 1, 0 }, .block_w = { 1, 2, 2 }, .block_h = { 1, 1, 1 },
1171           .hsub = 1, .vsub = 1, },
1172         { .format = DRM_FORMAT_XBGR8888, .depth = 24, .num_planes = 3,
1173           .char_per_block = { 4, 1, 0 }, .block_w = { 1, 2, 2 }, .block_h = { 1, 1, 1 },
1174           .hsub = 1, .vsub = 1, },
1175         { .format = DRM_FORMAT_ARGB8888, .depth = 32, .num_planes = 3,
1176           .char_per_block = { 4, 1, 0 }, .block_w = { 1, 2, 2 }, .block_h = { 1, 1, 1 },
1177           .hsub = 1, .vsub = 1, .has_alpha = true },
1178         { .format = DRM_FORMAT_ABGR8888, .depth = 32, .num_planes = 3,
1179           .char_per_block = { 4, 1, 0 }, .block_w = { 1, 2, 2 }, .block_h = { 1, 1, 1 },
1180           .hsub = 1, .vsub = 1, .has_alpha = true },
1181 };
1182
1183 static const struct drm_format_info *
1184 lookup_format_info(const struct drm_format_info formats[],
1185                    int num_formats, u32 format)
1186 {
1187         int i;
1188
1189         for (i = 0; i < num_formats; i++) {
1190                 if (formats[i].format == format)
1191                         return &formats[i];
1192         }
1193
1194         return NULL;
1195 }
1196
1197 static const struct drm_format_info *
1198 intel_get_format_info(const struct drm_mode_fb_cmd2 *cmd)
1199 {
1200         switch (cmd->modifier[0]) {
1201         case I915_FORMAT_MOD_Y_TILED_CCS:
1202         case I915_FORMAT_MOD_Yf_TILED_CCS:
1203                 return lookup_format_info(skl_ccs_formats,
1204                                           ARRAY_SIZE(skl_ccs_formats),
1205                                           cmd->pixel_format);
1206         case I915_FORMAT_MOD_Y_TILED_GEN12_RC_CCS:
1207         case I915_FORMAT_MOD_Y_TILED_GEN12_MC_CCS:
1208                 return lookup_format_info(gen12_ccs_formats,
1209                                           ARRAY_SIZE(gen12_ccs_formats),
1210                                           cmd->pixel_format);
1211         case I915_FORMAT_MOD_Y_TILED_GEN12_RC_CCS_CC:
1212                 return lookup_format_info(gen12_ccs_cc_formats,
1213                                           ARRAY_SIZE(gen12_ccs_cc_formats),
1214                                           cmd->pixel_format);
1215         default:
1216                 return NULL;
1217         }
1218 }
1219
1220 u32 intel_plane_fb_max_stride(struct drm_i915_private *dev_priv,
1221                               u32 pixel_format, u64 modifier)
1222 {
1223         struct intel_crtc *crtc;
1224         struct intel_plane *plane;
1225
1226         if (!HAS_DISPLAY(dev_priv))
1227                 return 0;
1228
1229         /*
1230          * We assume the primary plane for pipe A has
1231          * the highest stride limits of them all,
1232          * if in case pipe A is disabled, use the first pipe from pipe_mask.
1233          */
1234         crtc = intel_get_first_crtc(dev_priv);
1235         if (!crtc)
1236                 return 0;
1237
1238         plane = to_intel_plane(crtc->base.primary);
1239
1240         return plane->max_stride(plane, pixel_format, modifier,
1241                                  DRM_MODE_ROTATE_0);
1242 }
1243
1244 static struct i915_vma *
1245 initial_plane_vma(struct drm_i915_private *i915,
1246                   struct intel_initial_plane_config *plane_config)
1247 {
1248         struct drm_i915_gem_object *obj;
1249         struct i915_vma *vma;
1250         u32 base, size;
1251
1252         if (plane_config->size == 0)
1253                 return NULL;
1254
1255         base = round_down(plane_config->base,
1256                           I915_GTT_MIN_ALIGNMENT);
1257         size = round_up(plane_config->base + plane_config->size,
1258                         I915_GTT_MIN_ALIGNMENT);
1259         size -= base;
1260
1261         /*
1262          * If the FB is too big, just don't use it since fbdev is not very
1263          * important and we should probably use that space with FBC or other
1264          * features.
1265          */
1266         if (IS_ENABLED(CONFIG_FRAMEBUFFER_CONSOLE) &&
1267             size * 2 > i915->stolen_usable_size)
1268                 return NULL;
1269
1270         obj = i915_gem_object_create_stolen_for_preallocated(i915, base, size);
1271         if (IS_ERR(obj))
1272                 return NULL;
1273
1274         /*
1275          * Mark it WT ahead of time to avoid changing the
1276          * cache_level during fbdev initialization. The
1277          * unbind there would get stuck waiting for rcu.
1278          */
1279         i915_gem_object_set_cache_coherency(obj, HAS_WT(i915) ?
1280                                             I915_CACHE_WT : I915_CACHE_NONE);
1281
1282         switch (plane_config->tiling) {
1283         case I915_TILING_NONE:
1284                 break;
1285         case I915_TILING_X:
1286         case I915_TILING_Y:
1287                 obj->tiling_and_stride =
1288                         plane_config->fb->base.pitches[0] |
1289                         plane_config->tiling;
1290                 break;
1291         default:
1292                 MISSING_CASE(plane_config->tiling);
1293                 goto err_obj;
1294         }
1295
1296         vma = i915_vma_instance(obj, &i915->ggtt.vm, NULL);
1297         if (IS_ERR(vma))
1298                 goto err_obj;
1299
1300         if (i915_ggtt_pin(vma, NULL, 0, PIN_MAPPABLE | PIN_OFFSET_FIXED | base))
1301                 goto err_obj;
1302
1303         if (i915_gem_object_is_tiled(obj) &&
1304             !i915_vma_is_map_and_fenceable(vma))
1305                 goto err_obj;
1306
1307         return vma;
1308
1309 err_obj:
1310         i915_gem_object_put(obj);
1311         return NULL;
1312 }
1313
1314 static bool
1315 intel_alloc_initial_plane_obj(struct intel_crtc *crtc,
1316                               struct intel_initial_plane_config *plane_config)
1317 {
1318         struct drm_device *dev = crtc->base.dev;
1319         struct drm_i915_private *dev_priv = to_i915(dev);
1320         struct drm_mode_fb_cmd2 mode_cmd = { 0 };
1321         struct drm_framebuffer *fb = &plane_config->fb->base;
1322         struct i915_vma *vma;
1323
1324         switch (fb->modifier) {
1325         case DRM_FORMAT_MOD_LINEAR:
1326         case I915_FORMAT_MOD_X_TILED:
1327         case I915_FORMAT_MOD_Y_TILED:
1328                 break;
1329         default:
1330                 drm_dbg(&dev_priv->drm,
1331                         "Unsupported modifier for initial FB: 0x%llx\n",
1332                         fb->modifier);
1333                 return false;
1334         }
1335
1336         vma = initial_plane_vma(dev_priv, plane_config);
1337         if (!vma)
1338                 return false;
1339
1340         mode_cmd.pixel_format = fb->format->format;
1341         mode_cmd.width = fb->width;
1342         mode_cmd.height = fb->height;
1343         mode_cmd.pitches[0] = fb->pitches[0];
1344         mode_cmd.modifier[0] = fb->modifier;
1345         mode_cmd.flags = DRM_MODE_FB_MODIFIERS;
1346
1347         if (intel_framebuffer_init(to_intel_framebuffer(fb),
1348                                    vma->obj, &mode_cmd)) {
1349                 drm_dbg_kms(&dev_priv->drm, "intel fb init failed\n");
1350                 goto err_vma;
1351         }
1352
1353         plane_config->vma = vma;
1354         return true;
1355
1356 err_vma:
1357         i915_vma_put(vma);
1358         return false;
1359 }
1360
1361 static void
1362 intel_set_plane_visible(struct intel_crtc_state *crtc_state,
1363                         struct intel_plane_state *plane_state,
1364                         bool visible)
1365 {
1366         struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
1367
1368         plane_state->uapi.visible = visible;
1369
1370         if (visible)
1371                 crtc_state->uapi.plane_mask |= drm_plane_mask(&plane->base);
1372         else
1373                 crtc_state->uapi.plane_mask &= ~drm_plane_mask(&plane->base);
1374 }
1375
1376 static void fixup_plane_bitmasks(struct intel_crtc_state *crtc_state)
1377 {
1378         struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
1379         struct drm_plane *plane;
1380
1381         /*
1382          * Active_planes aliases if multiple "primary" or cursor planes
1383          * have been used on the same (or wrong) pipe. plane_mask uses
1384          * unique ids, hence we can use that to reconstruct active_planes.
1385          */
1386         crtc_state->enabled_planes = 0;
1387         crtc_state->active_planes = 0;
1388
1389         drm_for_each_plane_mask(plane, &dev_priv->drm,
1390                                 crtc_state->uapi.plane_mask) {
1391                 crtc_state->enabled_planes |= BIT(to_intel_plane(plane)->id);
1392                 crtc_state->active_planes |= BIT(to_intel_plane(plane)->id);
1393         }
1394 }
1395
1396 static void intel_plane_disable_noatomic(struct intel_crtc *crtc,
1397                                          struct intel_plane *plane)
1398 {
1399         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1400         struct intel_crtc_state *crtc_state =
1401                 to_intel_crtc_state(crtc->base.state);
1402         struct intel_plane_state *plane_state =
1403                 to_intel_plane_state(plane->base.state);
1404
1405         drm_dbg_kms(&dev_priv->drm,
1406                     "Disabling [PLANE:%d:%s] on [CRTC:%d:%s]\n",
1407                     plane->base.base.id, plane->base.name,
1408                     crtc->base.base.id, crtc->base.name);
1409
1410         intel_set_plane_visible(crtc_state, plane_state, false);
1411         fixup_plane_bitmasks(crtc_state);
1412         crtc_state->data_rate[plane->id] = 0;
1413         crtc_state->min_cdclk[plane->id] = 0;
1414
1415         if (plane->id == PLANE_PRIMARY)
1416                 hsw_disable_ips(crtc_state);
1417
1418         /*
1419          * Vblank time updates from the shadow to live plane control register
1420          * are blocked if the memory self-refresh mode is active at that
1421          * moment. So to make sure the plane gets truly disabled, disable
1422          * first the self-refresh mode. The self-refresh enable bit in turn
1423          * will be checked/applied by the HW only at the next frame start
1424          * event which is after the vblank start event, so we need to have a
1425          * wait-for-vblank between disabling the plane and the pipe.
1426          */
1427         if (HAS_GMCH(dev_priv) &&
1428             intel_set_memory_cxsr(dev_priv, false))
1429                 intel_wait_for_vblank(dev_priv, crtc->pipe);
1430
1431         /*
1432          * Gen2 reports pipe underruns whenever all planes are disabled.
1433          * So disable underrun reporting before all the planes get disabled.
1434          */
1435         if (DISPLAY_VER(dev_priv) == 2 && !crtc_state->active_planes)
1436                 intel_set_cpu_fifo_underrun_reporting(dev_priv, crtc->pipe, false);
1437
1438         intel_disable_plane(plane, crtc_state);
1439         intel_wait_for_vblank(dev_priv, crtc->pipe);
1440 }
1441
1442 static bool
1443 intel_reuse_initial_plane_obj(struct drm_i915_private *i915,
1444                               const struct intel_initial_plane_config *plane_config,
1445                               struct drm_framebuffer **fb,
1446                               struct i915_vma **vma)
1447 {
1448         struct intel_crtc *crtc;
1449
1450         for_each_intel_crtc(&i915->drm, crtc) {
1451                 struct intel_crtc_state *crtc_state =
1452                         to_intel_crtc_state(crtc->base.state);
1453                 struct intel_plane *plane =
1454                         to_intel_plane(crtc->base.primary);
1455                 struct intel_plane_state *plane_state =
1456                         to_intel_plane_state(plane->base.state);
1457
1458                 if (!crtc_state->uapi.active)
1459                         continue;
1460
1461                 if (!plane_state->ggtt_vma)
1462                         continue;
1463
1464                 if (intel_plane_ggtt_offset(plane_state) == plane_config->base) {
1465                         *fb = plane_state->hw.fb;
1466                         *vma = plane_state->ggtt_vma;
1467                         return true;
1468                 }
1469         }
1470
1471         return false;
1472 }
1473
1474 static void
1475 intel_find_initial_plane_obj(struct intel_crtc *crtc,
1476                              struct intel_initial_plane_config *plane_config)
1477 {
1478         struct drm_device *dev = crtc->base.dev;
1479         struct drm_i915_private *dev_priv = to_i915(dev);
1480         struct intel_crtc_state *crtc_state =
1481                 to_intel_crtc_state(crtc->base.state);
1482         struct intel_plane *plane =
1483                 to_intel_plane(crtc->base.primary);
1484         struct intel_plane_state *plane_state =
1485                 to_intel_plane_state(plane->base.state);
1486         struct drm_framebuffer *fb;
1487         struct i915_vma *vma;
1488
1489         /*
1490          * TODO:
1491          *   Disable planes if get_initial_plane_config() failed.
1492          *   Make sure things work if the surface base is not page aligned.
1493          */
1494         if (!plane_config->fb)
1495                 return;
1496
1497         if (intel_alloc_initial_plane_obj(crtc, plane_config)) {
1498                 fb = &plane_config->fb->base;
1499                 vma = plane_config->vma;
1500                 goto valid_fb;
1501         }
1502
1503         /*
1504          * Failed to alloc the obj, check to see if we should share
1505          * an fb with another CRTC instead
1506          */
1507         if (intel_reuse_initial_plane_obj(dev_priv, plane_config, &fb, &vma))
1508                 goto valid_fb;
1509
1510         /*
1511          * We've failed to reconstruct the BIOS FB.  Current display state
1512          * indicates that the primary plane is visible, but has a NULL FB,
1513          * which will lead to problems later if we don't fix it up.  The
1514          * simplest solution is to just disable the primary plane now and
1515          * pretend the BIOS never had it enabled.
1516          */
1517         intel_plane_disable_noatomic(crtc, plane);
1518         if (crtc_state->bigjoiner) {
1519                 struct intel_crtc *slave =
1520                         crtc_state->bigjoiner_linked_crtc;
1521                 intel_plane_disable_noatomic(slave, to_intel_plane(slave->base.primary));
1522         }
1523
1524         return;
1525
1526 valid_fb:
1527         plane_state->uapi.rotation = plane_config->rotation;
1528         intel_fb_fill_view(to_intel_framebuffer(fb),
1529                            plane_state->uapi.rotation, &plane_state->view);
1530
1531         __i915_vma_pin(vma);
1532         plane_state->ggtt_vma = i915_vma_get(vma);
1533         if (intel_plane_uses_fence(plane_state) &&
1534             i915_vma_pin_fence(vma) == 0 && vma->fence)
1535                 plane_state->flags |= PLANE_HAS_FENCE;
1536
1537         plane_state->uapi.src_x = 0;
1538         plane_state->uapi.src_y = 0;
1539         plane_state->uapi.src_w = fb->width << 16;
1540         plane_state->uapi.src_h = fb->height << 16;
1541
1542         plane_state->uapi.crtc_x = 0;
1543         plane_state->uapi.crtc_y = 0;
1544         plane_state->uapi.crtc_w = fb->width;
1545         plane_state->uapi.crtc_h = fb->height;
1546
1547         if (plane_config->tiling)
1548                 dev_priv->preserve_bios_swizzle = true;
1549
1550         plane_state->uapi.fb = fb;
1551         drm_framebuffer_get(fb);
1552
1553         plane_state->uapi.crtc = &crtc->base;
1554         intel_plane_copy_uapi_to_hw_state(plane_state, plane_state, crtc);
1555
1556         atomic_or(plane->frontbuffer_bit, &to_intel_frontbuffer(fb)->bits);
1557 }
1558
1559 unsigned int
1560 intel_plane_fence_y_offset(const struct intel_plane_state *plane_state)
1561 {
1562         int x = 0, y = 0;
1563
1564         intel_plane_adjust_aligned_offset(&x, &y, plane_state, 0,
1565                                           plane_state->view.color_plane[0].offset, 0);
1566
1567         return y;
1568 }
1569
1570 static int
1571 __intel_display_resume(struct drm_device *dev,
1572                        struct drm_atomic_state *state,
1573                        struct drm_modeset_acquire_ctx *ctx)
1574 {
1575         struct drm_crtc_state *crtc_state;
1576         struct drm_crtc *crtc;
1577         int i, ret;
1578
1579         intel_modeset_setup_hw_state(dev, ctx);
1580         intel_vga_redisable(to_i915(dev));
1581
1582         if (!state)
1583                 return 0;
1584
1585         /*
1586          * We've duplicated the state, pointers to the old state are invalid.
1587          *
1588          * Don't attempt to use the old state until we commit the duplicated state.
1589          */
1590         for_each_new_crtc_in_state(state, crtc, crtc_state, i) {
1591                 /*
1592                  * Force recalculation even if we restore
1593                  * current state. With fast modeset this may not result
1594                  * in a modeset when the state is compatible.
1595                  */
1596                 crtc_state->mode_changed = true;
1597         }
1598
1599         /* ignore any reset values/BIOS leftovers in the WM registers */
1600         if (!HAS_GMCH(to_i915(dev)))
1601                 to_intel_atomic_state(state)->skip_intermediate_wm = true;
1602
1603         ret = drm_atomic_helper_commit_duplicated_state(state, ctx);
1604
1605         drm_WARN_ON(dev, ret == -EDEADLK);
1606         return ret;
1607 }
1608
1609 static bool gpu_reset_clobbers_display(struct drm_i915_private *dev_priv)
1610 {
1611         return (INTEL_INFO(dev_priv)->gpu_reset_clobbers_display &&
1612                 intel_has_gpu_reset(&dev_priv->gt));
1613 }
1614
1615 void intel_display_prepare_reset(struct drm_i915_private *dev_priv)
1616 {
1617         struct drm_device *dev = &dev_priv->drm;
1618         struct drm_modeset_acquire_ctx *ctx = &dev_priv->reset_ctx;
1619         struct drm_atomic_state *state;
1620         int ret;
1621
1622         if (!HAS_DISPLAY(dev_priv))
1623                 return;
1624
1625         /* reset doesn't touch the display */
1626         if (!dev_priv->params.force_reset_modeset_test &&
1627             !gpu_reset_clobbers_display(dev_priv))
1628                 return;
1629
1630         /* We have a modeset vs reset deadlock, defensively unbreak it. */
1631         set_bit(I915_RESET_MODESET, &dev_priv->gt.reset.flags);
1632         smp_mb__after_atomic();
1633         wake_up_bit(&dev_priv->gt.reset.flags, I915_RESET_MODESET);
1634
1635         if (atomic_read(&dev_priv->gpu_error.pending_fb_pin)) {
1636                 drm_dbg_kms(&dev_priv->drm,
1637                             "Modeset potentially stuck, unbreaking through wedging\n");
1638                 intel_gt_set_wedged(&dev_priv->gt);
1639         }
1640
1641         /*
1642          * Need mode_config.mutex so that we don't
1643          * trample ongoing ->detect() and whatnot.
1644          */
1645         mutex_lock(&dev->mode_config.mutex);
1646         drm_modeset_acquire_init(ctx, 0);
1647         while (1) {
1648                 ret = drm_modeset_lock_all_ctx(dev, ctx);
1649                 if (ret != -EDEADLK)
1650                         break;
1651
1652                 drm_modeset_backoff(ctx);
1653         }
1654         /*
1655          * Disabling the crtcs gracefully seems nicer. Also the
1656          * g33 docs say we should at least disable all the planes.
1657          */
1658         state = drm_atomic_helper_duplicate_state(dev, ctx);
1659         if (IS_ERR(state)) {
1660                 ret = PTR_ERR(state);
1661                 drm_err(&dev_priv->drm, "Duplicating state failed with %i\n",
1662                         ret);
1663                 return;
1664         }
1665
1666         ret = drm_atomic_helper_disable_all(dev, ctx);
1667         if (ret) {
1668                 drm_err(&dev_priv->drm, "Suspending crtc's failed with %i\n",
1669                         ret);
1670                 drm_atomic_state_put(state);
1671                 return;
1672         }
1673
1674         dev_priv->modeset_restore_state = state;
1675         state->acquire_ctx = ctx;
1676 }
1677
1678 void intel_display_finish_reset(struct drm_i915_private *dev_priv)
1679 {
1680         struct drm_device *dev = &dev_priv->drm;
1681         struct drm_modeset_acquire_ctx *ctx = &dev_priv->reset_ctx;
1682         struct drm_atomic_state *state;
1683         int ret;
1684
1685         if (!HAS_DISPLAY(dev_priv))
1686                 return;
1687
1688         /* reset doesn't touch the display */
1689         if (!test_bit(I915_RESET_MODESET, &dev_priv->gt.reset.flags))
1690                 return;
1691
1692         state = fetch_and_zero(&dev_priv->modeset_restore_state);
1693         if (!state)
1694                 goto unlock;
1695
1696         /* reset doesn't touch the display */
1697         if (!gpu_reset_clobbers_display(dev_priv)) {
1698                 /* for testing only restore the display */
1699                 ret = __intel_display_resume(dev, state, ctx);
1700                 if (ret)
1701                         drm_err(&dev_priv->drm,
1702                                 "Restoring old state failed with %i\n", ret);
1703         } else {
1704                 /*
1705                  * The display has been reset as well,
1706                  * so need a full re-initialization.
1707                  */
1708                 intel_pps_unlock_regs_wa(dev_priv);
1709                 intel_modeset_init_hw(dev_priv);
1710                 intel_init_clock_gating(dev_priv);
1711                 intel_hpd_init(dev_priv);
1712
1713                 ret = __intel_display_resume(dev, state, ctx);
1714                 if (ret)
1715                         drm_err(&dev_priv->drm,
1716                                 "Restoring old state failed with %i\n", ret);
1717
1718                 intel_hpd_poll_disable(dev_priv);
1719         }
1720
1721         drm_atomic_state_put(state);
1722 unlock:
1723         drm_modeset_drop_locks(ctx);
1724         drm_modeset_acquire_fini(ctx);
1725         mutex_unlock(&dev->mode_config.mutex);
1726
1727         clear_bit_unlock(I915_RESET_MODESET, &dev_priv->gt.reset.flags);
1728 }
1729
1730 static bool underrun_recovery_supported(const struct intel_crtc_state *crtc_state)
1731 {
1732         if (crtc_state->pch_pfit.enabled &&
1733             (crtc_state->pipe_src_w > drm_rect_width(&crtc_state->pch_pfit.dst) ||
1734              crtc_state->pipe_src_h > drm_rect_height(&crtc_state->pch_pfit.dst) ||
1735              crtc_state->output_format == INTEL_OUTPUT_FORMAT_YCBCR420))
1736                 return false;
1737
1738         if (crtc_state->dsc.compression_enable)
1739                 return false;
1740
1741         if (crtc_state->has_psr2)
1742                 return false;
1743
1744         if (crtc_state->splitter.enable)
1745                 return false;
1746
1747         return true;
1748 }
1749
1750 static void icl_set_pipe_chicken(const struct intel_crtc_state *crtc_state)
1751 {
1752         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
1753         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1754         enum pipe pipe = crtc->pipe;
1755         u32 tmp;
1756
1757         tmp = intel_de_read(dev_priv, PIPE_CHICKEN(pipe));
1758
1759         /*
1760          * Display WA #1153: icl
1761          * enable hardware to bypass the alpha math
1762          * and rounding for per-pixel values 00 and 0xff
1763          */
1764         tmp |= PER_PIXEL_ALPHA_BYPASS_EN;
1765         /*
1766          * Display WA # 1605353570: icl
1767          * Set the pixel rounding bit to 1 for allowing
1768          * passthrough of Frame buffer pixels unmodified
1769          * across pipe
1770          */
1771         tmp |= PIXEL_ROUNDING_TRUNC_FB_PASSTHRU;
1772
1773         if (IS_DG2(dev_priv)) {
1774                 /*
1775                  * Underrun recovery must always be disabled on DG2.  However
1776                  * the chicken bit meaning is inverted compared to other
1777                  * platforms.
1778                  */
1779                 tmp &= ~UNDERRUN_RECOVERY_ENABLE_DG2;
1780         } else if (DISPLAY_VER(dev_priv) >= 13) {
1781                 if (underrun_recovery_supported(crtc_state))
1782                         tmp &= ~UNDERRUN_RECOVERY_DISABLE_ADLP;
1783                 else
1784                         tmp |= UNDERRUN_RECOVERY_DISABLE_ADLP;
1785         }
1786
1787         intel_de_write(dev_priv, PIPE_CHICKEN(pipe), tmp);
1788 }
1789
1790 bool intel_has_pending_fb_unpin(struct drm_i915_private *dev_priv)
1791 {
1792         struct drm_crtc *crtc;
1793         bool cleanup_done;
1794
1795         drm_for_each_crtc(crtc, &dev_priv->drm) {
1796                 struct drm_crtc_commit *commit;
1797                 spin_lock(&crtc->commit_lock);
1798                 commit = list_first_entry_or_null(&crtc->commit_list,
1799                                                   struct drm_crtc_commit, commit_entry);
1800                 cleanup_done = commit ?
1801                         try_wait_for_completion(&commit->cleanup_done) : true;
1802                 spin_unlock(&crtc->commit_lock);
1803
1804                 if (cleanup_done)
1805                         continue;
1806
1807                 drm_crtc_wait_one_vblank(crtc);
1808
1809                 return true;
1810         }
1811
1812         return false;
1813 }
1814
1815 void lpt_disable_iclkip(struct drm_i915_private *dev_priv)
1816 {
1817         u32 temp;
1818
1819         intel_de_write(dev_priv, PIXCLK_GATE, PIXCLK_GATE_GATE);
1820
1821         mutex_lock(&dev_priv->sb_lock);
1822
1823         temp = intel_sbi_read(dev_priv, SBI_SSCCTL6, SBI_ICLK);
1824         temp |= SBI_SSCCTL_DISABLE;
1825         intel_sbi_write(dev_priv, SBI_SSCCTL6, temp, SBI_ICLK);
1826
1827         mutex_unlock(&dev_priv->sb_lock);
1828 }
1829
1830 /* Program iCLKIP clock to the desired frequency */
1831 static void lpt_program_iclkip(const struct intel_crtc_state *crtc_state)
1832 {
1833         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
1834         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1835         int clock = crtc_state->hw.adjusted_mode.crtc_clock;
1836         u32 divsel, phaseinc, auxdiv, phasedir = 0;
1837         u32 temp;
1838
1839         lpt_disable_iclkip(dev_priv);
1840
1841         /* The iCLK virtual clock root frequency is in MHz,
1842          * but the adjusted_mode->crtc_clock in in KHz. To get the
1843          * divisors, it is necessary to divide one by another, so we
1844          * convert the virtual clock precision to KHz here for higher
1845          * precision.
1846          */
1847         for (auxdiv = 0; auxdiv < 2; auxdiv++) {
1848                 u32 iclk_virtual_root_freq = 172800 * 1000;
1849                 u32 iclk_pi_range = 64;
1850                 u32 desired_divisor;
1851
1852                 desired_divisor = DIV_ROUND_CLOSEST(iclk_virtual_root_freq,
1853                                                     clock << auxdiv);
1854                 divsel = (desired_divisor / iclk_pi_range) - 2;
1855                 phaseinc = desired_divisor % iclk_pi_range;
1856
1857                 /*
1858                  * Near 20MHz is a corner case which is
1859                  * out of range for the 7-bit divisor
1860                  */
1861                 if (divsel <= 0x7f)
1862                         break;
1863         }
1864
1865         /* This should not happen with any sane values */
1866         drm_WARN_ON(&dev_priv->drm, SBI_SSCDIVINTPHASE_DIVSEL(divsel) &
1867                     ~SBI_SSCDIVINTPHASE_DIVSEL_MASK);
1868         drm_WARN_ON(&dev_priv->drm, SBI_SSCDIVINTPHASE_DIR(phasedir) &
1869                     ~SBI_SSCDIVINTPHASE_INCVAL_MASK);
1870
1871         drm_dbg_kms(&dev_priv->drm,
1872                     "iCLKIP clock: found settings for %dKHz refresh rate: auxdiv=%x, divsel=%x, phasedir=%x, phaseinc=%x\n",
1873                     clock, auxdiv, divsel, phasedir, phaseinc);
1874
1875         mutex_lock(&dev_priv->sb_lock);
1876
1877         /* Program SSCDIVINTPHASE6 */
1878         temp = intel_sbi_read(dev_priv, SBI_SSCDIVINTPHASE6, SBI_ICLK);
1879         temp &= ~SBI_SSCDIVINTPHASE_DIVSEL_MASK;
1880         temp |= SBI_SSCDIVINTPHASE_DIVSEL(divsel);
1881         temp &= ~SBI_SSCDIVINTPHASE_INCVAL_MASK;
1882         temp |= SBI_SSCDIVINTPHASE_INCVAL(phaseinc);
1883         temp |= SBI_SSCDIVINTPHASE_DIR(phasedir);
1884         temp |= SBI_SSCDIVINTPHASE_PROPAGATE;
1885         intel_sbi_write(dev_priv, SBI_SSCDIVINTPHASE6, temp, SBI_ICLK);
1886
1887         /* Program SSCAUXDIV */
1888         temp = intel_sbi_read(dev_priv, SBI_SSCAUXDIV6, SBI_ICLK);
1889         temp &= ~SBI_SSCAUXDIV_FINALDIV2SEL(1);
1890         temp |= SBI_SSCAUXDIV_FINALDIV2SEL(auxdiv);
1891         intel_sbi_write(dev_priv, SBI_SSCAUXDIV6, temp, SBI_ICLK);
1892
1893         /* Enable modulator and associated divider */
1894         temp = intel_sbi_read(dev_priv, SBI_SSCCTL6, SBI_ICLK);
1895         temp &= ~SBI_SSCCTL_DISABLE;
1896         intel_sbi_write(dev_priv, SBI_SSCCTL6, temp, SBI_ICLK);
1897
1898         mutex_unlock(&dev_priv->sb_lock);
1899
1900         /* Wait for initialization time */
1901         udelay(24);
1902
1903         intel_de_write(dev_priv, PIXCLK_GATE, PIXCLK_GATE_UNGATE);
1904 }
1905
1906 int lpt_get_iclkip(struct drm_i915_private *dev_priv)
1907 {
1908         u32 divsel, phaseinc, auxdiv;
1909         u32 iclk_virtual_root_freq = 172800 * 1000;
1910         u32 iclk_pi_range = 64;
1911         u32 desired_divisor;
1912         u32 temp;
1913
1914         if ((intel_de_read(dev_priv, PIXCLK_GATE) & PIXCLK_GATE_UNGATE) == 0)
1915                 return 0;
1916
1917         mutex_lock(&dev_priv->sb_lock);
1918
1919         temp = intel_sbi_read(dev_priv, SBI_SSCCTL6, SBI_ICLK);
1920         if (temp & SBI_SSCCTL_DISABLE) {
1921                 mutex_unlock(&dev_priv->sb_lock);
1922                 return 0;
1923         }
1924
1925         temp = intel_sbi_read(dev_priv, SBI_SSCDIVINTPHASE6, SBI_ICLK);
1926         divsel = (temp & SBI_SSCDIVINTPHASE_DIVSEL_MASK) >>
1927                 SBI_SSCDIVINTPHASE_DIVSEL_SHIFT;
1928         phaseinc = (temp & SBI_SSCDIVINTPHASE_INCVAL_MASK) >>
1929                 SBI_SSCDIVINTPHASE_INCVAL_SHIFT;
1930
1931         temp = intel_sbi_read(dev_priv, SBI_SSCAUXDIV6, SBI_ICLK);
1932         auxdiv = (temp & SBI_SSCAUXDIV_FINALDIV2SEL_MASK) >>
1933                 SBI_SSCAUXDIV_FINALDIV2SEL_SHIFT;
1934
1935         mutex_unlock(&dev_priv->sb_lock);
1936
1937         desired_divisor = (divsel + 2) * iclk_pi_range + phaseinc;
1938
1939         return DIV_ROUND_CLOSEST(iclk_virtual_root_freq,
1940                                  desired_divisor << auxdiv);
1941 }
1942
1943 static void ilk_pch_transcoder_set_timings(const struct intel_crtc_state *crtc_state,
1944                                            enum pipe pch_transcoder)
1945 {
1946         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
1947         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1948         enum transcoder cpu_transcoder = crtc_state->cpu_transcoder;
1949
1950         intel_de_write(dev_priv, PCH_TRANS_HTOTAL(pch_transcoder),
1951                        intel_de_read(dev_priv, HTOTAL(cpu_transcoder)));
1952         intel_de_write(dev_priv, PCH_TRANS_HBLANK(pch_transcoder),
1953                        intel_de_read(dev_priv, HBLANK(cpu_transcoder)));
1954         intel_de_write(dev_priv, PCH_TRANS_HSYNC(pch_transcoder),
1955                        intel_de_read(dev_priv, HSYNC(cpu_transcoder)));
1956
1957         intel_de_write(dev_priv, PCH_TRANS_VTOTAL(pch_transcoder),
1958                        intel_de_read(dev_priv, VTOTAL(cpu_transcoder)));
1959         intel_de_write(dev_priv, PCH_TRANS_VBLANK(pch_transcoder),
1960                        intel_de_read(dev_priv, VBLANK(cpu_transcoder)));
1961         intel_de_write(dev_priv, PCH_TRANS_VSYNC(pch_transcoder),
1962                        intel_de_read(dev_priv, VSYNC(cpu_transcoder)));
1963         intel_de_write(dev_priv, PCH_TRANS_VSYNCSHIFT(pch_transcoder),
1964                        intel_de_read(dev_priv, VSYNCSHIFT(cpu_transcoder)));
1965 }
1966
1967 /*
1968  * Finds the encoder associated with the given CRTC. This can only be
1969  * used when we know that the CRTC isn't feeding multiple encoders!
1970  */
1971 struct intel_encoder *
1972 intel_get_crtc_new_encoder(const struct intel_atomic_state *state,
1973                            const struct intel_crtc_state *crtc_state)
1974 {
1975         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
1976         const struct drm_connector_state *connector_state;
1977         const struct drm_connector *connector;
1978         struct intel_encoder *encoder = NULL;
1979         int num_encoders = 0;
1980         int i;
1981
1982         for_each_new_connector_in_state(&state->base, connector, connector_state, i) {
1983                 if (connector_state->crtc != &crtc->base)
1984                         continue;
1985
1986                 encoder = to_intel_encoder(connector_state->best_encoder);
1987                 num_encoders++;
1988         }
1989
1990         drm_WARN(encoder->base.dev, num_encoders != 1,
1991                  "%d encoders for pipe %c\n",
1992                  num_encoders, pipe_name(crtc->pipe));
1993
1994         return encoder;
1995 }
1996
1997 /*
1998  * Enable PCH resources required for PCH ports:
1999  *   - PCH PLLs
2000  *   - FDI training & RX/TX
2001  *   - update transcoder timings
2002  *   - DP transcoding bits
2003  *   - transcoder
2004  */
2005 static void ilk_pch_enable(const struct intel_atomic_state *state,
2006                            const struct intel_crtc_state *crtc_state)
2007 {
2008         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
2009         struct drm_device *dev = crtc->base.dev;
2010         struct drm_i915_private *dev_priv = to_i915(dev);
2011         enum pipe pipe = crtc->pipe;
2012         u32 temp;
2013
2014         assert_pch_transcoder_disabled(dev_priv, pipe);
2015
2016         /* For PCH output, training FDI link */
2017         intel_fdi_link_train(crtc, crtc_state);
2018
2019         /* We need to program the right clock selection before writing the pixel
2020          * mutliplier into the DPLL. */
2021         if (HAS_PCH_CPT(dev_priv)) {
2022                 u32 sel;
2023
2024                 temp = intel_de_read(dev_priv, PCH_DPLL_SEL);
2025                 temp |= TRANS_DPLL_ENABLE(pipe);
2026                 sel = TRANS_DPLLB_SEL(pipe);
2027                 if (crtc_state->shared_dpll ==
2028                     intel_get_shared_dpll_by_id(dev_priv, DPLL_ID_PCH_PLL_B))
2029                         temp |= sel;
2030                 else
2031                         temp &= ~sel;
2032                 intel_de_write(dev_priv, PCH_DPLL_SEL, temp);
2033         }
2034
2035         /* XXX: pch pll's can be enabled any time before we enable the PCH
2036          * transcoder, and we actually should do this to not upset any PCH
2037          * transcoder that already use the clock when we share it.
2038          *
2039          * Note that enable_shared_dpll tries to do the right thing, but
2040          * get_shared_dpll unconditionally resets the pll - we need that to have
2041          * the right LVDS enable sequence. */
2042         intel_enable_shared_dpll(crtc_state);
2043
2044         /* set transcoder timing, panel must allow it */
2045         assert_pps_unlocked(dev_priv, pipe);
2046         ilk_pch_transcoder_set_timings(crtc_state, pipe);
2047
2048         intel_fdi_normal_train(crtc);
2049
2050         /* For PCH DP, enable TRANS_DP_CTL */
2051         if (HAS_PCH_CPT(dev_priv) &&
2052             intel_crtc_has_dp_encoder(crtc_state)) {
2053                 const struct drm_display_mode *adjusted_mode =
2054                         &crtc_state->hw.adjusted_mode;
2055                 u32 bpc = (intel_de_read(dev_priv, PIPECONF(pipe)) & PIPECONF_BPC_MASK) >> 5;
2056                 i915_reg_t reg = TRANS_DP_CTL(pipe);
2057                 enum port port;
2058
2059                 temp = intel_de_read(dev_priv, reg);
2060                 temp &= ~(TRANS_DP_PORT_SEL_MASK |
2061                           TRANS_DP_SYNC_MASK |
2062                           TRANS_DP_BPC_MASK);
2063                 temp |= TRANS_DP_OUTPUT_ENABLE;
2064                 temp |= bpc << 9; /* same format but at 11:9 */
2065
2066                 if (adjusted_mode->flags & DRM_MODE_FLAG_PHSYNC)
2067                         temp |= TRANS_DP_HSYNC_ACTIVE_HIGH;
2068                 if (adjusted_mode->flags & DRM_MODE_FLAG_PVSYNC)
2069                         temp |= TRANS_DP_VSYNC_ACTIVE_HIGH;
2070
2071                 port = intel_get_crtc_new_encoder(state, crtc_state)->port;
2072                 drm_WARN_ON(dev, port < PORT_B || port > PORT_D);
2073                 temp |= TRANS_DP_PORT_SEL(port);
2074
2075                 intel_de_write(dev_priv, reg, temp);
2076         }
2077
2078         ilk_enable_pch_transcoder(crtc_state);
2079 }
2080
2081 void lpt_pch_enable(const struct intel_crtc_state *crtc_state)
2082 {
2083         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
2084         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
2085         enum transcoder cpu_transcoder = crtc_state->cpu_transcoder;
2086
2087         assert_pch_transcoder_disabled(dev_priv, PIPE_A);
2088
2089         lpt_program_iclkip(crtc_state);
2090
2091         /* Set transcoder timing. */
2092         ilk_pch_transcoder_set_timings(crtc_state, PIPE_A);
2093
2094         lpt_enable_pch_transcoder(dev_priv, cpu_transcoder);
2095 }
2096
2097 static void cpt_verify_modeset(struct drm_i915_private *dev_priv,
2098                                enum pipe pipe)
2099 {
2100         i915_reg_t dslreg = PIPEDSL(pipe);
2101         u32 temp;
2102
2103         temp = intel_de_read(dev_priv, dslreg);
2104         udelay(500);
2105         if (wait_for(intel_de_read(dev_priv, dslreg) != temp, 5)) {
2106                 if (wait_for(intel_de_read(dev_priv, dslreg) != temp, 5))
2107                         drm_err(&dev_priv->drm,
2108                                 "mode set failed: pipe %c stuck\n",
2109                                 pipe_name(pipe));
2110         }
2111 }
2112
2113 static void ilk_pfit_enable(const struct intel_crtc_state *crtc_state)
2114 {
2115         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
2116         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
2117         const struct drm_rect *dst = &crtc_state->pch_pfit.dst;
2118         enum pipe pipe = crtc->pipe;
2119         int width = drm_rect_width(dst);
2120         int height = drm_rect_height(dst);
2121         int x = dst->x1;
2122         int y = dst->y1;
2123
2124         if (!crtc_state->pch_pfit.enabled)
2125                 return;
2126
2127         /* Force use of hard-coded filter coefficients
2128          * as some pre-programmed values are broken,
2129          * e.g. x201.
2130          */
2131         if (IS_IVYBRIDGE(dev_priv) || IS_HASWELL(dev_priv))
2132                 intel_de_write(dev_priv, PF_CTL(pipe), PF_ENABLE |
2133                                PF_FILTER_MED_3x3 | PF_PIPE_SEL_IVB(pipe));
2134         else
2135                 intel_de_write(dev_priv, PF_CTL(pipe), PF_ENABLE |
2136                                PF_FILTER_MED_3x3);
2137         intel_de_write(dev_priv, PF_WIN_POS(pipe), x << 16 | y);
2138         intel_de_write(dev_priv, PF_WIN_SZ(pipe), width << 16 | height);
2139 }
2140
2141 void hsw_enable_ips(const struct intel_crtc_state *crtc_state)
2142 {
2143         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
2144         struct drm_device *dev = crtc->base.dev;
2145         struct drm_i915_private *dev_priv = to_i915(dev);
2146
2147         if (!crtc_state->ips_enabled)
2148                 return;
2149
2150         /*
2151          * We can only enable IPS after we enable a plane and wait for a vblank
2152          * This function is called from post_plane_update, which is run after
2153          * a vblank wait.
2154          */
2155         drm_WARN_ON(dev, !(crtc_state->active_planes & ~BIT(PLANE_CURSOR)));
2156
2157         if (IS_BROADWELL(dev_priv)) {
2158                 drm_WARN_ON(dev, sandybridge_pcode_write(dev_priv, DISPLAY_IPS_CONTROL,
2159                                                          IPS_ENABLE | IPS_PCODE_CONTROL));
2160                 /* Quoting Art Runyan: "its not safe to expect any particular
2161                  * value in IPS_CTL bit 31 after enabling IPS through the
2162                  * mailbox." Moreover, the mailbox may return a bogus state,
2163                  * so we need to just enable it and continue on.
2164                  */
2165         } else {
2166                 intel_de_write(dev_priv, IPS_CTL, IPS_ENABLE);
2167                 /* The bit only becomes 1 in the next vblank, so this wait here
2168                  * is essentially intel_wait_for_vblank. If we don't have this
2169                  * and don't wait for vblanks until the end of crtc_enable, then
2170                  * the HW state readout code will complain that the expected
2171                  * IPS_CTL value is not the one we read. */
2172                 if (intel_de_wait_for_set(dev_priv, IPS_CTL, IPS_ENABLE, 50))
2173                         drm_err(&dev_priv->drm,
2174                                 "Timed out waiting for IPS enable\n");
2175         }
2176 }
2177
2178 void hsw_disable_ips(const struct intel_crtc_state *crtc_state)
2179 {
2180         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
2181         struct drm_device *dev = crtc->base.dev;
2182         struct drm_i915_private *dev_priv = to_i915(dev);
2183
2184         if (!crtc_state->ips_enabled)
2185                 return;
2186
2187         if (IS_BROADWELL(dev_priv)) {
2188                 drm_WARN_ON(dev,
2189                             sandybridge_pcode_write(dev_priv, DISPLAY_IPS_CONTROL, 0));
2190                 /*
2191                  * Wait for PCODE to finish disabling IPS. The BSpec specified
2192                  * 42ms timeout value leads to occasional timeouts so use 100ms
2193                  * instead.
2194                  */
2195                 if (intel_de_wait_for_clear(dev_priv, IPS_CTL, IPS_ENABLE, 100))
2196                         drm_err(&dev_priv->drm,
2197                                 "Timed out waiting for IPS disable\n");
2198         } else {
2199                 intel_de_write(dev_priv, IPS_CTL, 0);
2200                 intel_de_posting_read(dev_priv, IPS_CTL);
2201         }
2202
2203         /* We need to wait for a vblank before we can disable the plane. */
2204         intel_wait_for_vblank(dev_priv, crtc->pipe);
2205 }
2206
2207 static void intel_crtc_dpms_overlay_disable(struct intel_crtc *crtc)
2208 {
2209         if (crtc->overlay)
2210                 (void) intel_overlay_switch_off(crtc->overlay);
2211
2212         /* Let userspace switch the overlay on again. In most cases userspace
2213          * has to recompute where to put it anyway.
2214          */
2215 }
2216
2217 static bool hsw_pre_update_disable_ips(const struct intel_crtc_state *old_crtc_state,
2218                                        const struct intel_crtc_state *new_crtc_state)
2219 {
2220         struct intel_crtc *crtc = to_intel_crtc(new_crtc_state->uapi.crtc);
2221         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
2222
2223         if (!old_crtc_state->ips_enabled)
2224                 return false;
2225
2226         if (intel_crtc_needs_modeset(new_crtc_state))
2227                 return true;
2228
2229         /*
2230          * Workaround : Do not read or write the pipe palette/gamma data while
2231          * GAMMA_MODE is configured for split gamma and IPS_CTL has IPS enabled.
2232          *
2233          * Disable IPS before we program the LUT.
2234          */
2235         if (IS_HASWELL(dev_priv) &&
2236             (new_crtc_state->uapi.color_mgmt_changed ||
2237              new_crtc_state->update_pipe) &&
2238             new_crtc_state->gamma_mode == GAMMA_MODE_MODE_SPLIT)
2239                 return true;
2240
2241         return !new_crtc_state->ips_enabled;
2242 }
2243
2244 static bool hsw_post_update_enable_ips(const struct intel_crtc_state *old_crtc_state,
2245                                        const struct intel_crtc_state *new_crtc_state)
2246 {
2247         struct intel_crtc *crtc = to_intel_crtc(new_crtc_state->uapi.crtc);
2248         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
2249
2250         if (!new_crtc_state->ips_enabled)
2251                 return false;
2252
2253         if (intel_crtc_needs_modeset(new_crtc_state))
2254                 return true;
2255
2256         /*
2257          * Workaround : Do not read or write the pipe palette/gamma data while
2258          * GAMMA_MODE is configured for split gamma and IPS_CTL has IPS enabled.
2259          *
2260          * Re-enable IPS after the LUT has been programmed.
2261          */
2262         if (IS_HASWELL(dev_priv) &&
2263             (new_crtc_state->uapi.color_mgmt_changed ||
2264              new_crtc_state->update_pipe) &&
2265             new_crtc_state->gamma_mode == GAMMA_MODE_MODE_SPLIT)
2266                 return true;
2267
2268         /*
2269          * We can't read out IPS on broadwell, assume the worst and
2270          * forcibly enable IPS on the first fastset.
2271          */
2272         if (new_crtc_state->update_pipe && old_crtc_state->inherited)
2273                 return true;
2274
2275         return !old_crtc_state->ips_enabled;
2276 }
2277
2278 static bool needs_nv12_wa(const struct intel_crtc_state *crtc_state)
2279 {
2280         struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
2281
2282         if (!crtc_state->nv12_planes)
2283                 return false;
2284
2285         /* WA Display #0827: Gen9:all */
2286         if (DISPLAY_VER(dev_priv) == 9)
2287                 return true;
2288
2289         return false;
2290 }
2291
2292 static bool needs_scalerclk_wa(const struct intel_crtc_state *crtc_state)
2293 {
2294         struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
2295
2296         /* Wa_2006604312:icl,ehl */
2297         if (crtc_state->scaler_state.scaler_users > 0 && DISPLAY_VER(dev_priv) == 11)
2298                 return true;
2299
2300         return false;
2301 }
2302
2303 static bool needs_cursorclk_wa(const struct intel_crtc_state *crtc_state)
2304 {
2305         struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
2306
2307         /* Wa_1604331009:icl,jsl,ehl */
2308         if (is_hdr_mode(crtc_state) &&
2309             crtc_state->active_planes & BIT(PLANE_CURSOR) &&
2310             DISPLAY_VER(dev_priv) == 11)
2311                 return true;
2312
2313         return false;
2314 }
2315
2316 static bool planes_enabling(const struct intel_crtc_state *old_crtc_state,
2317                             const struct intel_crtc_state *new_crtc_state)
2318 {
2319         return (!old_crtc_state->active_planes || intel_crtc_needs_modeset(new_crtc_state)) &&
2320                 new_crtc_state->active_planes;
2321 }
2322
2323 static bool planes_disabling(const struct intel_crtc_state *old_crtc_state,
2324                              const struct intel_crtc_state *new_crtc_state)
2325 {
2326         return old_crtc_state->active_planes &&
2327                 (!new_crtc_state->active_planes || intel_crtc_needs_modeset(new_crtc_state));
2328 }
2329
2330 static void intel_post_plane_update(struct intel_atomic_state *state,
2331                                     struct intel_crtc *crtc)
2332 {
2333         struct drm_i915_private *dev_priv = to_i915(state->base.dev);
2334         const struct intel_crtc_state *old_crtc_state =
2335                 intel_atomic_get_old_crtc_state(state, crtc);
2336         const struct intel_crtc_state *new_crtc_state =
2337                 intel_atomic_get_new_crtc_state(state, crtc);
2338         enum pipe pipe = crtc->pipe;
2339
2340         intel_frontbuffer_flip(dev_priv, new_crtc_state->fb_bits);
2341
2342         if (new_crtc_state->update_wm_post && new_crtc_state->hw.active)
2343                 intel_update_watermarks(dev_priv);
2344
2345         if (hsw_post_update_enable_ips(old_crtc_state, new_crtc_state))
2346                 hsw_enable_ips(new_crtc_state);
2347
2348         intel_fbc_post_update(state, crtc);
2349         intel_drrs_page_flip(state, crtc);
2350
2351         if (needs_nv12_wa(old_crtc_state) &&
2352             !needs_nv12_wa(new_crtc_state))
2353                 skl_wa_827(dev_priv, pipe, false);
2354
2355         if (needs_scalerclk_wa(old_crtc_state) &&
2356             !needs_scalerclk_wa(new_crtc_state))
2357                 icl_wa_scalerclkgating(dev_priv, pipe, false);
2358
2359         if (needs_cursorclk_wa(old_crtc_state) &&
2360             !needs_cursorclk_wa(new_crtc_state))
2361                 icl_wa_cursorclkgating(dev_priv, pipe, false);
2362
2363 }
2364
2365 static void intel_crtc_enable_flip_done(struct intel_atomic_state *state,
2366                                         struct intel_crtc *crtc)
2367 {
2368         const struct intel_crtc_state *crtc_state =
2369                 intel_atomic_get_new_crtc_state(state, crtc);
2370         u8 update_planes = crtc_state->update_planes;
2371         const struct intel_plane_state *plane_state;
2372         struct intel_plane *plane;
2373         int i;
2374
2375         for_each_new_intel_plane_in_state(state, plane, plane_state, i) {
2376                 if (plane->enable_flip_done &&
2377                     plane->pipe == crtc->pipe &&
2378                     update_planes & BIT(plane->id))
2379                         plane->enable_flip_done(plane);
2380         }
2381 }
2382
2383 static void intel_crtc_disable_flip_done(struct intel_atomic_state *state,
2384                                          struct intel_crtc *crtc)
2385 {
2386         const struct intel_crtc_state *crtc_state =
2387                 intel_atomic_get_new_crtc_state(state, crtc);
2388         u8 update_planes = crtc_state->update_planes;
2389         const struct intel_plane_state *plane_state;
2390         struct intel_plane *plane;
2391         int i;
2392
2393         for_each_new_intel_plane_in_state(state, plane, plane_state, i) {
2394                 if (plane->disable_flip_done &&
2395                     plane->pipe == crtc->pipe &&
2396                     update_planes & BIT(plane->id))
2397                         plane->disable_flip_done(plane);
2398         }
2399 }
2400
2401 static void intel_crtc_async_flip_disable_wa(struct intel_atomic_state *state,
2402                                              struct intel_crtc *crtc)
2403 {
2404         struct drm_i915_private *i915 = to_i915(state->base.dev);
2405         const struct intel_crtc_state *old_crtc_state =
2406                 intel_atomic_get_old_crtc_state(state, crtc);
2407         const struct intel_crtc_state *new_crtc_state =
2408                 intel_atomic_get_new_crtc_state(state, crtc);
2409         u8 update_planes = new_crtc_state->update_planes;
2410         const struct intel_plane_state *old_plane_state;
2411         struct intel_plane *plane;
2412         bool need_vbl_wait = false;
2413         int i;
2414
2415         for_each_old_intel_plane_in_state(state, plane, old_plane_state, i) {
2416                 if (plane->need_async_flip_disable_wa &&
2417                     plane->pipe == crtc->pipe &&
2418                     update_planes & BIT(plane->id)) {
2419                         /*
2420                          * Apart from the async flip bit we want to
2421                          * preserve the old state for the plane.
2422                          */
2423                         plane->async_flip(plane, old_crtc_state,
2424                                           old_plane_state, false);
2425                         need_vbl_wait = true;
2426                 }
2427         }
2428
2429         if (need_vbl_wait)
2430                 intel_wait_for_vblank(i915, crtc->pipe);
2431 }
2432
2433 static void intel_pre_plane_update(struct intel_atomic_state *state,
2434                                    struct intel_crtc *crtc)
2435 {
2436         struct drm_i915_private *dev_priv = to_i915(state->base.dev);
2437         const struct intel_crtc_state *old_crtc_state =
2438                 intel_atomic_get_old_crtc_state(state, crtc);
2439         const struct intel_crtc_state *new_crtc_state =
2440                 intel_atomic_get_new_crtc_state(state, crtc);
2441         enum pipe pipe = crtc->pipe;
2442
2443         if (hsw_pre_update_disable_ips(old_crtc_state, new_crtc_state))
2444                 hsw_disable_ips(old_crtc_state);
2445
2446         if (intel_fbc_pre_update(state, crtc))
2447                 intel_wait_for_vblank(dev_priv, pipe);
2448
2449         /* Display WA 827 */
2450         if (!needs_nv12_wa(old_crtc_state) &&
2451             needs_nv12_wa(new_crtc_state))
2452                 skl_wa_827(dev_priv, pipe, true);
2453
2454         /* Wa_2006604312:icl,ehl */
2455         if (!needs_scalerclk_wa(old_crtc_state) &&
2456             needs_scalerclk_wa(new_crtc_state))
2457                 icl_wa_scalerclkgating(dev_priv, pipe, true);
2458
2459         /* Wa_1604331009:icl,jsl,ehl */
2460         if (!needs_cursorclk_wa(old_crtc_state) &&
2461             needs_cursorclk_wa(new_crtc_state))
2462                 icl_wa_cursorclkgating(dev_priv, pipe, true);
2463
2464         /*
2465          * Vblank time updates from the shadow to live plane control register
2466          * are blocked if the memory self-refresh mode is active at that
2467          * moment. So to make sure the plane gets truly disabled, disable
2468          * first the self-refresh mode. The self-refresh enable bit in turn
2469          * will be checked/applied by the HW only at the next frame start
2470          * event which is after the vblank start event, so we need to have a
2471          * wait-for-vblank between disabling the plane and the pipe.
2472          */
2473         if (HAS_GMCH(dev_priv) && old_crtc_state->hw.active &&
2474             new_crtc_state->disable_cxsr && intel_set_memory_cxsr(dev_priv, false))
2475                 intel_wait_for_vblank(dev_priv, pipe);
2476
2477         /*
2478          * IVB workaround: must disable low power watermarks for at least
2479          * one frame before enabling scaling.  LP watermarks can be re-enabled
2480          * when scaling is disabled.
2481          *
2482          * WaCxSRDisabledForSpriteScaling:ivb
2483          */
2484         if (old_crtc_state->hw.active &&
2485             new_crtc_state->disable_lp_wm && ilk_disable_lp_wm(dev_priv))
2486                 intel_wait_for_vblank(dev_priv, pipe);
2487
2488         /*
2489          * If we're doing a modeset we don't need to do any
2490          * pre-vblank watermark programming here.
2491          */
2492         if (!intel_crtc_needs_modeset(new_crtc_state)) {
2493                 /*
2494                  * For platforms that support atomic watermarks, program the
2495                  * 'intermediate' watermarks immediately.  On pre-gen9 platforms, these
2496                  * will be the intermediate values that are safe for both pre- and
2497                  * post- vblank; when vblank happens, the 'active' values will be set
2498                  * to the final 'target' values and we'll do this again to get the
2499                  * optimal watermarks.  For gen9+ platforms, the values we program here
2500                  * will be the final target values which will get automatically latched
2501                  * at vblank time; no further programming will be necessary.
2502                  *
2503                  * If a platform hasn't been transitioned to atomic watermarks yet,
2504                  * we'll continue to update watermarks the old way, if flags tell
2505                  * us to.
2506                  */
2507                 if (!intel_initial_watermarks(state, crtc))
2508                         if (new_crtc_state->update_wm_pre)
2509                                 intel_update_watermarks(dev_priv);
2510         }
2511
2512         /*
2513          * Gen2 reports pipe underruns whenever all planes are disabled.
2514          * So disable underrun reporting before all the planes get disabled.
2515          *
2516          * We do this after .initial_watermarks() so that we have a
2517          * chance of catching underruns with the intermediate watermarks
2518          * vs. the old plane configuration.
2519          */
2520         if (DISPLAY_VER(dev_priv) == 2 && planes_disabling(old_crtc_state, new_crtc_state))
2521                 intel_set_cpu_fifo_underrun_reporting(dev_priv, pipe, false);
2522
2523         /*
2524          * WA for platforms where async address update enable bit
2525          * is double buffered and only latched at start of vblank.
2526          */
2527         if (old_crtc_state->uapi.async_flip && !new_crtc_state->uapi.async_flip)
2528                 intel_crtc_async_flip_disable_wa(state, crtc);
2529 }
2530
2531 static void intel_crtc_disable_planes(struct intel_atomic_state *state,
2532                                       struct intel_crtc *crtc)
2533 {
2534         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
2535         const struct intel_crtc_state *new_crtc_state =
2536                 intel_atomic_get_new_crtc_state(state, crtc);
2537         unsigned int update_mask = new_crtc_state->update_planes;
2538         const struct intel_plane_state *old_plane_state;
2539         struct intel_plane *plane;
2540         unsigned fb_bits = 0;
2541         int i;
2542
2543         intel_crtc_dpms_overlay_disable(crtc);
2544
2545         for_each_old_intel_plane_in_state(state, plane, old_plane_state, i) {
2546                 if (crtc->pipe != plane->pipe ||
2547                     !(update_mask & BIT(plane->id)))
2548                         continue;
2549
2550                 intel_disable_plane(plane, new_crtc_state);
2551
2552                 if (old_plane_state->uapi.visible)
2553                         fb_bits |= plane->frontbuffer_bit;
2554         }
2555
2556         intel_frontbuffer_flip(dev_priv, fb_bits);
2557 }
2558
2559 /*
2560  * intel_connector_primary_encoder - get the primary encoder for a connector
2561  * @connector: connector for which to return the encoder
2562  *
2563  * Returns the primary encoder for a connector. There is a 1:1 mapping from
2564  * all connectors to their encoder, except for DP-MST connectors which have
2565  * both a virtual and a primary encoder. These DP-MST primary encoders can be
2566  * pointed to by as many DP-MST connectors as there are pipes.
2567  */
2568 static struct intel_encoder *
2569 intel_connector_primary_encoder(struct intel_connector *connector)
2570 {
2571         struct intel_encoder *encoder;
2572
2573         if (connector->mst_port)
2574                 return &dp_to_dig_port(connector->mst_port)->base;
2575
2576         encoder = intel_attached_encoder(connector);
2577         drm_WARN_ON(connector->base.dev, !encoder);
2578
2579         return encoder;
2580 }
2581
2582 static void intel_encoders_update_prepare(struct intel_atomic_state *state)
2583 {
2584         struct drm_connector_state *new_conn_state;
2585         struct drm_connector *connector;
2586         int i;
2587
2588         for_each_new_connector_in_state(&state->base, connector, new_conn_state,
2589                                         i) {
2590                 struct intel_connector *intel_connector;
2591                 struct intel_encoder *encoder;
2592                 struct intel_crtc *crtc;
2593
2594                 if (!intel_connector_needs_modeset(state, connector))
2595                         continue;
2596
2597                 intel_connector = to_intel_connector(connector);
2598                 encoder = intel_connector_primary_encoder(intel_connector);
2599                 if (!encoder->update_prepare)
2600                         continue;
2601
2602                 crtc = new_conn_state->crtc ?
2603                         to_intel_crtc(new_conn_state->crtc) : NULL;
2604                 encoder->update_prepare(state, encoder, crtc);
2605         }
2606 }
2607
2608 static void intel_encoders_update_complete(struct intel_atomic_state *state)
2609 {
2610         struct drm_connector_state *new_conn_state;
2611         struct drm_connector *connector;
2612         int i;
2613
2614         for_each_new_connector_in_state(&state->base, connector, new_conn_state,
2615                                         i) {
2616                 struct intel_connector *intel_connector;
2617                 struct intel_encoder *encoder;
2618                 struct intel_crtc *crtc;
2619
2620                 if (!intel_connector_needs_modeset(state, connector))
2621                         continue;
2622
2623                 intel_connector = to_intel_connector(connector);
2624                 encoder = intel_connector_primary_encoder(intel_connector);
2625                 if (!encoder->update_complete)
2626                         continue;
2627
2628                 crtc = new_conn_state->crtc ?
2629                         to_intel_crtc(new_conn_state->crtc) : NULL;
2630                 encoder->update_complete(state, encoder, crtc);
2631         }
2632 }
2633
2634 static void intel_encoders_pre_pll_enable(struct intel_atomic_state *state,
2635                                           struct intel_crtc *crtc)
2636 {
2637         const struct intel_crtc_state *crtc_state =
2638                 intel_atomic_get_new_crtc_state(state, crtc);
2639         const struct drm_connector_state *conn_state;
2640         struct drm_connector *conn;
2641         int i;
2642
2643         for_each_new_connector_in_state(&state->base, conn, conn_state, i) {
2644                 struct intel_encoder *encoder =
2645                         to_intel_encoder(conn_state->best_encoder);
2646
2647                 if (conn_state->crtc != &crtc->base)
2648                         continue;
2649
2650                 if (encoder->pre_pll_enable)
2651                         encoder->pre_pll_enable(state, encoder,
2652                                                 crtc_state, conn_state);
2653         }
2654 }
2655
2656 static void intel_encoders_pre_enable(struct intel_atomic_state *state,
2657                                       struct intel_crtc *crtc)
2658 {
2659         const struct intel_crtc_state *crtc_state =
2660                 intel_atomic_get_new_crtc_state(state, crtc);
2661         const struct drm_connector_state *conn_state;
2662         struct drm_connector *conn;
2663         int i;
2664
2665         for_each_new_connector_in_state(&state->base, conn, conn_state, i) {
2666                 struct intel_encoder *encoder =
2667                         to_intel_encoder(conn_state->best_encoder);
2668
2669                 if (conn_state->crtc != &crtc->base)
2670                         continue;
2671
2672                 if (encoder->pre_enable)
2673                         encoder->pre_enable(state, encoder,
2674                                             crtc_state, conn_state);
2675         }
2676 }
2677
2678 static void intel_encoders_enable(struct intel_atomic_state *state,
2679                                   struct intel_crtc *crtc)
2680 {
2681         const struct intel_crtc_state *crtc_state =
2682                 intel_atomic_get_new_crtc_state(state, crtc);
2683         const struct drm_connector_state *conn_state;
2684         struct drm_connector *conn;
2685         int i;
2686
2687         for_each_new_connector_in_state(&state->base, conn, conn_state, i) {
2688                 struct intel_encoder *encoder =
2689                         to_intel_encoder(conn_state->best_encoder);
2690
2691                 if (conn_state->crtc != &crtc->base)
2692                         continue;
2693
2694                 if (encoder->enable)
2695                         encoder->enable(state, encoder,
2696                                         crtc_state, conn_state);
2697                 intel_opregion_notify_encoder(encoder, true);
2698         }
2699 }
2700
2701 static void intel_encoders_pre_disable(struct intel_atomic_state *state,
2702                                        struct intel_crtc *crtc)
2703 {
2704         const struct intel_crtc_state *old_crtc_state =
2705                 intel_atomic_get_old_crtc_state(state, crtc);
2706         const struct drm_connector_state *old_conn_state;
2707         struct drm_connector *conn;
2708         int i;
2709
2710         for_each_old_connector_in_state(&state->base, conn, old_conn_state, i) {
2711                 struct intel_encoder *encoder =
2712                         to_intel_encoder(old_conn_state->best_encoder);
2713
2714                 if (old_conn_state->crtc != &crtc->base)
2715                         continue;
2716
2717                 if (encoder->pre_disable)
2718                         encoder->pre_disable(state, encoder, old_crtc_state,
2719                                              old_conn_state);
2720         }
2721 }
2722
2723 static void intel_encoders_disable(struct intel_atomic_state *state,
2724                                    struct intel_crtc *crtc)
2725 {
2726         const struct intel_crtc_state *old_crtc_state =
2727                 intel_atomic_get_old_crtc_state(state, crtc);
2728         const struct drm_connector_state *old_conn_state;
2729         struct drm_connector *conn;
2730         int i;
2731
2732         for_each_old_connector_in_state(&state->base, conn, old_conn_state, i) {
2733                 struct intel_encoder *encoder =
2734                         to_intel_encoder(old_conn_state->best_encoder);
2735
2736                 if (old_conn_state->crtc != &crtc->base)
2737                         continue;
2738
2739                 intel_opregion_notify_encoder(encoder, false);
2740                 if (encoder->disable)
2741                         encoder->disable(state, encoder,
2742                                          old_crtc_state, old_conn_state);
2743         }
2744 }
2745
2746 static void intel_encoders_post_disable(struct intel_atomic_state *state,
2747                                         struct intel_crtc *crtc)
2748 {
2749         const struct intel_crtc_state *old_crtc_state =
2750                 intel_atomic_get_old_crtc_state(state, crtc);
2751         const struct drm_connector_state *old_conn_state;
2752         struct drm_connector *conn;
2753         int i;
2754
2755         for_each_old_connector_in_state(&state->base, conn, old_conn_state, i) {
2756                 struct intel_encoder *encoder =
2757                         to_intel_encoder(old_conn_state->best_encoder);
2758
2759                 if (old_conn_state->crtc != &crtc->base)
2760                         continue;
2761
2762                 if (encoder->post_disable)
2763                         encoder->post_disable(state, encoder,
2764                                               old_crtc_state, old_conn_state);
2765         }
2766 }
2767
2768 static void intel_encoders_post_pll_disable(struct intel_atomic_state *state,
2769                                             struct intel_crtc *crtc)
2770 {
2771         const struct intel_crtc_state *old_crtc_state =
2772                 intel_atomic_get_old_crtc_state(state, crtc);
2773         const struct drm_connector_state *old_conn_state;
2774         struct drm_connector *conn;
2775         int i;
2776
2777         for_each_old_connector_in_state(&state->base, conn, old_conn_state, i) {
2778                 struct intel_encoder *encoder =
2779                         to_intel_encoder(old_conn_state->best_encoder);
2780
2781                 if (old_conn_state->crtc != &crtc->base)
2782                         continue;
2783
2784                 if (encoder->post_pll_disable)
2785                         encoder->post_pll_disable(state, encoder,
2786                                                   old_crtc_state, old_conn_state);
2787         }
2788 }
2789
2790 static void intel_encoders_update_pipe(struct intel_atomic_state *state,
2791                                        struct intel_crtc *crtc)
2792 {
2793         const struct intel_crtc_state *crtc_state =
2794                 intel_atomic_get_new_crtc_state(state, crtc);
2795         const struct drm_connector_state *conn_state;
2796         struct drm_connector *conn;
2797         int i;
2798
2799         for_each_new_connector_in_state(&state->base, conn, conn_state, i) {
2800                 struct intel_encoder *encoder =
2801                         to_intel_encoder(conn_state->best_encoder);
2802
2803                 if (conn_state->crtc != &crtc->base)
2804                         continue;
2805
2806                 if (encoder->update_pipe)
2807                         encoder->update_pipe(state, encoder,
2808                                              crtc_state, conn_state);
2809         }
2810 }
2811
2812 static void intel_disable_primary_plane(const struct intel_crtc_state *crtc_state)
2813 {
2814         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
2815         struct intel_plane *plane = to_intel_plane(crtc->base.primary);
2816
2817         plane->disable_plane(plane, crtc_state);
2818 }
2819
2820 static void ilk_crtc_enable(struct intel_atomic_state *state,
2821                             struct intel_crtc *crtc)
2822 {
2823         const struct intel_crtc_state *new_crtc_state =
2824                 intel_atomic_get_new_crtc_state(state, crtc);
2825         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
2826         enum pipe pipe = crtc->pipe;
2827
2828         if (drm_WARN_ON(&dev_priv->drm, crtc->active))
2829                 return;
2830
2831         /*
2832          * Sometimes spurious CPU pipe underruns happen during FDI
2833          * training, at least with VGA+HDMI cloning. Suppress them.
2834          *
2835          * On ILK we get an occasional spurious CPU pipe underruns
2836          * between eDP port A enable and vdd enable. Also PCH port
2837          * enable seems to result in the occasional CPU pipe underrun.
2838          *
2839          * Spurious PCH underruns also occur during PCH enabling.
2840          */
2841         intel_set_cpu_fifo_underrun_reporting(dev_priv, pipe, false);
2842         intel_set_pch_fifo_underrun_reporting(dev_priv, pipe, false);
2843
2844         if (intel_crtc_has_dp_encoder(new_crtc_state))
2845                 intel_dp_set_m_n(new_crtc_state, M1_N1);
2846
2847         intel_set_transcoder_timings(new_crtc_state);
2848         intel_set_pipe_src_size(new_crtc_state);
2849
2850         if (new_crtc_state->has_pch_encoder)
2851                 intel_cpu_transcoder_set_m_n(new_crtc_state,
2852                                              &new_crtc_state->fdi_m_n, NULL);
2853
2854         ilk_set_pipeconf(new_crtc_state);
2855
2856         crtc->active = true;
2857
2858         intel_encoders_pre_enable(state, crtc);
2859
2860         if (new_crtc_state->has_pch_encoder) {
2861                 /* Note: FDI PLL enabling _must_ be done before we enable the
2862                  * cpu pipes, hence this is separate from all the other fdi/pch
2863                  * enabling. */
2864                 ilk_fdi_pll_enable(new_crtc_state);
2865         } else {
2866                 assert_fdi_tx_disabled(dev_priv, pipe);
2867                 assert_fdi_rx_disabled(dev_priv, pipe);
2868         }
2869
2870         ilk_pfit_enable(new_crtc_state);
2871
2872         /*
2873          * On ILK+ LUT must be loaded before the pipe is running but with
2874          * clocks enabled
2875          */
2876         intel_color_load_luts(new_crtc_state);
2877         intel_color_commit(new_crtc_state);
2878         /* update DSPCNTR to configure gamma for pipe bottom color */
2879         intel_disable_primary_plane(new_crtc_state);
2880
2881         intel_initial_watermarks(state, crtc);
2882         intel_enable_transcoder(new_crtc_state);
2883
2884         if (new_crtc_state->has_pch_encoder)
2885                 ilk_pch_enable(state, new_crtc_state);
2886
2887         intel_crtc_vblank_on(new_crtc_state);
2888
2889         intel_encoders_enable(state, crtc);
2890
2891         if (HAS_PCH_CPT(dev_priv))
2892                 cpt_verify_modeset(dev_priv, pipe);
2893
2894         /*
2895          * Must wait for vblank to avoid spurious PCH FIFO underruns.
2896          * And a second vblank wait is needed at least on ILK with
2897          * some interlaced HDMI modes. Let's do the double wait always
2898          * in case there are more corner cases we don't know about.
2899          */
2900         if (new_crtc_state->has_pch_encoder) {
2901                 intel_wait_for_vblank(dev_priv, pipe);
2902                 intel_wait_for_vblank(dev_priv, pipe);
2903         }
2904         intel_set_cpu_fifo_underrun_reporting(dev_priv, pipe, true);
2905         intel_set_pch_fifo_underrun_reporting(dev_priv, pipe, true);
2906 }
2907
2908 /* IPS only exists on ULT machines and is tied to pipe A. */
2909 static bool hsw_crtc_supports_ips(struct intel_crtc *crtc)
2910 {
2911         return HAS_IPS(to_i915(crtc->base.dev)) && crtc->pipe == PIPE_A;
2912 }
2913
2914 static void glk_pipe_scaler_clock_gating_wa(struct drm_i915_private *dev_priv,
2915                                             enum pipe pipe, bool apply)
2916 {
2917         u32 val = intel_de_read(dev_priv, CLKGATE_DIS_PSL(pipe));
2918         u32 mask = DPF_GATING_DIS | DPF_RAM_GATING_DIS | DPFR_GATING_DIS;
2919
2920         if (apply)
2921                 val |= mask;
2922         else
2923                 val &= ~mask;
2924
2925         intel_de_write(dev_priv, CLKGATE_DIS_PSL(pipe), val);
2926 }
2927
2928 static void icl_pipe_mbus_enable(struct intel_crtc *crtc, bool joined_mbus)
2929 {
2930         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
2931         enum pipe pipe = crtc->pipe;
2932         u32 val;
2933
2934         /* Wa_22010947358:adl-p */
2935         if (IS_ALDERLAKE_P(dev_priv))
2936                 val = joined_mbus ? MBUS_DBOX_A_CREDIT(6) : MBUS_DBOX_A_CREDIT(4);
2937         else
2938                 val = MBUS_DBOX_A_CREDIT(2);
2939
2940         if (DISPLAY_VER(dev_priv) >= 12) {
2941                 val |= MBUS_DBOX_BW_CREDIT(2);
2942                 val |= MBUS_DBOX_B_CREDIT(12);
2943         } else {
2944                 val |= MBUS_DBOX_BW_CREDIT(1);
2945                 val |= MBUS_DBOX_B_CREDIT(8);
2946         }
2947
2948         intel_de_write(dev_priv, PIPE_MBUS_DBOX_CTL(pipe), val);
2949 }
2950
2951 static void hsw_set_linetime_wm(const struct intel_crtc_state *crtc_state)
2952 {
2953         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
2954         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
2955
2956         intel_de_write(dev_priv, WM_LINETIME(crtc->pipe),
2957                        HSW_LINETIME(crtc_state->linetime) |
2958                        HSW_IPS_LINETIME(crtc_state->ips_linetime));
2959 }
2960
2961 static void hsw_set_frame_start_delay(const struct intel_crtc_state *crtc_state)
2962 {
2963         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
2964         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
2965         i915_reg_t reg = CHICKEN_TRANS(crtc_state->cpu_transcoder);
2966         u32 val;
2967
2968         val = intel_de_read(dev_priv, reg);
2969         val &= ~HSW_FRAME_START_DELAY_MASK;
2970         val |= HSW_FRAME_START_DELAY(dev_priv->framestart_delay - 1);
2971         intel_de_write(dev_priv, reg, val);
2972 }
2973
2974 static void icl_ddi_bigjoiner_pre_enable(struct intel_atomic_state *state,
2975                                          const struct intel_crtc_state *crtc_state)
2976 {
2977         struct intel_crtc *master = to_intel_crtc(crtc_state->uapi.crtc);
2978         struct drm_i915_private *dev_priv = to_i915(master->base.dev);
2979         struct intel_crtc_state *master_crtc_state;
2980         struct drm_connector_state *conn_state;
2981         struct drm_connector *conn;
2982         struct intel_encoder *encoder = NULL;
2983         int i;
2984
2985         if (crtc_state->bigjoiner_slave)
2986                 master = crtc_state->bigjoiner_linked_crtc;
2987
2988         master_crtc_state = intel_atomic_get_new_crtc_state(state, master);
2989
2990         for_each_new_connector_in_state(&state->base, conn, conn_state, i) {
2991                 if (conn_state->crtc != &master->base)
2992                         continue;
2993
2994                 encoder = to_intel_encoder(conn_state->best_encoder);
2995                 break;
2996         }
2997
2998         if (!crtc_state->bigjoiner_slave) {
2999                 /* need to enable VDSC, which we skipped in pre-enable */
3000                 intel_dsc_enable(encoder, crtc_state);
3001         } else {
3002                 /*
3003                  * Enable sequence steps 1-7 on bigjoiner master
3004                  */
3005                 intel_encoders_pre_pll_enable(state, master);
3006                 if (master_crtc_state->shared_dpll)
3007                         intel_enable_shared_dpll(master_crtc_state);
3008                 intel_encoders_pre_enable(state, master);
3009
3010                 /* and DSC on slave */
3011                 intel_dsc_enable(NULL, crtc_state);
3012         }
3013
3014         if (DISPLAY_VER(dev_priv) >= 13)
3015                 intel_uncompressed_joiner_enable(crtc_state);
3016 }
3017
3018 static void hsw_crtc_enable(struct intel_atomic_state *state,
3019                             struct intel_crtc *crtc)
3020 {
3021         const struct intel_crtc_state *new_crtc_state =
3022                 intel_atomic_get_new_crtc_state(state, crtc);
3023         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
3024         enum pipe pipe = crtc->pipe, hsw_workaround_pipe;
3025         enum transcoder cpu_transcoder = new_crtc_state->cpu_transcoder;
3026         bool psl_clkgate_wa;
3027
3028         if (drm_WARN_ON(&dev_priv->drm, crtc->active))
3029                 return;
3030
3031         if (!new_crtc_state->bigjoiner) {
3032                 intel_encoders_pre_pll_enable(state, crtc);
3033
3034                 if (new_crtc_state->shared_dpll)
3035                         intel_enable_shared_dpll(new_crtc_state);
3036
3037                 intel_encoders_pre_enable(state, crtc);
3038         } else {
3039                 icl_ddi_bigjoiner_pre_enable(state, new_crtc_state);
3040         }
3041
3042         intel_set_pipe_src_size(new_crtc_state);
3043         if (DISPLAY_VER(dev_priv) >= 9 || IS_BROADWELL(dev_priv))
3044                 bdw_set_pipemisc(new_crtc_state);
3045
3046         if (!new_crtc_state->bigjoiner_slave && !transcoder_is_dsi(cpu_transcoder)) {
3047                 intel_set_transcoder_timings(new_crtc_state);
3048
3049                 if (cpu_transcoder != TRANSCODER_EDP)
3050                         intel_de_write(dev_priv, PIPE_MULT(cpu_transcoder),
3051                                        new_crtc_state->pixel_multiplier - 1);
3052
3053                 if (new_crtc_state->has_pch_encoder)
3054                         intel_cpu_transcoder_set_m_n(new_crtc_state,
3055                                                      &new_crtc_state->fdi_m_n, NULL);
3056
3057                 hsw_set_frame_start_delay(new_crtc_state);
3058
3059                 hsw_set_transconf(new_crtc_state);
3060         }
3061
3062         crtc->active = true;
3063
3064         /* Display WA #1180: WaDisableScalarClockGating: glk */
3065         psl_clkgate_wa = DISPLAY_VER(dev_priv) == 10 &&
3066                 new_crtc_state->pch_pfit.enabled;
3067         if (psl_clkgate_wa)
3068                 glk_pipe_scaler_clock_gating_wa(dev_priv, pipe, true);
3069
3070         if (DISPLAY_VER(dev_priv) >= 9)
3071                 skl_pfit_enable(new_crtc_state);
3072         else
3073                 ilk_pfit_enable(new_crtc_state);
3074
3075         /*
3076          * On ILK+ LUT must be loaded before the pipe is running but with
3077          * clocks enabled
3078          */
3079         intel_color_load_luts(new_crtc_state);
3080         intel_color_commit(new_crtc_state);
3081         /* update DSPCNTR to configure gamma/csc for pipe bottom color */
3082         if (DISPLAY_VER(dev_priv) < 9)
3083                 intel_disable_primary_plane(new_crtc_state);
3084
3085         hsw_set_linetime_wm(new_crtc_state);
3086
3087         if (DISPLAY_VER(dev_priv) >= 11)
3088                 icl_set_pipe_chicken(new_crtc_state);
3089
3090         intel_initial_watermarks(state, crtc);
3091
3092         if (DISPLAY_VER(dev_priv) >= 11) {
3093                 const struct intel_dbuf_state *dbuf_state =
3094                                 intel_atomic_get_new_dbuf_state(state);
3095
3096                 icl_pipe_mbus_enable(crtc, dbuf_state->joined_mbus);
3097         }
3098
3099         if (new_crtc_state->bigjoiner_slave)
3100                 intel_crtc_vblank_on(new_crtc_state);
3101
3102         intel_encoders_enable(state, crtc);
3103
3104         if (psl_clkgate_wa) {
3105                 intel_wait_for_vblank(dev_priv, pipe);
3106                 glk_pipe_scaler_clock_gating_wa(dev_priv, pipe, false);
3107         }
3108
3109         /* If we change the relative order between pipe/planes enabling, we need
3110          * to change the workaround. */
3111         hsw_workaround_pipe = new_crtc_state->hsw_workaround_pipe;
3112         if (IS_HASWELL(dev_priv) && hsw_workaround_pipe != INVALID_PIPE) {
3113                 intel_wait_for_vblank(dev_priv, hsw_workaround_pipe);
3114                 intel_wait_for_vblank(dev_priv, hsw_workaround_pipe);
3115         }
3116 }
3117
3118 void ilk_pfit_disable(const struct intel_crtc_state *old_crtc_state)
3119 {
3120         struct intel_crtc *crtc = to_intel_crtc(old_crtc_state->uapi.crtc);
3121         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
3122         enum pipe pipe = crtc->pipe;
3123
3124         /* To avoid upsetting the power well on haswell only disable the pfit if
3125          * it's in use. The hw state code will make sure we get this right. */
3126         if (!old_crtc_state->pch_pfit.enabled)
3127                 return;
3128
3129         intel_de_write(dev_priv, PF_CTL(pipe), 0);
3130         intel_de_write(dev_priv, PF_WIN_POS(pipe), 0);
3131         intel_de_write(dev_priv, PF_WIN_SZ(pipe), 0);
3132 }
3133
3134 static void ilk_crtc_disable(struct intel_atomic_state *state,
3135                              struct intel_crtc *crtc)
3136 {
3137         const struct intel_crtc_state *old_crtc_state =
3138                 intel_atomic_get_old_crtc_state(state, crtc);
3139         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
3140         enum pipe pipe = crtc->pipe;
3141
3142         /*
3143          * Sometimes spurious CPU pipe underruns happen when the
3144          * pipe is already disabled, but FDI RX/TX is still enabled.
3145          * Happens at least with VGA+HDMI cloning. Suppress them.
3146          */
3147         intel_set_cpu_fifo_underrun_reporting(dev_priv, pipe, false);
3148         intel_set_pch_fifo_underrun_reporting(dev_priv, pipe, false);
3149
3150         intel_encoders_disable(state, crtc);
3151
3152         intel_crtc_vblank_off(old_crtc_state);
3153
3154         intel_disable_transcoder(old_crtc_state);
3155
3156         ilk_pfit_disable(old_crtc_state);
3157
3158         if (old_crtc_state->has_pch_encoder)
3159                 ilk_fdi_disable(crtc);
3160
3161         intel_encoders_post_disable(state, crtc);
3162
3163         if (old_crtc_state->has_pch_encoder) {
3164                 ilk_disable_pch_transcoder(dev_priv, pipe);
3165
3166                 if (HAS_PCH_CPT(dev_priv)) {
3167                         i915_reg_t reg;
3168                         u32 temp;
3169
3170                         /* disable TRANS_DP_CTL */
3171                         reg = TRANS_DP_CTL(pipe);
3172                         temp = intel_de_read(dev_priv, reg);
3173                         temp &= ~(TRANS_DP_OUTPUT_ENABLE |
3174                                   TRANS_DP_PORT_SEL_MASK);
3175                         temp |= TRANS_DP_PORT_SEL_NONE;
3176                         intel_de_write(dev_priv, reg, temp);
3177
3178                         /* disable DPLL_SEL */
3179                         temp = intel_de_read(dev_priv, PCH_DPLL_SEL);
3180                         temp &= ~(TRANS_DPLL_ENABLE(pipe) | TRANS_DPLLB_SEL(pipe));
3181                         intel_de_write(dev_priv, PCH_DPLL_SEL, temp);
3182                 }
3183
3184                 ilk_fdi_pll_disable(crtc);
3185         }
3186
3187         intel_set_cpu_fifo_underrun_reporting(dev_priv, pipe, true);
3188         intel_set_pch_fifo_underrun_reporting(dev_priv, pipe, true);
3189 }
3190
3191 static void hsw_crtc_disable(struct intel_atomic_state *state,
3192                              struct intel_crtc *crtc)
3193 {
3194         /*
3195          * FIXME collapse everything to one hook.
3196          * Need care with mst->ddi interactions.
3197          */
3198         intel_encoders_disable(state, crtc);
3199         intel_encoders_post_disable(state, crtc);
3200 }
3201
3202 static void i9xx_pfit_enable(const struct intel_crtc_state *crtc_state)
3203 {
3204         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
3205         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
3206
3207         if (!crtc_state->gmch_pfit.control)
3208                 return;
3209
3210         /*
3211          * The panel fitter should only be adjusted whilst the pipe is disabled,
3212          * according to register description and PRM.
3213          */
3214         drm_WARN_ON(&dev_priv->drm,
3215                     intel_de_read(dev_priv, PFIT_CONTROL) & PFIT_ENABLE);
3216         assert_transcoder_disabled(dev_priv, crtc_state->cpu_transcoder);
3217
3218         intel_de_write(dev_priv, PFIT_PGM_RATIOS,
3219                        crtc_state->gmch_pfit.pgm_ratios);
3220         intel_de_write(dev_priv, PFIT_CONTROL, crtc_state->gmch_pfit.control);
3221
3222         /* Border color in case we don't scale up to the full screen. Black by
3223          * default, change to something else for debugging. */
3224         intel_de_write(dev_priv, BCLRPAT(crtc->pipe), 0);
3225 }
3226
3227 bool intel_phy_is_combo(struct drm_i915_private *dev_priv, enum phy phy)
3228 {
3229         if (phy == PHY_NONE)
3230                 return false;
3231         else if (IS_DG2(dev_priv))
3232                 /*
3233                  * DG2 outputs labelled as "combo PHY" in the bspec use
3234                  * SNPS PHYs with completely different programming,
3235                  * hence we always return false here.
3236                  */
3237                 return false;
3238         else if (IS_ALDERLAKE_S(dev_priv))
3239                 return phy <= PHY_E;
3240         else if (IS_DG1(dev_priv) || IS_ROCKETLAKE(dev_priv))
3241                 return phy <= PHY_D;
3242         else if (IS_JSL_EHL(dev_priv))
3243                 return phy <= PHY_C;
3244         else if (DISPLAY_VER(dev_priv) >= 11)
3245                 return phy <= PHY_B;
3246         else
3247                 return false;
3248 }
3249
3250 bool intel_phy_is_tc(struct drm_i915_private *dev_priv, enum phy phy)
3251 {
3252         if (IS_DG2(dev_priv))
3253                 /* DG2's "TC1" output uses a SNPS PHY */
3254                 return false;
3255         else if (IS_ALDERLAKE_P(dev_priv))
3256                 return phy >= PHY_F && phy <= PHY_I;
3257         else if (IS_TIGERLAKE(dev_priv))
3258                 return phy >= PHY_D && phy <= PHY_I;
3259         else if (IS_ICELAKE(dev_priv))
3260                 return phy >= PHY_C && phy <= PHY_F;
3261         else
3262                 return false;
3263 }
3264
3265 bool intel_phy_is_snps(struct drm_i915_private *dev_priv, enum phy phy)
3266 {
3267         if (phy == PHY_NONE)
3268                 return false;
3269         else if (IS_DG2(dev_priv))
3270                 /*
3271                  * All four "combo" ports and the TC1 port (PHY E) use
3272                  * Synopsis PHYs.
3273                  */
3274                 return phy <= PHY_E;
3275
3276         return false;
3277 }
3278
3279 enum phy intel_port_to_phy(struct drm_i915_private *i915, enum port port)
3280 {
3281         if (DISPLAY_VER(i915) >= 13 && port >= PORT_D_XELPD)
3282                 return PHY_D + port - PORT_D_XELPD;
3283         else if (DISPLAY_VER(i915) >= 13 && port >= PORT_TC1)
3284                 return PHY_F + port - PORT_TC1;
3285         else if (IS_ALDERLAKE_S(i915) && port >= PORT_TC1)
3286                 return PHY_B + port - PORT_TC1;
3287         else if ((IS_DG1(i915) || IS_ROCKETLAKE(i915)) && port >= PORT_TC1)
3288                 return PHY_C + port - PORT_TC1;
3289         else if (IS_JSL_EHL(i915) && port == PORT_D)
3290                 return PHY_A;
3291
3292         return PHY_A + port - PORT_A;
3293 }
3294
3295 enum tc_port intel_port_to_tc(struct drm_i915_private *dev_priv, enum port port)
3296 {
3297         if (!intel_phy_is_tc(dev_priv, intel_port_to_phy(dev_priv, port)))
3298                 return TC_PORT_NONE;
3299
3300         if (DISPLAY_VER(dev_priv) >= 12)
3301                 return TC_PORT_1 + port - PORT_TC1;
3302         else
3303                 return TC_PORT_1 + port - PORT_C;
3304 }
3305
3306 enum intel_display_power_domain intel_port_to_power_domain(enum port port)
3307 {
3308         switch (port) {
3309         case PORT_A:
3310                 return POWER_DOMAIN_PORT_DDI_A_LANES;
3311         case PORT_B:
3312                 return POWER_DOMAIN_PORT_DDI_B_LANES;
3313         case PORT_C:
3314                 return POWER_DOMAIN_PORT_DDI_C_LANES;
3315         case PORT_D:
3316                 return POWER_DOMAIN_PORT_DDI_D_LANES;
3317         case PORT_E:
3318                 return POWER_DOMAIN_PORT_DDI_E_LANES;
3319         case PORT_F:
3320                 return POWER_DOMAIN_PORT_DDI_F_LANES;
3321         case PORT_G:
3322                 return POWER_DOMAIN_PORT_DDI_G_LANES;
3323         case PORT_H:
3324                 return POWER_DOMAIN_PORT_DDI_H_LANES;
3325         case PORT_I:
3326                 return POWER_DOMAIN_PORT_DDI_I_LANES;
3327         default:
3328                 MISSING_CASE(port);
3329                 return POWER_DOMAIN_PORT_OTHER;
3330         }
3331 }
3332
3333 enum intel_display_power_domain
3334 intel_aux_power_domain(struct intel_digital_port *dig_port)
3335 {
3336         if (intel_tc_port_in_tbt_alt_mode(dig_port)) {
3337                 switch (dig_port->aux_ch) {
3338                 case AUX_CH_C:
3339                         return POWER_DOMAIN_AUX_C_TBT;
3340                 case AUX_CH_D:
3341                         return POWER_DOMAIN_AUX_D_TBT;
3342                 case AUX_CH_E:
3343                         return POWER_DOMAIN_AUX_E_TBT;
3344                 case AUX_CH_F:
3345                         return POWER_DOMAIN_AUX_F_TBT;
3346                 case AUX_CH_G:
3347                         return POWER_DOMAIN_AUX_G_TBT;
3348                 case AUX_CH_H:
3349                         return POWER_DOMAIN_AUX_H_TBT;
3350                 case AUX_CH_I:
3351                         return POWER_DOMAIN_AUX_I_TBT;
3352                 default:
3353                         MISSING_CASE(dig_port->aux_ch);
3354                         return POWER_DOMAIN_AUX_C_TBT;
3355                 }
3356         }
3357
3358         return intel_legacy_aux_to_power_domain(dig_port->aux_ch);
3359 }
3360
3361 /*
3362  * Converts aux_ch to power_domain without caring about TBT ports for that use
3363  * intel_aux_power_domain()
3364  */
3365 enum intel_display_power_domain
3366 intel_legacy_aux_to_power_domain(enum aux_ch aux_ch)
3367 {
3368         switch (aux_ch) {
3369         case AUX_CH_A:
3370                 return POWER_DOMAIN_AUX_A;
3371         case AUX_CH_B:
3372                 return POWER_DOMAIN_AUX_B;
3373         case AUX_CH_C:
3374                 return POWER_DOMAIN_AUX_C;
3375         case AUX_CH_D:
3376                 return POWER_DOMAIN_AUX_D;
3377         case AUX_CH_E:
3378                 return POWER_DOMAIN_AUX_E;
3379         case AUX_CH_F:
3380                 return POWER_DOMAIN_AUX_F;
3381         case AUX_CH_G:
3382                 return POWER_DOMAIN_AUX_G;
3383         case AUX_CH_H:
3384                 return POWER_DOMAIN_AUX_H;
3385         case AUX_CH_I:
3386                 return POWER_DOMAIN_AUX_I;
3387         default:
3388                 MISSING_CASE(aux_ch);
3389                 return POWER_DOMAIN_AUX_A;
3390         }
3391 }
3392
3393 static u64 get_crtc_power_domains(struct intel_crtc_state *crtc_state)
3394 {
3395         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
3396         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
3397         enum transcoder cpu_transcoder = crtc_state->cpu_transcoder;
3398         struct drm_encoder *encoder;
3399         enum pipe pipe = crtc->pipe;
3400         u64 mask;
3401
3402         if (!crtc_state->hw.active)
3403                 return 0;
3404
3405         mask = BIT_ULL(POWER_DOMAIN_PIPE(pipe));
3406         mask |= BIT_ULL(POWER_DOMAIN_TRANSCODER(cpu_transcoder));
3407         if (crtc_state->pch_pfit.enabled ||
3408             crtc_state->pch_pfit.force_thru)
3409                 mask |= BIT_ULL(POWER_DOMAIN_PIPE_PANEL_FITTER(pipe));
3410
3411         drm_for_each_encoder_mask(encoder, &dev_priv->drm,
3412                                   crtc_state->uapi.encoder_mask) {
3413                 struct intel_encoder *intel_encoder = to_intel_encoder(encoder);
3414
3415                 mask |= BIT_ULL(intel_encoder->power_domain);
3416         }
3417
3418         if (HAS_DDI(dev_priv) && crtc_state->has_audio)
3419                 mask |= BIT_ULL(POWER_DOMAIN_AUDIO_MMIO);
3420
3421         if (crtc_state->shared_dpll)
3422                 mask |= BIT_ULL(POWER_DOMAIN_DISPLAY_CORE);
3423
3424         if (crtc_state->dsc.compression_enable)
3425                 mask |= BIT_ULL(intel_dsc_power_domain(crtc, cpu_transcoder));
3426
3427         return mask;
3428 }
3429
3430 static u64
3431 modeset_get_crtc_power_domains(struct intel_crtc_state *crtc_state)
3432 {
3433         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
3434         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
3435         enum intel_display_power_domain domain;
3436         u64 domains, new_domains, old_domains;
3437
3438         domains = get_crtc_power_domains(crtc_state);
3439
3440         new_domains = domains & ~crtc->enabled_power_domains.mask;
3441         old_domains = crtc->enabled_power_domains.mask & ~domains;
3442
3443         for_each_power_domain(domain, new_domains)
3444                 intel_display_power_get_in_set(dev_priv,
3445                                                &crtc->enabled_power_domains,
3446                                                domain);
3447
3448         return old_domains;
3449 }
3450
3451 static void modeset_put_crtc_power_domains(struct intel_crtc *crtc,
3452                                            u64 domains)
3453 {
3454         intel_display_power_put_mask_in_set(to_i915(crtc->base.dev),
3455                                             &crtc->enabled_power_domains,
3456                                             domains);
3457 }
3458
3459 static void valleyview_crtc_enable(struct intel_atomic_state *state,
3460                                    struct intel_crtc *crtc)
3461 {
3462         const struct intel_crtc_state *new_crtc_state =
3463                 intel_atomic_get_new_crtc_state(state, crtc);
3464         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
3465         enum pipe pipe = crtc->pipe;
3466
3467         if (drm_WARN_ON(&dev_priv->drm, crtc->active))
3468                 return;
3469
3470         if (intel_crtc_has_dp_encoder(new_crtc_state))
3471                 intel_dp_set_m_n(new_crtc_state, M1_N1);
3472
3473         intel_set_transcoder_timings(new_crtc_state);
3474         intel_set_pipe_src_size(new_crtc_state);
3475
3476         if (IS_CHERRYVIEW(dev_priv) && pipe == PIPE_B) {
3477                 intel_de_write(dev_priv, CHV_BLEND(pipe), CHV_BLEND_LEGACY);
3478                 intel_de_write(dev_priv, CHV_CANVAS(pipe), 0);
3479         }
3480
3481         i9xx_set_pipeconf(new_crtc_state);
3482
3483         crtc->active = true;
3484
3485         intel_set_cpu_fifo_underrun_reporting(dev_priv, pipe, true);
3486
3487         intel_encoders_pre_pll_enable(state, crtc);
3488
3489         if (IS_CHERRYVIEW(dev_priv))
3490                 chv_enable_pll(new_crtc_state);
3491         else
3492                 vlv_enable_pll(new_crtc_state);
3493
3494         intel_encoders_pre_enable(state, crtc);
3495
3496         i9xx_pfit_enable(new_crtc_state);
3497
3498         intel_color_load_luts(new_crtc_state);
3499         intel_color_commit(new_crtc_state);
3500         /* update DSPCNTR to configure gamma for pipe bottom color */
3501         intel_disable_primary_plane(new_crtc_state);
3502
3503         intel_initial_watermarks(state, crtc);
3504         intel_enable_transcoder(new_crtc_state);
3505
3506         intel_crtc_vblank_on(new_crtc_state);
3507
3508         intel_encoders_enable(state, crtc);
3509 }
3510
3511 static void i9xx_crtc_enable(struct intel_atomic_state *state,
3512                              struct intel_crtc *crtc)
3513 {
3514         const struct intel_crtc_state *new_crtc_state =
3515                 intel_atomic_get_new_crtc_state(state, crtc);
3516         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
3517         enum pipe pipe = crtc->pipe;
3518
3519         if (drm_WARN_ON(&dev_priv->drm, crtc->active))
3520                 return;
3521
3522         if (intel_crtc_has_dp_encoder(new_crtc_state))
3523                 intel_dp_set_m_n(new_crtc_state, M1_N1);
3524
3525         intel_set_transcoder_timings(new_crtc_state);
3526         intel_set_pipe_src_size(new_crtc_state);
3527
3528         i9xx_set_pipeconf(new_crtc_state);
3529
3530         crtc->active = true;
3531
3532         if (DISPLAY_VER(dev_priv) != 2)
3533                 intel_set_cpu_fifo_underrun_reporting(dev_priv, pipe, true);
3534
3535         intel_encoders_pre_enable(state, crtc);
3536
3537         i9xx_enable_pll(new_crtc_state);
3538
3539         i9xx_pfit_enable(new_crtc_state);
3540
3541         intel_color_load_luts(new_crtc_state);
3542         intel_color_commit(new_crtc_state);
3543         /* update DSPCNTR to configure gamma for pipe bottom color */
3544         intel_disable_primary_plane(new_crtc_state);
3545
3546         if (!intel_initial_watermarks(state, crtc))
3547                 intel_update_watermarks(dev_priv);
3548         intel_enable_transcoder(new_crtc_state);
3549
3550         intel_crtc_vblank_on(new_crtc_state);
3551
3552         intel_encoders_enable(state, crtc);
3553
3554         /* prevents spurious underruns */
3555         if (DISPLAY_VER(dev_priv) == 2)
3556                 intel_wait_for_vblank(dev_priv, pipe);
3557 }
3558
3559 static void i9xx_pfit_disable(const struct intel_crtc_state *old_crtc_state)
3560 {
3561         struct intel_crtc *crtc = to_intel_crtc(old_crtc_state->uapi.crtc);
3562         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
3563
3564         if (!old_crtc_state->gmch_pfit.control)
3565                 return;
3566
3567         assert_transcoder_disabled(dev_priv, old_crtc_state->cpu_transcoder);
3568
3569         drm_dbg_kms(&dev_priv->drm, "disabling pfit, current: 0x%08x\n",
3570                     intel_de_read(dev_priv, PFIT_CONTROL));
3571         intel_de_write(dev_priv, PFIT_CONTROL, 0);
3572 }
3573
3574 static void i9xx_crtc_disable(struct intel_atomic_state *state,
3575                               struct intel_crtc *crtc)
3576 {
3577         struct intel_crtc_state *old_crtc_state =
3578                 intel_atomic_get_old_crtc_state(state, crtc);
3579         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
3580         enum pipe pipe = crtc->pipe;
3581
3582         /*
3583          * On gen2 planes are double buffered but the pipe isn't, so we must
3584          * wait for planes to fully turn off before disabling the pipe.
3585          */
3586         if (DISPLAY_VER(dev_priv) == 2)
3587                 intel_wait_for_vblank(dev_priv, pipe);
3588
3589         intel_encoders_disable(state, crtc);
3590
3591         intel_crtc_vblank_off(old_crtc_state);
3592
3593         intel_disable_transcoder(old_crtc_state);
3594
3595         i9xx_pfit_disable(old_crtc_state);
3596
3597         intel_encoders_post_disable(state, crtc);
3598
3599         if (!intel_crtc_has_type(old_crtc_state, INTEL_OUTPUT_DSI)) {
3600                 if (IS_CHERRYVIEW(dev_priv))
3601                         chv_disable_pll(dev_priv, pipe);
3602                 else if (IS_VALLEYVIEW(dev_priv))
3603                         vlv_disable_pll(dev_priv, pipe);
3604                 else
3605                         i9xx_disable_pll(old_crtc_state);
3606         }
3607
3608         intel_encoders_post_pll_disable(state, crtc);
3609
3610         if (DISPLAY_VER(dev_priv) != 2)
3611                 intel_set_cpu_fifo_underrun_reporting(dev_priv, pipe, false);
3612
3613         if (!dev_priv->wm_disp->initial_watermarks)
3614                 intel_update_watermarks(dev_priv);
3615
3616         /* clock the pipe down to 640x480@60 to potentially save power */
3617         if (IS_I830(dev_priv))
3618                 i830_enable_pipe(dev_priv, pipe);
3619 }
3620
3621 static void intel_crtc_disable_noatomic(struct intel_crtc *crtc,
3622                                         struct drm_modeset_acquire_ctx *ctx)
3623 {
3624         struct intel_encoder *encoder;
3625         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
3626         struct intel_bw_state *bw_state =
3627                 to_intel_bw_state(dev_priv->bw_obj.state);
3628         struct intel_cdclk_state *cdclk_state =
3629                 to_intel_cdclk_state(dev_priv->cdclk.obj.state);
3630         struct intel_dbuf_state *dbuf_state =
3631                 to_intel_dbuf_state(dev_priv->dbuf.obj.state);
3632         struct intel_crtc_state *crtc_state =
3633                 to_intel_crtc_state(crtc->base.state);
3634         struct intel_plane *plane;
3635         struct drm_atomic_state *state;
3636         struct intel_crtc_state *temp_crtc_state;
3637         enum pipe pipe = crtc->pipe;
3638         int ret;
3639
3640         if (!crtc_state->hw.active)
3641                 return;
3642
3643         for_each_intel_plane_on_crtc(&dev_priv->drm, crtc, plane) {
3644                 const struct intel_plane_state *plane_state =
3645                         to_intel_plane_state(plane->base.state);
3646
3647                 if (plane_state->uapi.visible)
3648                         intel_plane_disable_noatomic(crtc, plane);
3649         }
3650
3651         state = drm_atomic_state_alloc(&dev_priv->drm);
3652         if (!state) {
3653                 drm_dbg_kms(&dev_priv->drm,
3654                             "failed to disable [CRTC:%d:%s], out of memory",
3655                             crtc->base.base.id, crtc->base.name);
3656                 return;
3657         }
3658
3659         state->acquire_ctx = ctx;
3660
3661         /* Everything's already locked, -EDEADLK can't happen. */
3662         temp_crtc_state = intel_atomic_get_crtc_state(state, crtc);
3663         ret = drm_atomic_add_affected_connectors(state, &crtc->base);
3664
3665         drm_WARN_ON(&dev_priv->drm, IS_ERR(temp_crtc_state) || ret);
3666
3667         dev_priv->display->crtc_disable(to_intel_atomic_state(state), crtc);
3668
3669         drm_atomic_state_put(state);
3670
3671         drm_dbg_kms(&dev_priv->drm,
3672                     "[CRTC:%d:%s] hw state adjusted, was enabled, now disabled\n",
3673                     crtc->base.base.id, crtc->base.name);
3674
3675         crtc->active = false;
3676         crtc->base.enabled = false;
3677
3678         drm_WARN_ON(&dev_priv->drm,
3679                     drm_atomic_set_mode_for_crtc(&crtc_state->uapi, NULL) < 0);
3680         crtc_state->uapi.active = false;
3681         crtc_state->uapi.connector_mask = 0;
3682         crtc_state->uapi.encoder_mask = 0;
3683         intel_crtc_free_hw_state(crtc_state);
3684         memset(&crtc_state->hw, 0, sizeof(crtc_state->hw));
3685
3686         for_each_encoder_on_crtc(&dev_priv->drm, &crtc->base, encoder)
3687                 encoder->base.crtc = NULL;
3688
3689         intel_fbc_disable(crtc);
3690         intel_update_watermarks(dev_priv);
3691         intel_disable_shared_dpll(crtc_state);
3692
3693         intel_display_power_put_all_in_set(dev_priv, &crtc->enabled_power_domains);
3694
3695         cdclk_state->min_cdclk[pipe] = 0;
3696         cdclk_state->min_voltage_level[pipe] = 0;
3697         cdclk_state->active_pipes &= ~BIT(pipe);
3698
3699         dbuf_state->active_pipes &= ~BIT(pipe);
3700
3701         bw_state->data_rate[pipe] = 0;
3702         bw_state->num_active_planes[pipe] = 0;
3703 }
3704
3705 /*
3706  * turn all crtc's off, but do not adjust state
3707  * This has to be paired with a call to intel_modeset_setup_hw_state.
3708  */
3709 int intel_display_suspend(struct drm_device *dev)
3710 {
3711         struct drm_i915_private *dev_priv = to_i915(dev);
3712         struct drm_atomic_state *state;
3713         int ret;
3714
3715         if (!HAS_DISPLAY(dev_priv))
3716                 return 0;
3717
3718         state = drm_atomic_helper_suspend(dev);
3719         ret = PTR_ERR_OR_ZERO(state);
3720         if (ret)
3721                 drm_err(&dev_priv->drm, "Suspending crtc's failed with %i\n",
3722                         ret);
3723         else
3724                 dev_priv->modeset_restore_state = state;
3725         return ret;
3726 }
3727
3728 void intel_encoder_destroy(struct drm_encoder *encoder)
3729 {
3730         struct intel_encoder *intel_encoder = to_intel_encoder(encoder);
3731
3732         drm_encoder_cleanup(encoder);
3733         kfree(intel_encoder);
3734 }
3735
3736 /* Cross check the actual hw state with our own modeset state tracking (and it's
3737  * internal consistency). */
3738 static void intel_connector_verify_state(struct intel_crtc_state *crtc_state,
3739                                          struct drm_connector_state *conn_state)
3740 {
3741         struct intel_connector *connector = to_intel_connector(conn_state->connector);
3742         struct drm_i915_private *i915 = to_i915(connector->base.dev);
3743
3744         drm_dbg_kms(&i915->drm, "[CONNECTOR:%d:%s]\n",
3745                     connector->base.base.id, connector->base.name);
3746
3747         if (connector->get_hw_state(connector)) {
3748                 struct intel_encoder *encoder = intel_attached_encoder(connector);
3749
3750                 I915_STATE_WARN(!crtc_state,
3751                          "connector enabled without attached crtc\n");
3752
3753                 if (!crtc_state)
3754                         return;
3755
3756                 I915_STATE_WARN(!crtc_state->hw.active,
3757                                 "connector is active, but attached crtc isn't\n");
3758
3759                 if (!encoder || encoder->type == INTEL_OUTPUT_DP_MST)
3760                         return;
3761
3762                 I915_STATE_WARN(conn_state->best_encoder != &encoder->base,
3763                         "atomic encoder doesn't match attached encoder\n");
3764
3765                 I915_STATE_WARN(conn_state->crtc != encoder->base.crtc,
3766                         "attached encoder crtc differs from connector crtc\n");
3767         } else {
3768                 I915_STATE_WARN(crtc_state && crtc_state->hw.active,
3769                                 "attached crtc is active, but connector isn't\n");
3770                 I915_STATE_WARN(!crtc_state && conn_state->best_encoder,
3771                         "best encoder set without crtc!\n");
3772         }
3773 }
3774
3775 bool hsw_crtc_state_ips_capable(const struct intel_crtc_state *crtc_state)
3776 {
3777         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
3778         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
3779
3780         /* IPS only exists on ULT machines and is tied to pipe A. */
3781         if (!hsw_crtc_supports_ips(crtc))
3782                 return false;
3783
3784         if (!dev_priv->params.enable_ips)
3785                 return false;
3786
3787         if (crtc_state->pipe_bpp > 24)
3788                 return false;
3789
3790         /*
3791          * We compare against max which means we must take
3792          * the increased cdclk requirement into account when
3793          * calculating the new cdclk.
3794          *
3795          * Should measure whether using a lower cdclk w/o IPS
3796          */
3797         if (IS_BROADWELL(dev_priv) &&
3798             crtc_state->pixel_rate > dev_priv->max_cdclk_freq * 95 / 100)
3799                 return false;
3800
3801         return true;
3802 }
3803
3804 static int hsw_compute_ips_config(struct intel_crtc_state *crtc_state)
3805 {
3806         struct drm_i915_private *dev_priv =
3807                 to_i915(crtc_state->uapi.crtc->dev);
3808         struct intel_atomic_state *state =
3809                 to_intel_atomic_state(crtc_state->uapi.state);
3810
3811         crtc_state->ips_enabled = false;
3812
3813         if (!hsw_crtc_state_ips_capable(crtc_state))
3814                 return 0;
3815
3816         /*
3817          * When IPS gets enabled, the pipe CRC changes. Since IPS gets
3818          * enabled and disabled dynamically based on package C states,
3819          * user space can't make reliable use of the CRCs, so let's just
3820          * completely disable it.
3821          */
3822         if (crtc_state->crc_enabled)
3823                 return 0;
3824
3825         /* IPS should be fine as long as at least one plane is enabled. */
3826         if (!(crtc_state->active_planes & ~BIT(PLANE_CURSOR)))
3827                 return 0;
3828
3829         if (IS_BROADWELL(dev_priv)) {
3830                 const struct intel_cdclk_state *cdclk_state;
3831
3832                 cdclk_state = intel_atomic_get_cdclk_state(state);
3833                 if (IS_ERR(cdclk_state))
3834                         return PTR_ERR(cdclk_state);
3835
3836                 /* pixel rate mustn't exceed 95% of cdclk with IPS on BDW */
3837                 if (crtc_state->pixel_rate > cdclk_state->logical.cdclk * 95 / 100)
3838                         return 0;
3839         }
3840
3841         crtc_state->ips_enabled = true;
3842
3843         return 0;
3844 }
3845
3846 static bool intel_crtc_supports_double_wide(const struct intel_crtc *crtc)
3847 {
3848         const struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
3849
3850         /* GDG double wide on either pipe, otherwise pipe A only */
3851         return DISPLAY_VER(dev_priv) < 4 &&
3852                 (crtc->pipe == PIPE_A || IS_I915G(dev_priv));
3853 }
3854
3855 static u32 ilk_pipe_pixel_rate(const struct intel_crtc_state *crtc_state)
3856 {
3857         u32 pixel_rate = crtc_state->hw.pipe_mode.crtc_clock;
3858         struct drm_rect src;
3859
3860         /*
3861          * We only use IF-ID interlacing. If we ever use
3862          * PF-ID we'll need to adjust the pixel_rate here.
3863          */
3864
3865         if (!crtc_state->pch_pfit.enabled)
3866                 return pixel_rate;
3867
3868         drm_rect_init(&src, 0, 0,
3869                       crtc_state->pipe_src_w << 16,
3870                       crtc_state->pipe_src_h << 16);
3871
3872         return intel_adjusted_rate(&src, &crtc_state->pch_pfit.dst,
3873                                    pixel_rate);
3874 }
3875
3876 static void intel_mode_from_crtc_timings(struct drm_display_mode *mode,
3877                                          const struct drm_display_mode *timings)
3878 {
3879         mode->hdisplay = timings->crtc_hdisplay;
3880         mode->htotal = timings->crtc_htotal;
3881         mode->hsync_start = timings->crtc_hsync_start;
3882         mode->hsync_end = timings->crtc_hsync_end;
3883
3884         mode->vdisplay = timings->crtc_vdisplay;
3885         mode->vtotal = timings->crtc_vtotal;
3886         mode->vsync_start = timings->crtc_vsync_start;
3887         mode->vsync_end = timings->crtc_vsync_end;
3888
3889         mode->flags = timings->flags;
3890         mode->type = DRM_MODE_TYPE_DRIVER;
3891
3892         mode->clock = timings->crtc_clock;
3893
3894         drm_mode_set_name(mode);
3895 }
3896
3897 static void intel_crtc_compute_pixel_rate(struct intel_crtc_state *crtc_state)
3898 {
3899         struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
3900
3901         if (HAS_GMCH(dev_priv))
3902                 /* FIXME calculate proper pipe pixel rate for GMCH pfit */
3903                 crtc_state->pixel_rate =
3904                         crtc_state->hw.pipe_mode.crtc_clock;
3905         else
3906                 crtc_state->pixel_rate =
3907                         ilk_pipe_pixel_rate(crtc_state);
3908 }
3909
3910 static void intel_crtc_readout_derived_state(struct intel_crtc_state *crtc_state)
3911 {
3912         struct drm_display_mode *mode = &crtc_state->hw.mode;
3913         struct drm_display_mode *pipe_mode = &crtc_state->hw.pipe_mode;
3914         struct drm_display_mode *adjusted_mode = &crtc_state->hw.adjusted_mode;
3915
3916         drm_mode_copy(pipe_mode, adjusted_mode);
3917
3918         if (crtc_state->bigjoiner) {
3919                 /*
3920                  * transcoder is programmed to the full mode,
3921                  * but pipe timings are half of the transcoder mode
3922                  */
3923                 pipe_mode->crtc_hdisplay /= 2;
3924                 pipe_mode->crtc_hblank_start /= 2;
3925                 pipe_mode->crtc_hblank_end /= 2;
3926                 pipe_mode->crtc_hsync_start /= 2;
3927                 pipe_mode->crtc_hsync_end /= 2;
3928                 pipe_mode->crtc_htotal /= 2;
3929                 pipe_mode->crtc_clock /= 2;
3930         }
3931
3932         if (crtc_state->splitter.enable) {
3933                 int n = crtc_state->splitter.link_count;
3934                 int overlap = crtc_state->splitter.pixel_overlap;
3935
3936                 /*
3937                  * eDP MSO uses segment timings from EDID for transcoder
3938                  * timings, but full mode for everything else.
3939                  *
3940                  * h_full = (h_segment - pixel_overlap) * link_count
3941                  */
3942                 pipe_mode->crtc_hdisplay = (pipe_mode->crtc_hdisplay - overlap) * n;
3943                 pipe_mode->crtc_hblank_start = (pipe_mode->crtc_hblank_start - overlap) * n;
3944                 pipe_mode->crtc_hblank_end = (pipe_mode->crtc_hblank_end - overlap) * n;
3945                 pipe_mode->crtc_hsync_start = (pipe_mode->crtc_hsync_start - overlap) * n;
3946                 pipe_mode->crtc_hsync_end = (pipe_mode->crtc_hsync_end - overlap) * n;
3947                 pipe_mode->crtc_htotal = (pipe_mode->crtc_htotal - overlap) * n;
3948                 pipe_mode->crtc_clock *= n;
3949
3950                 intel_mode_from_crtc_timings(pipe_mode, pipe_mode);
3951                 intel_mode_from_crtc_timings(adjusted_mode, pipe_mode);
3952         } else {
3953                 intel_mode_from_crtc_timings(pipe_mode, pipe_mode);
3954                 intel_mode_from_crtc_timings(adjusted_mode, adjusted_mode);
3955         }
3956
3957         intel_crtc_compute_pixel_rate(crtc_state);
3958
3959         drm_mode_copy(mode, adjusted_mode);
3960         mode->hdisplay = crtc_state->pipe_src_w << crtc_state->bigjoiner;
3961         mode->vdisplay = crtc_state->pipe_src_h;
3962 }
3963
3964 static void intel_encoder_get_config(struct intel_encoder *encoder,
3965                                      struct intel_crtc_state *crtc_state)
3966 {
3967         encoder->get_config(encoder, crtc_state);
3968
3969         intel_crtc_readout_derived_state(crtc_state);
3970 }
3971
3972 static int intel_crtc_compute_config(struct intel_crtc *crtc,
3973                                      struct intel_crtc_state *pipe_config)
3974 {
3975         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
3976         struct drm_display_mode *pipe_mode = &pipe_config->hw.pipe_mode;
3977         int clock_limit = dev_priv->max_dotclk_freq;
3978
3979         drm_mode_copy(pipe_mode, &pipe_config->hw.adjusted_mode);
3980
3981         /* Adjust pipe_mode for bigjoiner, with half the horizontal mode */
3982         if (pipe_config->bigjoiner) {
3983                 pipe_mode->crtc_clock /= 2;
3984                 pipe_mode->crtc_hdisplay /= 2;
3985                 pipe_mode->crtc_hblank_start /= 2;
3986                 pipe_mode->crtc_hblank_end /= 2;
3987                 pipe_mode->crtc_hsync_start /= 2;
3988                 pipe_mode->crtc_hsync_end /= 2;
3989                 pipe_mode->crtc_htotal /= 2;
3990                 pipe_config->pipe_src_w /= 2;
3991         }
3992
3993         if (pipe_config->splitter.enable) {
3994                 int n = pipe_config->splitter.link_count;
3995                 int overlap = pipe_config->splitter.pixel_overlap;
3996
3997                 pipe_mode->crtc_hdisplay = (pipe_mode->crtc_hdisplay - overlap) * n;
3998                 pipe_mode->crtc_hblank_start = (pipe_mode->crtc_hblank_start - overlap) * n;
3999                 pipe_mode->crtc_hblank_end = (pipe_mode->crtc_hblank_end - overlap) * n;
4000                 pipe_mode->crtc_hsync_start = (pipe_mode->crtc_hsync_start - overlap) * n;
4001                 pipe_mode->crtc_hsync_end = (pipe_mode->crtc_hsync_end - overlap) * n;
4002                 pipe_mode->crtc_htotal = (pipe_mode->crtc_htotal - overlap) * n;
4003                 pipe_mode->crtc_clock *= n;
4004         }
4005
4006         intel_mode_from_crtc_timings(pipe_mode, pipe_mode);
4007
4008         if (DISPLAY_VER(dev_priv) < 4) {
4009                 clock_limit = dev_priv->max_cdclk_freq * 9 / 10;
4010
4011                 /*
4012                  * Enable double wide mode when the dot clock
4013                  * is > 90% of the (display) core speed.
4014                  */
4015                 if (intel_crtc_supports_double_wide(crtc) &&
4016                     pipe_mode->crtc_clock > clock_limit) {
4017                         clock_limit = dev_priv->max_dotclk_freq;
4018                         pipe_config->double_wide = true;
4019                 }
4020         }
4021
4022         if (pipe_mode->crtc_clock > clock_limit) {
4023                 drm_dbg_kms(&dev_priv->drm,
4024                             "requested pixel clock (%d kHz) too high (max: %d kHz, double wide: %s)\n",
4025                             pipe_mode->crtc_clock, clock_limit,
4026                             yesno(pipe_config->double_wide));
4027                 return -EINVAL;
4028         }
4029
4030         /*
4031          * Pipe horizontal size must be even in:
4032          * - DVO ganged mode
4033          * - LVDS dual channel mode
4034          * - Double wide pipe
4035          */
4036         if (pipe_config->pipe_src_w & 1) {
4037                 if (pipe_config->double_wide) {
4038                         drm_dbg_kms(&dev_priv->drm,
4039                                     "Odd pipe source width not supported with double wide pipe\n");
4040                         return -EINVAL;
4041                 }
4042
4043                 if (intel_crtc_has_type(pipe_config, INTEL_OUTPUT_LVDS) &&
4044                     intel_is_dual_link_lvds(dev_priv)) {
4045                         drm_dbg_kms(&dev_priv->drm,
4046                                     "Odd pipe source width not supported with dual link LVDS\n");
4047                         return -EINVAL;
4048                 }
4049         }
4050
4051         intel_crtc_compute_pixel_rate(pipe_config);
4052
4053         if (pipe_config->has_pch_encoder)
4054                 return ilk_fdi_compute_config(crtc, pipe_config);
4055
4056         return 0;
4057 }
4058
4059 static void
4060 intel_reduce_m_n_ratio(u32 *num, u32 *den)
4061 {
4062         while (*num > DATA_LINK_M_N_MASK ||
4063                *den > DATA_LINK_M_N_MASK) {
4064                 *num >>= 1;
4065                 *den >>= 1;
4066         }
4067 }
4068
4069 static void compute_m_n(unsigned int m, unsigned int n,
4070                         u32 *ret_m, u32 *ret_n,
4071                         bool constant_n)
4072 {
4073         /*
4074          * Several DP dongles in particular seem to be fussy about
4075          * too large link M/N values. Give N value as 0x8000 that
4076          * should be acceptable by specific devices. 0x8000 is the
4077          * specified fixed N value for asynchronous clock mode,
4078          * which the devices expect also in synchronous clock mode.
4079          */
4080         if (constant_n)
4081                 *ret_n = DP_LINK_CONSTANT_N_VALUE;
4082         else
4083                 *ret_n = min_t(unsigned int, roundup_pow_of_two(n), DATA_LINK_N_MAX);
4084
4085         *ret_m = div_u64(mul_u32_u32(m, *ret_n), n);
4086         intel_reduce_m_n_ratio(ret_m, ret_n);
4087 }
4088
4089 void
4090 intel_link_compute_m_n(u16 bits_per_pixel, int nlanes,
4091                        int pixel_clock, int link_clock,
4092                        struct intel_link_m_n *m_n,
4093                        bool constant_n, bool fec_enable)
4094 {
4095         u32 data_clock = bits_per_pixel * pixel_clock;
4096
4097         if (fec_enable)
4098                 data_clock = intel_dp_mode_to_fec_clock(data_clock);
4099
4100         m_n->tu = 64;
4101         compute_m_n(data_clock,
4102                     link_clock * nlanes * 8,
4103                     &m_n->gmch_m, &m_n->gmch_n,
4104                     constant_n);
4105
4106         compute_m_n(pixel_clock, link_clock,
4107                     &m_n->link_m, &m_n->link_n,
4108                     constant_n);
4109 }
4110
4111 static void intel_panel_sanitize_ssc(struct drm_i915_private *dev_priv)
4112 {
4113         /*
4114          * There may be no VBT; and if the BIOS enabled SSC we can
4115          * just keep using it to avoid unnecessary flicker.  Whereas if the
4116          * BIOS isn't using it, don't assume it will work even if the VBT
4117          * indicates as much.
4118          */
4119         if (HAS_PCH_IBX(dev_priv) || HAS_PCH_CPT(dev_priv)) {
4120                 bool bios_lvds_use_ssc = intel_de_read(dev_priv,
4121                                                        PCH_DREF_CONTROL) &
4122                         DREF_SSC1_ENABLE;
4123
4124                 if (dev_priv->vbt.lvds_use_ssc != bios_lvds_use_ssc) {
4125                         drm_dbg_kms(&dev_priv->drm,
4126                                     "SSC %s by BIOS, overriding VBT which says %s\n",
4127                                     enableddisabled(bios_lvds_use_ssc),
4128                                     enableddisabled(dev_priv->vbt.lvds_use_ssc));
4129                         dev_priv->vbt.lvds_use_ssc = bios_lvds_use_ssc;
4130                 }
4131         }
4132 }
4133
4134 static void intel_pch_transcoder_set_m_n(const struct intel_crtc_state *crtc_state,
4135                                          const struct intel_link_m_n *m_n)
4136 {
4137         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
4138         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
4139         enum pipe pipe = crtc->pipe;
4140
4141         intel_de_write(dev_priv, PCH_TRANS_DATA_M1(pipe),
4142                        TU_SIZE(m_n->tu) | m_n->gmch_m);
4143         intel_de_write(dev_priv, PCH_TRANS_DATA_N1(pipe), m_n->gmch_n);
4144         intel_de_write(dev_priv, PCH_TRANS_LINK_M1(pipe), m_n->link_m);
4145         intel_de_write(dev_priv, PCH_TRANS_LINK_N1(pipe), m_n->link_n);
4146 }
4147
4148 static bool transcoder_has_m2_n2(struct drm_i915_private *dev_priv,
4149                                  enum transcoder transcoder)
4150 {
4151         if (IS_HASWELL(dev_priv))
4152                 return transcoder == TRANSCODER_EDP;
4153
4154         /*
4155          * Strictly speaking some registers are available before
4156          * gen7, but we only support DRRS on gen7+
4157          */
4158         return DISPLAY_VER(dev_priv) == 7 || IS_CHERRYVIEW(dev_priv);
4159 }
4160
4161 static void intel_cpu_transcoder_set_m_n(const struct intel_crtc_state *crtc_state,
4162                                          const struct intel_link_m_n *m_n,
4163                                          const struct intel_link_m_n *m2_n2)
4164 {
4165         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
4166         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
4167         enum pipe pipe = crtc->pipe;
4168         enum transcoder transcoder = crtc_state->cpu_transcoder;
4169
4170         if (DISPLAY_VER(dev_priv) >= 5) {
4171                 intel_de_write(dev_priv, PIPE_DATA_M1(transcoder),
4172                                TU_SIZE(m_n->tu) | m_n->gmch_m);
4173                 intel_de_write(dev_priv, PIPE_DATA_N1(transcoder),
4174                                m_n->gmch_n);
4175                 intel_de_write(dev_priv, PIPE_LINK_M1(transcoder),
4176                                m_n->link_m);
4177                 intel_de_write(dev_priv, PIPE_LINK_N1(transcoder),
4178                                m_n->link_n);
4179                 /*
4180                  *  M2_N2 registers are set only if DRRS is supported
4181                  * (to make sure the registers are not unnecessarily accessed).
4182                  */
4183                 if (m2_n2 && crtc_state->has_drrs &&
4184                     transcoder_has_m2_n2(dev_priv, transcoder)) {
4185                         intel_de_write(dev_priv, PIPE_DATA_M2(transcoder),
4186                                        TU_SIZE(m2_n2->tu) | m2_n2->gmch_m);
4187                         intel_de_write(dev_priv, PIPE_DATA_N2(transcoder),
4188                                        m2_n2->gmch_n);
4189                         intel_de_write(dev_priv, PIPE_LINK_M2(transcoder),
4190                                        m2_n2->link_m);
4191                         intel_de_write(dev_priv, PIPE_LINK_N2(transcoder),
4192                                        m2_n2->link_n);
4193                 }
4194         } else {
4195                 intel_de_write(dev_priv, PIPE_DATA_M_G4X(pipe),
4196                                TU_SIZE(m_n->tu) | m_n->gmch_m);
4197                 intel_de_write(dev_priv, PIPE_DATA_N_G4X(pipe), m_n->gmch_n);
4198                 intel_de_write(dev_priv, PIPE_LINK_M_G4X(pipe), m_n->link_m);
4199                 intel_de_write(dev_priv, PIPE_LINK_N_G4X(pipe), m_n->link_n);
4200         }
4201 }
4202
4203 void intel_dp_set_m_n(const struct intel_crtc_state *crtc_state, enum link_m_n_set m_n)
4204 {
4205         const struct intel_link_m_n *dp_m_n, *dp_m2_n2 = NULL;
4206         struct drm_i915_private *i915 = to_i915(crtc_state->uapi.crtc->dev);
4207
4208         if (m_n == M1_N1) {
4209                 dp_m_n = &crtc_state->dp_m_n;
4210                 dp_m2_n2 = &crtc_state->dp_m2_n2;
4211         } else if (m_n == M2_N2) {
4212
4213                 /*
4214                  * M2_N2 registers are not supported. Hence m2_n2 divider value
4215                  * needs to be programmed into M1_N1.
4216                  */
4217                 dp_m_n = &crtc_state->dp_m2_n2;
4218         } else {
4219                 drm_err(&i915->drm, "Unsupported divider value\n");
4220                 return;
4221         }
4222
4223         if (crtc_state->has_pch_encoder)
4224                 intel_pch_transcoder_set_m_n(crtc_state, &crtc_state->dp_m_n);
4225         else
4226                 intel_cpu_transcoder_set_m_n(crtc_state, dp_m_n, dp_m2_n2);
4227 }
4228
4229 static void intel_set_transcoder_timings(const struct intel_crtc_state *crtc_state)
4230 {
4231         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
4232         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
4233         enum pipe pipe = crtc->pipe;
4234         enum transcoder cpu_transcoder = crtc_state->cpu_transcoder;
4235         const struct drm_display_mode *adjusted_mode = &crtc_state->hw.adjusted_mode;
4236         u32 crtc_vtotal, crtc_vblank_end;
4237         int vsyncshift = 0;
4238
4239         /* We need to be careful not to changed the adjusted mode, for otherwise
4240          * the hw state checker will get angry at the mismatch. */
4241         crtc_vtotal = adjusted_mode->crtc_vtotal;
4242         crtc_vblank_end = adjusted_mode->crtc_vblank_end;
4243
4244         if (adjusted_mode->flags & DRM_MODE_FLAG_INTERLACE) {
4245                 /* the chip adds 2 halflines automatically */
4246                 crtc_vtotal -= 1;
4247                 crtc_vblank_end -= 1;
4248
4249                 if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_SDVO))
4250                         vsyncshift = (adjusted_mode->crtc_htotal - 1) / 2;
4251                 else
4252                         vsyncshift = adjusted_mode->crtc_hsync_start -
4253                                 adjusted_mode->crtc_htotal / 2;
4254                 if (vsyncshift < 0)
4255                         vsyncshift += adjusted_mode->crtc_htotal;
4256         }
4257
4258         if (DISPLAY_VER(dev_priv) > 3)
4259                 intel_de_write(dev_priv, VSYNCSHIFT(cpu_transcoder),
4260                                vsyncshift);
4261
4262         intel_de_write(dev_priv, HTOTAL(cpu_transcoder),
4263                        (adjusted_mode->crtc_hdisplay - 1) | ((adjusted_mode->crtc_htotal - 1) << 16));
4264         intel_de_write(dev_priv, HBLANK(cpu_transcoder),
4265                        (adjusted_mode->crtc_hblank_start - 1) | ((adjusted_mode->crtc_hblank_end - 1) << 16));
4266         intel_de_write(dev_priv, HSYNC(cpu_transcoder),
4267                        (adjusted_mode->crtc_hsync_start - 1) | ((adjusted_mode->crtc_hsync_end - 1) << 16));
4268
4269         intel_de_write(dev_priv, VTOTAL(cpu_transcoder),
4270                        (adjusted_mode->crtc_vdisplay - 1) | ((crtc_vtotal - 1) << 16));
4271         intel_de_write(dev_priv, VBLANK(cpu_transcoder),
4272                        (adjusted_mode->crtc_vblank_start - 1) | ((crtc_vblank_end - 1) << 16));
4273         intel_de_write(dev_priv, VSYNC(cpu_transcoder),
4274                        (adjusted_mode->crtc_vsync_start - 1) | ((adjusted_mode->crtc_vsync_end - 1) << 16));
4275
4276         /* Workaround: when the EDP input selection is B, the VTOTAL_B must be
4277          * programmed with the VTOTAL_EDP value. Same for VTOTAL_C. This is
4278          * documented on the DDI_FUNC_CTL register description, EDP Input Select
4279          * bits. */
4280         if (IS_HASWELL(dev_priv) && cpu_transcoder == TRANSCODER_EDP &&
4281             (pipe == PIPE_B || pipe == PIPE_C))
4282                 intel_de_write(dev_priv, VTOTAL(pipe),
4283                                intel_de_read(dev_priv, VTOTAL(cpu_transcoder)));
4284
4285 }
4286
4287 static void intel_set_pipe_src_size(const struct intel_crtc_state *crtc_state)
4288 {
4289         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
4290         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
4291         enum pipe pipe = crtc->pipe;
4292
4293         /* pipesrc controls the size that is scaled from, which should
4294          * always be the user's requested size.
4295          */
4296         intel_de_write(dev_priv, PIPESRC(pipe),
4297                        ((crtc_state->pipe_src_w - 1) << 16) | (crtc_state->pipe_src_h - 1));
4298 }
4299
4300 static bool intel_pipe_is_interlaced(const struct intel_crtc_state *crtc_state)
4301 {
4302         struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
4303         enum transcoder cpu_transcoder = crtc_state->cpu_transcoder;
4304
4305         if (DISPLAY_VER(dev_priv) == 2)
4306                 return false;
4307
4308         if (DISPLAY_VER(dev_priv) >= 9 ||
4309             IS_BROADWELL(dev_priv) || IS_HASWELL(dev_priv))
4310                 return intel_de_read(dev_priv, PIPECONF(cpu_transcoder)) & PIPECONF_INTERLACE_MASK_HSW;
4311         else
4312                 return intel_de_read(dev_priv, PIPECONF(cpu_transcoder)) & PIPECONF_INTERLACE_MASK;
4313 }
4314
4315 static void intel_get_transcoder_timings(struct intel_crtc *crtc,
4316                                          struct intel_crtc_state *pipe_config)
4317 {
4318         struct drm_device *dev = crtc->base.dev;
4319         struct drm_i915_private *dev_priv = to_i915(dev);
4320         enum transcoder cpu_transcoder = pipe_config->cpu_transcoder;
4321         u32 tmp;
4322
4323         tmp = intel_de_read(dev_priv, HTOTAL(cpu_transcoder));
4324         pipe_config->hw.adjusted_mode.crtc_hdisplay = (tmp & 0xffff) + 1;
4325         pipe_config->hw.adjusted_mode.crtc_htotal = ((tmp >> 16) & 0xffff) + 1;
4326
4327         if (!transcoder_is_dsi(cpu_transcoder)) {
4328                 tmp = intel_de_read(dev_priv, HBLANK(cpu_transcoder));
4329                 pipe_config->hw.adjusted_mode.crtc_hblank_start =
4330                                                         (tmp & 0xffff) + 1;
4331                 pipe_config->hw.adjusted_mode.crtc_hblank_end =
4332                                                 ((tmp >> 16) & 0xffff) + 1;
4333         }
4334         tmp = intel_de_read(dev_priv, HSYNC(cpu_transcoder));
4335         pipe_config->hw.adjusted_mode.crtc_hsync_start = (tmp & 0xffff) + 1;
4336         pipe_config->hw.adjusted_mode.crtc_hsync_end = ((tmp >> 16) & 0xffff) + 1;
4337
4338         tmp = intel_de_read(dev_priv, VTOTAL(cpu_transcoder));
4339         pipe_config->hw.adjusted_mode.crtc_vdisplay = (tmp & 0xffff) + 1;
4340         pipe_config->hw.adjusted_mode.crtc_vtotal = ((tmp >> 16) & 0xffff) + 1;
4341
4342         if (!transcoder_is_dsi(cpu_transcoder)) {
4343                 tmp = intel_de_read(dev_priv, VBLANK(cpu_transcoder));
4344                 pipe_config->hw.adjusted_mode.crtc_vblank_start =
4345                                                         (tmp & 0xffff) + 1;
4346                 pipe_config->hw.adjusted_mode.crtc_vblank_end =
4347                                                 ((tmp >> 16) & 0xffff) + 1;
4348         }
4349         tmp = intel_de_read(dev_priv, VSYNC(cpu_transcoder));
4350         pipe_config->hw.adjusted_mode.crtc_vsync_start = (tmp & 0xffff) + 1;
4351         pipe_config->hw.adjusted_mode.crtc_vsync_end = ((tmp >> 16) & 0xffff) + 1;
4352
4353         if (intel_pipe_is_interlaced(pipe_config)) {
4354                 pipe_config->hw.adjusted_mode.flags |= DRM_MODE_FLAG_INTERLACE;
4355                 pipe_config->hw.adjusted_mode.crtc_vtotal += 1;
4356                 pipe_config->hw.adjusted_mode.crtc_vblank_end += 1;
4357         }
4358 }
4359
4360 static void intel_get_pipe_src_size(struct intel_crtc *crtc,
4361                                     struct intel_crtc_state *pipe_config)
4362 {
4363         struct drm_device *dev = crtc->base.dev;
4364         struct drm_i915_private *dev_priv = to_i915(dev);
4365         u32 tmp;
4366
4367         tmp = intel_de_read(dev_priv, PIPESRC(crtc->pipe));
4368         pipe_config->pipe_src_h = (tmp & 0xffff) + 1;
4369         pipe_config->pipe_src_w = ((tmp >> 16) & 0xffff) + 1;
4370 }
4371
4372 static void i9xx_set_pipeconf(const struct intel_crtc_state *crtc_state)
4373 {
4374         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
4375         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
4376         u32 pipeconf;
4377
4378         pipeconf = 0;
4379
4380         /* we keep both pipes enabled on 830 */
4381         if (IS_I830(dev_priv))
4382                 pipeconf |= intel_de_read(dev_priv, PIPECONF(crtc->pipe)) & PIPECONF_ENABLE;
4383
4384         if (crtc_state->double_wide)
4385                 pipeconf |= PIPECONF_DOUBLE_WIDE;
4386
4387         /* only g4x and later have fancy bpc/dither controls */
4388         if (IS_G4X(dev_priv) || IS_VALLEYVIEW(dev_priv) ||
4389             IS_CHERRYVIEW(dev_priv)) {
4390                 /* Bspec claims that we can't use dithering for 30bpp pipes. */
4391                 if (crtc_state->dither && crtc_state->pipe_bpp != 30)
4392                         pipeconf |= PIPECONF_DITHER_EN |
4393                                     PIPECONF_DITHER_TYPE_SP;
4394
4395                 switch (crtc_state->pipe_bpp) {
4396                 case 18:
4397                         pipeconf |= PIPECONF_6BPC;
4398                         break;
4399                 case 24:
4400                         pipeconf |= PIPECONF_8BPC;
4401                         break;
4402                 case 30:
4403                         pipeconf |= PIPECONF_10BPC;
4404                         break;
4405                 default:
4406                         /* Case prevented by intel_choose_pipe_bpp_dither. */
4407                         BUG();
4408                 }
4409         }
4410
4411         if (crtc_state->hw.adjusted_mode.flags & DRM_MODE_FLAG_INTERLACE) {
4412                 if (DISPLAY_VER(dev_priv) < 4 ||
4413                     intel_crtc_has_type(crtc_state, INTEL_OUTPUT_SDVO))
4414                         pipeconf |= PIPECONF_INTERLACE_W_FIELD_INDICATION;
4415                 else
4416                         pipeconf |= PIPECONF_INTERLACE_W_SYNC_SHIFT;
4417         } else {
4418                 pipeconf |= PIPECONF_PROGRESSIVE;
4419         }
4420
4421         if ((IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) &&
4422              crtc_state->limited_color_range)
4423                 pipeconf |= PIPECONF_COLOR_RANGE_SELECT;
4424
4425         pipeconf |= PIPECONF_GAMMA_MODE(crtc_state->gamma_mode);
4426
4427         pipeconf |= PIPECONF_FRAME_START_DELAY(dev_priv->framestart_delay - 1);
4428
4429         intel_de_write(dev_priv, PIPECONF(crtc->pipe), pipeconf);
4430         intel_de_posting_read(dev_priv, PIPECONF(crtc->pipe));
4431 }
4432
4433 static bool i9xx_has_pfit(struct drm_i915_private *dev_priv)
4434 {
4435         if (IS_I830(dev_priv))
4436                 return false;
4437
4438         return DISPLAY_VER(dev_priv) >= 4 ||
4439                 IS_PINEVIEW(dev_priv) || IS_MOBILE(dev_priv);
4440 }
4441
4442 static void i9xx_get_pfit_config(struct intel_crtc_state *crtc_state)
4443 {
4444         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
4445         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
4446         u32 tmp;
4447
4448         if (!i9xx_has_pfit(dev_priv))
4449                 return;
4450
4451         tmp = intel_de_read(dev_priv, PFIT_CONTROL);
4452         if (!(tmp & PFIT_ENABLE))
4453                 return;
4454
4455         /* Check whether the pfit is attached to our pipe. */
4456         if (DISPLAY_VER(dev_priv) < 4) {
4457                 if (crtc->pipe != PIPE_B)
4458                         return;
4459         } else {
4460                 if ((tmp & PFIT_PIPE_MASK) != (crtc->pipe << PFIT_PIPE_SHIFT))
4461                         return;
4462         }
4463
4464         crtc_state->gmch_pfit.control = tmp;
4465         crtc_state->gmch_pfit.pgm_ratios =
4466                 intel_de_read(dev_priv, PFIT_PGM_RATIOS);
4467 }
4468
4469 static void vlv_crtc_clock_get(struct intel_crtc *crtc,
4470                                struct intel_crtc_state *pipe_config)
4471 {
4472         struct drm_device *dev = crtc->base.dev;
4473         struct drm_i915_private *dev_priv = to_i915(dev);
4474         enum pipe pipe = crtc->pipe;
4475         struct dpll clock;
4476         u32 mdiv;
4477         int refclk = 100000;
4478
4479         /* In case of DSI, DPLL will not be used */
4480         if ((pipe_config->dpll_hw_state.dpll & DPLL_VCO_ENABLE) == 0)
4481                 return;
4482
4483         vlv_dpio_get(dev_priv);
4484         mdiv = vlv_dpio_read(dev_priv, pipe, VLV_PLL_DW3(pipe));
4485         vlv_dpio_put(dev_priv);
4486
4487         clock.m1 = (mdiv >> DPIO_M1DIV_SHIFT) & 7;
4488         clock.m2 = mdiv & DPIO_M2DIV_MASK;
4489         clock.n = (mdiv >> DPIO_N_SHIFT) & 0xf;
4490         clock.p1 = (mdiv >> DPIO_P1_SHIFT) & 7;
4491         clock.p2 = (mdiv >> DPIO_P2_SHIFT) & 0x1f;
4492
4493         pipe_config->port_clock = vlv_calc_dpll_params(refclk, &clock);
4494 }
4495
4496 static void chv_crtc_clock_get(struct intel_crtc *crtc,
4497                                struct intel_crtc_state *pipe_config)
4498 {
4499         struct drm_device *dev = crtc->base.dev;
4500         struct drm_i915_private *dev_priv = to_i915(dev);
4501         enum pipe pipe = crtc->pipe;
4502         enum dpio_channel port = vlv_pipe_to_channel(pipe);
4503         struct dpll clock;
4504         u32 cmn_dw13, pll_dw0, pll_dw1, pll_dw2, pll_dw3;
4505         int refclk = 100000;
4506
4507         /* In case of DSI, DPLL will not be used */
4508         if ((pipe_config->dpll_hw_state.dpll & DPLL_VCO_ENABLE) == 0)
4509                 return;
4510
4511         vlv_dpio_get(dev_priv);
4512         cmn_dw13 = vlv_dpio_read(dev_priv, pipe, CHV_CMN_DW13(port));
4513         pll_dw0 = vlv_dpio_read(dev_priv, pipe, CHV_PLL_DW0(port));
4514         pll_dw1 = vlv_dpio_read(dev_priv, pipe, CHV_PLL_DW1(port));
4515         pll_dw2 = vlv_dpio_read(dev_priv, pipe, CHV_PLL_DW2(port));
4516         pll_dw3 = vlv_dpio_read(dev_priv, pipe, CHV_PLL_DW3(port));
4517         vlv_dpio_put(dev_priv);
4518
4519         clock.m1 = (pll_dw1 & 0x7) == DPIO_CHV_M1_DIV_BY_2 ? 2 : 0;
4520         clock.m2 = (pll_dw0 & 0xff) << 22;
4521         if (pll_dw3 & DPIO_CHV_FRAC_DIV_EN)
4522                 clock.m2 |= pll_dw2 & 0x3fffff;
4523         clock.n = (pll_dw1 >> DPIO_CHV_N_DIV_SHIFT) & 0xf;
4524         clock.p1 = (cmn_dw13 >> DPIO_CHV_P1_DIV_SHIFT) & 0x7;
4525         clock.p2 = (cmn_dw13 >> DPIO_CHV_P2_DIV_SHIFT) & 0x1f;
4526
4527         pipe_config->port_clock = chv_calc_dpll_params(refclk, &clock);
4528 }
4529
4530 static enum intel_output_format
4531 bdw_get_pipemisc_output_format(struct intel_crtc *crtc)
4532 {
4533         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
4534         u32 tmp;
4535
4536         tmp = intel_de_read(dev_priv, PIPEMISC(crtc->pipe));
4537
4538         if (tmp & PIPEMISC_YUV420_ENABLE) {
4539                 /* We support 4:2:0 in full blend mode only */
4540                 drm_WARN_ON(&dev_priv->drm,
4541                             (tmp & PIPEMISC_YUV420_MODE_FULL_BLEND) == 0);
4542
4543                 return INTEL_OUTPUT_FORMAT_YCBCR420;
4544         } else if (tmp & PIPEMISC_OUTPUT_COLORSPACE_YUV) {
4545                 return INTEL_OUTPUT_FORMAT_YCBCR444;
4546         } else {
4547                 return INTEL_OUTPUT_FORMAT_RGB;
4548         }
4549 }
4550
4551 static void i9xx_get_pipe_color_config(struct intel_crtc_state *crtc_state)
4552 {
4553         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
4554         struct intel_plane *plane = to_intel_plane(crtc->base.primary);
4555         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
4556         enum i9xx_plane_id i9xx_plane = plane->i9xx_plane;
4557         u32 tmp;
4558
4559         tmp = intel_de_read(dev_priv, DSPCNTR(i9xx_plane));
4560
4561         if (tmp & DISPPLANE_GAMMA_ENABLE)
4562                 crtc_state->gamma_enable = true;
4563
4564         if (!HAS_GMCH(dev_priv) &&
4565             tmp & DISPPLANE_PIPE_CSC_ENABLE)
4566                 crtc_state->csc_enable = true;
4567 }
4568
4569 static bool i9xx_get_pipe_config(struct intel_crtc *crtc,
4570                                  struct intel_crtc_state *pipe_config)
4571 {
4572         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
4573         enum intel_display_power_domain power_domain;
4574         intel_wakeref_t wakeref;
4575         u32 tmp;
4576         bool ret;
4577
4578         power_domain = POWER_DOMAIN_PIPE(crtc->pipe);
4579         wakeref = intel_display_power_get_if_enabled(dev_priv, power_domain);
4580         if (!wakeref)
4581                 return false;
4582
4583         pipe_config->output_format = INTEL_OUTPUT_FORMAT_RGB;
4584         pipe_config->cpu_transcoder = (enum transcoder) crtc->pipe;
4585         pipe_config->shared_dpll = NULL;
4586
4587         ret = false;
4588
4589         tmp = intel_de_read(dev_priv, PIPECONF(crtc->pipe));
4590         if (!(tmp & PIPECONF_ENABLE))
4591                 goto out;
4592
4593         if (IS_G4X(dev_priv) || IS_VALLEYVIEW(dev_priv) ||
4594             IS_CHERRYVIEW(dev_priv)) {
4595                 switch (tmp & PIPECONF_BPC_MASK) {
4596                 case PIPECONF_6BPC:
4597                         pipe_config->pipe_bpp = 18;
4598                         break;
4599                 case PIPECONF_8BPC:
4600                         pipe_config->pipe_bpp = 24;
4601                         break;
4602                 case PIPECONF_10BPC:
4603                         pipe_config->pipe_bpp = 30;
4604                         break;
4605                 default:
4606                         break;
4607                 }
4608         }
4609
4610         if ((IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) &&
4611             (tmp & PIPECONF_COLOR_RANGE_SELECT))
4612                 pipe_config->limited_color_range = true;
4613
4614         pipe_config->gamma_mode = (tmp & PIPECONF_GAMMA_MODE_MASK_I9XX) >>
4615                 PIPECONF_GAMMA_MODE_SHIFT;
4616
4617         if (IS_CHERRYVIEW(dev_priv))
4618                 pipe_config->cgm_mode = intel_de_read(dev_priv,
4619                                                       CGM_PIPE_MODE(crtc->pipe));
4620
4621         i9xx_get_pipe_color_config(pipe_config);
4622         intel_color_get_config(pipe_config);
4623
4624         if (DISPLAY_VER(dev_priv) < 4)
4625                 pipe_config->double_wide = tmp & PIPECONF_DOUBLE_WIDE;
4626
4627         intel_get_transcoder_timings(crtc, pipe_config);
4628         intel_get_pipe_src_size(crtc, pipe_config);
4629
4630         i9xx_get_pfit_config(pipe_config);
4631
4632         if (DISPLAY_VER(dev_priv) >= 4) {
4633                 /* No way to read it out on pipes B and C */
4634                 if (IS_CHERRYVIEW(dev_priv) && crtc->pipe != PIPE_A)
4635                         tmp = dev_priv->chv_dpll_md[crtc->pipe];
4636                 else
4637                         tmp = intel_de_read(dev_priv, DPLL_MD(crtc->pipe));
4638                 pipe_config->pixel_multiplier =
4639                         ((tmp & DPLL_MD_UDI_MULTIPLIER_MASK)
4640                          >> DPLL_MD_UDI_MULTIPLIER_SHIFT) + 1;
4641                 pipe_config->dpll_hw_state.dpll_md = tmp;
4642         } else if (IS_I945G(dev_priv) || IS_I945GM(dev_priv) ||
4643                    IS_G33(dev_priv) || IS_PINEVIEW(dev_priv)) {
4644                 tmp = intel_de_read(dev_priv, DPLL(crtc->pipe));
4645                 pipe_config->pixel_multiplier =
4646                         ((tmp & SDVO_MULTIPLIER_MASK)
4647                          >> SDVO_MULTIPLIER_SHIFT_HIRES) + 1;
4648         } else {
4649                 /* Note that on i915G/GM the pixel multiplier is in the sdvo
4650                  * port and will be fixed up in the encoder->get_config
4651                  * function. */
4652                 pipe_config->pixel_multiplier = 1;
4653         }
4654         pipe_config->dpll_hw_state.dpll = intel_de_read(dev_priv,
4655                                                         DPLL(crtc->pipe));
4656         if (!IS_VALLEYVIEW(dev_priv) && !IS_CHERRYVIEW(dev_priv)) {
4657                 pipe_config->dpll_hw_state.fp0 = intel_de_read(dev_priv,
4658                                                                FP0(crtc->pipe));
4659                 pipe_config->dpll_hw_state.fp1 = intel_de_read(dev_priv,
4660                                                                FP1(crtc->pipe));
4661         } else {
4662                 /* Mask out read-only status bits. */
4663                 pipe_config->dpll_hw_state.dpll &= ~(DPLL_LOCK_VLV |
4664                                                      DPLL_PORTC_READY_MASK |
4665                                                      DPLL_PORTB_READY_MASK);
4666         }
4667
4668         if (IS_CHERRYVIEW(dev_priv))
4669                 chv_crtc_clock_get(crtc, pipe_config);
4670         else if (IS_VALLEYVIEW(dev_priv))
4671                 vlv_crtc_clock_get(crtc, pipe_config);
4672         else
4673                 i9xx_crtc_clock_get(crtc, pipe_config);
4674
4675         /*
4676          * Normally the dotclock is filled in by the encoder .get_config()
4677          * but in case the pipe is enabled w/o any ports we need a sane
4678          * default.
4679          */
4680         pipe_config->hw.adjusted_mode.crtc_clock =
4681                 pipe_config->port_clock / pipe_config->pixel_multiplier;
4682
4683         ret = true;
4684
4685 out:
4686         intel_display_power_put(dev_priv, power_domain, wakeref);
4687
4688         return ret;
4689 }
4690
4691 static void ilk_init_pch_refclk(struct drm_i915_private *dev_priv)
4692 {
4693         struct intel_encoder *encoder;
4694         int i;
4695         u32 val, final;
4696         bool has_lvds = false;
4697         bool has_cpu_edp = false;
4698         bool has_panel = false;
4699         bool has_ck505 = false;
4700         bool can_ssc = false;
4701         bool using_ssc_source = false;
4702
4703         /* We need to take the global config into account */
4704         for_each_intel_encoder(&dev_priv->drm, encoder) {
4705                 switch (encoder->type) {
4706                 case INTEL_OUTPUT_LVDS:
4707                         has_panel = true;
4708                         has_lvds = true;
4709                         break;
4710                 case INTEL_OUTPUT_EDP:
4711                         has_panel = true;
4712                         if (encoder->port == PORT_A)
4713                                 has_cpu_edp = true;
4714                         break;
4715                 default:
4716                         break;
4717                 }
4718         }
4719
4720         if (HAS_PCH_IBX(dev_priv)) {
4721                 has_ck505 = dev_priv->vbt.display_clock_mode;
4722                 can_ssc = has_ck505;
4723         } else {
4724                 has_ck505 = false;
4725                 can_ssc = true;
4726         }
4727
4728         /* Check if any DPLLs are using the SSC source */
4729         for (i = 0; i < dev_priv->dpll.num_shared_dpll; i++) {
4730                 u32 temp = intel_de_read(dev_priv, PCH_DPLL(i));
4731
4732                 if (!(temp & DPLL_VCO_ENABLE))
4733                         continue;
4734
4735                 if ((temp & PLL_REF_INPUT_MASK) ==
4736                     PLLB_REF_INPUT_SPREADSPECTRUMIN) {
4737                         using_ssc_source = true;
4738                         break;
4739                 }
4740         }
4741
4742         drm_dbg_kms(&dev_priv->drm,
4743                     "has_panel %d has_lvds %d has_ck505 %d using_ssc_source %d\n",
4744                     has_panel, has_lvds, has_ck505, using_ssc_source);
4745
4746         /* Ironlake: try to setup display ref clock before DPLL
4747          * enabling. This is only under driver's control after
4748          * PCH B stepping, previous chipset stepping should be
4749          * ignoring this setting.
4750          */
4751         val = intel_de_read(dev_priv, PCH_DREF_CONTROL);
4752
4753         /* As we must carefully and slowly disable/enable each source in turn,
4754          * compute the final state we want first and check if we need to
4755          * make any changes at all.
4756          */
4757         final = val;
4758         final &= ~DREF_NONSPREAD_SOURCE_MASK;
4759         if (has_ck505)
4760                 final |= DREF_NONSPREAD_CK505_ENABLE;
4761         else
4762                 final |= DREF_NONSPREAD_SOURCE_ENABLE;
4763
4764         final &= ~DREF_SSC_SOURCE_MASK;
4765         final &= ~DREF_CPU_SOURCE_OUTPUT_MASK;
4766         final &= ~DREF_SSC1_ENABLE;
4767
4768         if (has_panel) {
4769                 final |= DREF_SSC_SOURCE_ENABLE;
4770
4771                 if (intel_panel_use_ssc(dev_priv) && can_ssc)
4772                         final |= DREF_SSC1_ENABLE;
4773
4774                 if (has_cpu_edp) {
4775                         if (intel_panel_use_ssc(dev_priv) && can_ssc)
4776                                 final |= DREF_CPU_SOURCE_OUTPUT_DOWNSPREAD;
4777                         else
4778                                 final |= DREF_CPU_SOURCE_OUTPUT_NONSPREAD;
4779                 } else
4780                         final |= DREF_CPU_SOURCE_OUTPUT_DISABLE;
4781         } else if (using_ssc_source) {
4782                 final |= DREF_SSC_SOURCE_ENABLE;
4783                 final |= DREF_SSC1_ENABLE;
4784         }
4785
4786         if (final == val)
4787                 return;
4788
4789         /* Always enable nonspread source */
4790         val &= ~DREF_NONSPREAD_SOURCE_MASK;
4791
4792         if (has_ck505)
4793                 val |= DREF_NONSPREAD_CK505_ENABLE;
4794         else
4795                 val |= DREF_NONSPREAD_SOURCE_ENABLE;
4796
4797         if (has_panel) {
4798                 val &= ~DREF_SSC_SOURCE_MASK;
4799                 val |= DREF_SSC_SOURCE_ENABLE;
4800
4801                 /* SSC must be turned on before enabling the CPU output  */
4802                 if (intel_panel_use_ssc(dev_priv) && can_ssc) {
4803                         drm_dbg_kms(&dev_priv->drm, "Using SSC on panel\n");
4804                         val |= DREF_SSC1_ENABLE;
4805                 } else
4806                         val &= ~DREF_SSC1_ENABLE;
4807
4808                 /* Get SSC going before enabling the outputs */
4809                 intel_de_write(dev_priv, PCH_DREF_CONTROL, val);
4810                 intel_de_posting_read(dev_priv, PCH_DREF_CONTROL);
4811                 udelay(200);
4812
4813                 val &= ~DREF_CPU_SOURCE_OUTPUT_MASK;
4814
4815                 /* Enable CPU source on CPU attached eDP */
4816                 if (has_cpu_edp) {
4817                         if (intel_panel_use_ssc(dev_priv) && can_ssc) {
4818                                 drm_dbg_kms(&dev_priv->drm,
4819                                             "Using SSC on eDP\n");
4820                                 val |= DREF_CPU_SOURCE_OUTPUT_DOWNSPREAD;
4821                         } else
4822                                 val |= DREF_CPU_SOURCE_OUTPUT_NONSPREAD;
4823                 } else
4824                         val |= DREF_CPU_SOURCE_OUTPUT_DISABLE;
4825
4826                 intel_de_write(dev_priv, PCH_DREF_CONTROL, val);
4827                 intel_de_posting_read(dev_priv, PCH_DREF_CONTROL);
4828                 udelay(200);
4829         } else {
4830                 drm_dbg_kms(&dev_priv->drm, "Disabling CPU source output\n");
4831
4832                 val &= ~DREF_CPU_SOURCE_OUTPUT_MASK;
4833
4834                 /* Turn off CPU output */
4835                 val |= DREF_CPU_SOURCE_OUTPUT_DISABLE;
4836
4837                 intel_de_write(dev_priv, PCH_DREF_CONTROL, val);
4838                 intel_de_posting_read(dev_priv, PCH_DREF_CONTROL);
4839                 udelay(200);
4840
4841                 if (!using_ssc_source) {
4842                         drm_dbg_kms(&dev_priv->drm, "Disabling SSC source\n");
4843
4844                         /* Turn off the SSC source */
4845                         val &= ~DREF_SSC_SOURCE_MASK;
4846                         val |= DREF_SSC_SOURCE_DISABLE;
4847
4848                         /* Turn off SSC1 */
4849                         val &= ~DREF_SSC1_ENABLE;
4850
4851                         intel_de_write(dev_priv, PCH_DREF_CONTROL, val);
4852                         intel_de_posting_read(dev_priv, PCH_DREF_CONTROL);
4853                         udelay(200);
4854                 }
4855         }
4856
4857         BUG_ON(val != final);
4858 }
4859
4860 /* Implements 3 different sequences from BSpec chapter "Display iCLK
4861  * Programming" based on the parameters passed:
4862  * - Sequence to enable CLKOUT_DP
4863  * - Sequence to enable CLKOUT_DP without spread
4864  * - Sequence to enable CLKOUT_DP for FDI usage and configure PCH FDI I/O
4865  */
4866 static void lpt_enable_clkout_dp(struct drm_i915_private *dev_priv,
4867                                  bool with_spread, bool with_fdi)
4868 {
4869         u32 reg, tmp;
4870
4871         if (drm_WARN(&dev_priv->drm, with_fdi && !with_spread,
4872                      "FDI requires downspread\n"))
4873                 with_spread = true;
4874         if (drm_WARN(&dev_priv->drm, HAS_PCH_LPT_LP(dev_priv) &&
4875                      with_fdi, "LP PCH doesn't have FDI\n"))
4876                 with_fdi = false;
4877
4878         mutex_lock(&dev_priv->sb_lock);
4879
4880         tmp = intel_sbi_read(dev_priv, SBI_SSCCTL, SBI_ICLK);
4881         tmp &= ~SBI_SSCCTL_DISABLE;
4882         tmp |= SBI_SSCCTL_PATHALT;
4883         intel_sbi_write(dev_priv, SBI_SSCCTL, tmp, SBI_ICLK);
4884
4885         udelay(24);
4886
4887         if (with_spread) {
4888                 tmp = intel_sbi_read(dev_priv, SBI_SSCCTL, SBI_ICLK);
4889                 tmp &= ~SBI_SSCCTL_PATHALT;
4890                 intel_sbi_write(dev_priv, SBI_SSCCTL, tmp, SBI_ICLK);
4891
4892                 if (with_fdi)
4893                         lpt_fdi_program_mphy(dev_priv);
4894         }
4895
4896         reg = HAS_PCH_LPT_LP(dev_priv) ? SBI_GEN0 : SBI_DBUFF0;
4897         tmp = intel_sbi_read(dev_priv, reg, SBI_ICLK);
4898         tmp |= SBI_GEN0_CFG_BUFFENABLE_DISABLE;
4899         intel_sbi_write(dev_priv, reg, tmp, SBI_ICLK);
4900
4901         mutex_unlock(&dev_priv->sb_lock);
4902 }
4903
4904 /* Sequence to disable CLKOUT_DP */
4905 void lpt_disable_clkout_dp(struct drm_i915_private *dev_priv)
4906 {
4907         u32 reg, tmp;
4908
4909         mutex_lock(&dev_priv->sb_lock);
4910
4911         reg = HAS_PCH_LPT_LP(dev_priv) ? SBI_GEN0 : SBI_DBUFF0;
4912         tmp = intel_sbi_read(dev_priv, reg, SBI_ICLK);
4913         tmp &= ~SBI_GEN0_CFG_BUFFENABLE_DISABLE;
4914         intel_sbi_write(dev_priv, reg, tmp, SBI_ICLK);
4915
4916         tmp = intel_sbi_read(dev_priv, SBI_SSCCTL, SBI_ICLK);
4917         if (!(tmp & SBI_SSCCTL_DISABLE)) {
4918                 if (!(tmp & SBI_SSCCTL_PATHALT)) {
4919                         tmp |= SBI_SSCCTL_PATHALT;
4920                         intel_sbi_write(dev_priv, SBI_SSCCTL, tmp, SBI_ICLK);
4921                         udelay(32);
4922                 }
4923                 tmp |= SBI_SSCCTL_DISABLE;
4924                 intel_sbi_write(dev_priv, SBI_SSCCTL, tmp, SBI_ICLK);
4925         }
4926
4927         mutex_unlock(&dev_priv->sb_lock);
4928 }
4929
4930 #define BEND_IDX(steps) ((50 + (steps)) / 5)
4931
4932 static const u16 sscdivintphase[] = {
4933         [BEND_IDX( 50)] = 0x3B23,
4934         [BEND_IDX( 45)] = 0x3B23,
4935         [BEND_IDX( 40)] = 0x3C23,
4936         [BEND_IDX( 35)] = 0x3C23,
4937         [BEND_IDX( 30)] = 0x3D23,
4938         [BEND_IDX( 25)] = 0x3D23,
4939         [BEND_IDX( 20)] = 0x3E23,
4940         [BEND_IDX( 15)] = 0x3E23,
4941         [BEND_IDX( 10)] = 0x3F23,
4942         [BEND_IDX(  5)] = 0x3F23,
4943         [BEND_IDX(  0)] = 0x0025,
4944         [BEND_IDX( -5)] = 0x0025,
4945         [BEND_IDX(-10)] = 0x0125,
4946         [BEND_IDX(-15)] = 0x0125,
4947         [BEND_IDX(-20)] = 0x0225,
4948         [BEND_IDX(-25)] = 0x0225,
4949         [BEND_IDX(-30)] = 0x0325,
4950         [BEND_IDX(-35)] = 0x0325,
4951         [BEND_IDX(-40)] = 0x0425,
4952         [BEND_IDX(-45)] = 0x0425,
4953         [BEND_IDX(-50)] = 0x0525,
4954 };
4955
4956 /*
4957  * Bend CLKOUT_DP
4958  * steps -50 to 50 inclusive, in steps of 5
4959  * < 0 slow down the clock, > 0 speed up the clock, 0 == no bend (135MHz)
4960  * change in clock period = -(steps / 10) * 5.787 ps
4961  */
4962 static void lpt_bend_clkout_dp(struct drm_i915_private *dev_priv, int steps)
4963 {
4964         u32 tmp;
4965         int idx = BEND_IDX(steps);
4966
4967         if (drm_WARN_ON(&dev_priv->drm, steps % 5 != 0))
4968                 return;
4969
4970         if (drm_WARN_ON(&dev_priv->drm, idx >= ARRAY_SIZE(sscdivintphase)))
4971                 return;
4972
4973         mutex_lock(&dev_priv->sb_lock);
4974
4975         if (steps % 10 != 0)
4976                 tmp = 0xAAAAAAAB;
4977         else
4978                 tmp = 0x00000000;
4979         intel_sbi_write(dev_priv, SBI_SSCDITHPHASE, tmp, SBI_ICLK);
4980
4981         tmp = intel_sbi_read(dev_priv, SBI_SSCDIVINTPHASE, SBI_ICLK);
4982         tmp &= 0xffff0000;
4983         tmp |= sscdivintphase[idx];
4984         intel_sbi_write(dev_priv, SBI_SSCDIVINTPHASE, tmp, SBI_ICLK);
4985
4986         mutex_unlock(&dev_priv->sb_lock);
4987 }
4988
4989 #undef BEND_IDX
4990
4991 static bool spll_uses_pch_ssc(struct drm_i915_private *dev_priv)
4992 {
4993         u32 fuse_strap = intel_de_read(dev_priv, FUSE_STRAP);
4994         u32 ctl = intel_de_read(dev_priv, SPLL_CTL);
4995
4996         if ((ctl & SPLL_PLL_ENABLE) == 0)
4997                 return false;
4998
4999         if ((ctl & SPLL_REF_MASK) == SPLL_REF_MUXED_SSC &&
5000             (fuse_strap & HSW_CPU_SSC_ENABLE) == 0)
5001                 return true;
5002
5003         if (IS_BROADWELL(dev_priv) &&
5004             (ctl & SPLL_REF_MASK) == SPLL_REF_PCH_SSC_BDW)
5005                 return true;
5006
5007         return false;
5008 }
5009
5010 static bool wrpll_uses_pch_ssc(struct drm_i915_private *dev_priv,
5011                                enum intel_dpll_id id)
5012 {
5013         u32 fuse_strap = intel_de_read(dev_priv, FUSE_STRAP);
5014         u32 ctl = intel_de_read(dev_priv, WRPLL_CTL(id));
5015
5016         if ((ctl & WRPLL_PLL_ENABLE) == 0)
5017                 return false;
5018
5019         if ((ctl & WRPLL_REF_MASK) == WRPLL_REF_PCH_SSC)
5020                 return true;
5021
5022         if ((IS_BROADWELL(dev_priv) || IS_HSW_ULT(dev_priv)) &&
5023             (ctl & WRPLL_REF_MASK) == WRPLL_REF_MUXED_SSC_BDW &&
5024             (fuse_strap & HSW_CPU_SSC_ENABLE) == 0)
5025                 return true;
5026
5027         return false;
5028 }
5029
5030 static void lpt_init_pch_refclk(struct drm_i915_private *dev_priv)
5031 {
5032         struct intel_encoder *encoder;
5033         bool has_fdi = false;
5034
5035         for_each_intel_encoder(&dev_priv->drm, encoder) {
5036                 switch (encoder->type) {
5037                 case INTEL_OUTPUT_ANALOG:
5038                         has_fdi = true;
5039                         break;
5040                 default:
5041                         break;
5042                 }
5043         }
5044
5045         /*
5046          * The BIOS may have decided to use the PCH SSC
5047          * reference so we must not disable it until the
5048          * relevant PLLs have stopped relying on it. We'll
5049          * just leave the PCH SSC reference enabled in case
5050          * any active PLL is using it. It will get disabled
5051          * after runtime suspend if we don't have FDI.
5052          *
5053          * TODO: Move the whole reference clock handling
5054          * to the modeset sequence proper so that we can
5055          * actually enable/disable/reconfigure these things
5056          * safely. To do that we need to introduce a real
5057          * clock hierarchy. That would also allow us to do
5058          * clock bending finally.
5059          */
5060         dev_priv->pch_ssc_use = 0;
5061
5062         if (spll_uses_pch_ssc(dev_priv)) {
5063                 drm_dbg_kms(&dev_priv->drm, "SPLL using PCH SSC\n");
5064                 dev_priv->pch_ssc_use |= BIT(DPLL_ID_SPLL);
5065         }
5066
5067         if (wrpll_uses_pch_ssc(dev_priv, DPLL_ID_WRPLL1)) {
5068                 drm_dbg_kms(&dev_priv->drm, "WRPLL1 using PCH SSC\n");
5069                 dev_priv->pch_ssc_use |= BIT(DPLL_ID_WRPLL1);
5070         }
5071
5072         if (wrpll_uses_pch_ssc(dev_priv, DPLL_ID_WRPLL2)) {
5073                 drm_dbg_kms(&dev_priv->drm, "WRPLL2 using PCH SSC\n");
5074                 dev_priv->pch_ssc_use |= BIT(DPLL_ID_WRPLL2);
5075         }
5076
5077         if (dev_priv->pch_ssc_use)
5078                 return;
5079
5080         if (has_fdi) {
5081                 lpt_bend_clkout_dp(dev_priv, 0);
5082                 lpt_enable_clkout_dp(dev_priv, true, true);
5083         } else {
5084                 lpt_disable_clkout_dp(dev_priv);
5085         }
5086 }
5087
5088 /*
5089  * Initialize reference clocks when the driver loads
5090  */
5091 void intel_init_pch_refclk(struct drm_i915_private *dev_priv)
5092 {
5093         if (HAS_PCH_IBX(dev_priv) || HAS_PCH_CPT(dev_priv))
5094                 ilk_init_pch_refclk(dev_priv);
5095         else if (HAS_PCH_LPT(dev_priv))
5096                 lpt_init_pch_refclk(dev_priv);
5097 }
5098
5099 static void ilk_set_pipeconf(const struct intel_crtc_state *crtc_state)
5100 {
5101         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
5102         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
5103         enum pipe pipe = crtc->pipe;
5104         u32 val;
5105
5106         val = 0;
5107
5108         switch (crtc_state->pipe_bpp) {
5109         case 18:
5110                 val |= PIPECONF_6BPC;
5111                 break;
5112         case 24:
5113                 val |= PIPECONF_8BPC;
5114                 break;
5115         case 30:
5116                 val |= PIPECONF_10BPC;
5117                 break;
5118         case 36:
5119                 val |= PIPECONF_12BPC;
5120                 break;
5121         default:
5122                 /* Case prevented by intel_choose_pipe_bpp_dither. */
5123                 BUG();
5124         }
5125
5126         if (crtc_state->dither)
5127                 val |= (PIPECONF_DITHER_EN | PIPECONF_DITHER_TYPE_SP);
5128
5129         if (crtc_state->hw.adjusted_mode.flags & DRM_MODE_FLAG_INTERLACE)
5130                 val |= PIPECONF_INTERLACED_ILK;
5131         else
5132                 val |= PIPECONF_PROGRESSIVE;
5133
5134         /*
5135          * This would end up with an odd purple hue over
5136          * the entire display. Make sure we don't do it.
5137          */
5138         drm_WARN_ON(&dev_priv->drm, crtc_state->limited_color_range &&
5139                     crtc_state->output_format != INTEL_OUTPUT_FORMAT_RGB);
5140
5141         if (crtc_state->limited_color_range &&
5142             !intel_crtc_has_type(crtc_state, INTEL_OUTPUT_SDVO))
5143                 val |= PIPECONF_COLOR_RANGE_SELECT;
5144
5145         if (crtc_state->output_format != INTEL_OUTPUT_FORMAT_RGB)
5146                 val |= PIPECONF_OUTPUT_COLORSPACE_YUV709;
5147
5148         val |= PIPECONF_GAMMA_MODE(crtc_state->gamma_mode);
5149
5150         val |= PIPECONF_FRAME_START_DELAY(dev_priv->framestart_delay - 1);
5151
5152         intel_de_write(dev_priv, PIPECONF(pipe), val);
5153         intel_de_posting_read(dev_priv, PIPECONF(pipe));
5154 }
5155
5156 static void hsw_set_transconf(const struct intel_crtc_state *crtc_state)
5157 {
5158         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
5159         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
5160         enum transcoder cpu_transcoder = crtc_state->cpu_transcoder;
5161         u32 val = 0;
5162
5163         if (IS_HASWELL(dev_priv) && crtc_state->dither)
5164                 val |= (PIPECONF_DITHER_EN | PIPECONF_DITHER_TYPE_SP);
5165
5166         if (crtc_state->hw.adjusted_mode.flags & DRM_MODE_FLAG_INTERLACE)
5167                 val |= PIPECONF_INTERLACED_ILK;
5168         else
5169                 val |= PIPECONF_PROGRESSIVE;
5170
5171         if (IS_HASWELL(dev_priv) &&
5172             crtc_state->output_format != INTEL_OUTPUT_FORMAT_RGB)
5173                 val |= PIPECONF_OUTPUT_COLORSPACE_YUV_HSW;
5174
5175         intel_de_write(dev_priv, PIPECONF(cpu_transcoder), val);
5176         intel_de_posting_read(dev_priv, PIPECONF(cpu_transcoder));
5177 }
5178
5179 static void bdw_set_pipemisc(const struct intel_crtc_state *crtc_state)
5180 {
5181         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
5182         const struct intel_crtc_scaler_state *scaler_state =
5183                 &crtc_state->scaler_state;
5184
5185         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
5186         u32 val = 0;
5187         int i;
5188
5189         switch (crtc_state->pipe_bpp) {
5190         case 18:
5191                 val |= PIPEMISC_6_BPC;
5192                 break;
5193         case 24:
5194                 val |= PIPEMISC_8_BPC;
5195                 break;
5196         case 30:
5197                 val |= PIPEMISC_10_BPC;
5198                 break;
5199         case 36:
5200                 /* Port output 12BPC defined for ADLP+ */
5201                 if (DISPLAY_VER(dev_priv) > 12)
5202                         val |= PIPEMISC_12_BPC_ADLP;
5203                 break;
5204         default:
5205                 MISSING_CASE(crtc_state->pipe_bpp);
5206                 break;
5207         }
5208
5209         if (crtc_state->dither)
5210                 val |= PIPEMISC_DITHER_ENABLE | PIPEMISC_DITHER_TYPE_SP;
5211
5212         if (crtc_state->output_format == INTEL_OUTPUT_FORMAT_YCBCR420 ||
5213             crtc_state->output_format == INTEL_OUTPUT_FORMAT_YCBCR444)
5214                 val |= PIPEMISC_OUTPUT_COLORSPACE_YUV;
5215
5216         if (crtc_state->output_format == INTEL_OUTPUT_FORMAT_YCBCR420)
5217                 val |= PIPEMISC_YUV420_ENABLE |
5218                         PIPEMISC_YUV420_MODE_FULL_BLEND;
5219
5220         if (DISPLAY_VER(dev_priv) >= 11 && is_hdr_mode(crtc_state))
5221                 val |= PIPEMISC_HDR_MODE_PRECISION;
5222
5223         if (DISPLAY_VER(dev_priv) >= 12)
5224                 val |= PIPEMISC_PIXEL_ROUNDING_TRUNC;
5225
5226         if (IS_ALDERLAKE_P(dev_priv)) {
5227                 bool scaler_in_use = false;
5228
5229                 for (i = 0; i < crtc->num_scalers; i++) {
5230                         if (!scaler_state->scalers[i].in_use)
5231                                 continue;
5232
5233                         scaler_in_use = true;
5234                         break;
5235                 }
5236
5237                 intel_de_rmw(dev_priv, PIPE_MISC2(crtc->pipe),
5238                              PIPE_MISC2_UNDERRUN_BUBBLE_COUNTER_MASK,
5239                              scaler_in_use ? PIPE_MISC2_BUBBLE_COUNTER_SCALER_EN :
5240                              PIPE_MISC2_BUBBLE_COUNTER_SCALER_DIS);
5241         }
5242
5243         intel_de_write(dev_priv, PIPEMISC(crtc->pipe), val);
5244 }
5245
5246 int bdw_get_pipemisc_bpp(struct intel_crtc *crtc)
5247 {
5248         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
5249         u32 tmp;
5250
5251         tmp = intel_de_read(dev_priv, PIPEMISC(crtc->pipe));
5252
5253         switch (tmp & PIPEMISC_BPC_MASK) {
5254         case PIPEMISC_6_BPC:
5255                 return 18;
5256         case PIPEMISC_8_BPC:
5257                 return 24;
5258         case PIPEMISC_10_BPC:
5259                 return 30;
5260         /*
5261          * PORT OUTPUT 12 BPC defined for ADLP+.
5262          *
5263          * TODO:
5264          * For previous platforms with DSI interface, bits 5:7
5265          * are used for storing pipe_bpp irrespective of dithering.
5266          * Since the value of 12 BPC is not defined for these bits
5267          * on older platforms, need to find a workaround for 12 BPC
5268          * MIPI DSI HW readout.
5269          */
5270         case PIPEMISC_12_BPC_ADLP:
5271                 if (DISPLAY_VER(dev_priv) > 12)
5272                         return 36;
5273                 fallthrough;
5274         default:
5275                 MISSING_CASE(tmp);
5276                 return 0;
5277         }
5278 }
5279
5280 int ilk_get_lanes_required(int target_clock, int link_bw, int bpp)
5281 {
5282         /*
5283          * Account for spread spectrum to avoid
5284          * oversubscribing the link. Max center spread
5285          * is 2.5%; use 5% for safety's sake.
5286          */
5287         u32 bps = target_clock * bpp * 21 / 20;
5288         return DIV_ROUND_UP(bps, link_bw * 8);
5289 }
5290
5291 static void intel_pch_transcoder_get_m_n(struct intel_crtc *crtc,
5292                                          struct intel_link_m_n *m_n)
5293 {
5294         struct drm_device *dev = crtc->base.dev;
5295         struct drm_i915_private *dev_priv = to_i915(dev);
5296         enum pipe pipe = crtc->pipe;
5297
5298         m_n->link_m = intel_de_read(dev_priv, PCH_TRANS_LINK_M1(pipe));
5299         m_n->link_n = intel_de_read(dev_priv, PCH_TRANS_LINK_N1(pipe));
5300         m_n->gmch_m = intel_de_read(dev_priv, PCH_TRANS_DATA_M1(pipe))
5301                 & ~TU_SIZE_MASK;
5302         m_n->gmch_n = intel_de_read(dev_priv, PCH_TRANS_DATA_N1(pipe));
5303         m_n->tu = ((intel_de_read(dev_priv, PCH_TRANS_DATA_M1(pipe))
5304                     & TU_SIZE_MASK) >> TU_SIZE_SHIFT) + 1;
5305 }
5306
5307 static void intel_cpu_transcoder_get_m_n(struct intel_crtc *crtc,
5308                                          enum transcoder transcoder,
5309                                          struct intel_link_m_n *m_n,
5310                                          struct intel_link_m_n *m2_n2)
5311 {
5312         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
5313         enum pipe pipe = crtc->pipe;
5314
5315         if (DISPLAY_VER(dev_priv) >= 5) {
5316                 m_n->link_m = intel_de_read(dev_priv,
5317                                             PIPE_LINK_M1(transcoder));
5318                 m_n->link_n = intel_de_read(dev_priv,
5319                                             PIPE_LINK_N1(transcoder));
5320                 m_n->gmch_m = intel_de_read(dev_priv,
5321                                             PIPE_DATA_M1(transcoder))
5322                         & ~TU_SIZE_MASK;
5323                 m_n->gmch_n = intel_de_read(dev_priv,
5324                                             PIPE_DATA_N1(transcoder));
5325                 m_n->tu = ((intel_de_read(dev_priv, PIPE_DATA_M1(transcoder))
5326                             & TU_SIZE_MASK) >> TU_SIZE_SHIFT) + 1;
5327
5328                 if (m2_n2 && transcoder_has_m2_n2(dev_priv, transcoder)) {
5329                         m2_n2->link_m = intel_de_read(dev_priv,
5330                                                       PIPE_LINK_M2(transcoder));
5331                         m2_n2->link_n = intel_de_read(dev_priv,
5332                                                              PIPE_LINK_N2(transcoder));
5333                         m2_n2->gmch_m = intel_de_read(dev_priv,
5334                                                              PIPE_DATA_M2(transcoder))
5335                                         & ~TU_SIZE_MASK;
5336                         m2_n2->gmch_n = intel_de_read(dev_priv,
5337                                                              PIPE_DATA_N2(transcoder));
5338                         m2_n2->tu = ((intel_de_read(dev_priv, PIPE_DATA_M2(transcoder))
5339                                         & TU_SIZE_MASK) >> TU_SIZE_SHIFT) + 1;
5340                 }
5341         } else {
5342                 m_n->link_m = intel_de_read(dev_priv, PIPE_LINK_M_G4X(pipe));
5343                 m_n->link_n = intel_de_read(dev_priv, PIPE_LINK_N_G4X(pipe));
5344                 m_n->gmch_m = intel_de_read(dev_priv, PIPE_DATA_M_G4X(pipe))
5345                         & ~TU_SIZE_MASK;
5346                 m_n->gmch_n = intel_de_read(dev_priv, PIPE_DATA_N_G4X(pipe));
5347                 m_n->tu = ((intel_de_read(dev_priv, PIPE_DATA_M_G4X(pipe))
5348                             & TU_SIZE_MASK) >> TU_SIZE_SHIFT) + 1;
5349         }
5350 }
5351
5352 void intel_dp_get_m_n(struct intel_crtc *crtc,
5353                       struct intel_crtc_state *pipe_config)
5354 {
5355         if (pipe_config->has_pch_encoder)
5356                 intel_pch_transcoder_get_m_n(crtc, &pipe_config->dp_m_n);
5357         else
5358                 intel_cpu_transcoder_get_m_n(crtc, pipe_config->cpu_transcoder,
5359                                              &pipe_config->dp_m_n,
5360                                              &pipe_config->dp_m2_n2);
5361 }
5362
5363 static void ilk_get_fdi_m_n_config(struct intel_crtc *crtc,
5364                                    struct intel_crtc_state *pipe_config)
5365 {
5366         intel_cpu_transcoder_get_m_n(crtc, pipe_config->cpu_transcoder,
5367                                      &pipe_config->fdi_m_n, NULL);
5368 }
5369
5370 static void ilk_get_pfit_pos_size(struct intel_crtc_state *crtc_state,
5371                                   u32 pos, u32 size)
5372 {
5373         drm_rect_init(&crtc_state->pch_pfit.dst,
5374                       pos >> 16, pos & 0xffff,
5375                       size >> 16, size & 0xffff);
5376 }
5377
5378 static void skl_get_pfit_config(struct intel_crtc_state *crtc_state)
5379 {
5380         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
5381         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
5382         struct intel_crtc_scaler_state *scaler_state = &crtc_state->scaler_state;
5383         int id = -1;
5384         int i;
5385
5386         /* find scaler attached to this pipe */
5387         for (i = 0; i < crtc->num_scalers; i++) {
5388                 u32 ctl, pos, size;
5389
5390                 ctl = intel_de_read(dev_priv, SKL_PS_CTRL(crtc->pipe, i));
5391                 if ((ctl & (PS_SCALER_EN | PS_PLANE_SEL_MASK)) != PS_SCALER_EN)
5392                         continue;
5393
5394                 id = i;
5395                 crtc_state->pch_pfit.enabled = true;
5396
5397                 pos = intel_de_read(dev_priv, SKL_PS_WIN_POS(crtc->pipe, i));
5398                 size = intel_de_read(dev_priv, SKL_PS_WIN_SZ(crtc->pipe, i));
5399
5400                 ilk_get_pfit_pos_size(crtc_state, pos, size);
5401
5402                 scaler_state->scalers[i].in_use = true;
5403                 break;
5404         }
5405
5406         scaler_state->scaler_id = id;
5407         if (id >= 0)
5408                 scaler_state->scaler_users |= (1 << SKL_CRTC_INDEX);
5409         else
5410                 scaler_state->scaler_users &= ~(1 << SKL_CRTC_INDEX);
5411 }
5412
5413 static void ilk_get_pfit_config(struct intel_crtc_state *crtc_state)
5414 {
5415         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
5416         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
5417         u32 ctl, pos, size;
5418
5419         ctl = intel_de_read(dev_priv, PF_CTL(crtc->pipe));
5420         if ((ctl & PF_ENABLE) == 0)
5421                 return;
5422
5423         crtc_state->pch_pfit.enabled = true;
5424
5425         pos = intel_de_read(dev_priv, PF_WIN_POS(crtc->pipe));
5426         size = intel_de_read(dev_priv, PF_WIN_SZ(crtc->pipe));
5427
5428         ilk_get_pfit_pos_size(crtc_state, pos, size);
5429
5430         /*
5431          * We currently do not free assignements of panel fitters on
5432          * ivb/hsw (since we don't use the higher upscaling modes which
5433          * differentiates them) so just WARN about this case for now.
5434          */
5435         drm_WARN_ON(&dev_priv->drm, DISPLAY_VER(dev_priv) == 7 &&
5436                     (ctl & PF_PIPE_SEL_MASK_IVB) != PF_PIPE_SEL_IVB(crtc->pipe));
5437 }
5438
5439 static bool ilk_get_pipe_config(struct intel_crtc *crtc,
5440                                 struct intel_crtc_state *pipe_config)
5441 {
5442         struct drm_device *dev = crtc->base.dev;
5443         struct drm_i915_private *dev_priv = to_i915(dev);
5444         enum intel_display_power_domain power_domain;
5445         intel_wakeref_t wakeref;
5446         u32 tmp;
5447         bool ret;
5448
5449         power_domain = POWER_DOMAIN_PIPE(crtc->pipe);
5450         wakeref = intel_display_power_get_if_enabled(dev_priv, power_domain);
5451         if (!wakeref)
5452                 return false;
5453
5454         pipe_config->cpu_transcoder = (enum transcoder) crtc->pipe;
5455         pipe_config->shared_dpll = NULL;
5456
5457         ret = false;
5458         tmp = intel_de_read(dev_priv, PIPECONF(crtc->pipe));
5459         if (!(tmp & PIPECONF_ENABLE))
5460                 goto out;
5461
5462         switch (tmp & PIPECONF_BPC_MASK) {
5463         case PIPECONF_6BPC:
5464                 pipe_config->pipe_bpp = 18;
5465                 break;
5466         case PIPECONF_8BPC:
5467                 pipe_config->pipe_bpp = 24;
5468                 break;
5469         case PIPECONF_10BPC:
5470                 pipe_config->pipe_bpp = 30;
5471                 break;
5472         case PIPECONF_12BPC:
5473                 pipe_config->pipe_bpp = 36;
5474                 break;
5475         default:
5476                 break;
5477         }
5478
5479         if (tmp & PIPECONF_COLOR_RANGE_SELECT)
5480                 pipe_config->limited_color_range = true;
5481
5482         switch (tmp & PIPECONF_OUTPUT_COLORSPACE_MASK) {
5483         case PIPECONF_OUTPUT_COLORSPACE_YUV601:
5484         case PIPECONF_OUTPUT_COLORSPACE_YUV709:
5485                 pipe_config->output_format = INTEL_OUTPUT_FORMAT_YCBCR444;
5486                 break;
5487         default:
5488                 pipe_config->output_format = INTEL_OUTPUT_FORMAT_RGB;
5489                 break;
5490         }
5491
5492         pipe_config->gamma_mode = (tmp & PIPECONF_GAMMA_MODE_MASK_ILK) >>
5493                 PIPECONF_GAMMA_MODE_SHIFT;
5494
5495         pipe_config->csc_mode = intel_de_read(dev_priv,
5496                                               PIPE_CSC_MODE(crtc->pipe));
5497
5498         i9xx_get_pipe_color_config(pipe_config);
5499         intel_color_get_config(pipe_config);
5500
5501         if (intel_de_read(dev_priv, PCH_TRANSCONF(crtc->pipe)) & TRANS_ENABLE) {
5502                 struct intel_shared_dpll *pll;
5503                 enum intel_dpll_id pll_id;
5504                 bool pll_active;
5505
5506                 pipe_config->has_pch_encoder = true;
5507
5508                 tmp = intel_de_read(dev_priv, FDI_RX_CTL(crtc->pipe));
5509                 pipe_config->fdi_lanes = ((FDI_DP_PORT_WIDTH_MASK & tmp) >>
5510                                           FDI_DP_PORT_WIDTH_SHIFT) + 1;
5511
5512                 ilk_get_fdi_m_n_config(crtc, pipe_config);
5513
5514                 if (HAS_PCH_IBX(dev_priv)) {
5515                         /*
5516                          * The pipe->pch transcoder and pch transcoder->pll
5517                          * mapping is fixed.
5518                          */
5519                         pll_id = (enum intel_dpll_id) crtc->pipe;
5520                 } else {
5521                         tmp = intel_de_read(dev_priv, PCH_DPLL_SEL);
5522                         if (tmp & TRANS_DPLLB_SEL(crtc->pipe))
5523                                 pll_id = DPLL_ID_PCH_PLL_B;
5524                         else
5525                                 pll_id= DPLL_ID_PCH_PLL_A;
5526                 }
5527
5528                 pipe_config->shared_dpll =
5529                         intel_get_shared_dpll_by_id(dev_priv, pll_id);
5530                 pll = pipe_config->shared_dpll;
5531
5532                 pll_active = intel_dpll_get_hw_state(dev_priv, pll,
5533                                                      &pipe_config->dpll_hw_state);
5534                 drm_WARN_ON(dev, !pll_active);
5535
5536                 tmp = pipe_config->dpll_hw_state.dpll;
5537                 pipe_config->pixel_multiplier =
5538                         ((tmp & PLL_REF_SDVO_HDMI_MULTIPLIER_MASK)
5539                          >> PLL_REF_SDVO_HDMI_MULTIPLIER_SHIFT) + 1;
5540
5541                 ilk_pch_clock_get(crtc, pipe_config);
5542         } else {
5543                 pipe_config->pixel_multiplier = 1;
5544         }
5545
5546         intel_get_transcoder_timings(crtc, pipe_config);
5547         intel_get_pipe_src_size(crtc, pipe_config);
5548
5549         ilk_get_pfit_config(pipe_config);
5550
5551         ret = true;
5552
5553 out:
5554         intel_display_power_put(dev_priv, power_domain, wakeref);
5555
5556         return ret;
5557 }
5558
5559 static bool transcoder_ddi_func_is_enabled(struct drm_i915_private *dev_priv,
5560                                            enum transcoder cpu_transcoder)
5561 {
5562         enum intel_display_power_domain power_domain;
5563         intel_wakeref_t wakeref;
5564         u32 tmp = 0;
5565
5566         power_domain = POWER_DOMAIN_TRANSCODER(cpu_transcoder);
5567
5568         with_intel_display_power_if_enabled(dev_priv, power_domain, wakeref)
5569                 tmp = intel_de_read(dev_priv, TRANS_DDI_FUNC_CTL(cpu_transcoder));
5570
5571         return tmp & TRANS_DDI_FUNC_ENABLE;
5572 }
5573
5574 static u8 hsw_panel_transcoders(struct drm_i915_private *i915)
5575 {
5576         u8 panel_transcoder_mask = BIT(TRANSCODER_EDP);
5577
5578         if (DISPLAY_VER(i915) >= 11)
5579                 panel_transcoder_mask |= BIT(TRANSCODER_DSI_0) | BIT(TRANSCODER_DSI_1);
5580
5581         return panel_transcoder_mask;
5582 }
5583
5584 static u8 hsw_enabled_transcoders(struct intel_crtc *crtc)
5585 {
5586         struct drm_device *dev = crtc->base.dev;
5587         struct drm_i915_private *dev_priv = to_i915(dev);
5588         u8 panel_transcoder_mask = hsw_panel_transcoders(dev_priv);
5589         enum transcoder cpu_transcoder;
5590         u8 enabled_transcoders = 0;
5591
5592         /*
5593          * XXX: Do intel_display_power_get_if_enabled before reading this (for
5594          * consistency and less surprising code; it's in always on power).
5595          */
5596         for_each_cpu_transcoder_masked(dev_priv, cpu_transcoder,
5597                                        panel_transcoder_mask) {
5598                 enum intel_display_power_domain power_domain;
5599                 intel_wakeref_t wakeref;
5600                 enum pipe trans_pipe;
5601                 u32 tmp = 0;
5602
5603                 power_domain = POWER_DOMAIN_TRANSCODER(cpu_transcoder);
5604                 with_intel_display_power_if_enabled(dev_priv, power_domain, wakeref)
5605                         tmp = intel_de_read(dev_priv, TRANS_DDI_FUNC_CTL(cpu_transcoder));
5606
5607                 if (!(tmp & TRANS_DDI_FUNC_ENABLE))
5608                         continue;
5609
5610                 switch (tmp & TRANS_DDI_EDP_INPUT_MASK) {
5611                 default:
5612                         drm_WARN(dev, 1,
5613                                  "unknown pipe linked to transcoder %s\n",
5614                                  transcoder_name(cpu_transcoder));
5615                         fallthrough;
5616                 case TRANS_DDI_EDP_INPUT_A_ONOFF:
5617                 case TRANS_DDI_EDP_INPUT_A_ON:
5618                         trans_pipe = PIPE_A;
5619                         break;
5620                 case TRANS_DDI_EDP_INPUT_B_ONOFF:
5621                         trans_pipe = PIPE_B;
5622                         break;
5623                 case TRANS_DDI_EDP_INPUT_C_ONOFF:
5624                         trans_pipe = PIPE_C;
5625                         break;
5626                 case TRANS_DDI_EDP_INPUT_D_ONOFF:
5627                         trans_pipe = PIPE_D;
5628                         break;
5629                 }
5630
5631                 if (trans_pipe == crtc->pipe)
5632                         enabled_transcoders |= BIT(cpu_transcoder);
5633         }
5634
5635         cpu_transcoder = (enum transcoder) crtc->pipe;
5636         if (transcoder_ddi_func_is_enabled(dev_priv, cpu_transcoder))
5637                 enabled_transcoders |= BIT(cpu_transcoder);
5638
5639         return enabled_transcoders;
5640 }
5641
5642 static bool has_edp_transcoders(u8 enabled_transcoders)
5643 {
5644         return enabled_transcoders & BIT(TRANSCODER_EDP);
5645 }
5646
5647 static bool has_dsi_transcoders(u8 enabled_transcoders)
5648 {
5649         return enabled_transcoders & (BIT(TRANSCODER_DSI_0) |
5650                                       BIT(TRANSCODER_DSI_1));
5651 }
5652
5653 static bool has_pipe_transcoders(u8 enabled_transcoders)
5654 {
5655         return enabled_transcoders & ~(BIT(TRANSCODER_EDP) |
5656                                        BIT(TRANSCODER_DSI_0) |
5657                                        BIT(TRANSCODER_DSI_1));
5658 }
5659
5660 static void assert_enabled_transcoders(struct drm_i915_private *i915,
5661                                        u8 enabled_transcoders)
5662 {
5663         /* Only one type of transcoder please */
5664         drm_WARN_ON(&i915->drm,
5665                     has_edp_transcoders(enabled_transcoders) +
5666                     has_dsi_transcoders(enabled_transcoders) +
5667                     has_pipe_transcoders(enabled_transcoders) > 1);
5668
5669         /* Only DSI transcoders can be ganged */
5670         drm_WARN_ON(&i915->drm,
5671                     !has_dsi_transcoders(enabled_transcoders) &&
5672                     !is_power_of_2(enabled_transcoders));
5673 }
5674
5675 static bool hsw_get_transcoder_state(struct intel_crtc *crtc,
5676                                      struct intel_crtc_state *pipe_config,
5677                                      struct intel_display_power_domain_set *power_domain_set)
5678 {
5679         struct drm_device *dev = crtc->base.dev;
5680         struct drm_i915_private *dev_priv = to_i915(dev);
5681         unsigned long enabled_transcoders;
5682         u32 tmp;
5683
5684         enabled_transcoders = hsw_enabled_transcoders(crtc);
5685         if (!enabled_transcoders)
5686                 return false;
5687
5688         assert_enabled_transcoders(dev_priv, enabled_transcoders);
5689
5690         /*
5691          * With the exception of DSI we should only ever have
5692          * a single enabled transcoder. With DSI let's just
5693          * pick the first one.
5694          */
5695         pipe_config->cpu_transcoder = ffs(enabled_transcoders) - 1;
5696
5697         if (!intel_display_power_get_in_set_if_enabled(dev_priv, power_domain_set,
5698                                                        POWER_DOMAIN_TRANSCODER(pipe_config->cpu_transcoder)))
5699                 return false;
5700
5701         if (hsw_panel_transcoders(dev_priv) & BIT(pipe_config->cpu_transcoder)) {
5702                 tmp = intel_de_read(dev_priv, TRANS_DDI_FUNC_CTL(pipe_config->cpu_transcoder));
5703
5704                 if ((tmp & TRANS_DDI_EDP_INPUT_MASK) == TRANS_DDI_EDP_INPUT_A_ONOFF)
5705                         pipe_config->pch_pfit.force_thru = true;
5706         }
5707
5708         tmp = intel_de_read(dev_priv, PIPECONF(pipe_config->cpu_transcoder));
5709
5710         return tmp & PIPECONF_ENABLE;
5711 }
5712
5713 static bool bxt_get_dsi_transcoder_state(struct intel_crtc *crtc,
5714                                          struct intel_crtc_state *pipe_config,
5715                                          struct intel_display_power_domain_set *power_domain_set)
5716 {
5717         struct drm_device *dev = crtc->base.dev;
5718         struct drm_i915_private *dev_priv = to_i915(dev);
5719         enum transcoder cpu_transcoder;
5720         enum port port;
5721         u32 tmp;
5722
5723         for_each_port_masked(port, BIT(PORT_A) | BIT(PORT_C)) {
5724                 if (port == PORT_A)
5725                         cpu_transcoder = TRANSCODER_DSI_A;
5726                 else
5727                         cpu_transcoder = TRANSCODER_DSI_C;
5728
5729                 if (!intel_display_power_get_in_set_if_enabled(dev_priv, power_domain_set,
5730                                                                POWER_DOMAIN_TRANSCODER(cpu_transcoder)))
5731                         continue;
5732
5733                 /*
5734                  * The PLL needs to be enabled with a valid divider
5735                  * configuration, otherwise accessing DSI registers will hang
5736                  * the machine. See BSpec North Display Engine
5737                  * registers/MIPI[BXT]. We can break out here early, since we
5738                  * need the same DSI PLL to be enabled for both DSI ports.
5739                  */
5740                 if (!bxt_dsi_pll_is_enabled(dev_priv))
5741                         break;
5742
5743                 /* XXX: this works for video mode only */
5744                 tmp = intel_de_read(dev_priv, BXT_MIPI_PORT_CTRL(port));
5745                 if (!(tmp & DPI_ENABLE))
5746                         continue;
5747
5748                 tmp = intel_de_read(dev_priv, MIPI_CTRL(port));
5749                 if ((tmp & BXT_PIPE_SELECT_MASK) != BXT_PIPE_SELECT(crtc->pipe))
5750                         continue;
5751
5752                 pipe_config->cpu_transcoder = cpu_transcoder;
5753                 break;
5754         }
5755
5756         return transcoder_is_dsi(pipe_config->cpu_transcoder);
5757 }
5758
5759 static void hsw_get_ddi_port_state(struct intel_crtc *crtc,
5760                                    struct intel_crtc_state *pipe_config)
5761 {
5762         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
5763         enum transcoder cpu_transcoder = pipe_config->cpu_transcoder;
5764         enum port port;
5765         u32 tmp;
5766
5767         if (transcoder_is_dsi(cpu_transcoder)) {
5768                 port = (cpu_transcoder == TRANSCODER_DSI_A) ?
5769                                                 PORT_A : PORT_B;
5770         } else {
5771                 tmp = intel_de_read(dev_priv,
5772                                     TRANS_DDI_FUNC_CTL(cpu_transcoder));
5773                 if (!(tmp & TRANS_DDI_FUNC_ENABLE))
5774                         return;
5775                 if (DISPLAY_VER(dev_priv) >= 12)
5776                         port = TGL_TRANS_DDI_FUNC_CTL_VAL_TO_PORT(tmp);
5777                 else
5778                         port = TRANS_DDI_FUNC_CTL_VAL_TO_PORT(tmp);
5779         }
5780
5781         /*
5782          * Haswell has only FDI/PCH transcoder A. It is which is connected to
5783          * DDI E. So just check whether this pipe is wired to DDI E and whether
5784          * the PCH transcoder is on.
5785          */
5786         if (DISPLAY_VER(dev_priv) < 9 &&
5787             (port == PORT_E) && intel_de_read(dev_priv, LPT_TRANSCONF) & TRANS_ENABLE) {
5788                 pipe_config->has_pch_encoder = true;
5789
5790                 tmp = intel_de_read(dev_priv, FDI_RX_CTL(PIPE_A));
5791                 pipe_config->fdi_lanes = ((FDI_DP_PORT_WIDTH_MASK & tmp) >>
5792                                           FDI_DP_PORT_WIDTH_SHIFT) + 1;
5793
5794                 ilk_get_fdi_m_n_config(crtc, pipe_config);
5795         }
5796 }
5797
5798 static bool hsw_get_pipe_config(struct intel_crtc *crtc,
5799                                 struct intel_crtc_state *pipe_config)
5800 {
5801         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
5802         struct intel_display_power_domain_set power_domain_set = { };
5803         bool active;
5804         u32 tmp;
5805
5806         if (!intel_display_power_get_in_set_if_enabled(dev_priv, &power_domain_set,
5807                                                        POWER_DOMAIN_PIPE(crtc->pipe)))
5808                 return false;
5809
5810         pipe_config->shared_dpll = NULL;
5811
5812         active = hsw_get_transcoder_state(crtc, pipe_config, &power_domain_set);
5813
5814         if ((IS_GEMINILAKE(dev_priv) || IS_BROXTON(dev_priv)) &&
5815             bxt_get_dsi_transcoder_state(crtc, pipe_config, &power_domain_set)) {
5816                 drm_WARN_ON(&dev_priv->drm, active);
5817                 active = true;
5818         }
5819
5820         intel_dsc_get_config(pipe_config);
5821         if (DISPLAY_VER(dev_priv) >= 13 && !pipe_config->dsc.compression_enable)
5822                 intel_uncompressed_joiner_get_config(pipe_config);
5823
5824         if (!active) {
5825                 /* bigjoiner slave doesn't enable transcoder */
5826                 if (!pipe_config->bigjoiner_slave)
5827                         goto out;
5828
5829                 active = true;
5830                 pipe_config->pixel_multiplier = 1;
5831
5832                 /* we cannot read out most state, so don't bother.. */
5833                 pipe_config->quirks |= PIPE_CONFIG_QUIRK_BIGJOINER_SLAVE;
5834         } else if (!transcoder_is_dsi(pipe_config->cpu_transcoder) ||
5835             DISPLAY_VER(dev_priv) >= 11) {
5836                 hsw_get_ddi_port_state(crtc, pipe_config);
5837                 intel_get_transcoder_timings(crtc, pipe_config);
5838         }
5839
5840         if (HAS_VRR(dev_priv) && !transcoder_is_dsi(pipe_config->cpu_transcoder))
5841                 intel_vrr_get_config(crtc, pipe_config);
5842
5843         intel_get_pipe_src_size(crtc, pipe_config);
5844
5845         if (IS_HASWELL(dev_priv)) {
5846                 u32 tmp = intel_de_read(dev_priv,
5847                                         PIPECONF(pipe_config->cpu_transcoder));
5848
5849                 if (tmp & PIPECONF_OUTPUT_COLORSPACE_YUV_HSW)
5850                         pipe_config->output_format = INTEL_OUTPUT_FORMAT_YCBCR444;
5851                 else
5852                         pipe_config->output_format = INTEL_OUTPUT_FORMAT_RGB;
5853         } else {
5854                 pipe_config->output_format =
5855                         bdw_get_pipemisc_output_format(crtc);
5856         }
5857
5858         pipe_config->gamma_mode = intel_de_read(dev_priv,
5859                                                 GAMMA_MODE(crtc->pipe));
5860
5861         pipe_config->csc_mode = intel_de_read(dev_priv,
5862                                               PIPE_CSC_MODE(crtc->pipe));
5863
5864         if (DISPLAY_VER(dev_priv) >= 9) {
5865                 tmp = intel_de_read(dev_priv, SKL_BOTTOM_COLOR(crtc->pipe));
5866
5867                 if (tmp & SKL_BOTTOM_COLOR_GAMMA_ENABLE)
5868                         pipe_config->gamma_enable = true;
5869
5870                 if (tmp & SKL_BOTTOM_COLOR_CSC_ENABLE)
5871                         pipe_config->csc_enable = true;
5872         } else {
5873                 i9xx_get_pipe_color_config(pipe_config);
5874         }
5875
5876         intel_color_get_config(pipe_config);
5877
5878         tmp = intel_de_read(dev_priv, WM_LINETIME(crtc->pipe));
5879         pipe_config->linetime = REG_FIELD_GET(HSW_LINETIME_MASK, tmp);
5880         if (IS_BROADWELL(dev_priv) || IS_HASWELL(dev_priv))
5881                 pipe_config->ips_linetime =
5882                         REG_FIELD_GET(HSW_IPS_LINETIME_MASK, tmp);
5883
5884         if (intel_display_power_get_in_set_if_enabled(dev_priv, &power_domain_set,
5885                                                       POWER_DOMAIN_PIPE_PANEL_FITTER(crtc->pipe))) {
5886                 if (DISPLAY_VER(dev_priv) >= 9)
5887                         skl_get_pfit_config(pipe_config);
5888                 else
5889                         ilk_get_pfit_config(pipe_config);
5890         }
5891
5892         if (hsw_crtc_supports_ips(crtc)) {
5893                 if (IS_HASWELL(dev_priv))
5894                         pipe_config->ips_enabled = intel_de_read(dev_priv,
5895                                                                  IPS_CTL) & IPS_ENABLE;
5896                 else {
5897                         /*
5898                          * We cannot readout IPS state on broadwell, set to
5899                          * true so we can set it to a defined state on first
5900                          * commit.
5901                          */
5902                         pipe_config->ips_enabled = true;
5903                 }
5904         }
5905
5906         if (pipe_config->bigjoiner_slave) {
5907                 /* Cannot be read out as a slave, set to 0. */
5908                 pipe_config->pixel_multiplier = 0;
5909         } else if (pipe_config->cpu_transcoder != TRANSCODER_EDP &&
5910             !transcoder_is_dsi(pipe_config->cpu_transcoder)) {
5911                 pipe_config->pixel_multiplier =
5912                         intel_de_read(dev_priv,
5913                                       PIPE_MULT(pipe_config->cpu_transcoder)) + 1;
5914         } else {
5915                 pipe_config->pixel_multiplier = 1;
5916         }
5917
5918 out:
5919         intel_display_power_put_all_in_set(dev_priv, &power_domain_set);
5920
5921         return active;
5922 }
5923
5924 static bool intel_crtc_get_pipe_config(struct intel_crtc_state *crtc_state)
5925 {
5926         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
5927         struct drm_i915_private *i915 = to_i915(crtc->base.dev);
5928
5929         if (!i915->display->get_pipe_config(crtc, crtc_state))
5930                 return false;
5931
5932         crtc_state->hw.active = true;
5933
5934         intel_crtc_readout_derived_state(crtc_state);
5935
5936         return true;
5937 }
5938
5939 /* VESA 640x480x72Hz mode to set on the pipe */
5940 static const struct drm_display_mode load_detect_mode = {
5941         DRM_MODE("640x480", DRM_MODE_TYPE_DEFAULT, 31500, 640, 664,
5942                  704, 832, 0, 480, 489, 491, 520, 0, DRM_MODE_FLAG_NHSYNC | DRM_MODE_FLAG_NVSYNC),
5943 };
5944
5945 static int intel_modeset_disable_planes(struct drm_atomic_state *state,
5946                                         struct drm_crtc *crtc)
5947 {
5948         struct drm_plane *plane;
5949         struct drm_plane_state *plane_state;
5950         int ret, i;
5951
5952         ret = drm_atomic_add_affected_planes(state, crtc);
5953         if (ret)
5954                 return ret;
5955
5956         for_each_new_plane_in_state(state, plane, plane_state, i) {
5957                 if (plane_state->crtc != crtc)
5958                         continue;
5959
5960                 ret = drm_atomic_set_crtc_for_plane(plane_state, NULL);
5961                 if (ret)
5962                         return ret;
5963
5964                 drm_atomic_set_fb_for_plane(plane_state, NULL);
5965         }
5966
5967         return 0;
5968 }
5969
5970 int intel_get_load_detect_pipe(struct drm_connector *connector,
5971                                struct intel_load_detect_pipe *old,
5972                                struct drm_modeset_acquire_ctx *ctx)
5973 {
5974         struct intel_encoder *encoder =
5975                 intel_attached_encoder(to_intel_connector(connector));
5976         struct intel_crtc *possible_crtc;
5977         struct intel_crtc *crtc = NULL;
5978         struct drm_device *dev = encoder->base.dev;
5979         struct drm_i915_private *dev_priv = to_i915(dev);
5980         struct drm_mode_config *config = &dev->mode_config;
5981         struct drm_atomic_state *state = NULL, *restore_state = NULL;
5982         struct drm_connector_state *connector_state;
5983         struct intel_crtc_state *crtc_state;
5984         int ret;
5985
5986         drm_dbg_kms(&dev_priv->drm, "[CONNECTOR:%d:%s], [ENCODER:%d:%s]\n",
5987                     connector->base.id, connector->name,
5988                     encoder->base.base.id, encoder->base.name);
5989
5990         old->restore_state = NULL;
5991
5992         drm_WARN_ON(dev, !drm_modeset_is_locked(&config->connection_mutex));
5993
5994         /*
5995          * Algorithm gets a little messy:
5996          *
5997          *   - if the connector already has an assigned crtc, use it (but make
5998          *     sure it's on first)
5999          *
6000          *   - try to find the first unused crtc that can drive this connector,
6001          *     and use that if we find one
6002          */
6003
6004         /* See if we already have a CRTC for this connector */
6005         if (connector->state->crtc) {
6006                 crtc = to_intel_crtc(connector->state->crtc);
6007
6008                 ret = drm_modeset_lock(&crtc->base.mutex, ctx);
6009                 if (ret)
6010                         goto fail;
6011
6012                 /* Make sure the crtc and connector are running */
6013                 goto found;
6014         }
6015
6016         /* Find an unused one (if possible) */
6017         for_each_intel_crtc(dev, possible_crtc) {
6018                 if (!(encoder->base.possible_crtcs &
6019                       drm_crtc_mask(&possible_crtc->base)))
6020                         continue;
6021
6022                 ret = drm_modeset_lock(&possible_crtc->base.mutex, ctx);
6023                 if (ret)
6024                         goto fail;
6025
6026                 if (possible_crtc->base.state->enable) {
6027                         drm_modeset_unlock(&possible_crtc->base.mutex);
6028                         continue;
6029                 }
6030
6031                 crtc = possible_crtc;
6032                 break;
6033         }
6034
6035         /*
6036          * If we didn't find an unused CRTC, don't use any.
6037          */
6038         if (!crtc) {
6039                 drm_dbg_kms(&dev_priv->drm,
6040                             "no pipe available for load-detect\n");
6041                 ret = -ENODEV;
6042                 goto fail;
6043         }
6044
6045 found:
6046         state = drm_atomic_state_alloc(dev);
6047         restore_state = drm_atomic_state_alloc(dev);
6048         if (!state || !restore_state) {
6049                 ret = -ENOMEM;
6050                 goto fail;
6051         }
6052
6053         state->acquire_ctx = ctx;
6054         restore_state->acquire_ctx = ctx;
6055
6056         connector_state = drm_atomic_get_connector_state(state, connector);
6057         if (IS_ERR(connector_state)) {
6058                 ret = PTR_ERR(connector_state);
6059                 goto fail;
6060         }
6061
6062         ret = drm_atomic_set_crtc_for_connector(connector_state, &crtc->base);
6063         if (ret)
6064                 goto fail;
6065
6066         crtc_state = intel_atomic_get_crtc_state(state, crtc);
6067         if (IS_ERR(crtc_state)) {
6068                 ret = PTR_ERR(crtc_state);
6069                 goto fail;
6070         }
6071
6072         crtc_state->uapi.active = true;
6073
6074         ret = drm_atomic_set_mode_for_crtc(&crtc_state->uapi,
6075                                            &load_detect_mode);
6076         if (ret)
6077                 goto fail;
6078
6079         ret = intel_modeset_disable_planes(state, &crtc->base);
6080         if (ret)
6081                 goto fail;
6082
6083         ret = PTR_ERR_OR_ZERO(drm_atomic_get_connector_state(restore_state, connector));
6084         if (!ret)
6085                 ret = PTR_ERR_OR_ZERO(drm_atomic_get_crtc_state(restore_state, &crtc->base));
6086         if (!ret)
6087                 ret = drm_atomic_add_affected_planes(restore_state, &crtc->base);
6088         if (ret) {
6089                 drm_dbg_kms(&dev_priv->drm,
6090                             "Failed to create a copy of old state to restore: %i\n",
6091                             ret);
6092                 goto fail;
6093         }
6094
6095         ret = drm_atomic_commit(state);
6096         if (ret) {
6097                 drm_dbg_kms(&dev_priv->drm,
6098                             "failed to set mode on load-detect pipe\n");
6099                 goto fail;
6100         }
6101
6102         old->restore_state = restore_state;
6103         drm_atomic_state_put(state);
6104
6105         /* let the connector get through one full cycle before testing */
6106         intel_wait_for_vblank(dev_priv, crtc->pipe);
6107         return true;
6108
6109 fail:
6110         if (state) {
6111                 drm_atomic_state_put(state);
6112                 state = NULL;
6113         }
6114         if (restore_state) {
6115                 drm_atomic_state_put(restore_state);
6116                 restore_state = NULL;
6117         }
6118
6119         if (ret == -EDEADLK)
6120                 return ret;
6121
6122         return false;
6123 }
6124
6125 void intel_release_load_detect_pipe(struct drm_connector *connector,
6126                                     struct intel_load_detect_pipe *old,
6127                                     struct drm_modeset_acquire_ctx *ctx)
6128 {
6129         struct intel_encoder *intel_encoder =
6130                 intel_attached_encoder(to_intel_connector(connector));
6131         struct drm_i915_private *i915 = to_i915(intel_encoder->base.dev);
6132         struct drm_encoder *encoder = &intel_encoder->base;
6133         struct drm_atomic_state *state = old->restore_state;
6134         int ret;
6135
6136         drm_dbg_kms(&i915->drm, "[CONNECTOR:%d:%s], [ENCODER:%d:%s]\n",
6137                     connector->base.id, connector->name,
6138                     encoder->base.id, encoder->name);
6139
6140         if (!state)
6141                 return;
6142
6143         ret = drm_atomic_helper_commit_duplicated_state(state, ctx);
6144         if (ret)
6145                 drm_dbg_kms(&i915->drm,
6146                             "Couldn't release load detect pipe: %i\n", ret);
6147         drm_atomic_state_put(state);
6148 }
6149
6150 static int i9xx_pll_refclk(struct drm_device *dev,
6151                            const struct intel_crtc_state *pipe_config)
6152 {
6153         struct drm_i915_private *dev_priv = to_i915(dev);
6154         u32 dpll = pipe_config->dpll_hw_state.dpll;
6155
6156         if ((dpll & PLL_REF_INPUT_MASK) == PLLB_REF_INPUT_SPREADSPECTRUMIN)
6157                 return dev_priv->vbt.lvds_ssc_freq;
6158         else if (HAS_PCH_SPLIT(dev_priv))
6159                 return 120000;
6160         else if (DISPLAY_VER(dev_priv) != 2)
6161                 return 96000;
6162         else
6163                 return 48000;
6164 }
6165
6166 /* Returns the clock of the currently programmed mode of the given pipe. */
6167 static void i9xx_crtc_clock_get(struct intel_crtc *crtc,
6168                                 struct intel_crtc_state *pipe_config)
6169 {
6170         struct drm_device *dev = crtc->base.dev;
6171         struct drm_i915_private *dev_priv = to_i915(dev);
6172         u32 dpll = pipe_config->dpll_hw_state.dpll;
6173         u32 fp;
6174         struct dpll clock;
6175         int port_clock;
6176         int refclk = i9xx_pll_refclk(dev, pipe_config);
6177
6178         if ((dpll & DISPLAY_RATE_SELECT_FPA1) == 0)
6179                 fp = pipe_config->dpll_hw_state.fp0;
6180         else
6181                 fp = pipe_config->dpll_hw_state.fp1;
6182
6183         clock.m1 = (fp & FP_M1_DIV_MASK) >> FP_M1_DIV_SHIFT;
6184         if (IS_PINEVIEW(dev_priv)) {
6185                 clock.n = ffs((fp & FP_N_PINEVIEW_DIV_MASK) >> FP_N_DIV_SHIFT) - 1;
6186                 clock.m2 = (fp & FP_M2_PINEVIEW_DIV_MASK) >> FP_M2_DIV_SHIFT;
6187         } else {
6188                 clock.n = (fp & FP_N_DIV_MASK) >> FP_N_DIV_SHIFT;
6189                 clock.m2 = (fp & FP_M2_DIV_MASK) >> FP_M2_DIV_SHIFT;
6190         }
6191
6192         if (DISPLAY_VER(dev_priv) != 2) {
6193                 if (IS_PINEVIEW(dev_priv))
6194                         clock.p1 = ffs((dpll & DPLL_FPA01_P1_POST_DIV_MASK_PINEVIEW) >>
6195                                 DPLL_FPA01_P1_POST_DIV_SHIFT_PINEVIEW);
6196                 else
6197                         clock.p1 = ffs((dpll & DPLL_FPA01_P1_POST_DIV_MASK) >>
6198                                DPLL_FPA01_P1_POST_DIV_SHIFT);
6199
6200                 switch (dpll & DPLL_MODE_MASK) {
6201                 case DPLLB_MODE_DAC_SERIAL:
6202                         clock.p2 = dpll & DPLL_DAC_SERIAL_P2_CLOCK_DIV_5 ?
6203                                 5 : 10;
6204                         break;
6205                 case DPLLB_MODE_LVDS:
6206                         clock.p2 = dpll & DPLLB_LVDS_P2_CLOCK_DIV_7 ?
6207                                 7 : 14;
6208                         break;
6209                 default:
6210                         drm_dbg_kms(&dev_priv->drm,
6211                                     "Unknown DPLL mode %08x in programmed "
6212                                     "mode\n", (int)(dpll & DPLL_MODE_MASK));
6213                         return;
6214                 }
6215
6216                 if (IS_PINEVIEW(dev_priv))
6217                         port_clock = pnv_calc_dpll_params(refclk, &clock);
6218                 else
6219                         port_clock = i9xx_calc_dpll_params(refclk, &clock);
6220         } else {
6221                 enum pipe lvds_pipe;
6222
6223                 if (IS_I85X(dev_priv) &&
6224                     intel_lvds_port_enabled(dev_priv, LVDS, &lvds_pipe) &&
6225                     lvds_pipe == crtc->pipe) {
6226                         u32 lvds = intel_de_read(dev_priv, LVDS);
6227
6228                         clock.p1 = ffs((dpll & DPLL_FPA01_P1_POST_DIV_MASK_I830_LVDS) >>
6229                                        DPLL_FPA01_P1_POST_DIV_SHIFT);
6230
6231                         if (lvds & LVDS_CLKB_POWER_UP)
6232                                 clock.p2 = 7;
6233                         else
6234                                 clock.p2 = 14;
6235                 } else {
6236                         if (dpll & PLL_P1_DIVIDE_BY_TWO)
6237                                 clock.p1 = 2;
6238                         else {
6239                                 clock.p1 = ((dpll & DPLL_FPA01_P1_POST_DIV_MASK_I830) >>
6240                                             DPLL_FPA01_P1_POST_DIV_SHIFT) + 2;
6241                         }
6242                         if (dpll & PLL_P2_DIVIDE_BY_4)
6243                                 clock.p2 = 4;
6244                         else
6245                                 clock.p2 = 2;
6246                 }
6247
6248                 port_clock = i9xx_calc_dpll_params(refclk, &clock);
6249         }
6250
6251         /*
6252          * This value includes pixel_multiplier. We will use
6253          * port_clock to compute adjusted_mode.crtc_clock in the
6254          * encoder's get_config() function.
6255          */
6256         pipe_config->port_clock = port_clock;
6257 }
6258
6259 int intel_dotclock_calculate(int link_freq,
6260                              const struct intel_link_m_n *m_n)
6261 {
6262         /*
6263          * The calculation for the data clock is:
6264          * pixel_clock = ((m/n)*(link_clock * nr_lanes))/bpp
6265          * But we want to avoid losing precison if possible, so:
6266          * pixel_clock = ((m * link_clock * nr_lanes)/(n*bpp))
6267          *
6268          * and the link clock is simpler:
6269          * link_clock = (m * link_clock) / n
6270          */
6271
6272         if (!m_n->link_n)
6273                 return 0;
6274
6275         return div_u64(mul_u32_u32(m_n->link_m, link_freq), m_n->link_n);
6276 }
6277
6278 static void ilk_pch_clock_get(struct intel_crtc *crtc,
6279                               struct intel_crtc_state *pipe_config)
6280 {
6281         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
6282
6283         /* read out port_clock from the DPLL */
6284         i9xx_crtc_clock_get(crtc, pipe_config);
6285
6286         /*
6287          * In case there is an active pipe without active ports,
6288          * we may need some idea for the dotclock anyway.
6289          * Calculate one based on the FDI configuration.
6290          */
6291         pipe_config->hw.adjusted_mode.crtc_clock =
6292                 intel_dotclock_calculate(intel_fdi_link_freq(dev_priv, pipe_config),
6293                                          &pipe_config->fdi_m_n);
6294 }
6295
6296 /* Returns the currently programmed mode of the given encoder. */
6297 struct drm_display_mode *
6298 intel_encoder_current_mode(struct intel_encoder *encoder)
6299 {
6300         struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
6301         struct intel_crtc_state *crtc_state;
6302         struct drm_display_mode *mode;
6303         struct intel_crtc *crtc;
6304         enum pipe pipe;
6305
6306         if (!encoder->get_hw_state(encoder, &pipe))
6307                 return NULL;
6308
6309         crtc = intel_get_crtc_for_pipe(dev_priv, pipe);
6310
6311         mode = kzalloc(sizeof(*mode), GFP_KERNEL);
6312         if (!mode)
6313                 return NULL;
6314
6315         crtc_state = intel_crtc_state_alloc(crtc);
6316         if (!crtc_state) {
6317                 kfree(mode);
6318                 return NULL;
6319         }
6320
6321         if (!intel_crtc_get_pipe_config(crtc_state)) {
6322                 kfree(crtc_state);
6323                 kfree(mode);
6324                 return NULL;
6325         }
6326
6327         intel_encoder_get_config(encoder, crtc_state);
6328
6329         intel_mode_from_crtc_timings(mode, &crtc_state->hw.adjusted_mode);
6330
6331         kfree(crtc_state);
6332
6333         return mode;
6334 }
6335
6336 /**
6337  * intel_wm_need_update - Check whether watermarks need updating
6338  * @cur: current plane state
6339  * @new: new plane state
6340  *
6341  * Check current plane state versus the new one to determine whether
6342  * watermarks need to be recalculated.
6343  *
6344  * Returns true or false.
6345  */
6346 static bool intel_wm_need_update(const struct intel_plane_state *cur,
6347                                  struct intel_plane_state *new)
6348 {
6349         /* Update watermarks on tiling or size changes. */
6350         if (new->uapi.visible != cur->uapi.visible)
6351                 return true;
6352
6353         if (!cur->hw.fb || !new->hw.fb)
6354                 return false;
6355
6356         if (cur->hw.fb->modifier != new->hw.fb->modifier ||
6357             cur->hw.rotation != new->hw.rotation ||
6358             drm_rect_width(&new->uapi.src) != drm_rect_width(&cur->uapi.src) ||
6359             drm_rect_height(&new->uapi.src) != drm_rect_height(&cur->uapi.src) ||
6360             drm_rect_width(&new->uapi.dst) != drm_rect_width(&cur->uapi.dst) ||
6361             drm_rect_height(&new->uapi.dst) != drm_rect_height(&cur->uapi.dst))
6362                 return true;
6363
6364         return false;
6365 }
6366
6367 static bool needs_scaling(const struct intel_plane_state *state)
6368 {
6369         int src_w = drm_rect_width(&state->uapi.src) >> 16;
6370         int src_h = drm_rect_height(&state->uapi.src) >> 16;
6371         int dst_w = drm_rect_width(&state->uapi.dst);
6372         int dst_h = drm_rect_height(&state->uapi.dst);
6373
6374         return (src_w != dst_w || src_h != dst_h);
6375 }
6376
6377 int intel_plane_atomic_calc_changes(const struct intel_crtc_state *old_crtc_state,
6378                                     struct intel_crtc_state *new_crtc_state,
6379                                     const struct intel_plane_state *old_plane_state,
6380                                     struct intel_plane_state *new_plane_state)
6381 {
6382         struct intel_crtc *crtc = to_intel_crtc(new_crtc_state->uapi.crtc);
6383         struct intel_plane *plane = to_intel_plane(new_plane_state->uapi.plane);
6384         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
6385         bool mode_changed = intel_crtc_needs_modeset(new_crtc_state);
6386         bool was_crtc_enabled = old_crtc_state->hw.active;
6387         bool is_crtc_enabled = new_crtc_state->hw.active;
6388         bool turn_off, turn_on, visible, was_visible;
6389         int ret;
6390
6391         if (DISPLAY_VER(dev_priv) >= 9 && plane->id != PLANE_CURSOR) {
6392                 ret = skl_update_scaler_plane(new_crtc_state, new_plane_state);
6393                 if (ret)
6394                         return ret;
6395         }
6396
6397         was_visible = old_plane_state->uapi.visible;
6398         visible = new_plane_state->uapi.visible;
6399
6400         if (!was_crtc_enabled && drm_WARN_ON(&dev_priv->drm, was_visible))
6401                 was_visible = false;
6402
6403         /*
6404          * Visibility is calculated as if the crtc was on, but
6405          * after scaler setup everything depends on it being off
6406          * when the crtc isn't active.
6407          *
6408          * FIXME this is wrong for watermarks. Watermarks should also
6409          * be computed as if the pipe would be active. Perhaps move
6410          * per-plane wm computation to the .check_plane() hook, and
6411          * only combine the results from all planes in the current place?
6412          */
6413         if (!is_crtc_enabled) {
6414                 intel_plane_set_invisible(new_crtc_state, new_plane_state);
6415                 visible = false;
6416         }
6417
6418         if (!was_visible && !visible)
6419                 return 0;
6420
6421         turn_off = was_visible && (!visible || mode_changed);
6422         turn_on = visible && (!was_visible || mode_changed);
6423
6424         drm_dbg_atomic(&dev_priv->drm,
6425                        "[CRTC:%d:%s] with [PLANE:%d:%s] visible %i -> %i, off %i, on %i, ms %i\n",
6426                        crtc->base.base.id, crtc->base.name,
6427                        plane->base.base.id, plane->base.name,
6428                        was_visible, visible,
6429                        turn_off, turn_on, mode_changed);
6430
6431         if (turn_on) {
6432                 if (DISPLAY_VER(dev_priv) < 5 && !IS_G4X(dev_priv))
6433                         new_crtc_state->update_wm_pre = true;
6434
6435                 /* must disable cxsr around plane enable/disable */
6436                 if (plane->id != PLANE_CURSOR)
6437                         new_crtc_state->disable_cxsr = true;
6438         } else if (turn_off) {
6439                 if (DISPLAY_VER(dev_priv) < 5 && !IS_G4X(dev_priv))
6440                         new_crtc_state->update_wm_post = true;
6441
6442                 /* must disable cxsr around plane enable/disable */
6443                 if (plane->id != PLANE_CURSOR)
6444                         new_crtc_state->disable_cxsr = true;
6445         } else if (intel_wm_need_update(old_plane_state, new_plane_state)) {
6446                 if (DISPLAY_VER(dev_priv) < 5 && !IS_G4X(dev_priv)) {
6447                         /* FIXME bollocks */
6448                         new_crtc_state->update_wm_pre = true;
6449                         new_crtc_state->update_wm_post = true;
6450                 }
6451         }
6452
6453         if (visible || was_visible)
6454                 new_crtc_state->fb_bits |= plane->frontbuffer_bit;
6455
6456         /*
6457          * ILK/SNB DVSACNTR/Sprite Enable
6458          * IVB SPR_CTL/Sprite Enable
6459          * "When in Self Refresh Big FIFO mode, a write to enable the
6460          *  plane will be internally buffered and delayed while Big FIFO
6461          *  mode is exiting."
6462          *
6463          * Which means that enabling the sprite can take an extra frame
6464          * when we start in big FIFO mode (LP1+). Thus we need to drop
6465          * down to LP0 and wait for vblank in order to make sure the
6466          * sprite gets enabled on the next vblank after the register write.
6467          * Doing otherwise would risk enabling the sprite one frame after
6468          * we've already signalled flip completion. We can resume LP1+
6469          * once the sprite has been enabled.
6470          *
6471          *
6472          * WaCxSRDisabledForSpriteScaling:ivb
6473          * IVB SPR_SCALE/Scaling Enable
6474          * "Low Power watermarks must be disabled for at least one
6475          *  frame before enabling sprite scaling, and kept disabled
6476          *  until sprite scaling is disabled."
6477          *
6478          * ILK/SNB DVSASCALE/Scaling Enable
6479          * "When in Self Refresh Big FIFO mode, scaling enable will be
6480          *  masked off while Big FIFO mode is exiting."
6481          *
6482          * Despite the w/a only being listed for IVB we assume that
6483          * the ILK/SNB note has similar ramifications, hence we apply
6484          * the w/a on all three platforms.
6485          *
6486          * With experimental results seems this is needed also for primary
6487          * plane, not only sprite plane.
6488          */
6489         if (plane->id != PLANE_CURSOR &&
6490             (IS_IRONLAKE(dev_priv) || IS_SANDYBRIDGE(dev_priv) ||
6491              IS_IVYBRIDGE(dev_priv)) &&
6492             (turn_on || (!needs_scaling(old_plane_state) &&
6493                          needs_scaling(new_plane_state))))
6494                 new_crtc_state->disable_lp_wm = true;
6495
6496         return 0;
6497 }
6498
6499 static bool encoders_cloneable(const struct intel_encoder *a,
6500                                const struct intel_encoder *b)
6501 {
6502         /* masks could be asymmetric, so check both ways */
6503         return a == b || (a->cloneable & (1 << b->type) &&
6504                           b->cloneable & (1 << a->type));
6505 }
6506
6507 static bool check_single_encoder_cloning(struct intel_atomic_state *state,
6508                                          struct intel_crtc *crtc,
6509                                          struct intel_encoder *encoder)
6510 {
6511         struct intel_encoder *source_encoder;
6512         struct drm_connector *connector;
6513         struct drm_connector_state *connector_state;
6514         int i;
6515
6516         for_each_new_connector_in_state(&state->base, connector, connector_state, i) {
6517                 if (connector_state->crtc != &crtc->base)
6518                         continue;
6519
6520                 source_encoder =
6521                         to_intel_encoder(connector_state->best_encoder);
6522                 if (!encoders_cloneable(encoder, source_encoder))
6523                         return false;
6524         }
6525
6526         return true;
6527 }
6528
6529 static int icl_add_linked_planes(struct intel_atomic_state *state)
6530 {
6531         struct intel_plane *plane, *linked;
6532         struct intel_plane_state *plane_state, *linked_plane_state;
6533         int i;
6534
6535         for_each_new_intel_plane_in_state(state, plane, plane_state, i) {
6536                 linked = plane_state->planar_linked_plane;
6537
6538                 if (!linked)
6539                         continue;
6540
6541                 linked_plane_state = intel_atomic_get_plane_state(state, linked);
6542                 if (IS_ERR(linked_plane_state))
6543                         return PTR_ERR(linked_plane_state);
6544
6545                 drm_WARN_ON(state->base.dev,
6546                             linked_plane_state->planar_linked_plane != plane);
6547                 drm_WARN_ON(state->base.dev,
6548                             linked_plane_state->planar_slave == plane_state->planar_slave);
6549         }
6550
6551         return 0;
6552 }
6553
6554 static int icl_check_nv12_planes(struct intel_crtc_state *crtc_state)
6555 {
6556         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
6557         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
6558         struct intel_atomic_state *state = to_intel_atomic_state(crtc_state->uapi.state);
6559         struct intel_plane *plane, *linked;
6560         struct intel_plane_state *plane_state;
6561         int i;
6562
6563         if (DISPLAY_VER(dev_priv) < 11)
6564                 return 0;
6565
6566         /*
6567          * Destroy all old plane links and make the slave plane invisible
6568          * in the crtc_state->active_planes mask.
6569          */
6570         for_each_new_intel_plane_in_state(state, plane, plane_state, i) {
6571                 if (plane->pipe != crtc->pipe || !plane_state->planar_linked_plane)
6572                         continue;
6573
6574                 plane_state->planar_linked_plane = NULL;
6575                 if (plane_state->planar_slave && !plane_state->uapi.visible) {
6576                         crtc_state->enabled_planes &= ~BIT(plane->id);
6577                         crtc_state->active_planes &= ~BIT(plane->id);
6578                         crtc_state->update_planes |= BIT(plane->id);
6579                 }
6580
6581                 plane_state->planar_slave = false;
6582         }
6583
6584         if (!crtc_state->nv12_planes)
6585                 return 0;
6586
6587         for_each_new_intel_plane_in_state(state, plane, plane_state, i) {
6588                 struct intel_plane_state *linked_state = NULL;
6589
6590                 if (plane->pipe != crtc->pipe ||
6591                     !(crtc_state->nv12_planes & BIT(plane->id)))
6592                         continue;
6593
6594                 for_each_intel_plane_on_crtc(&dev_priv->drm, crtc, linked) {
6595                         if (!icl_is_nv12_y_plane(dev_priv, linked->id))
6596                                 continue;
6597
6598                         if (crtc_state->active_planes & BIT(linked->id))
6599                                 continue;
6600
6601                         linked_state = intel_atomic_get_plane_state(state, linked);
6602                         if (IS_ERR(linked_state))
6603                                 return PTR_ERR(linked_state);
6604
6605                         break;
6606                 }
6607
6608                 if (!linked_state) {
6609                         drm_dbg_kms(&dev_priv->drm,
6610                                     "Need %d free Y planes for planar YUV\n",
6611                                     hweight8(crtc_state->nv12_planes));
6612
6613                         return -EINVAL;
6614                 }
6615
6616                 plane_state->planar_linked_plane = linked;
6617
6618                 linked_state->planar_slave = true;
6619                 linked_state->planar_linked_plane = plane;
6620                 crtc_state->enabled_planes |= BIT(linked->id);
6621                 crtc_state->active_planes |= BIT(linked->id);
6622                 crtc_state->update_planes |= BIT(linked->id);
6623                 drm_dbg_kms(&dev_priv->drm, "Using %s as Y plane for %s\n",
6624                             linked->base.name, plane->base.name);
6625
6626                 /* Copy parameters to slave plane */
6627                 linked_state->ctl = plane_state->ctl | PLANE_CTL_YUV420_Y_PLANE;
6628                 linked_state->color_ctl = plane_state->color_ctl;
6629                 linked_state->view = plane_state->view;
6630
6631                 intel_plane_copy_hw_state(linked_state, plane_state);
6632                 linked_state->uapi.src = plane_state->uapi.src;
6633                 linked_state->uapi.dst = plane_state->uapi.dst;
6634
6635                 if (icl_is_hdr_plane(dev_priv, plane->id)) {
6636                         if (linked->id == PLANE_SPRITE5)
6637                                 plane_state->cus_ctl |= PLANE_CUS_PLANE_7;
6638                         else if (linked->id == PLANE_SPRITE4)
6639                                 plane_state->cus_ctl |= PLANE_CUS_PLANE_6;
6640                         else if (linked->id == PLANE_SPRITE3)
6641                                 plane_state->cus_ctl |= PLANE_CUS_PLANE_5_RKL;
6642                         else if (linked->id == PLANE_SPRITE2)
6643                                 plane_state->cus_ctl |= PLANE_CUS_PLANE_4_RKL;
6644                         else
6645                                 MISSING_CASE(linked->id);
6646                 }
6647         }
6648
6649         return 0;
6650 }
6651
6652 static bool c8_planes_changed(const struct intel_crtc_state *new_crtc_state)
6653 {
6654         struct intel_crtc *crtc = to_intel_crtc(new_crtc_state->uapi.crtc);
6655         struct intel_atomic_state *state =
6656                 to_intel_atomic_state(new_crtc_state->uapi.state);
6657         const struct intel_crtc_state *old_crtc_state =
6658                 intel_atomic_get_old_crtc_state(state, crtc);
6659
6660         return !old_crtc_state->c8_planes != !new_crtc_state->c8_planes;
6661 }
6662
6663 static u16 hsw_linetime_wm(const struct intel_crtc_state *crtc_state)
6664 {
6665         const struct drm_display_mode *pipe_mode =
6666                 &crtc_state->hw.pipe_mode;
6667         int linetime_wm;
6668
6669         if (!crtc_state->hw.enable)
6670                 return 0;
6671
6672         linetime_wm = DIV_ROUND_CLOSEST(pipe_mode->crtc_htotal * 1000 * 8,
6673                                         pipe_mode->crtc_clock);
6674
6675         return min(linetime_wm, 0x1ff);
6676 }
6677
6678 static u16 hsw_ips_linetime_wm(const struct intel_crtc_state *crtc_state,
6679                                const struct intel_cdclk_state *cdclk_state)
6680 {
6681         const struct drm_display_mode *pipe_mode =
6682                 &crtc_state->hw.pipe_mode;
6683         int linetime_wm;
6684
6685         if (!crtc_state->hw.enable)
6686                 return 0;
6687
6688         linetime_wm = DIV_ROUND_CLOSEST(pipe_mode->crtc_htotal * 1000 * 8,
6689                                         cdclk_state->logical.cdclk);
6690
6691         return min(linetime_wm, 0x1ff);
6692 }
6693
6694 static u16 skl_linetime_wm(const struct intel_crtc_state *crtc_state)
6695 {
6696         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
6697         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
6698         const struct drm_display_mode *pipe_mode =
6699                 &crtc_state->hw.pipe_mode;
6700         int linetime_wm;
6701
6702         if (!crtc_state->hw.enable)
6703                 return 0;
6704
6705         linetime_wm = DIV_ROUND_UP(pipe_mode->crtc_htotal * 1000 * 8,
6706                                    crtc_state->pixel_rate);
6707
6708         /* Display WA #1135: BXT:ALL GLK:ALL */
6709         if ((IS_GEMINILAKE(dev_priv) || IS_BROXTON(dev_priv)) &&
6710             dev_priv->ipc_enabled)
6711                 linetime_wm /= 2;
6712
6713         return min(linetime_wm, 0x1ff);
6714 }
6715
6716 static int hsw_compute_linetime_wm(struct intel_atomic_state *state,
6717                                    struct intel_crtc *crtc)
6718 {
6719         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
6720         struct intel_crtc_state *crtc_state =
6721                 intel_atomic_get_new_crtc_state(state, crtc);
6722         const struct intel_cdclk_state *cdclk_state;
6723
6724         if (DISPLAY_VER(dev_priv) >= 9)
6725                 crtc_state->linetime = skl_linetime_wm(crtc_state);
6726         else
6727                 crtc_state->linetime = hsw_linetime_wm(crtc_state);
6728
6729         if (!hsw_crtc_supports_ips(crtc))
6730                 return 0;
6731
6732         cdclk_state = intel_atomic_get_cdclk_state(state);
6733         if (IS_ERR(cdclk_state))
6734                 return PTR_ERR(cdclk_state);
6735
6736         crtc_state->ips_linetime = hsw_ips_linetime_wm(crtc_state,
6737                                                        cdclk_state);
6738
6739         return 0;
6740 }
6741
6742 static int intel_crtc_atomic_check(struct intel_atomic_state *state,
6743                                    struct intel_crtc *crtc)
6744 {
6745         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
6746         struct intel_crtc_state *crtc_state =
6747                 intel_atomic_get_new_crtc_state(state, crtc);
6748         bool mode_changed = intel_crtc_needs_modeset(crtc_state);
6749         int ret;
6750
6751         if (DISPLAY_VER(dev_priv) < 5 && !IS_G4X(dev_priv) &&
6752             mode_changed && !crtc_state->hw.active)
6753                 crtc_state->update_wm_post = true;
6754
6755         if (mode_changed && crtc_state->hw.enable &&
6756             dev_priv->dpll_funcs &&
6757             !crtc_state->bigjoiner_slave &&
6758             !drm_WARN_ON(&dev_priv->drm, crtc_state->shared_dpll)) {
6759                 ret = dev_priv->dpll_funcs->crtc_compute_clock(crtc_state);
6760                 if (ret)
6761                         return ret;
6762         }
6763
6764         /*
6765          * May need to update pipe gamma enable bits
6766          * when C8 planes are getting enabled/disabled.
6767          */
6768         if (c8_planes_changed(crtc_state))
6769                 crtc_state->uapi.color_mgmt_changed = true;
6770
6771         if (mode_changed || crtc_state->update_pipe ||
6772             crtc_state->uapi.color_mgmt_changed) {
6773                 ret = intel_color_check(crtc_state);
6774                 if (ret)
6775                         return ret;
6776         }
6777
6778         ret = intel_compute_pipe_wm(state, crtc);
6779         if (ret) {
6780                 drm_dbg_kms(&dev_priv->drm,
6781                             "Target pipe watermarks are invalid\n");
6782                 return ret;
6783         }
6784
6785         /*
6786          * Calculate 'intermediate' watermarks that satisfy both the
6787          * old state and the new state.  We can program these
6788          * immediately.
6789          */
6790         ret = intel_compute_intermediate_wm(state, crtc);
6791         if (ret) {
6792                 drm_dbg_kms(&dev_priv->drm,
6793                             "No valid intermediate pipe watermarks are possible\n");
6794                 return ret;
6795         }
6796
6797         if (DISPLAY_VER(dev_priv) >= 9) {
6798                 if (mode_changed || crtc_state->update_pipe) {
6799                         ret = skl_update_scaler_crtc(crtc_state);
6800                         if (ret)
6801                                 return ret;
6802                 }
6803
6804                 ret = intel_atomic_setup_scalers(dev_priv, crtc, crtc_state);
6805                 if (ret)
6806                         return ret;
6807         }
6808
6809         if (HAS_IPS(dev_priv)) {
6810                 ret = hsw_compute_ips_config(crtc_state);
6811                 if (ret)
6812                         return ret;
6813         }
6814
6815         if (DISPLAY_VER(dev_priv) >= 9 ||
6816             IS_BROADWELL(dev_priv) || IS_HASWELL(dev_priv)) {
6817                 ret = hsw_compute_linetime_wm(state, crtc);
6818                 if (ret)
6819                         return ret;
6820
6821         }
6822
6823         ret = intel_psr2_sel_fetch_update(state, crtc);
6824         if (ret)
6825                 return ret;
6826
6827         return 0;
6828 }
6829
6830 static void intel_modeset_update_connector_atomic_state(struct drm_device *dev)
6831 {
6832         struct intel_connector *connector;
6833         struct drm_connector_list_iter conn_iter;
6834
6835         drm_connector_list_iter_begin(dev, &conn_iter);
6836         for_each_intel_connector_iter(connector, &conn_iter) {
6837                 struct drm_connector_state *conn_state = connector->base.state;
6838                 struct intel_encoder *encoder =
6839                         to_intel_encoder(connector->base.encoder);
6840
6841                 if (conn_state->crtc)
6842                         drm_connector_put(&connector->base);
6843
6844                 if (encoder) {
6845                         struct intel_crtc *crtc =
6846                                 to_intel_crtc(encoder->base.crtc);
6847                         const struct intel_crtc_state *crtc_state =
6848                                 to_intel_crtc_state(crtc->base.state);
6849
6850                         conn_state->best_encoder = &encoder->base;
6851                         conn_state->crtc = &crtc->base;
6852                         conn_state->max_bpc = (crtc_state->pipe_bpp ?: 24) / 3;
6853
6854                         drm_connector_get(&connector->base);
6855                 } else {
6856                         conn_state->best_encoder = NULL;
6857                         conn_state->crtc = NULL;
6858                 }
6859         }
6860         drm_connector_list_iter_end(&conn_iter);
6861 }
6862
6863 static int
6864 compute_sink_pipe_bpp(const struct drm_connector_state *conn_state,
6865                       struct intel_crtc_state *pipe_config)
6866 {
6867         struct drm_connector *connector = conn_state->connector;
6868         struct drm_i915_private *i915 = to_i915(pipe_config->uapi.crtc->dev);
6869         const struct drm_display_info *info = &connector->display_info;
6870         int bpp;
6871
6872         switch (conn_state->max_bpc) {
6873         case 6 ... 7:
6874                 bpp = 6 * 3;
6875                 break;
6876         case 8 ... 9:
6877                 bpp = 8 * 3;
6878                 break;
6879         case 10 ... 11:
6880                 bpp = 10 * 3;
6881                 break;
6882         case 12 ... 16:
6883                 bpp = 12 * 3;
6884                 break;
6885         default:
6886                 MISSING_CASE(conn_state->max_bpc);
6887                 return -EINVAL;
6888         }
6889
6890         if (bpp < pipe_config->pipe_bpp) {
6891                 drm_dbg_kms(&i915->drm,
6892                             "[CONNECTOR:%d:%s] Limiting display bpp to %d instead of "
6893                             "EDID bpp %d, requested bpp %d, max platform bpp %d\n",
6894                             connector->base.id, connector->name,
6895                             bpp, 3 * info->bpc,
6896                             3 * conn_state->max_requested_bpc,
6897                             pipe_config->pipe_bpp);
6898
6899                 pipe_config->pipe_bpp = bpp;
6900         }
6901
6902         return 0;
6903 }
6904
6905 static int
6906 compute_baseline_pipe_bpp(struct intel_crtc *crtc,
6907                           struct intel_crtc_state *pipe_config)
6908 {
6909         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
6910         struct drm_atomic_state *state = pipe_config->uapi.state;
6911         struct drm_connector *connector;
6912         struct drm_connector_state *connector_state;
6913         int bpp, i;
6914
6915         if ((IS_G4X(dev_priv) || IS_VALLEYVIEW(dev_priv) ||
6916             IS_CHERRYVIEW(dev_priv)))
6917                 bpp = 10*3;
6918         else if (DISPLAY_VER(dev_priv) >= 5)
6919                 bpp = 12*3;
6920         else
6921                 bpp = 8*3;
6922
6923         pipe_config->pipe_bpp = bpp;
6924
6925         /* Clamp display bpp to connector max bpp */
6926         for_each_new_connector_in_state(state, connector, connector_state, i) {
6927                 int ret;
6928
6929                 if (connector_state->crtc != &crtc->base)
6930                         continue;
6931
6932                 ret = compute_sink_pipe_bpp(connector_state, pipe_config);
6933                 if (ret)
6934                         return ret;
6935         }
6936
6937         return 0;
6938 }
6939
6940 static void intel_dump_crtc_timings(struct drm_i915_private *i915,
6941                                     const struct drm_display_mode *mode)
6942 {
6943         drm_dbg_kms(&i915->drm, "crtc timings: %d %d %d %d %d %d %d %d %d, "
6944                     "type: 0x%x flags: 0x%x\n",
6945                     mode->crtc_clock,
6946                     mode->crtc_hdisplay, mode->crtc_hsync_start,
6947                     mode->crtc_hsync_end, mode->crtc_htotal,
6948                     mode->crtc_vdisplay, mode->crtc_vsync_start,
6949                     mode->crtc_vsync_end, mode->crtc_vtotal,
6950                     mode->type, mode->flags);
6951 }
6952
6953 static void
6954 intel_dump_m_n_config(const struct intel_crtc_state *pipe_config,
6955                       const char *id, unsigned int lane_count,
6956                       const struct intel_link_m_n *m_n)
6957 {
6958         struct drm_i915_private *i915 = to_i915(pipe_config->uapi.crtc->dev);
6959
6960         drm_dbg_kms(&i915->drm,
6961                     "%s: lanes: %i; gmch_m: %u, gmch_n: %u, link_m: %u, link_n: %u, tu: %u\n",
6962                     id, lane_count,
6963                     m_n->gmch_m, m_n->gmch_n,
6964                     m_n->link_m, m_n->link_n, m_n->tu);
6965 }
6966
6967 static void
6968 intel_dump_infoframe(struct drm_i915_private *dev_priv,
6969                      const union hdmi_infoframe *frame)
6970 {
6971         if (!drm_debug_enabled(DRM_UT_KMS))
6972                 return;
6973
6974         hdmi_infoframe_log(KERN_DEBUG, dev_priv->drm.dev, frame);
6975 }
6976
6977 static void
6978 intel_dump_dp_vsc_sdp(struct drm_i915_private *dev_priv,
6979                       const struct drm_dp_vsc_sdp *vsc)
6980 {
6981         if (!drm_debug_enabled(DRM_UT_KMS))
6982                 return;
6983
6984         drm_dp_vsc_sdp_log(KERN_DEBUG, dev_priv->drm.dev, vsc);
6985 }
6986
6987 #define OUTPUT_TYPE(x) [INTEL_OUTPUT_ ## x] = #x
6988
6989 static const char * const output_type_str[] = {
6990         OUTPUT_TYPE(UNUSED),
6991         OUTPUT_TYPE(ANALOG),
6992         OUTPUT_TYPE(DVO),
6993         OUTPUT_TYPE(SDVO),
6994         OUTPUT_TYPE(LVDS),
6995         OUTPUT_TYPE(TVOUT),
6996         OUTPUT_TYPE(HDMI),
6997         OUTPUT_TYPE(DP),
6998         OUTPUT_TYPE(EDP),
6999         OUTPUT_TYPE(DSI),
7000         OUTPUT_TYPE(DDI),
7001         OUTPUT_TYPE(DP_MST),
7002 };
7003
7004 #undef OUTPUT_TYPE
7005
7006 static void snprintf_output_types(char *buf, size_t len,
7007                                   unsigned int output_types)
7008 {
7009         char *str = buf;
7010         int i;
7011
7012         str[0] = '\0';
7013
7014         for (i = 0; i < ARRAY_SIZE(output_type_str); i++) {
7015                 int r;
7016
7017                 if ((output_types & BIT(i)) == 0)
7018                         continue;
7019
7020                 r = snprintf(str, len, "%s%s",
7021                              str != buf ? "," : "", output_type_str[i]);
7022                 if (r >= len)
7023                         break;
7024                 str += r;
7025                 len -= r;
7026
7027                 output_types &= ~BIT(i);
7028         }
7029
7030         WARN_ON_ONCE(output_types != 0);
7031 }
7032
7033 static const char * const output_format_str[] = {
7034         [INTEL_OUTPUT_FORMAT_RGB] = "RGB",
7035         [INTEL_OUTPUT_FORMAT_YCBCR420] = "YCBCR4:2:0",
7036         [INTEL_OUTPUT_FORMAT_YCBCR444] = "YCBCR4:4:4",
7037 };
7038
7039 static const char *output_formats(enum intel_output_format format)
7040 {
7041         if (format >= ARRAY_SIZE(output_format_str))
7042                 return "invalid";
7043         return output_format_str[format];
7044 }
7045
7046 static void intel_dump_plane_state(const struct intel_plane_state *plane_state)
7047 {
7048         struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
7049         struct drm_i915_private *i915 = to_i915(plane->base.dev);
7050         const struct drm_framebuffer *fb = plane_state->hw.fb;
7051
7052         if (!fb) {
7053                 drm_dbg_kms(&i915->drm,
7054                             "[PLANE:%d:%s] fb: [NOFB], visible: %s\n",
7055                             plane->base.base.id, plane->base.name,
7056                             yesno(plane_state->uapi.visible));
7057                 return;
7058         }
7059
7060         drm_dbg_kms(&i915->drm,
7061                     "[PLANE:%d:%s] fb: [FB:%d] %ux%u format = %p4cc modifier = 0x%llx, visible: %s\n",
7062                     plane->base.base.id, plane->base.name,
7063                     fb->base.id, fb->width, fb->height, &fb->format->format,
7064                     fb->modifier, yesno(plane_state->uapi.visible));
7065         drm_dbg_kms(&i915->drm, "\trotation: 0x%x, scaler: %d\n",
7066                     plane_state->hw.rotation, plane_state->scaler_id);
7067         if (plane_state->uapi.visible)
7068                 drm_dbg_kms(&i915->drm,
7069                             "\tsrc: " DRM_RECT_FP_FMT " dst: " DRM_RECT_FMT "\n",
7070                             DRM_RECT_FP_ARG(&plane_state->uapi.src),
7071                             DRM_RECT_ARG(&plane_state->uapi.dst));
7072 }
7073
7074 static void intel_dump_pipe_config(const struct intel_crtc_state *pipe_config,
7075                                    struct intel_atomic_state *state,
7076                                    const char *context)
7077 {
7078         struct intel_crtc *crtc = to_intel_crtc(pipe_config->uapi.crtc);
7079         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
7080         const struct intel_plane_state *plane_state;
7081         struct intel_plane *plane;
7082         char buf[64];
7083         int i;
7084
7085         drm_dbg_kms(&dev_priv->drm, "[CRTC:%d:%s] enable: %s %s\n",
7086                     crtc->base.base.id, crtc->base.name,
7087                     yesno(pipe_config->hw.enable), context);
7088
7089         if (!pipe_config->hw.enable)
7090                 goto dump_planes;
7091
7092         snprintf_output_types(buf, sizeof(buf), pipe_config->output_types);
7093         drm_dbg_kms(&dev_priv->drm,
7094                     "active: %s, output_types: %s (0x%x), output format: %s\n",
7095                     yesno(pipe_config->hw.active),
7096                     buf, pipe_config->output_types,
7097                     output_formats(pipe_config->output_format));
7098
7099         drm_dbg_kms(&dev_priv->drm,
7100                     "cpu_transcoder: %s, pipe bpp: %i, dithering: %i\n",
7101                     transcoder_name(pipe_config->cpu_transcoder),
7102                     pipe_config->pipe_bpp, pipe_config->dither);
7103
7104         drm_dbg_kms(&dev_priv->drm, "MST master transcoder: %s\n",
7105                     transcoder_name(pipe_config->mst_master_transcoder));
7106
7107         drm_dbg_kms(&dev_priv->drm,
7108                     "port sync: master transcoder: %s, slave transcoder bitmask = 0x%x\n",
7109                     transcoder_name(pipe_config->master_transcoder),
7110                     pipe_config->sync_mode_slaves_mask);
7111
7112         drm_dbg_kms(&dev_priv->drm, "bigjoiner: %s\n",
7113                     pipe_config->bigjoiner_slave ? "slave" :
7114                     pipe_config->bigjoiner ? "master" : "no");
7115
7116         drm_dbg_kms(&dev_priv->drm, "splitter: %s, link count %d, overlap %d\n",
7117                     enableddisabled(pipe_config->splitter.enable),
7118                     pipe_config->splitter.link_count,
7119                     pipe_config->splitter.pixel_overlap);
7120
7121         if (pipe_config->has_pch_encoder)
7122                 intel_dump_m_n_config(pipe_config, "fdi",
7123                                       pipe_config->fdi_lanes,
7124                                       &pipe_config->fdi_m_n);
7125
7126         if (intel_crtc_has_dp_encoder(pipe_config)) {
7127                 intel_dump_m_n_config(pipe_config, "dp m_n",
7128                                 pipe_config->lane_count, &pipe_config->dp_m_n);
7129                 if (pipe_config->has_drrs)
7130                         intel_dump_m_n_config(pipe_config, "dp m2_n2",
7131                                               pipe_config->lane_count,
7132                                               &pipe_config->dp_m2_n2);
7133         }
7134
7135         drm_dbg_kms(&dev_priv->drm,
7136                     "audio: %i, infoframes: %i, infoframes enabled: 0x%x\n",
7137                     pipe_config->has_audio, pipe_config->has_infoframe,
7138                     pipe_config->infoframes.enable);
7139
7140         if (pipe_config->infoframes.enable &
7141             intel_hdmi_infoframe_enable(HDMI_PACKET_TYPE_GENERAL_CONTROL))
7142                 drm_dbg_kms(&dev_priv->drm, "GCP: 0x%x\n",
7143                             pipe_config->infoframes.gcp);
7144         if (pipe_config->infoframes.enable &
7145             intel_hdmi_infoframe_enable(HDMI_INFOFRAME_TYPE_AVI))
7146                 intel_dump_infoframe(dev_priv, &pipe_config->infoframes.avi);
7147         if (pipe_config->infoframes.enable &
7148             intel_hdmi_infoframe_enable(HDMI_INFOFRAME_TYPE_SPD))
7149                 intel_dump_infoframe(dev_priv, &pipe_config->infoframes.spd);
7150         if (pipe_config->infoframes.enable &
7151             intel_hdmi_infoframe_enable(HDMI_INFOFRAME_TYPE_VENDOR))
7152                 intel_dump_infoframe(dev_priv, &pipe_config->infoframes.hdmi);
7153         if (pipe_config->infoframes.enable &
7154             intel_hdmi_infoframe_enable(HDMI_INFOFRAME_TYPE_DRM))
7155                 intel_dump_infoframe(dev_priv, &pipe_config->infoframes.drm);
7156         if (pipe_config->infoframes.enable &
7157             intel_hdmi_infoframe_enable(HDMI_PACKET_TYPE_GAMUT_METADATA))
7158                 intel_dump_infoframe(dev_priv, &pipe_config->infoframes.drm);
7159         if (pipe_config->infoframes.enable &
7160             intel_hdmi_infoframe_enable(DP_SDP_VSC))
7161                 intel_dump_dp_vsc_sdp(dev_priv, &pipe_config->infoframes.vsc);
7162
7163         drm_dbg_kms(&dev_priv->drm, "vrr: %s, vmin: %d, vmax: %d, pipeline full: %d, guardband: %d flipline: %d, vmin vblank: %d, vmax vblank: %d\n",
7164                     yesno(pipe_config->vrr.enable),
7165                     pipe_config->vrr.vmin, pipe_config->vrr.vmax,
7166                     pipe_config->vrr.pipeline_full, pipe_config->vrr.guardband,
7167                     pipe_config->vrr.flipline,
7168                     intel_vrr_vmin_vblank_start(pipe_config),
7169                     intel_vrr_vmax_vblank_start(pipe_config));
7170
7171         drm_dbg_kms(&dev_priv->drm, "requested mode:\n");
7172         drm_mode_debug_printmodeline(&pipe_config->hw.mode);
7173         drm_dbg_kms(&dev_priv->drm, "adjusted mode:\n");
7174         drm_mode_debug_printmodeline(&pipe_config->hw.adjusted_mode);
7175         intel_dump_crtc_timings(dev_priv, &pipe_config->hw.adjusted_mode);
7176         drm_dbg_kms(&dev_priv->drm, "pipe mode:\n");
7177         drm_mode_debug_printmodeline(&pipe_config->hw.pipe_mode);
7178         intel_dump_crtc_timings(dev_priv, &pipe_config->hw.pipe_mode);
7179         drm_dbg_kms(&dev_priv->drm,
7180                     "port clock: %d, pipe src size: %dx%d, pixel rate %d\n",
7181                     pipe_config->port_clock,
7182                     pipe_config->pipe_src_w, pipe_config->pipe_src_h,
7183                     pipe_config->pixel_rate);
7184
7185         drm_dbg_kms(&dev_priv->drm, "linetime: %d, ips linetime: %d\n",
7186                     pipe_config->linetime, pipe_config->ips_linetime);
7187
7188         if (DISPLAY_VER(dev_priv) >= 9)
7189                 drm_dbg_kms(&dev_priv->drm,
7190                             "num_scalers: %d, scaler_users: 0x%x, scaler_id: %d\n",
7191                             crtc->num_scalers,
7192                             pipe_config->scaler_state.scaler_users,
7193                             pipe_config->scaler_state.scaler_id);
7194
7195         if (HAS_GMCH(dev_priv))
7196                 drm_dbg_kms(&dev_priv->drm,
7197                             "gmch pfit: control: 0x%08x, ratios: 0x%08x, lvds border: 0x%08x\n",
7198                             pipe_config->gmch_pfit.control,
7199                             pipe_config->gmch_pfit.pgm_ratios,
7200                             pipe_config->gmch_pfit.lvds_border_bits);
7201         else
7202                 drm_dbg_kms(&dev_priv->drm,
7203                             "pch pfit: " DRM_RECT_FMT ", %s, force thru: %s\n",
7204                             DRM_RECT_ARG(&pipe_config->pch_pfit.dst),
7205                             enableddisabled(pipe_config->pch_pfit.enabled),
7206                             yesno(pipe_config->pch_pfit.force_thru));
7207
7208         drm_dbg_kms(&dev_priv->drm, "ips: %i, double wide: %i\n",
7209                     pipe_config->ips_enabled, pipe_config->double_wide);
7210
7211         intel_dpll_dump_hw_state(dev_priv, &pipe_config->dpll_hw_state);
7212
7213         if (IS_CHERRYVIEW(dev_priv))
7214                 drm_dbg_kms(&dev_priv->drm,
7215                             "cgm_mode: 0x%x gamma_mode: 0x%x gamma_enable: %d csc_enable: %d\n",
7216                             pipe_config->cgm_mode, pipe_config->gamma_mode,
7217                             pipe_config->gamma_enable, pipe_config->csc_enable);
7218         else
7219                 drm_dbg_kms(&dev_priv->drm,
7220                             "csc_mode: 0x%x gamma_mode: 0x%x gamma_enable: %d csc_enable: %d\n",
7221                             pipe_config->csc_mode, pipe_config->gamma_mode,
7222                             pipe_config->gamma_enable, pipe_config->csc_enable);
7223
7224         drm_dbg_kms(&dev_priv->drm, "degamma lut: %d entries, gamma lut: %d entries\n",
7225                     pipe_config->hw.degamma_lut ?
7226                     drm_color_lut_size(pipe_config->hw.degamma_lut) : 0,
7227                     pipe_config->hw.gamma_lut ?
7228                     drm_color_lut_size(pipe_config->hw.gamma_lut) : 0);
7229
7230 dump_planes:
7231         if (!state)
7232                 return;
7233
7234         for_each_new_intel_plane_in_state(state, plane, plane_state, i) {
7235                 if (plane->pipe == crtc->pipe)
7236                         intel_dump_plane_state(plane_state);
7237         }
7238 }
7239
7240 static bool check_digital_port_conflicts(struct intel_atomic_state *state)
7241 {
7242         struct drm_device *dev = state->base.dev;
7243         struct drm_connector *connector;
7244         struct drm_connector_list_iter conn_iter;
7245         unsigned int used_ports = 0;
7246         unsigned int used_mst_ports = 0;
7247         bool ret = true;
7248
7249         /*
7250          * We're going to peek into connector->state,
7251          * hence connection_mutex must be held.
7252          */
7253         drm_modeset_lock_assert_held(&dev->mode_config.connection_mutex);
7254
7255         /*
7256          * Walk the connector list instead of the encoder
7257          * list to detect the problem on ddi platforms
7258          * where there's just one encoder per digital port.
7259          */
7260         drm_connector_list_iter_begin(dev, &conn_iter);
7261         drm_for_each_connector_iter(connector, &conn_iter) {
7262                 struct drm_connector_state *connector_state;
7263                 struct intel_encoder *encoder;
7264
7265                 connector_state =
7266                         drm_atomic_get_new_connector_state(&state->base,
7267                                                            connector);
7268                 if (!connector_state)
7269                         connector_state = connector->state;
7270
7271                 if (!connector_state->best_encoder)
7272                         continue;
7273
7274                 encoder = to_intel_encoder(connector_state->best_encoder);
7275
7276                 drm_WARN_ON(dev, !connector_state->crtc);
7277
7278                 switch (encoder->type) {
7279                 case INTEL_OUTPUT_DDI:
7280                         if (drm_WARN_ON(dev, !HAS_DDI(to_i915(dev))))
7281                                 break;
7282                         fallthrough;
7283                 case INTEL_OUTPUT_DP:
7284                 case INTEL_OUTPUT_HDMI:
7285                 case INTEL_OUTPUT_EDP:
7286                         /* the same port mustn't appear more than once */
7287                         if (used_ports & BIT(encoder->port))
7288                                 ret = false;
7289
7290                         used_ports |= BIT(encoder->port);
7291                         break;
7292                 case INTEL_OUTPUT_DP_MST:
7293                         used_mst_ports |=
7294                                 1 << encoder->port;
7295                         break;
7296                 default:
7297                         break;
7298                 }
7299         }
7300         drm_connector_list_iter_end(&conn_iter);
7301
7302         /* can't mix MST and SST/HDMI on the same port */
7303         if (used_ports & used_mst_ports)
7304                 return false;
7305
7306         return ret;
7307 }
7308
7309 static void
7310 intel_crtc_copy_uapi_to_hw_state_nomodeset(struct intel_atomic_state *state,
7311                                            struct intel_crtc_state *crtc_state)
7312 {
7313         const struct intel_crtc_state *from_crtc_state = crtc_state;
7314
7315         if (crtc_state->bigjoiner_slave) {
7316                 from_crtc_state = intel_atomic_get_new_crtc_state(state,
7317                                                                   crtc_state->bigjoiner_linked_crtc);
7318
7319                 /* No need to copy state if the master state is unchanged */
7320                 if (!from_crtc_state)
7321                         return;
7322         }
7323
7324         intel_crtc_copy_color_blobs(crtc_state, from_crtc_state);
7325 }
7326
7327 static void
7328 intel_crtc_copy_uapi_to_hw_state(struct intel_atomic_state *state,
7329                                  struct intel_crtc_state *crtc_state)
7330 {
7331         crtc_state->hw.enable = crtc_state->uapi.enable;
7332         crtc_state->hw.active = crtc_state->uapi.active;
7333         crtc_state->hw.mode = crtc_state->uapi.mode;
7334         crtc_state->hw.adjusted_mode = crtc_state->uapi.adjusted_mode;
7335         crtc_state->hw.scaling_filter = crtc_state->uapi.scaling_filter;
7336
7337         intel_crtc_copy_uapi_to_hw_state_nomodeset(state, crtc_state);
7338 }
7339
7340 static void intel_crtc_copy_hw_to_uapi_state(struct intel_crtc_state *crtc_state)
7341 {
7342         if (crtc_state->bigjoiner_slave)
7343                 return;
7344
7345         crtc_state->uapi.enable = crtc_state->hw.enable;
7346         crtc_state->uapi.active = crtc_state->hw.active;
7347         drm_WARN_ON(crtc_state->uapi.crtc->dev,
7348                     drm_atomic_set_mode_for_crtc(&crtc_state->uapi, &crtc_state->hw.mode) < 0);
7349
7350         crtc_state->uapi.adjusted_mode = crtc_state->hw.adjusted_mode;
7351         crtc_state->uapi.scaling_filter = crtc_state->hw.scaling_filter;
7352
7353         /* copy color blobs to uapi */
7354         drm_property_replace_blob(&crtc_state->uapi.degamma_lut,
7355                                   crtc_state->hw.degamma_lut);
7356         drm_property_replace_blob(&crtc_state->uapi.gamma_lut,
7357                                   crtc_state->hw.gamma_lut);
7358         drm_property_replace_blob(&crtc_state->uapi.ctm,
7359                                   crtc_state->hw.ctm);
7360 }
7361
7362 static int
7363 copy_bigjoiner_crtc_state(struct intel_crtc_state *crtc_state,
7364                           const struct intel_crtc_state *from_crtc_state)
7365 {
7366         struct intel_crtc_state *saved_state;
7367         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
7368
7369         saved_state = kmemdup(from_crtc_state, sizeof(*saved_state), GFP_KERNEL);
7370         if (!saved_state)
7371                 return -ENOMEM;
7372
7373         saved_state->uapi = crtc_state->uapi;
7374         saved_state->scaler_state = crtc_state->scaler_state;
7375         saved_state->shared_dpll = crtc_state->shared_dpll;
7376         saved_state->dpll_hw_state = crtc_state->dpll_hw_state;
7377         saved_state->crc_enabled = crtc_state->crc_enabled;
7378
7379         intel_crtc_free_hw_state(crtc_state);
7380         memcpy(crtc_state, saved_state, sizeof(*crtc_state));
7381         kfree(saved_state);
7382
7383         /* Re-init hw state */
7384         memset(&crtc_state->hw, 0, sizeof(saved_state->hw));
7385         crtc_state->hw.enable = from_crtc_state->hw.enable;
7386         crtc_state->hw.active = from_crtc_state->hw.active;
7387         crtc_state->hw.pipe_mode = from_crtc_state->hw.pipe_mode;
7388         crtc_state->hw.adjusted_mode = from_crtc_state->hw.adjusted_mode;
7389
7390         /* Some fixups */
7391         crtc_state->uapi.mode_changed = from_crtc_state->uapi.mode_changed;
7392         crtc_state->uapi.connectors_changed = from_crtc_state->uapi.connectors_changed;
7393         crtc_state->uapi.active_changed = from_crtc_state->uapi.active_changed;
7394         crtc_state->nv12_planes = crtc_state->c8_planes = crtc_state->update_planes = 0;
7395         crtc_state->bigjoiner_linked_crtc = to_intel_crtc(from_crtc_state->uapi.crtc);
7396         crtc_state->bigjoiner_slave = true;
7397         crtc_state->cpu_transcoder = (enum transcoder)crtc->pipe;
7398         crtc_state->has_audio = false;
7399
7400         return 0;
7401 }
7402
7403 static int
7404 intel_crtc_prepare_cleared_state(struct intel_atomic_state *state,
7405                                  struct intel_crtc_state *crtc_state)
7406 {
7407         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
7408         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
7409         struct intel_crtc_state *saved_state;
7410
7411         saved_state = intel_crtc_state_alloc(crtc);
7412         if (!saved_state)
7413                 return -ENOMEM;
7414
7415         /* free the old crtc_state->hw members */
7416         intel_crtc_free_hw_state(crtc_state);
7417
7418         /* FIXME: before the switch to atomic started, a new pipe_config was
7419          * kzalloc'd. Code that depends on any field being zero should be
7420          * fixed, so that the crtc_state can be safely duplicated. For now,
7421          * only fields that are know to not cause problems are preserved. */
7422
7423         saved_state->uapi = crtc_state->uapi;
7424         saved_state->scaler_state = crtc_state->scaler_state;
7425         saved_state->shared_dpll = crtc_state->shared_dpll;
7426         saved_state->dpll_hw_state = crtc_state->dpll_hw_state;
7427         memcpy(saved_state->icl_port_dplls, crtc_state->icl_port_dplls,
7428                sizeof(saved_state->icl_port_dplls));
7429         saved_state->crc_enabled = crtc_state->crc_enabled;
7430         if (IS_G4X(dev_priv) ||
7431             IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
7432                 saved_state->wm = crtc_state->wm;
7433
7434         memcpy(crtc_state, saved_state, sizeof(*crtc_state));
7435         kfree(saved_state);
7436
7437         intel_crtc_copy_uapi_to_hw_state(state, crtc_state);
7438
7439         return 0;
7440 }
7441
7442 static int
7443 intel_modeset_pipe_config(struct intel_atomic_state *state,
7444                           struct intel_crtc_state *pipe_config)
7445 {
7446         struct drm_crtc *crtc = pipe_config->uapi.crtc;
7447         struct drm_i915_private *i915 = to_i915(pipe_config->uapi.crtc->dev);
7448         struct drm_connector *connector;
7449         struct drm_connector_state *connector_state;
7450         int base_bpp, ret, i;
7451         bool retry = true;
7452
7453         pipe_config->cpu_transcoder =
7454                 (enum transcoder) to_intel_crtc(crtc)->pipe;
7455
7456         /*
7457          * Sanitize sync polarity flags based on requested ones. If neither
7458          * positive or negative polarity is requested, treat this as meaning
7459          * negative polarity.
7460          */
7461         if (!(pipe_config->hw.adjusted_mode.flags &
7462               (DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_NHSYNC)))
7463                 pipe_config->hw.adjusted_mode.flags |= DRM_MODE_FLAG_NHSYNC;
7464
7465         if (!(pipe_config->hw.adjusted_mode.flags &
7466               (DRM_MODE_FLAG_PVSYNC | DRM_MODE_FLAG_NVSYNC)))
7467                 pipe_config->hw.adjusted_mode.flags |= DRM_MODE_FLAG_NVSYNC;
7468
7469         ret = compute_baseline_pipe_bpp(to_intel_crtc(crtc),
7470                                         pipe_config);
7471         if (ret)
7472                 return ret;
7473
7474         base_bpp = pipe_config->pipe_bpp;
7475
7476         /*
7477          * Determine the real pipe dimensions. Note that stereo modes can
7478          * increase the actual pipe size due to the frame doubling and
7479          * insertion of additional space for blanks between the frame. This
7480          * is stored in the crtc timings. We use the requested mode to do this
7481          * computation to clearly distinguish it from the adjusted mode, which
7482          * can be changed by the connectors in the below retry loop.
7483          */
7484         drm_mode_get_hv_timing(&pipe_config->hw.mode,
7485                                &pipe_config->pipe_src_w,
7486                                &pipe_config->pipe_src_h);
7487
7488         for_each_new_connector_in_state(&state->base, connector, connector_state, i) {
7489                 struct intel_encoder *encoder =
7490                         to_intel_encoder(connector_state->best_encoder);
7491
7492                 if (connector_state->crtc != crtc)
7493                         continue;
7494
7495                 if (!check_single_encoder_cloning(state, to_intel_crtc(crtc), encoder)) {
7496                         drm_dbg_kms(&i915->drm,
7497                                     "rejecting invalid cloning configuration\n");
7498                         return -EINVAL;
7499                 }
7500
7501                 /*
7502                  * Determine output_types before calling the .compute_config()
7503                  * hooks so that the hooks can use this information safely.
7504                  */
7505                 if (encoder->compute_output_type)
7506                         pipe_config->output_types |=
7507                                 BIT(encoder->compute_output_type(encoder, pipe_config,
7508                                                                  connector_state));
7509                 else
7510                         pipe_config->output_types |= BIT(encoder->type);
7511         }
7512
7513 encoder_retry:
7514         /* Ensure the port clock defaults are reset when retrying. */
7515         pipe_config->port_clock = 0;
7516         pipe_config->pixel_multiplier = 1;
7517
7518         /* Fill in default crtc timings, allow encoders to overwrite them. */
7519         drm_mode_set_crtcinfo(&pipe_config->hw.adjusted_mode,
7520                               CRTC_STEREO_DOUBLE);
7521
7522         /* Pass our mode to the connectors and the CRTC to give them a chance to
7523          * adjust it according to limitations or connector properties, and also
7524          * a chance to reject the mode entirely.
7525          */
7526         for_each_new_connector_in_state(&state->base, connector, connector_state, i) {
7527                 struct intel_encoder *encoder =
7528                         to_intel_encoder(connector_state->best_encoder);
7529
7530                 if (connector_state->crtc != crtc)
7531                         continue;
7532
7533                 ret = encoder->compute_config(encoder, pipe_config,
7534                                               connector_state);
7535                 if (ret == -EDEADLK)
7536                         return ret;
7537                 if (ret < 0) {
7538                         drm_dbg_kms(&i915->drm, "Encoder config failure: %d\n", ret);
7539                         return ret;
7540                 }
7541         }
7542
7543         /* Set default port clock if not overwritten by the encoder. Needs to be
7544          * done afterwards in case the encoder adjusts the mode. */
7545         if (!pipe_config->port_clock)
7546                 pipe_config->port_clock = pipe_config->hw.adjusted_mode.crtc_clock
7547                         * pipe_config->pixel_multiplier;
7548
7549         ret = intel_crtc_compute_config(to_intel_crtc(crtc), pipe_config);
7550         if (ret == -EDEADLK)
7551                 return ret;
7552         if (ret == -EAGAIN) {
7553                 if (drm_WARN(&i915->drm, !retry,
7554                              "loop in pipe configuration computation\n"))
7555                         return -EINVAL;
7556
7557                 drm_dbg_kms(&i915->drm, "CRTC bw constrained, retrying\n");
7558                 retry = false;
7559                 goto encoder_retry;
7560         }
7561         if (ret < 0) {
7562                 drm_dbg_kms(&i915->drm, "CRTC config failure: %d\n", ret);
7563                 return ret;
7564         }
7565
7566         /* Dithering seems to not pass-through bits correctly when it should, so
7567          * only enable it on 6bpc panels and when its not a compliance
7568          * test requesting 6bpc video pattern.
7569          */
7570         pipe_config->dither = (pipe_config->pipe_bpp == 6*3) &&
7571                 !pipe_config->dither_force_disable;
7572         drm_dbg_kms(&i915->drm,
7573                     "hw max bpp: %i, pipe bpp: %i, dithering: %i\n",
7574                     base_bpp, pipe_config->pipe_bpp, pipe_config->dither);
7575
7576         return 0;
7577 }
7578
7579 static int
7580 intel_modeset_pipe_config_late(struct intel_crtc_state *crtc_state)
7581 {
7582         struct intel_atomic_state *state =
7583                 to_intel_atomic_state(crtc_state->uapi.state);
7584         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
7585         struct drm_connector_state *conn_state;
7586         struct drm_connector *connector;
7587         int i;
7588
7589         for_each_new_connector_in_state(&state->base, connector,
7590                                         conn_state, i) {
7591                 struct intel_encoder *encoder =
7592                         to_intel_encoder(conn_state->best_encoder);
7593                 int ret;
7594
7595                 if (conn_state->crtc != &crtc->base ||
7596                     !encoder->compute_config_late)
7597                         continue;
7598
7599                 ret = encoder->compute_config_late(encoder, crtc_state,
7600                                                    conn_state);
7601                 if (ret)
7602                         return ret;
7603         }
7604
7605         return 0;
7606 }
7607
7608 bool intel_fuzzy_clock_check(int clock1, int clock2)
7609 {
7610         int diff;
7611
7612         if (clock1 == clock2)
7613                 return true;
7614
7615         if (!clock1 || !clock2)
7616                 return false;
7617
7618         diff = abs(clock1 - clock2);
7619
7620         if (((((diff + clock1 + clock2) * 100)) / (clock1 + clock2)) < 105)
7621                 return true;
7622
7623         return false;
7624 }
7625
7626 static bool
7627 intel_compare_m_n(unsigned int m, unsigned int n,
7628                   unsigned int m2, unsigned int n2,
7629                   bool exact)
7630 {
7631         if (m == m2 && n == n2)
7632                 return true;
7633
7634         if (exact || !m || !n || !m2 || !n2)
7635                 return false;
7636
7637         BUILD_BUG_ON(DATA_LINK_M_N_MASK > INT_MAX);
7638
7639         if (n > n2) {
7640                 while (n > n2) {
7641                         m2 <<= 1;
7642                         n2 <<= 1;
7643                 }
7644         } else if (n < n2) {
7645                 while (n < n2) {
7646                         m <<= 1;
7647                         n <<= 1;
7648                 }
7649         }
7650
7651         if (n != n2)
7652                 return false;
7653
7654         return intel_fuzzy_clock_check(m, m2);
7655 }
7656
7657 static bool
7658 intel_compare_link_m_n(const struct intel_link_m_n *m_n,
7659                        const struct intel_link_m_n *m2_n2,
7660                        bool exact)
7661 {
7662         return m_n->tu == m2_n2->tu &&
7663                 intel_compare_m_n(m_n->gmch_m, m_n->gmch_n,
7664                                   m2_n2->gmch_m, m2_n2->gmch_n, exact) &&
7665                 intel_compare_m_n(m_n->link_m, m_n->link_n,
7666                                   m2_n2->link_m, m2_n2->link_n, exact);
7667 }
7668
7669 static bool
7670 intel_compare_infoframe(const union hdmi_infoframe *a,
7671                         const union hdmi_infoframe *b)
7672 {
7673         return memcmp(a, b, sizeof(*a)) == 0;
7674 }
7675
7676 static bool
7677 intel_compare_dp_vsc_sdp(const struct drm_dp_vsc_sdp *a,
7678                          const struct drm_dp_vsc_sdp *b)
7679 {
7680         return memcmp(a, b, sizeof(*a)) == 0;
7681 }
7682
7683 static void
7684 pipe_config_infoframe_mismatch(struct drm_i915_private *dev_priv,
7685                                bool fastset, const char *name,
7686                                const union hdmi_infoframe *a,
7687                                const union hdmi_infoframe *b)
7688 {
7689         if (fastset) {
7690                 if (!drm_debug_enabled(DRM_UT_KMS))
7691                         return;
7692
7693                 drm_dbg_kms(&dev_priv->drm,
7694                             "fastset mismatch in %s infoframe\n", name);
7695                 drm_dbg_kms(&dev_priv->drm, "expected:\n");
7696                 hdmi_infoframe_log(KERN_DEBUG, dev_priv->drm.dev, a);
7697                 drm_dbg_kms(&dev_priv->drm, "found:\n");
7698                 hdmi_infoframe_log(KERN_DEBUG, dev_priv->drm.dev, b);
7699         } else {
7700                 drm_err(&dev_priv->drm, "mismatch in %s infoframe\n", name);
7701                 drm_err(&dev_priv->drm, "expected:\n");
7702                 hdmi_infoframe_log(KERN_ERR, dev_priv->drm.dev, a);
7703                 drm_err(&dev_priv->drm, "found:\n");
7704                 hdmi_infoframe_log(KERN_ERR, dev_priv->drm.dev, b);
7705         }
7706 }
7707
7708 static void
7709 pipe_config_dp_vsc_sdp_mismatch(struct drm_i915_private *dev_priv,
7710                                 bool fastset, const char *name,
7711                                 const struct drm_dp_vsc_sdp *a,
7712                                 const struct drm_dp_vsc_sdp *b)
7713 {
7714         if (fastset) {
7715                 if (!drm_debug_enabled(DRM_UT_KMS))
7716                         return;
7717
7718                 drm_dbg_kms(&dev_priv->drm,
7719                             "fastset mismatch in %s dp sdp\n", name);
7720                 drm_dbg_kms(&dev_priv->drm, "expected:\n");
7721                 drm_dp_vsc_sdp_log(KERN_DEBUG, dev_priv->drm.dev, a);
7722                 drm_dbg_kms(&dev_priv->drm, "found:\n");
7723                 drm_dp_vsc_sdp_log(KERN_DEBUG, dev_priv->drm.dev, b);
7724         } else {
7725                 drm_err(&dev_priv->drm, "mismatch in %s dp sdp\n", name);
7726                 drm_err(&dev_priv->drm, "expected:\n");
7727                 drm_dp_vsc_sdp_log(KERN_ERR, dev_priv->drm.dev, a);
7728                 drm_err(&dev_priv->drm, "found:\n");
7729                 drm_dp_vsc_sdp_log(KERN_ERR, dev_priv->drm.dev, b);
7730         }
7731 }
7732
7733 static void __printf(4, 5)
7734 pipe_config_mismatch(bool fastset, const struct intel_crtc *crtc,
7735                      const char *name, const char *format, ...)
7736 {
7737         struct drm_i915_private *i915 = to_i915(crtc->base.dev);
7738         struct va_format vaf;
7739         va_list args;
7740
7741         va_start(args, format);
7742         vaf.fmt = format;
7743         vaf.va = &args;
7744
7745         if (fastset)
7746                 drm_dbg_kms(&i915->drm,
7747                             "[CRTC:%d:%s] fastset mismatch in %s %pV\n",
7748                             crtc->base.base.id, crtc->base.name, name, &vaf);
7749         else
7750                 drm_err(&i915->drm, "[CRTC:%d:%s] mismatch in %s %pV\n",
7751                         crtc->base.base.id, crtc->base.name, name, &vaf);
7752
7753         va_end(args);
7754 }
7755
7756 static bool fastboot_enabled(struct drm_i915_private *dev_priv)
7757 {
7758         if (dev_priv->params.fastboot != -1)
7759                 return dev_priv->params.fastboot;
7760
7761         /* Enable fastboot by default on Skylake and newer */
7762         if (DISPLAY_VER(dev_priv) >= 9)
7763                 return true;
7764
7765         /* Enable fastboot by default on VLV and CHV */
7766         if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
7767                 return true;
7768
7769         /* Disabled by default on all others */
7770         return false;
7771 }
7772
7773 static bool
7774 intel_pipe_config_compare(const struct intel_crtc_state *current_config,
7775                           const struct intel_crtc_state *pipe_config,
7776                           bool fastset)
7777 {
7778         struct drm_i915_private *dev_priv = to_i915(current_config->uapi.crtc->dev);
7779         struct intel_crtc *crtc = to_intel_crtc(pipe_config->uapi.crtc);
7780         bool ret = true;
7781         u32 bp_gamma = 0;
7782         bool fixup_inherited = fastset &&
7783                 current_config->inherited && !pipe_config->inherited;
7784
7785         if (fixup_inherited && !fastboot_enabled(dev_priv)) {
7786                 drm_dbg_kms(&dev_priv->drm,
7787                             "initial modeset and fastboot not set\n");
7788                 ret = false;
7789         }
7790
7791 #define PIPE_CONF_CHECK_X(name) do { \
7792         if (current_config->name != pipe_config->name) { \
7793                 pipe_config_mismatch(fastset, crtc, __stringify(name), \
7794                                      "(expected 0x%08x, found 0x%08x)", \
7795                                      current_config->name, \
7796                                      pipe_config->name); \
7797                 ret = false; \
7798         } \
7799 } while (0)
7800
7801 #define PIPE_CONF_CHECK_X_WITH_MASK(name, mask) do { \
7802         if ((current_config->name & (mask)) != (pipe_config->name & (mask))) { \
7803                 pipe_config_mismatch(fastset, crtc, __stringify(name), \
7804                                      "(expected 0x%08x, found 0x%08x)", \
7805                                      current_config->name & (mask), \
7806                                      pipe_config->name & (mask)); \
7807                 ret = false; \
7808         } \
7809 } while (0)
7810
7811 #define PIPE_CONF_CHECK_I(name) do { \
7812         if (current_config->name != pipe_config->name) { \
7813                 pipe_config_mismatch(fastset, crtc, __stringify(name), \
7814                                      "(expected %i, found %i)", \
7815                                      current_config->name, \
7816                                      pipe_config->name); \
7817                 ret = false; \
7818         } \
7819 } while (0)
7820
7821 #define PIPE_CONF_CHECK_BOOL(name) do { \
7822         if (current_config->name != pipe_config->name) { \
7823                 pipe_config_mismatch(fastset, crtc,  __stringify(name), \
7824                                      "(expected %s, found %s)", \
7825                                      yesno(current_config->name), \
7826                                      yesno(pipe_config->name)); \
7827                 ret = false; \
7828         } \
7829 } while (0)
7830
7831 /*
7832  * Checks state where we only read out the enabling, but not the entire
7833  * state itself (like full infoframes or ELD for audio). These states
7834  * require a full modeset on bootup to fix up.
7835  */
7836 #define PIPE_CONF_CHECK_BOOL_INCOMPLETE(name) do { \
7837         if (!fixup_inherited || (!current_config->name && !pipe_config->name)) { \
7838                 PIPE_CONF_CHECK_BOOL(name); \
7839         } else { \
7840                 pipe_config_mismatch(fastset, crtc, __stringify(name), \
7841                                      "unable to verify whether state matches exactly, forcing modeset (expected %s, found %s)", \
7842                                      yesno(current_config->name), \
7843                                      yesno(pipe_config->name)); \
7844                 ret = false; \
7845         } \
7846 } while (0)
7847
7848 #define PIPE_CONF_CHECK_P(name) do { \
7849         if (current_config->name != pipe_config->name) { \
7850                 pipe_config_mismatch(fastset, crtc, __stringify(name), \
7851                                      "(expected %p, found %p)", \
7852                                      current_config->name, \
7853                                      pipe_config->name); \
7854                 ret = false; \
7855         } \
7856 } while (0)
7857
7858 #define PIPE_CONF_CHECK_M_N(name) do { \
7859         if (!intel_compare_link_m_n(&current_config->name, \
7860                                     &pipe_config->name,\
7861                                     !fastset)) { \
7862                 pipe_config_mismatch(fastset, crtc, __stringify(name), \
7863                                      "(expected tu %i gmch %i/%i link %i/%i, " \
7864                                      "found tu %i, gmch %i/%i link %i/%i)", \
7865                                      current_config->name.tu, \
7866                                      current_config->name.gmch_m, \
7867                                      current_config->name.gmch_n, \
7868                                      current_config->name.link_m, \
7869                                      current_config->name.link_n, \
7870                                      pipe_config->name.tu, \
7871                                      pipe_config->name.gmch_m, \
7872                                      pipe_config->name.gmch_n, \
7873                                      pipe_config->name.link_m, \
7874                                      pipe_config->name.link_n); \
7875                 ret = false; \
7876         } \
7877 } while (0)
7878
7879 /* This is required for BDW+ where there is only one set of registers for
7880  * switching between high and low RR.
7881  * This macro can be used whenever a comparison has to be made between one
7882  * hw state and multiple sw state variables.
7883  */
7884 #define PIPE_CONF_CHECK_M_N_ALT(name, alt_name) do { \
7885         if (!intel_compare_link_m_n(&current_config->name, \
7886                                     &pipe_config->name, !fastset) && \
7887             !intel_compare_link_m_n(&current_config->alt_name, \
7888                                     &pipe_config->name, !fastset)) { \
7889                 pipe_config_mismatch(fastset, crtc, __stringify(name), \
7890                                      "(expected tu %i gmch %i/%i link %i/%i, " \
7891                                      "or tu %i gmch %i/%i link %i/%i, " \
7892                                      "found tu %i, gmch %i/%i link %i/%i)", \
7893                                      current_config->name.tu, \
7894                                      current_config->name.gmch_m, \
7895                                      current_config->name.gmch_n, \
7896                                      current_config->name.link_m, \
7897                                      current_config->name.link_n, \
7898                                      current_config->alt_name.tu, \
7899                                      current_config->alt_name.gmch_m, \
7900                                      current_config->alt_name.gmch_n, \
7901                                      current_config->alt_name.link_m, \
7902                                      current_config->alt_name.link_n, \
7903                                      pipe_config->name.tu, \
7904                                      pipe_config->name.gmch_m, \
7905                                      pipe_config->name.gmch_n, \
7906                                      pipe_config->name.link_m, \
7907                                      pipe_config->name.link_n); \
7908                 ret = false; \
7909         } \
7910 } while (0)
7911
7912 #define PIPE_CONF_CHECK_FLAGS(name, mask) do { \
7913         if ((current_config->name ^ pipe_config->name) & (mask)) { \
7914                 pipe_config_mismatch(fastset, crtc, __stringify(name), \
7915                                      "(%x) (expected %i, found %i)", \
7916                                      (mask), \
7917                                      current_config->name & (mask), \
7918                                      pipe_config->name & (mask)); \
7919                 ret = false; \
7920         } \
7921 } while (0)
7922
7923 #define PIPE_CONF_CHECK_CLOCK_FUZZY(name) do { \
7924         if (!intel_fuzzy_clock_check(current_config->name, pipe_config->name)) { \
7925                 pipe_config_mismatch(fastset, crtc, __stringify(name), \
7926                                      "(expected %i, found %i)", \
7927                                      current_config->name, \
7928                                      pipe_config->name); \
7929                 ret = false; \
7930         } \
7931 } while (0)
7932
7933 #define PIPE_CONF_CHECK_INFOFRAME(name) do { \
7934         if (!intel_compare_infoframe(&current_config->infoframes.name, \
7935                                      &pipe_config->infoframes.name)) { \
7936                 pipe_config_infoframe_mismatch(dev_priv, fastset, __stringify(name), \
7937                                                &current_config->infoframes.name, \
7938                                                &pipe_config->infoframes.name); \
7939                 ret = false; \
7940         } \
7941 } while (0)
7942
7943 #define PIPE_CONF_CHECK_DP_VSC_SDP(name) do { \
7944         if (!current_config->has_psr && !pipe_config->has_psr && \
7945             !intel_compare_dp_vsc_sdp(&current_config->infoframes.name, \
7946                                       &pipe_config->infoframes.name)) { \
7947                 pipe_config_dp_vsc_sdp_mismatch(dev_priv, fastset, __stringify(name), \
7948                                                 &current_config->infoframes.name, \
7949                                                 &pipe_config->infoframes.name); \
7950                 ret = false; \
7951         } \
7952 } while (0)
7953
7954 #define PIPE_CONF_CHECK_COLOR_LUT(name1, name2, bit_precision) do { \
7955         if (current_config->name1 != pipe_config->name1) { \
7956                 pipe_config_mismatch(fastset, crtc, __stringify(name1), \
7957                                 "(expected %i, found %i, won't compare lut values)", \
7958                                 current_config->name1, \
7959                                 pipe_config->name1); \
7960                 ret = false;\
7961         } else { \
7962                 if (!intel_color_lut_equal(current_config->name2, \
7963                                         pipe_config->name2, pipe_config->name1, \
7964                                         bit_precision)) { \
7965                         pipe_config_mismatch(fastset, crtc, __stringify(name2), \
7966                                         "hw_state doesn't match sw_state"); \
7967                         ret = false; \
7968                 } \
7969         } \
7970 } while (0)
7971
7972 #define PIPE_CONF_QUIRK(quirk) \
7973         ((current_config->quirks | pipe_config->quirks) & (quirk))
7974
7975         PIPE_CONF_CHECK_I(cpu_transcoder);
7976
7977         PIPE_CONF_CHECK_BOOL(has_pch_encoder);
7978         PIPE_CONF_CHECK_I(fdi_lanes);
7979         PIPE_CONF_CHECK_M_N(fdi_m_n);
7980
7981         PIPE_CONF_CHECK_I(lane_count);
7982         PIPE_CONF_CHECK_X(lane_lat_optim_mask);
7983
7984         if (DISPLAY_VER(dev_priv) < 8) {
7985                 PIPE_CONF_CHECK_M_N(dp_m_n);
7986
7987                 if (current_config->has_drrs)
7988                         PIPE_CONF_CHECK_M_N(dp_m2_n2);
7989         } else
7990                 PIPE_CONF_CHECK_M_N_ALT(dp_m_n, dp_m2_n2);
7991
7992         PIPE_CONF_CHECK_X(output_types);
7993
7994         /* FIXME do the readout properly and get rid of this quirk */
7995         if (!PIPE_CONF_QUIRK(PIPE_CONFIG_QUIRK_BIGJOINER_SLAVE)) {
7996                 PIPE_CONF_CHECK_I(hw.pipe_mode.crtc_hdisplay);
7997                 PIPE_CONF_CHECK_I(hw.pipe_mode.crtc_htotal);
7998                 PIPE_CONF_CHECK_I(hw.pipe_mode.crtc_hblank_start);
7999                 PIPE_CONF_CHECK_I(hw.pipe_mode.crtc_hblank_end);
8000                 PIPE_CONF_CHECK_I(hw.pipe_mode.crtc_hsync_start);
8001                 PIPE_CONF_CHECK_I(hw.pipe_mode.crtc_hsync_end);
8002
8003                 PIPE_CONF_CHECK_I(hw.pipe_mode.crtc_vdisplay);
8004                 PIPE_CONF_CHECK_I(hw.pipe_mode.crtc_vtotal);
8005                 PIPE_CONF_CHECK_I(hw.pipe_mode.crtc_vblank_start);
8006                 PIPE_CONF_CHECK_I(hw.pipe_mode.crtc_vblank_end);
8007                 PIPE_CONF_CHECK_I(hw.pipe_mode.crtc_vsync_start);
8008                 PIPE_CONF_CHECK_I(hw.pipe_mode.crtc_vsync_end);
8009
8010                 PIPE_CONF_CHECK_I(hw.adjusted_mode.crtc_hdisplay);
8011                 PIPE_CONF_CHECK_I(hw.adjusted_mode.crtc_htotal);
8012                 PIPE_CONF_CHECK_I(hw.adjusted_mode.crtc_hblank_start);
8013                 PIPE_CONF_CHECK_I(hw.adjusted_mode.crtc_hblank_end);
8014                 PIPE_CONF_CHECK_I(hw.adjusted_mode.crtc_hsync_start);
8015                 PIPE_CONF_CHECK_I(hw.adjusted_mode.crtc_hsync_end);
8016
8017                 PIPE_CONF_CHECK_I(hw.adjusted_mode.crtc_vdisplay);
8018                 PIPE_CONF_CHECK_I(hw.adjusted_mode.crtc_vtotal);
8019                 PIPE_CONF_CHECK_I(hw.adjusted_mode.crtc_vblank_start);
8020                 PIPE_CONF_CHECK_I(hw.adjusted_mode.crtc_vblank_end);
8021                 PIPE_CONF_CHECK_I(hw.adjusted_mode.crtc_vsync_start);
8022                 PIPE_CONF_CHECK_I(hw.adjusted_mode.crtc_vsync_end);
8023
8024                 PIPE_CONF_CHECK_I(pixel_multiplier);
8025
8026                 PIPE_CONF_CHECK_FLAGS(hw.adjusted_mode.flags,
8027                                       DRM_MODE_FLAG_INTERLACE);
8028
8029                 if (!PIPE_CONF_QUIRK(PIPE_CONFIG_QUIRK_MODE_SYNC_FLAGS)) {
8030                         PIPE_CONF_CHECK_FLAGS(hw.adjusted_mode.flags,
8031                                               DRM_MODE_FLAG_PHSYNC);
8032                         PIPE_CONF_CHECK_FLAGS(hw.adjusted_mode.flags,
8033                                               DRM_MODE_FLAG_NHSYNC);
8034                         PIPE_CONF_CHECK_FLAGS(hw.adjusted_mode.flags,
8035                                               DRM_MODE_FLAG_PVSYNC);
8036                         PIPE_CONF_CHECK_FLAGS(hw.adjusted_mode.flags,
8037                                               DRM_MODE_FLAG_NVSYNC);
8038                 }
8039         }
8040
8041         PIPE_CONF_CHECK_I(output_format);
8042         PIPE_CONF_CHECK_BOOL(has_hdmi_sink);
8043         if ((DISPLAY_VER(dev_priv) < 8 && !IS_HASWELL(dev_priv)) ||
8044             IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
8045                 PIPE_CONF_CHECK_BOOL(limited_color_range);
8046
8047         PIPE_CONF_CHECK_BOOL(hdmi_scrambling);
8048         PIPE_CONF_CHECK_BOOL(hdmi_high_tmds_clock_ratio);
8049         PIPE_CONF_CHECK_BOOL(has_infoframe);
8050         /* FIXME do the readout properly and get rid of this quirk */
8051         if (!PIPE_CONF_QUIRK(PIPE_CONFIG_QUIRK_BIGJOINER_SLAVE))
8052                 PIPE_CONF_CHECK_BOOL(fec_enable);
8053
8054         PIPE_CONF_CHECK_BOOL_INCOMPLETE(has_audio);
8055
8056         PIPE_CONF_CHECK_X(gmch_pfit.control);
8057         /* pfit ratios are autocomputed by the hw on gen4+ */
8058         if (DISPLAY_VER(dev_priv) < 4)
8059                 PIPE_CONF_CHECK_X(gmch_pfit.pgm_ratios);
8060         PIPE_CONF_CHECK_X(gmch_pfit.lvds_border_bits);
8061
8062         /*
8063          * Changing the EDP transcoder input mux
8064          * (A_ONOFF vs. A_ON) requires a full modeset.
8065          */
8066         PIPE_CONF_CHECK_BOOL(pch_pfit.force_thru);
8067
8068         if (!fastset) {
8069                 PIPE_CONF_CHECK_I(pipe_src_w);
8070                 PIPE_CONF_CHECK_I(pipe_src_h);
8071
8072                 PIPE_CONF_CHECK_BOOL(pch_pfit.enabled);
8073                 if (current_config->pch_pfit.enabled) {
8074                         PIPE_CONF_CHECK_I(pch_pfit.dst.x1);
8075                         PIPE_CONF_CHECK_I(pch_pfit.dst.y1);
8076                         PIPE_CONF_CHECK_I(pch_pfit.dst.x2);
8077                         PIPE_CONF_CHECK_I(pch_pfit.dst.y2);
8078                 }
8079
8080                 PIPE_CONF_CHECK_I(scaler_state.scaler_id);
8081                 /* FIXME do the readout properly and get rid of this quirk */
8082                 if (!PIPE_CONF_QUIRK(PIPE_CONFIG_QUIRK_BIGJOINER_SLAVE))
8083                         PIPE_CONF_CHECK_CLOCK_FUZZY(pixel_rate);
8084
8085                 PIPE_CONF_CHECK_X(gamma_mode);
8086                 if (IS_CHERRYVIEW(dev_priv))
8087                         PIPE_CONF_CHECK_X(cgm_mode);
8088                 else
8089                         PIPE_CONF_CHECK_X(csc_mode);
8090                 PIPE_CONF_CHECK_BOOL(gamma_enable);
8091                 PIPE_CONF_CHECK_BOOL(csc_enable);
8092
8093                 PIPE_CONF_CHECK_I(linetime);
8094                 PIPE_CONF_CHECK_I(ips_linetime);
8095
8096                 bp_gamma = intel_color_get_gamma_bit_precision(pipe_config);
8097                 if (bp_gamma)
8098                         PIPE_CONF_CHECK_COLOR_LUT(gamma_mode, hw.gamma_lut, bp_gamma);
8099
8100                 if (current_config->active_planes) {
8101                         PIPE_CONF_CHECK_BOOL(has_psr);
8102                         PIPE_CONF_CHECK_BOOL(has_psr2);
8103                         PIPE_CONF_CHECK_BOOL(enable_psr2_sel_fetch);
8104                         PIPE_CONF_CHECK_I(dc3co_exitline);
8105                 }
8106         }
8107
8108         PIPE_CONF_CHECK_BOOL(double_wide);
8109
8110         if (dev_priv->dpll.mgr)
8111                 PIPE_CONF_CHECK_P(shared_dpll);
8112
8113         /* FIXME do the readout properly and get rid of this quirk */
8114         if (dev_priv->dpll.mgr && !PIPE_CONF_QUIRK(PIPE_CONFIG_QUIRK_BIGJOINER_SLAVE)) {
8115                 PIPE_CONF_CHECK_X(dpll_hw_state.dpll);
8116                 PIPE_CONF_CHECK_X(dpll_hw_state.dpll_md);
8117                 PIPE_CONF_CHECK_X(dpll_hw_state.fp0);
8118                 PIPE_CONF_CHECK_X(dpll_hw_state.fp1);
8119                 PIPE_CONF_CHECK_X(dpll_hw_state.wrpll);
8120                 PIPE_CONF_CHECK_X(dpll_hw_state.spll);
8121                 PIPE_CONF_CHECK_X(dpll_hw_state.ctrl1);
8122                 PIPE_CONF_CHECK_X(dpll_hw_state.cfgcr1);
8123                 PIPE_CONF_CHECK_X(dpll_hw_state.cfgcr2);
8124                 PIPE_CONF_CHECK_X(dpll_hw_state.cfgcr0);
8125                 PIPE_CONF_CHECK_X(dpll_hw_state.ebb0);
8126                 PIPE_CONF_CHECK_X(dpll_hw_state.ebb4);
8127                 PIPE_CONF_CHECK_X(dpll_hw_state.pll0);
8128                 PIPE_CONF_CHECK_X(dpll_hw_state.pll1);
8129                 PIPE_CONF_CHECK_X(dpll_hw_state.pll2);
8130                 PIPE_CONF_CHECK_X(dpll_hw_state.pll3);
8131                 PIPE_CONF_CHECK_X(dpll_hw_state.pll6);
8132                 PIPE_CONF_CHECK_X(dpll_hw_state.pll8);
8133                 PIPE_CONF_CHECK_X(dpll_hw_state.pll9);
8134                 PIPE_CONF_CHECK_X(dpll_hw_state.pll10);
8135                 PIPE_CONF_CHECK_X(dpll_hw_state.pcsdw12);
8136                 PIPE_CONF_CHECK_X(dpll_hw_state.mg_refclkin_ctl);
8137                 PIPE_CONF_CHECK_X(dpll_hw_state.mg_clktop2_coreclkctl1);
8138                 PIPE_CONF_CHECK_X(dpll_hw_state.mg_clktop2_hsclkctl);
8139                 PIPE_CONF_CHECK_X(dpll_hw_state.mg_pll_div0);
8140                 PIPE_CONF_CHECK_X(dpll_hw_state.mg_pll_div1);
8141                 PIPE_CONF_CHECK_X(dpll_hw_state.mg_pll_lf);
8142                 PIPE_CONF_CHECK_X(dpll_hw_state.mg_pll_frac_lock);
8143                 PIPE_CONF_CHECK_X(dpll_hw_state.mg_pll_ssc);
8144                 PIPE_CONF_CHECK_X(dpll_hw_state.mg_pll_bias);
8145                 PIPE_CONF_CHECK_X(dpll_hw_state.mg_pll_tdc_coldst_bias);
8146         }
8147
8148         if (!PIPE_CONF_QUIRK(PIPE_CONFIG_QUIRK_BIGJOINER_SLAVE)) {
8149                 PIPE_CONF_CHECK_X(dsi_pll.ctrl);
8150                 PIPE_CONF_CHECK_X(dsi_pll.div);
8151
8152                 if (IS_G4X(dev_priv) || DISPLAY_VER(dev_priv) >= 5)
8153                         PIPE_CONF_CHECK_I(pipe_bpp);
8154
8155                 PIPE_CONF_CHECK_CLOCK_FUZZY(hw.pipe_mode.crtc_clock);
8156                 PIPE_CONF_CHECK_CLOCK_FUZZY(hw.adjusted_mode.crtc_clock);
8157                 PIPE_CONF_CHECK_CLOCK_FUZZY(port_clock);
8158
8159                 PIPE_CONF_CHECK_I(min_voltage_level);
8160         }
8161
8162         if (current_config->has_psr || pipe_config->has_psr)
8163                 PIPE_CONF_CHECK_X_WITH_MASK(infoframes.enable,
8164                                             ~intel_hdmi_infoframe_enable(DP_SDP_VSC));
8165         else
8166                 PIPE_CONF_CHECK_X(infoframes.enable);
8167
8168         PIPE_CONF_CHECK_X(infoframes.gcp);
8169         PIPE_CONF_CHECK_INFOFRAME(avi);
8170         PIPE_CONF_CHECK_INFOFRAME(spd);
8171         PIPE_CONF_CHECK_INFOFRAME(hdmi);
8172         PIPE_CONF_CHECK_INFOFRAME(drm);
8173         PIPE_CONF_CHECK_DP_VSC_SDP(vsc);
8174
8175         PIPE_CONF_CHECK_X(sync_mode_slaves_mask);
8176         PIPE_CONF_CHECK_I(master_transcoder);
8177         PIPE_CONF_CHECK_BOOL(bigjoiner);
8178         PIPE_CONF_CHECK_BOOL(bigjoiner_slave);
8179         PIPE_CONF_CHECK_P(bigjoiner_linked_crtc);
8180
8181         PIPE_CONF_CHECK_I(dsc.compression_enable);
8182         PIPE_CONF_CHECK_I(dsc.dsc_split);
8183         PIPE_CONF_CHECK_I(dsc.compressed_bpp);
8184
8185         PIPE_CONF_CHECK_BOOL(splitter.enable);
8186         PIPE_CONF_CHECK_I(splitter.link_count);
8187         PIPE_CONF_CHECK_I(splitter.pixel_overlap);
8188
8189         PIPE_CONF_CHECK_I(mst_master_transcoder);
8190
8191         PIPE_CONF_CHECK_BOOL(vrr.enable);
8192         PIPE_CONF_CHECK_I(vrr.vmin);
8193         PIPE_CONF_CHECK_I(vrr.vmax);
8194         PIPE_CONF_CHECK_I(vrr.flipline);
8195         PIPE_CONF_CHECK_I(vrr.pipeline_full);
8196         PIPE_CONF_CHECK_I(vrr.guardband);
8197
8198 #undef PIPE_CONF_CHECK_X
8199 #undef PIPE_CONF_CHECK_I
8200 #undef PIPE_CONF_CHECK_BOOL
8201 #undef PIPE_CONF_CHECK_BOOL_INCOMPLETE
8202 #undef PIPE_CONF_CHECK_P
8203 #undef PIPE_CONF_CHECK_FLAGS
8204 #undef PIPE_CONF_CHECK_CLOCK_FUZZY
8205 #undef PIPE_CONF_CHECK_COLOR_LUT
8206 #undef PIPE_CONF_QUIRK
8207
8208         return ret;
8209 }
8210
8211 static void intel_pipe_config_sanity_check(struct drm_i915_private *dev_priv,
8212                                            const struct intel_crtc_state *pipe_config)
8213 {
8214         if (pipe_config->has_pch_encoder) {
8215                 int fdi_dotclock = intel_dotclock_calculate(intel_fdi_link_freq(dev_priv, pipe_config),
8216                                                             &pipe_config->fdi_m_n);
8217                 int dotclock = pipe_config->hw.adjusted_mode.crtc_clock;
8218
8219                 /*
8220                  * FDI already provided one idea for the dotclock.
8221                  * Yell if the encoder disagrees.
8222                  */
8223                 drm_WARN(&dev_priv->drm,
8224                          !intel_fuzzy_clock_check(fdi_dotclock, dotclock),
8225                          "FDI dotclock and encoder dotclock mismatch, fdi: %i, encoder: %i\n",
8226                          fdi_dotclock, dotclock);
8227         }
8228 }
8229
8230 static void verify_wm_state(struct intel_crtc *crtc,
8231                             struct intel_crtc_state *new_crtc_state)
8232 {
8233         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
8234         struct skl_hw_state {
8235                 struct skl_ddb_entry ddb_y[I915_MAX_PLANES];
8236                 struct skl_ddb_entry ddb_uv[I915_MAX_PLANES];
8237                 struct skl_pipe_wm wm;
8238         } *hw;
8239         const struct skl_pipe_wm *sw_wm = &new_crtc_state->wm.skl.optimal;
8240         int level, max_level = ilk_wm_max_level(dev_priv);
8241         struct intel_plane *plane;
8242         u8 hw_enabled_slices;
8243
8244         if (DISPLAY_VER(dev_priv) < 9 || !new_crtc_state->hw.active)
8245                 return;
8246
8247         hw = kzalloc(sizeof(*hw), GFP_KERNEL);
8248         if (!hw)
8249                 return;
8250
8251         skl_pipe_wm_get_hw_state(crtc, &hw->wm);
8252
8253         skl_pipe_ddb_get_hw_state(crtc, hw->ddb_y, hw->ddb_uv);
8254
8255         hw_enabled_slices = intel_enabled_dbuf_slices_mask(dev_priv);
8256
8257         if (DISPLAY_VER(dev_priv) >= 11 &&
8258             hw_enabled_slices != dev_priv->dbuf.enabled_slices)
8259                 drm_err(&dev_priv->drm,
8260                         "mismatch in DBUF Slices (expected 0x%x, got 0x%x)\n",
8261                         dev_priv->dbuf.enabled_slices,
8262                         hw_enabled_slices);
8263
8264         for_each_intel_plane_on_crtc(&dev_priv->drm, crtc, plane) {
8265                 const struct skl_ddb_entry *hw_ddb_entry, *sw_ddb_entry;
8266                 const struct skl_wm_level *hw_wm_level, *sw_wm_level;
8267
8268                 /* Watermarks */
8269                 for (level = 0; level <= max_level; level++) {
8270                         hw_wm_level = &hw->wm.planes[plane->id].wm[level];
8271                         sw_wm_level = skl_plane_wm_level(sw_wm, plane->id, level);
8272
8273                         if (skl_wm_level_equals(hw_wm_level, sw_wm_level))
8274                                 continue;
8275
8276                         drm_err(&dev_priv->drm,
8277                                 "[PLANE:%d:%s] mismatch in WM%d (expected e=%d b=%u l=%u, got e=%d b=%u l=%u)\n",
8278                                 plane->base.base.id, plane->base.name, level,
8279                                 sw_wm_level->enable,
8280                                 sw_wm_level->blocks,
8281                                 sw_wm_level->lines,
8282                                 hw_wm_level->enable,
8283                                 hw_wm_level->blocks,
8284                                 hw_wm_level->lines);
8285                 }
8286
8287                 hw_wm_level = &hw->wm.planes[plane->id].trans_wm;
8288                 sw_wm_level = skl_plane_trans_wm(sw_wm, plane->id);
8289
8290                 if (!skl_wm_level_equals(hw_wm_level, sw_wm_level)) {
8291                         drm_err(&dev_priv->drm,
8292                                 "[PLANE:%d:%s] mismatch in trans WM (expected e=%d b=%u l=%u, got e=%d b=%u l=%u)\n",
8293                                 plane->base.base.id, plane->base.name,
8294                                 sw_wm_level->enable,
8295                                 sw_wm_level->blocks,
8296                                 sw_wm_level->lines,
8297                                 hw_wm_level->enable,
8298                                 hw_wm_level->blocks,
8299                                 hw_wm_level->lines);
8300                 }
8301
8302                 hw_wm_level = &hw->wm.planes[plane->id].sagv.wm0;
8303                 sw_wm_level = &sw_wm->planes[plane->id].sagv.wm0;
8304
8305                 if (HAS_HW_SAGV_WM(dev_priv) &&
8306                     !skl_wm_level_equals(hw_wm_level, sw_wm_level)) {
8307                         drm_err(&dev_priv->drm,
8308                                 "[PLANE:%d:%s] mismatch in SAGV WM (expected e=%d b=%u l=%u, got e=%d b=%u l=%u)\n",
8309                                 plane->base.base.id, plane->base.name,
8310                                 sw_wm_level->enable,
8311                                 sw_wm_level->blocks,
8312                                 sw_wm_level->lines,
8313                                 hw_wm_level->enable,
8314                                 hw_wm_level->blocks,
8315                                 hw_wm_level->lines);
8316                 }
8317
8318                 hw_wm_level = &hw->wm.planes[plane->id].sagv.trans_wm;
8319                 sw_wm_level = &sw_wm->planes[plane->id].sagv.trans_wm;
8320
8321                 if (HAS_HW_SAGV_WM(dev_priv) &&
8322                     !skl_wm_level_equals(hw_wm_level, sw_wm_level)) {
8323                         drm_err(&dev_priv->drm,
8324                                 "[PLANE:%d:%s] mismatch in SAGV trans WM (expected e=%d b=%u l=%u, got e=%d b=%u l=%u)\n",
8325                                 plane->base.base.id, plane->base.name,
8326                                 sw_wm_level->enable,
8327                                 sw_wm_level->blocks,
8328                                 sw_wm_level->lines,
8329                                 hw_wm_level->enable,
8330                                 hw_wm_level->blocks,
8331                                 hw_wm_level->lines);
8332                 }
8333
8334                 /* DDB */
8335                 hw_ddb_entry = &hw->ddb_y[plane->id];
8336                 sw_ddb_entry = &new_crtc_state->wm.skl.plane_ddb_y[plane->id];
8337
8338                 if (!skl_ddb_entry_equal(hw_ddb_entry, sw_ddb_entry)) {
8339                         drm_err(&dev_priv->drm,
8340                                 "[PLANE:%d:%s] mismatch in DDB (expected (%u,%u), found (%u,%u))\n",
8341                                 plane->base.base.id, plane->base.name,
8342                                 sw_ddb_entry->start, sw_ddb_entry->end,
8343                                 hw_ddb_entry->start, hw_ddb_entry->end);
8344                 }
8345         }
8346
8347         kfree(hw);
8348 }
8349
8350 static void
8351 verify_connector_state(struct intel_atomic_state *state,
8352                        struct intel_crtc *crtc)
8353 {
8354         struct drm_connector *connector;
8355         struct drm_connector_state *new_conn_state;
8356         int i;
8357
8358         for_each_new_connector_in_state(&state->base, connector, new_conn_state, i) {
8359                 struct drm_encoder *encoder = connector->encoder;
8360                 struct intel_crtc_state *crtc_state = NULL;
8361
8362                 if (new_conn_state->crtc != &crtc->base)
8363                         continue;
8364
8365                 if (crtc)
8366                         crtc_state = intel_atomic_get_new_crtc_state(state, crtc);
8367
8368                 intel_connector_verify_state(crtc_state, new_conn_state);
8369
8370                 I915_STATE_WARN(new_conn_state->best_encoder != encoder,
8371                      "connector's atomic encoder doesn't match legacy encoder\n");
8372         }
8373 }
8374
8375 static void
8376 verify_encoder_state(struct drm_i915_private *dev_priv, struct intel_atomic_state *state)
8377 {
8378         struct intel_encoder *encoder;
8379         struct drm_connector *connector;
8380         struct drm_connector_state *old_conn_state, *new_conn_state;
8381         int i;
8382
8383         for_each_intel_encoder(&dev_priv->drm, encoder) {
8384                 bool enabled = false, found = false;
8385                 enum pipe pipe;
8386
8387                 drm_dbg_kms(&dev_priv->drm, "[ENCODER:%d:%s]\n",
8388                             encoder->base.base.id,
8389                             encoder->base.name);
8390
8391                 for_each_oldnew_connector_in_state(&state->base, connector, old_conn_state,
8392                                                    new_conn_state, i) {
8393                         if (old_conn_state->best_encoder == &encoder->base)
8394                                 found = true;
8395
8396                         if (new_conn_state->best_encoder != &encoder->base)
8397                                 continue;
8398                         found = enabled = true;
8399
8400                         I915_STATE_WARN(new_conn_state->crtc !=
8401                                         encoder->base.crtc,
8402                              "connector's crtc doesn't match encoder crtc\n");
8403                 }
8404
8405                 if (!found)
8406                         continue;
8407
8408                 I915_STATE_WARN(!!encoder->base.crtc != enabled,
8409                      "encoder's enabled state mismatch "
8410                      "(expected %i, found %i)\n",
8411                      !!encoder->base.crtc, enabled);
8412
8413                 if (!encoder->base.crtc) {
8414                         bool active;
8415
8416                         active = encoder->get_hw_state(encoder, &pipe);
8417                         I915_STATE_WARN(active,
8418                              "encoder detached but still enabled on pipe %c.\n",
8419                              pipe_name(pipe));
8420                 }
8421         }
8422 }
8423
8424 static void
8425 verify_crtc_state(struct intel_crtc *crtc,
8426                   struct intel_crtc_state *old_crtc_state,
8427                   struct intel_crtc_state *new_crtc_state)
8428 {
8429         struct drm_device *dev = crtc->base.dev;
8430         struct drm_i915_private *dev_priv = to_i915(dev);
8431         struct intel_encoder *encoder;
8432         struct intel_crtc_state *pipe_config = old_crtc_state;
8433         struct drm_atomic_state *state = old_crtc_state->uapi.state;
8434         struct intel_crtc *master = crtc;
8435
8436         __drm_atomic_helper_crtc_destroy_state(&old_crtc_state->uapi);
8437         intel_crtc_free_hw_state(old_crtc_state);
8438         intel_crtc_state_reset(old_crtc_state, crtc);
8439         old_crtc_state->uapi.state = state;
8440
8441         drm_dbg_kms(&dev_priv->drm, "[CRTC:%d:%s]\n", crtc->base.base.id,
8442                     crtc->base.name);
8443
8444         pipe_config->hw.enable = new_crtc_state->hw.enable;
8445
8446         intel_crtc_get_pipe_config(pipe_config);
8447
8448         /* we keep both pipes enabled on 830 */
8449         if (IS_I830(dev_priv) && pipe_config->hw.active)
8450                 pipe_config->hw.active = new_crtc_state->hw.active;
8451
8452         I915_STATE_WARN(new_crtc_state->hw.active != pipe_config->hw.active,
8453                         "crtc active state doesn't match with hw state "
8454                         "(expected %i, found %i)\n",
8455                         new_crtc_state->hw.active, pipe_config->hw.active);
8456
8457         I915_STATE_WARN(crtc->active != new_crtc_state->hw.active,
8458                         "transitional active state does not match atomic hw state "
8459                         "(expected %i, found %i)\n",
8460                         new_crtc_state->hw.active, crtc->active);
8461
8462         if (new_crtc_state->bigjoiner_slave)
8463                 master = new_crtc_state->bigjoiner_linked_crtc;
8464
8465         for_each_encoder_on_crtc(dev, &master->base, encoder) {
8466                 enum pipe pipe;
8467                 bool active;
8468
8469                 active = encoder->get_hw_state(encoder, &pipe);
8470                 I915_STATE_WARN(active != new_crtc_state->hw.active,
8471                                 "[ENCODER:%i] active %i with crtc active %i\n",
8472                                 encoder->base.base.id, active,
8473                                 new_crtc_state->hw.active);
8474
8475                 I915_STATE_WARN(active && master->pipe != pipe,
8476                                 "Encoder connected to wrong pipe %c\n",
8477                                 pipe_name(pipe));
8478
8479                 if (active)
8480                         intel_encoder_get_config(encoder, pipe_config);
8481         }
8482
8483         if (!new_crtc_state->hw.active)
8484                 return;
8485
8486         if (new_crtc_state->bigjoiner_slave)
8487                 /* No PLLs set for slave */
8488                 pipe_config->shared_dpll = NULL;
8489
8490         intel_pipe_config_sanity_check(dev_priv, pipe_config);
8491
8492         if (!intel_pipe_config_compare(new_crtc_state,
8493                                        pipe_config, false)) {
8494                 I915_STATE_WARN(1, "pipe state doesn't match!\n");
8495                 intel_dump_pipe_config(pipe_config, NULL, "[hw state]");
8496                 intel_dump_pipe_config(new_crtc_state, NULL, "[sw state]");
8497         }
8498 }
8499
8500 static void
8501 intel_verify_planes(struct intel_atomic_state *state)
8502 {
8503         struct intel_plane *plane;
8504         const struct intel_plane_state *plane_state;
8505         int i;
8506
8507         for_each_new_intel_plane_in_state(state, plane,
8508                                           plane_state, i)
8509                 assert_plane(plane, plane_state->planar_slave ||
8510                              plane_state->uapi.visible);
8511 }
8512
8513 static void
8514 verify_single_dpll_state(struct drm_i915_private *dev_priv,
8515                          struct intel_shared_dpll *pll,
8516                          struct intel_crtc *crtc,
8517                          struct intel_crtc_state *new_crtc_state)
8518 {
8519         struct intel_dpll_hw_state dpll_hw_state;
8520         u8 pipe_mask;
8521         bool active;
8522
8523         memset(&dpll_hw_state, 0, sizeof(dpll_hw_state));
8524
8525         drm_dbg_kms(&dev_priv->drm, "%s\n", pll->info->name);
8526
8527         active = intel_dpll_get_hw_state(dev_priv, pll, &dpll_hw_state);
8528
8529         if (!(pll->info->flags & INTEL_DPLL_ALWAYS_ON)) {
8530                 I915_STATE_WARN(!pll->on && pll->active_mask,
8531                      "pll in active use but not on in sw tracking\n");
8532                 I915_STATE_WARN(pll->on && !pll->active_mask,
8533                      "pll is on but not used by any active pipe\n");
8534                 I915_STATE_WARN(pll->on != active,
8535                      "pll on state mismatch (expected %i, found %i)\n",
8536                      pll->on, active);
8537         }
8538
8539         if (!crtc) {
8540                 I915_STATE_WARN(pll->active_mask & ~pll->state.pipe_mask,
8541                                 "more active pll users than references: 0x%x vs 0x%x\n",
8542                                 pll->active_mask, pll->state.pipe_mask);
8543
8544                 return;
8545         }
8546
8547         pipe_mask = BIT(crtc->pipe);
8548
8549         if (new_crtc_state->hw.active)
8550                 I915_STATE_WARN(!(pll->active_mask & pipe_mask),
8551                                 "pll active mismatch (expected pipe %c in active mask 0x%x)\n",
8552                                 pipe_name(crtc->pipe), pll->active_mask);
8553         else
8554                 I915_STATE_WARN(pll->active_mask & pipe_mask,
8555                                 "pll active mismatch (didn't expect pipe %c in active mask 0x%x)\n",
8556                                 pipe_name(crtc->pipe), pll->active_mask);
8557
8558         I915_STATE_WARN(!(pll->state.pipe_mask & pipe_mask),
8559                         "pll enabled crtcs mismatch (expected 0x%x in 0x%x)\n",
8560                         pipe_mask, pll->state.pipe_mask);
8561
8562         I915_STATE_WARN(pll->on && memcmp(&pll->state.hw_state,
8563                                           &dpll_hw_state,
8564                                           sizeof(dpll_hw_state)),
8565                         "pll hw state mismatch\n");
8566 }
8567
8568 static void
8569 verify_shared_dpll_state(struct intel_crtc *crtc,
8570                          struct intel_crtc_state *old_crtc_state,
8571                          struct intel_crtc_state *new_crtc_state)
8572 {
8573         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
8574
8575         if (new_crtc_state->shared_dpll)
8576                 verify_single_dpll_state(dev_priv, new_crtc_state->shared_dpll, crtc, new_crtc_state);
8577
8578         if (old_crtc_state->shared_dpll &&
8579             old_crtc_state->shared_dpll != new_crtc_state->shared_dpll) {
8580                 u8 pipe_mask = BIT(crtc->pipe);
8581                 struct intel_shared_dpll *pll = old_crtc_state->shared_dpll;
8582
8583                 I915_STATE_WARN(pll->active_mask & pipe_mask,
8584                                 "pll active mismatch (didn't expect pipe %c in active mask (0x%x))\n",
8585                                 pipe_name(crtc->pipe), pll->active_mask);
8586                 I915_STATE_WARN(pll->state.pipe_mask & pipe_mask,
8587                                 "pll enabled crtcs mismatch (found %x in enabled mask (0x%x))\n",
8588                                 pipe_name(crtc->pipe), pll->state.pipe_mask);
8589         }
8590 }
8591
8592 static void
8593 verify_mpllb_state(struct intel_atomic_state *state,
8594                    struct intel_crtc_state *new_crtc_state)
8595 {
8596         struct drm_i915_private *i915 = to_i915(state->base.dev);
8597         struct intel_mpllb_state mpllb_hw_state = { 0 };
8598         struct intel_mpllb_state *mpllb_sw_state = &new_crtc_state->mpllb_state;
8599         struct intel_crtc *crtc = to_intel_crtc(new_crtc_state->uapi.crtc);
8600         struct intel_encoder *encoder;
8601
8602         if (!IS_DG2(i915))
8603                 return;
8604
8605         if (!new_crtc_state->hw.active)
8606                 return;
8607
8608         if (new_crtc_state->bigjoiner_slave)
8609                 return;
8610
8611         encoder = intel_get_crtc_new_encoder(state, new_crtc_state);
8612         intel_mpllb_readout_hw_state(encoder, &mpllb_hw_state);
8613
8614 #define MPLLB_CHECK(name) do { \
8615         if (mpllb_sw_state->name != mpllb_hw_state.name) { \
8616                 pipe_config_mismatch(false, crtc, "MPLLB:" __stringify(name), \
8617                                      "(expected 0x%08x, found 0x%08x)", \
8618                                      mpllb_sw_state->name, \
8619                                      mpllb_hw_state.name); \
8620         } \
8621 } while (0)
8622
8623         MPLLB_CHECK(mpllb_cp);
8624         MPLLB_CHECK(mpllb_div);
8625         MPLLB_CHECK(mpllb_div2);
8626         MPLLB_CHECK(mpllb_fracn1);
8627         MPLLB_CHECK(mpllb_fracn2);
8628         MPLLB_CHECK(mpllb_sscen);
8629         MPLLB_CHECK(mpllb_sscstep);
8630
8631         /*
8632          * ref_control is handled by the hardware/firemware and never
8633          * programmed by the software, but the proper values are supplied
8634          * in the bspec for verification purposes.
8635          */
8636         MPLLB_CHECK(ref_control);
8637
8638 #undef MPLLB_CHECK
8639 }
8640
8641 static void
8642 intel_modeset_verify_crtc(struct intel_crtc *crtc,
8643                           struct intel_atomic_state *state,
8644                           struct intel_crtc_state *old_crtc_state,
8645                           struct intel_crtc_state *new_crtc_state)
8646 {
8647         if (!intel_crtc_needs_modeset(new_crtc_state) && !new_crtc_state->update_pipe)
8648                 return;
8649
8650         verify_wm_state(crtc, new_crtc_state);
8651         verify_connector_state(state, crtc);
8652         verify_crtc_state(crtc, old_crtc_state, new_crtc_state);
8653         verify_shared_dpll_state(crtc, old_crtc_state, new_crtc_state);
8654         verify_mpllb_state(state, new_crtc_state);
8655 }
8656
8657 static void
8658 verify_disabled_dpll_state(struct drm_i915_private *dev_priv)
8659 {
8660         int i;
8661
8662         for (i = 0; i < dev_priv->dpll.num_shared_dpll; i++)
8663                 verify_single_dpll_state(dev_priv,
8664                                          &dev_priv->dpll.shared_dplls[i],
8665                                          NULL, NULL);
8666 }
8667
8668 static void
8669 intel_modeset_verify_disabled(struct drm_i915_private *dev_priv,
8670                               struct intel_atomic_state *state)
8671 {
8672         verify_encoder_state(dev_priv, state);
8673         verify_connector_state(state, NULL);
8674         verify_disabled_dpll_state(dev_priv);
8675 }
8676
8677 int intel_modeset_all_pipes(struct intel_atomic_state *state)
8678 {
8679         struct drm_i915_private *dev_priv = to_i915(state->base.dev);
8680         struct intel_crtc *crtc;
8681
8682         /*
8683          * Add all pipes to the state, and force
8684          * a modeset on all the active ones.
8685          */
8686         for_each_intel_crtc(&dev_priv->drm, crtc) {
8687                 struct intel_crtc_state *crtc_state;
8688                 int ret;
8689
8690                 crtc_state = intel_atomic_get_crtc_state(&state->base, crtc);
8691                 if (IS_ERR(crtc_state))
8692                         return PTR_ERR(crtc_state);
8693
8694                 if (!crtc_state->hw.active ||
8695                     drm_atomic_crtc_needs_modeset(&crtc_state->uapi))
8696                         continue;
8697
8698                 crtc_state->uapi.mode_changed = true;
8699
8700                 ret = drm_atomic_add_affected_connectors(&state->base,
8701                                                          &crtc->base);
8702                 if (ret)
8703                         return ret;
8704
8705                 ret = intel_atomic_add_affected_planes(state, crtc);
8706                 if (ret)
8707                         return ret;
8708
8709                 crtc_state->update_planes |= crtc_state->active_planes;
8710         }
8711
8712         return 0;
8713 }
8714
8715 static void
8716 intel_crtc_update_active_timings(const struct intel_crtc_state *crtc_state)
8717 {
8718         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
8719         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
8720         struct drm_display_mode adjusted_mode =
8721                 crtc_state->hw.adjusted_mode;
8722
8723         if (crtc_state->vrr.enable) {
8724                 adjusted_mode.crtc_vtotal = crtc_state->vrr.vmax;
8725                 adjusted_mode.crtc_vblank_end = crtc_state->vrr.vmax;
8726                 adjusted_mode.crtc_vblank_start = intel_vrr_vmin_vblank_start(crtc_state);
8727                 crtc->vmax_vblank_start = intel_vrr_vmax_vblank_start(crtc_state);
8728         }
8729
8730         drm_calc_timestamping_constants(&crtc->base, &adjusted_mode);
8731
8732         crtc->mode_flags = crtc_state->mode_flags;
8733
8734         /*
8735          * The scanline counter increments at the leading edge of hsync.
8736          *
8737          * On most platforms it starts counting from vtotal-1 on the
8738          * first active line. That means the scanline counter value is
8739          * always one less than what we would expect. Ie. just after
8740          * start of vblank, which also occurs at start of hsync (on the
8741          * last active line), the scanline counter will read vblank_start-1.
8742          *
8743          * On gen2 the scanline counter starts counting from 1 instead
8744          * of vtotal-1, so we have to subtract one (or rather add vtotal-1
8745          * to keep the value positive), instead of adding one.
8746          *
8747          * On HSW+ the behaviour of the scanline counter depends on the output
8748          * type. For DP ports it behaves like most other platforms, but on HDMI
8749          * there's an extra 1 line difference. So we need to add two instead of
8750          * one to the value.
8751          *
8752          * On VLV/CHV DSI the scanline counter would appear to increment
8753          * approx. 1/3 of a scanline before start of vblank. Unfortunately
8754          * that means we can't tell whether we're in vblank or not while
8755          * we're on that particular line. We must still set scanline_offset
8756          * to 1 so that the vblank timestamps come out correct when we query
8757          * the scanline counter from within the vblank interrupt handler.
8758          * However if queried just before the start of vblank we'll get an
8759          * answer that's slightly in the future.
8760          */
8761         if (DISPLAY_VER(dev_priv) == 2) {
8762                 int vtotal;
8763
8764                 vtotal = adjusted_mode.crtc_vtotal;
8765                 if (adjusted_mode.flags & DRM_MODE_FLAG_INTERLACE)
8766                         vtotal /= 2;
8767
8768                 crtc->scanline_offset = vtotal - 1;
8769         } else if (HAS_DDI(dev_priv) &&
8770                    intel_crtc_has_type(crtc_state, INTEL_OUTPUT_HDMI)) {
8771                 crtc->scanline_offset = 2;
8772         } else {
8773                 crtc->scanline_offset = 1;
8774         }
8775 }
8776
8777 static void intel_modeset_clear_plls(struct intel_atomic_state *state)
8778 {
8779         struct drm_i915_private *dev_priv = to_i915(state->base.dev);
8780         struct intel_crtc_state *new_crtc_state;
8781         struct intel_crtc *crtc;
8782         int i;
8783
8784         if (!dev_priv->dpll_funcs)
8785                 return;
8786
8787         for_each_new_intel_crtc_in_state(state, crtc, new_crtc_state, i) {
8788                 if (!intel_crtc_needs_modeset(new_crtc_state))
8789                         continue;
8790
8791                 intel_release_shared_dplls(state, crtc);
8792         }
8793 }
8794
8795 /*
8796  * This implements the workaround described in the "notes" section of the mode
8797  * set sequence documentation. When going from no pipes or single pipe to
8798  * multiple pipes, and planes are enabled after the pipe, we need to wait at
8799  * least 2 vblanks on the first pipe before enabling planes on the second pipe.
8800  */
8801 static int hsw_mode_set_planes_workaround(struct intel_atomic_state *state)
8802 {
8803         struct intel_crtc_state *crtc_state;
8804         struct intel_crtc *crtc;
8805         struct intel_crtc_state *first_crtc_state = NULL;
8806         struct intel_crtc_state *other_crtc_state = NULL;
8807         enum pipe first_pipe = INVALID_PIPE, enabled_pipe = INVALID_PIPE;
8808         int i;
8809
8810         /* look at all crtc's that are going to be enabled in during modeset */
8811         for_each_new_intel_crtc_in_state(state, crtc, crtc_state, i) {
8812                 if (!crtc_state->hw.active ||
8813                     !intel_crtc_needs_modeset(crtc_state))
8814                         continue;
8815
8816                 if (first_crtc_state) {
8817                         other_crtc_state = crtc_state;
8818                         break;
8819                 } else {
8820                         first_crtc_state = crtc_state;
8821                         first_pipe = crtc->pipe;
8822                 }
8823         }
8824
8825         /* No workaround needed? */
8826         if (!first_crtc_state)
8827                 return 0;
8828
8829         /* w/a possibly needed, check how many crtc's are already enabled. */
8830         for_each_intel_crtc(state->base.dev, crtc) {
8831                 crtc_state = intel_atomic_get_crtc_state(&state->base, crtc);
8832                 if (IS_ERR(crtc_state))
8833                         return PTR_ERR(crtc_state);
8834
8835                 crtc_state->hsw_workaround_pipe = INVALID_PIPE;
8836
8837                 if (!crtc_state->hw.active ||
8838                     intel_crtc_needs_modeset(crtc_state))
8839                         continue;
8840
8841                 /* 2 or more enabled crtcs means no need for w/a */
8842                 if (enabled_pipe != INVALID_PIPE)
8843                         return 0;
8844
8845                 enabled_pipe = crtc->pipe;
8846         }
8847
8848         if (enabled_pipe != INVALID_PIPE)
8849                 first_crtc_state->hsw_workaround_pipe = enabled_pipe;
8850         else if (other_crtc_state)
8851                 other_crtc_state->hsw_workaround_pipe = first_pipe;
8852
8853         return 0;
8854 }
8855
8856 u8 intel_calc_active_pipes(struct intel_atomic_state *state,
8857                            u8 active_pipes)
8858 {
8859         const struct intel_crtc_state *crtc_state;
8860         struct intel_crtc *crtc;
8861         int i;
8862
8863         for_each_new_intel_crtc_in_state(state, crtc, crtc_state, i) {
8864                 if (crtc_state->hw.active)
8865                         active_pipes |= BIT(crtc->pipe);
8866                 else
8867                         active_pipes &= ~BIT(crtc->pipe);
8868         }
8869
8870         return active_pipes;
8871 }
8872
8873 static int intel_modeset_checks(struct intel_atomic_state *state)
8874 {
8875         struct drm_i915_private *dev_priv = to_i915(state->base.dev);
8876
8877         state->modeset = true;
8878
8879         if (IS_HASWELL(dev_priv))
8880                 return hsw_mode_set_planes_workaround(state);
8881
8882         return 0;
8883 }
8884
8885 static void intel_crtc_check_fastset(const struct intel_crtc_state *old_crtc_state,
8886                                      struct intel_crtc_state *new_crtc_state)
8887 {
8888         if (!intel_pipe_config_compare(old_crtc_state, new_crtc_state, true))
8889                 return;
8890
8891         new_crtc_state->uapi.mode_changed = false;
8892         new_crtc_state->update_pipe = true;
8893 }
8894
8895 static void intel_crtc_copy_fastset(const struct intel_crtc_state *old_crtc_state,
8896                                     struct intel_crtc_state *new_crtc_state)
8897 {
8898         /*
8899          * If we're not doing the full modeset we want to
8900          * keep the current M/N values as they may be
8901          * sufficiently different to the computed values
8902          * to cause problems.
8903          *
8904          * FIXME: should really copy more fuzzy state here
8905          */
8906         new_crtc_state->fdi_m_n = old_crtc_state->fdi_m_n;
8907         new_crtc_state->dp_m_n = old_crtc_state->dp_m_n;
8908         new_crtc_state->dp_m2_n2 = old_crtc_state->dp_m2_n2;
8909         new_crtc_state->has_drrs = old_crtc_state->has_drrs;
8910 }
8911
8912 static int intel_crtc_add_planes_to_state(struct intel_atomic_state *state,
8913                                           struct intel_crtc *crtc,
8914                                           u8 plane_ids_mask)
8915 {
8916         struct drm_i915_private *dev_priv = to_i915(state->base.dev);
8917         struct intel_plane *plane;
8918
8919         for_each_intel_plane_on_crtc(&dev_priv->drm, crtc, plane) {
8920                 struct intel_plane_state *plane_state;
8921
8922                 if ((plane_ids_mask & BIT(plane->id)) == 0)
8923                         continue;
8924
8925                 plane_state = intel_atomic_get_plane_state(state, plane);
8926                 if (IS_ERR(plane_state))
8927                         return PTR_ERR(plane_state);
8928         }
8929
8930         return 0;
8931 }
8932
8933 int intel_atomic_add_affected_planes(struct intel_atomic_state *state,
8934                                      struct intel_crtc *crtc)
8935 {
8936         const struct intel_crtc_state *old_crtc_state =
8937                 intel_atomic_get_old_crtc_state(state, crtc);
8938         const struct intel_crtc_state *new_crtc_state =
8939                 intel_atomic_get_new_crtc_state(state, crtc);
8940
8941         return intel_crtc_add_planes_to_state(state, crtc,
8942                                               old_crtc_state->enabled_planes |
8943                                               new_crtc_state->enabled_planes);
8944 }
8945
8946 static bool active_planes_affects_min_cdclk(struct drm_i915_private *dev_priv)
8947 {
8948         /* See {hsw,vlv,ivb}_plane_ratio() */
8949         return IS_BROADWELL(dev_priv) || IS_HASWELL(dev_priv) ||
8950                 IS_CHERRYVIEW(dev_priv) || IS_VALLEYVIEW(dev_priv) ||
8951                 IS_IVYBRIDGE(dev_priv);
8952 }
8953
8954 static int intel_crtc_add_bigjoiner_planes(struct intel_atomic_state *state,
8955                                            struct intel_crtc *crtc,
8956                                            struct intel_crtc *other)
8957 {
8958         const struct intel_plane_state *plane_state;
8959         struct intel_plane *plane;
8960         u8 plane_ids = 0;
8961         int i;
8962
8963         for_each_new_intel_plane_in_state(state, plane, plane_state, i) {
8964                 if (plane->pipe == crtc->pipe)
8965                         plane_ids |= BIT(plane->id);
8966         }
8967
8968         return intel_crtc_add_planes_to_state(state, other, plane_ids);
8969 }
8970
8971 static int intel_bigjoiner_add_affected_planes(struct intel_atomic_state *state)
8972 {
8973         const struct intel_crtc_state *crtc_state;
8974         struct intel_crtc *crtc;
8975         int i;
8976
8977         for_each_new_intel_crtc_in_state(state, crtc, crtc_state, i) {
8978                 int ret;
8979
8980                 if (!crtc_state->bigjoiner)
8981                         continue;
8982
8983                 ret = intel_crtc_add_bigjoiner_planes(state, crtc,
8984                                                       crtc_state->bigjoiner_linked_crtc);
8985                 if (ret)
8986                         return ret;
8987         }
8988
8989         return 0;
8990 }
8991
8992 static bool bo_has_valid_encryption(struct drm_i915_gem_object *obj)
8993 {
8994         struct drm_i915_private *i915 = to_i915(obj->base.dev);
8995
8996         return intel_pxp_key_check(&i915->gt.pxp, obj, false) == 0;
8997 }
8998
8999 static bool pxp_is_borked(struct drm_i915_gem_object *obj)
9000 {
9001         return i915_gem_object_is_protected(obj) && !bo_has_valid_encryption(obj);
9002 }
9003
9004 static int intel_atomic_check_planes(struct intel_atomic_state *state)
9005 {
9006         struct drm_i915_private *dev_priv = to_i915(state->base.dev);
9007         struct intel_crtc_state *old_crtc_state, *new_crtc_state;
9008         struct intel_plane_state *plane_state;
9009         struct intel_plane *plane;
9010         struct intel_plane_state *new_plane_state;
9011         struct intel_plane_state *old_plane_state;
9012         struct intel_crtc *crtc;
9013         const struct drm_framebuffer *fb;
9014         int i, ret;
9015
9016         ret = icl_add_linked_planes(state);
9017         if (ret)
9018                 return ret;
9019
9020         ret = intel_bigjoiner_add_affected_planes(state);
9021         if (ret)
9022                 return ret;
9023
9024         for_each_new_intel_plane_in_state(state, plane, plane_state, i) {
9025                 ret = intel_plane_atomic_check(state, plane);
9026                 if (ret) {
9027                         drm_dbg_atomic(&dev_priv->drm,
9028                                        "[PLANE:%d:%s] atomic driver check failed\n",
9029                                        plane->base.base.id, plane->base.name);
9030                         return ret;
9031                 }
9032         }
9033
9034         for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state,
9035                                             new_crtc_state, i) {
9036                 u8 old_active_planes, new_active_planes;
9037
9038                 ret = icl_check_nv12_planes(new_crtc_state);
9039                 if (ret)
9040                         return ret;
9041
9042                 /*
9043                  * On some platforms the number of active planes affects
9044                  * the planes' minimum cdclk calculation. Add such planes
9045                  * to the state before we compute the minimum cdclk.
9046                  */
9047                 if (!active_planes_affects_min_cdclk(dev_priv))
9048                         continue;
9049
9050                 old_active_planes = old_crtc_state->active_planes & ~BIT(PLANE_CURSOR);
9051                 new_active_planes = new_crtc_state->active_planes & ~BIT(PLANE_CURSOR);
9052
9053                 if (hweight8(old_active_planes) == hweight8(new_active_planes))
9054                         continue;
9055
9056                 ret = intel_crtc_add_planes_to_state(state, crtc, new_active_planes);
9057                 if (ret)
9058                         return ret;
9059         }
9060
9061         for_each_new_intel_plane_in_state(state, plane, plane_state, i) {
9062                 new_plane_state = intel_atomic_get_new_plane_state(state, plane);
9063                 old_plane_state = intel_atomic_get_old_plane_state(state, plane);
9064                 fb = new_plane_state->hw.fb;
9065                 if (fb) {
9066                         new_plane_state->decrypt = bo_has_valid_encryption(intel_fb_obj(fb));
9067                         new_plane_state->force_black = pxp_is_borked(intel_fb_obj(fb));
9068                 } else {
9069                         new_plane_state->decrypt = old_plane_state->decrypt;
9070                         new_plane_state->force_black = old_plane_state->force_black;
9071                 }
9072         }
9073
9074         return 0;
9075 }
9076
9077 static int intel_atomic_check_cdclk(struct intel_atomic_state *state,
9078                                     bool *need_cdclk_calc)
9079 {
9080         struct drm_i915_private *dev_priv = to_i915(state->base.dev);
9081         const struct intel_cdclk_state *old_cdclk_state;
9082         const struct intel_cdclk_state *new_cdclk_state;
9083         struct intel_plane_state *plane_state;
9084         struct intel_bw_state *new_bw_state;
9085         struct intel_plane *plane;
9086         int min_cdclk = 0;
9087         enum pipe pipe;
9088         int ret;
9089         int i;
9090         /*
9091          * active_planes bitmask has been updated, and potentially
9092          * affected planes are part of the state. We can now
9093          * compute the minimum cdclk for each plane.
9094          */
9095         for_each_new_intel_plane_in_state(state, plane, plane_state, i) {
9096                 ret = intel_plane_calc_min_cdclk(state, plane, need_cdclk_calc);
9097                 if (ret)
9098                         return ret;
9099         }
9100
9101         old_cdclk_state = intel_atomic_get_old_cdclk_state(state);
9102         new_cdclk_state = intel_atomic_get_new_cdclk_state(state);
9103
9104         if (new_cdclk_state &&
9105             old_cdclk_state->force_min_cdclk != new_cdclk_state->force_min_cdclk)
9106                 *need_cdclk_calc = true;
9107
9108         ret = intel_cdclk_bw_calc_min_cdclk(state);
9109         if (ret)
9110                 return ret;
9111
9112         new_bw_state = intel_atomic_get_new_bw_state(state);
9113
9114         if (!new_cdclk_state || !new_bw_state)
9115                 return 0;
9116
9117         for_each_pipe(dev_priv, pipe) {
9118                 min_cdclk = max(new_cdclk_state->min_cdclk[pipe], min_cdclk);
9119
9120                 /*
9121                  * Currently do this change only if we need to increase
9122                  */
9123                 if (new_bw_state->min_cdclk > min_cdclk)
9124                         *need_cdclk_calc = true;
9125         }
9126
9127         return 0;
9128 }
9129
9130 static int intel_atomic_check_crtcs(struct intel_atomic_state *state)
9131 {
9132         struct intel_crtc_state *crtc_state;
9133         struct intel_crtc *crtc;
9134         int i;
9135
9136         for_each_new_intel_crtc_in_state(state, crtc, crtc_state, i) {
9137                 struct drm_i915_private *i915 = to_i915(crtc->base.dev);
9138                 int ret;
9139
9140                 ret = intel_crtc_atomic_check(state, crtc);
9141                 if (ret) {
9142                         drm_dbg_atomic(&i915->drm,
9143                                        "[CRTC:%d:%s] atomic driver check failed\n",
9144                                        crtc->base.base.id, crtc->base.name);
9145                         return ret;
9146                 }
9147         }
9148
9149         return 0;
9150 }
9151
9152 static bool intel_cpu_transcoders_need_modeset(struct intel_atomic_state *state,
9153                                                u8 transcoders)
9154 {
9155         const struct intel_crtc_state *new_crtc_state;
9156         struct intel_crtc *crtc;
9157         int i;
9158
9159         for_each_new_intel_crtc_in_state(state, crtc, new_crtc_state, i) {
9160                 if (new_crtc_state->hw.enable &&
9161                     transcoders & BIT(new_crtc_state->cpu_transcoder) &&
9162                     intel_crtc_needs_modeset(new_crtc_state))
9163                         return true;
9164         }
9165
9166         return false;
9167 }
9168
9169 static int intel_atomic_check_bigjoiner(struct intel_atomic_state *state,
9170                                         struct intel_crtc *crtc,
9171                                         struct intel_crtc_state *old_crtc_state,
9172                                         struct intel_crtc_state *new_crtc_state)
9173 {
9174         struct intel_crtc_state *slave_crtc_state, *master_crtc_state;
9175         struct intel_crtc *slave, *master;
9176
9177         /* slave being enabled, is master is still claiming this crtc? */
9178         if (old_crtc_state->bigjoiner_slave) {
9179                 slave = crtc;
9180                 master = old_crtc_state->bigjoiner_linked_crtc;
9181                 master_crtc_state = intel_atomic_get_new_crtc_state(state, master);
9182                 if (!master_crtc_state || !intel_crtc_needs_modeset(master_crtc_state))
9183                         goto claimed;
9184         }
9185
9186         if (!new_crtc_state->bigjoiner)
9187                 return 0;
9188
9189         slave = intel_dsc_get_bigjoiner_secondary(crtc);
9190         if (!slave) {
9191                 DRM_DEBUG_KMS("[CRTC:%d:%s] Big joiner configuration requires "
9192                               "CRTC + 1 to be used, doesn't exist\n",
9193                               crtc->base.base.id, crtc->base.name);
9194                 return -EINVAL;
9195         }
9196
9197         new_crtc_state->bigjoiner_linked_crtc = slave;
9198         slave_crtc_state = intel_atomic_get_crtc_state(&state->base, slave);
9199         master = crtc;
9200         if (IS_ERR(slave_crtc_state))
9201                 return PTR_ERR(slave_crtc_state);
9202
9203         /* master being enabled, slave was already configured? */
9204         if (slave_crtc_state->uapi.enable)
9205                 goto claimed;
9206
9207         DRM_DEBUG_KMS("[CRTC:%d:%s] Used as slave for big joiner\n",
9208                       slave->base.base.id, slave->base.name);
9209
9210         return copy_bigjoiner_crtc_state(slave_crtc_state, new_crtc_state);
9211
9212 claimed:
9213         DRM_DEBUG_KMS("[CRTC:%d:%s] Slave is enabled as normal CRTC, but "
9214                       "[CRTC:%d:%s] claiming this CRTC for bigjoiner.\n",
9215                       slave->base.base.id, slave->base.name,
9216                       master->base.base.id, master->base.name);
9217         return -EINVAL;
9218 }
9219
9220 static void kill_bigjoiner_slave(struct intel_atomic_state *state,
9221                                  struct intel_crtc_state *master_crtc_state)
9222 {
9223         struct intel_crtc_state *slave_crtc_state =
9224                 intel_atomic_get_new_crtc_state(state, master_crtc_state->bigjoiner_linked_crtc);
9225
9226         slave_crtc_state->bigjoiner = master_crtc_state->bigjoiner = false;
9227         slave_crtc_state->bigjoiner_slave = master_crtc_state->bigjoiner_slave = false;
9228         slave_crtc_state->bigjoiner_linked_crtc = master_crtc_state->bigjoiner_linked_crtc = NULL;
9229         intel_crtc_copy_uapi_to_hw_state(state, slave_crtc_state);
9230 }
9231
9232 /**
9233  * DOC: asynchronous flip implementation
9234  *
9235  * Asynchronous page flip is the implementation for the DRM_MODE_PAGE_FLIP_ASYNC
9236  * flag. Currently async flip is only supported via the drmModePageFlip IOCTL.
9237  * Correspondingly, support is currently added for primary plane only.
9238  *
9239  * Async flip can only change the plane surface address, so anything else
9240  * changing is rejected from the intel_atomic_check_async() function.
9241  * Once this check is cleared, flip done interrupt is enabled using
9242  * the intel_crtc_enable_flip_done() function.
9243  *
9244  * As soon as the surface address register is written, flip done interrupt is
9245  * generated and the requested events are sent to the usersapce in the interrupt
9246  * handler itself. The timestamp and sequence sent during the flip done event
9247  * correspond to the last vblank and have no relation to the actual time when
9248  * the flip done event was sent.
9249  */
9250 static int intel_atomic_check_async(struct intel_atomic_state *state)
9251 {
9252         struct drm_i915_private *i915 = to_i915(state->base.dev);
9253         const struct intel_crtc_state *old_crtc_state, *new_crtc_state;
9254         const struct intel_plane_state *new_plane_state, *old_plane_state;
9255         struct intel_crtc *crtc;
9256         struct intel_plane *plane;
9257         int i;
9258
9259         for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state,
9260                                             new_crtc_state, i) {
9261                 if (intel_crtc_needs_modeset(new_crtc_state)) {
9262                         drm_dbg_kms(&i915->drm, "Modeset Required. Async flip not supported\n");
9263                         return -EINVAL;
9264                 }
9265
9266                 if (!new_crtc_state->hw.active) {
9267                         drm_dbg_kms(&i915->drm, "CRTC inactive\n");
9268                         return -EINVAL;
9269                 }
9270                 if (old_crtc_state->active_planes != new_crtc_state->active_planes) {
9271                         drm_dbg_kms(&i915->drm,
9272                                     "Active planes cannot be changed during async flip\n");
9273                         return -EINVAL;
9274                 }
9275         }
9276
9277         for_each_oldnew_intel_plane_in_state(state, plane, old_plane_state,
9278                                              new_plane_state, i) {
9279                 /*
9280                  * TODO: Async flip is only supported through the page flip IOCTL
9281                  * as of now. So support currently added for primary plane only.
9282                  * Support for other planes on platforms on which supports
9283                  * this(vlv/chv and icl+) should be added when async flip is
9284                  * enabled in the atomic IOCTL path.
9285                  */
9286                 if (!plane->async_flip)
9287                         return -EINVAL;
9288
9289                 /*
9290                  * FIXME: This check is kept generic for all platforms.
9291                  * Need to verify this for all gen9 platforms to enable
9292                  * this selectively if required.
9293                  */
9294                 switch (new_plane_state->hw.fb->modifier) {
9295                 case I915_FORMAT_MOD_X_TILED:
9296                 case I915_FORMAT_MOD_Y_TILED:
9297                 case I915_FORMAT_MOD_Yf_TILED:
9298                         break;
9299                 default:
9300                         drm_dbg_kms(&i915->drm,
9301                                     "Linear memory/CCS does not support async flips\n");
9302                         return -EINVAL;
9303                 }
9304
9305                 if (old_plane_state->view.color_plane[0].stride !=
9306                     new_plane_state->view.color_plane[0].stride) {
9307                         drm_dbg_kms(&i915->drm, "Stride cannot be changed in async flip\n");
9308                         return -EINVAL;
9309                 }
9310
9311                 if (old_plane_state->hw.fb->modifier !=
9312                     new_plane_state->hw.fb->modifier) {
9313                         drm_dbg_kms(&i915->drm,
9314                                     "Framebuffer modifiers cannot be changed in async flip\n");
9315                         return -EINVAL;
9316                 }
9317
9318                 if (old_plane_state->hw.fb->format !=
9319                     new_plane_state->hw.fb->format) {
9320                         drm_dbg_kms(&i915->drm,
9321                                     "Framebuffer format cannot be changed in async flip\n");
9322                         return -EINVAL;
9323                 }
9324
9325                 if (old_plane_state->hw.rotation !=
9326                     new_plane_state->hw.rotation) {
9327                         drm_dbg_kms(&i915->drm, "Rotation cannot be changed in async flip\n");
9328                         return -EINVAL;
9329                 }
9330
9331                 if (!drm_rect_equals(&old_plane_state->uapi.src, &new_plane_state->uapi.src) ||
9332                     !drm_rect_equals(&old_plane_state->uapi.dst, &new_plane_state->uapi.dst)) {
9333                         drm_dbg_kms(&i915->drm,
9334                                     "Plane size/co-ordinates cannot be changed in async flip\n");
9335                         return -EINVAL;
9336                 }
9337
9338                 if (old_plane_state->hw.alpha != new_plane_state->hw.alpha) {
9339                         drm_dbg_kms(&i915->drm, "Alpha value cannot be changed in async flip\n");
9340                         return -EINVAL;
9341                 }
9342
9343                 if (old_plane_state->hw.pixel_blend_mode !=
9344                     new_plane_state->hw.pixel_blend_mode) {
9345                         drm_dbg_kms(&i915->drm,
9346                                     "Pixel blend mode cannot be changed in async flip\n");
9347                         return -EINVAL;
9348                 }
9349
9350                 if (old_plane_state->hw.color_encoding != new_plane_state->hw.color_encoding) {
9351                         drm_dbg_kms(&i915->drm,
9352                                     "Color encoding cannot be changed in async flip\n");
9353                         return -EINVAL;
9354                 }
9355
9356                 if (old_plane_state->hw.color_range != new_plane_state->hw.color_range) {
9357                         drm_dbg_kms(&i915->drm, "Color range cannot be changed in async flip\n");
9358                         return -EINVAL;
9359                 }
9360
9361                 /* plane decryption is allow to change only in synchronous flips */
9362                 if (old_plane_state->decrypt != new_plane_state->decrypt)
9363                         return -EINVAL;
9364         }
9365
9366         return 0;
9367 }
9368
9369 static int intel_bigjoiner_add_affected_crtcs(struct intel_atomic_state *state)
9370 {
9371         struct intel_crtc_state *crtc_state;
9372         struct intel_crtc *crtc;
9373         int i;
9374
9375         for_each_new_intel_crtc_in_state(state, crtc, crtc_state, i) {
9376                 struct intel_crtc_state *linked_crtc_state;
9377                 struct intel_crtc *linked_crtc;
9378                 int ret;
9379
9380                 if (!crtc_state->bigjoiner)
9381                         continue;
9382
9383                 linked_crtc = crtc_state->bigjoiner_linked_crtc;
9384                 linked_crtc_state = intel_atomic_get_crtc_state(&state->base, linked_crtc);
9385                 if (IS_ERR(linked_crtc_state))
9386                         return PTR_ERR(linked_crtc_state);
9387
9388                 if (!intel_crtc_needs_modeset(crtc_state))
9389                         continue;
9390
9391                 linked_crtc_state->uapi.mode_changed = true;
9392
9393                 ret = drm_atomic_add_affected_connectors(&state->base,
9394                                                          &linked_crtc->base);
9395                 if (ret)
9396                         return ret;
9397
9398                 ret = intel_atomic_add_affected_planes(state, linked_crtc);
9399                 if (ret)
9400                         return ret;
9401         }
9402
9403         for_each_new_intel_crtc_in_state(state, crtc, crtc_state, i) {
9404                 /* Kill old bigjoiner link, we may re-establish afterwards */
9405                 if (intel_crtc_needs_modeset(crtc_state) &&
9406                     crtc_state->bigjoiner && !crtc_state->bigjoiner_slave)
9407                         kill_bigjoiner_slave(state, crtc_state);
9408         }
9409
9410         return 0;
9411 }
9412
9413 /**
9414  * intel_atomic_check - validate state object
9415  * @dev: drm device
9416  * @_state: state to validate
9417  */
9418 static int intel_atomic_check(struct drm_device *dev,
9419                               struct drm_atomic_state *_state)
9420 {
9421         struct drm_i915_private *dev_priv = to_i915(dev);
9422         struct intel_atomic_state *state = to_intel_atomic_state(_state);
9423         struct intel_crtc_state *old_crtc_state, *new_crtc_state;
9424         struct intel_crtc *crtc;
9425         int ret, i;
9426         bool any_ms = false;
9427
9428         for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state,
9429                                             new_crtc_state, i) {
9430                 if (new_crtc_state->inherited != old_crtc_state->inherited)
9431                         new_crtc_state->uapi.mode_changed = true;
9432         }
9433
9434         intel_vrr_check_modeset(state);
9435
9436         ret = drm_atomic_helper_check_modeset(dev, &state->base);
9437         if (ret)
9438                 goto fail;
9439
9440         ret = intel_bigjoiner_add_affected_crtcs(state);
9441         if (ret)
9442                 goto fail;
9443
9444         for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state,
9445                                             new_crtc_state, i) {
9446                 if (!intel_crtc_needs_modeset(new_crtc_state)) {
9447                         /* Light copy */
9448                         intel_crtc_copy_uapi_to_hw_state_nomodeset(state, new_crtc_state);
9449
9450                         continue;
9451                 }
9452
9453                 if (!new_crtc_state->uapi.enable) {
9454                         if (!new_crtc_state->bigjoiner_slave) {
9455                                 intel_crtc_copy_uapi_to_hw_state(state, new_crtc_state);
9456                                 any_ms = true;
9457                         }
9458                         continue;
9459                 }
9460
9461                 ret = intel_crtc_prepare_cleared_state(state, new_crtc_state);
9462                 if (ret)
9463                         goto fail;
9464
9465                 ret = intel_modeset_pipe_config(state, new_crtc_state);
9466                 if (ret)
9467                         goto fail;
9468
9469                 ret = intel_atomic_check_bigjoiner(state, crtc, old_crtc_state,
9470                                                    new_crtc_state);
9471                 if (ret)
9472                         goto fail;
9473         }
9474
9475         for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state,
9476                                             new_crtc_state, i) {
9477                 if (!intel_crtc_needs_modeset(new_crtc_state))
9478                         continue;
9479
9480                 ret = intel_modeset_pipe_config_late(new_crtc_state);
9481                 if (ret)
9482                         goto fail;
9483
9484                 intel_crtc_check_fastset(old_crtc_state, new_crtc_state);
9485         }
9486
9487         /**
9488          * Check if fastset is allowed by external dependencies like other
9489          * pipes and transcoders.
9490          *
9491          * Right now it only forces a fullmodeset when the MST master
9492          * transcoder did not changed but the pipe of the master transcoder
9493          * needs a fullmodeset so all slaves also needs to do a fullmodeset or
9494          * in case of port synced crtcs, if one of the synced crtcs
9495          * needs a full modeset, all other synced crtcs should be
9496          * forced a full modeset.
9497          */
9498         for_each_new_intel_crtc_in_state(state, crtc, new_crtc_state, i) {
9499                 if (!new_crtc_state->hw.enable || intel_crtc_needs_modeset(new_crtc_state))
9500                         continue;
9501
9502                 if (intel_dp_mst_is_slave_trans(new_crtc_state)) {
9503                         enum transcoder master = new_crtc_state->mst_master_transcoder;
9504
9505                         if (intel_cpu_transcoders_need_modeset(state, BIT(master))) {
9506                                 new_crtc_state->uapi.mode_changed = true;
9507                                 new_crtc_state->update_pipe = false;
9508                         }
9509                 }
9510
9511                 if (is_trans_port_sync_mode(new_crtc_state)) {
9512                         u8 trans = new_crtc_state->sync_mode_slaves_mask;
9513
9514                         if (new_crtc_state->master_transcoder != INVALID_TRANSCODER)
9515                                 trans |= BIT(new_crtc_state->master_transcoder);
9516
9517                         if (intel_cpu_transcoders_need_modeset(state, trans)) {
9518                                 new_crtc_state->uapi.mode_changed = true;
9519                                 new_crtc_state->update_pipe = false;
9520                         }
9521                 }
9522
9523                 if (new_crtc_state->bigjoiner) {
9524                         struct intel_crtc_state *linked_crtc_state =
9525                                 intel_atomic_get_new_crtc_state(state, new_crtc_state->bigjoiner_linked_crtc);
9526
9527                         if (intel_crtc_needs_modeset(linked_crtc_state)) {
9528                                 new_crtc_state->uapi.mode_changed = true;
9529                                 new_crtc_state->update_pipe = false;
9530                         }
9531                 }
9532         }
9533
9534         for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state,
9535                                             new_crtc_state, i) {
9536                 if (intel_crtc_needs_modeset(new_crtc_state)) {
9537                         any_ms = true;
9538                         continue;
9539                 }
9540
9541                 if (!new_crtc_state->update_pipe)
9542                         continue;
9543
9544                 intel_crtc_copy_fastset(old_crtc_state, new_crtc_state);
9545         }
9546
9547         if (any_ms && !check_digital_port_conflicts(state)) {
9548                 drm_dbg_kms(&dev_priv->drm,
9549                             "rejecting conflicting digital port configuration\n");
9550                 ret = -EINVAL;
9551                 goto fail;
9552         }
9553
9554         ret = drm_dp_mst_atomic_check(&state->base);
9555         if (ret)
9556                 goto fail;
9557
9558         ret = intel_atomic_check_planes(state);
9559         if (ret)
9560                 goto fail;
9561
9562         intel_fbc_choose_crtc(dev_priv, state);
9563         ret = intel_compute_global_watermarks(state);
9564         if (ret)
9565                 goto fail;
9566
9567         ret = intel_bw_atomic_check(state);
9568         if (ret)
9569                 goto fail;
9570
9571         ret = intel_atomic_check_cdclk(state, &any_ms);
9572         if (ret)
9573                 goto fail;
9574
9575         if (intel_any_crtc_needs_modeset(state))
9576                 any_ms = true;
9577
9578         if (any_ms) {
9579                 ret = intel_modeset_checks(state);
9580                 if (ret)
9581                         goto fail;
9582
9583                 ret = intel_modeset_calc_cdclk(state);
9584                 if (ret)
9585                         return ret;
9586
9587                 intel_modeset_clear_plls(state);
9588         }
9589
9590         ret = intel_atomic_check_crtcs(state);
9591         if (ret)
9592                 goto fail;
9593
9594         for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state,
9595                                             new_crtc_state, i) {
9596                 if (new_crtc_state->uapi.async_flip) {
9597                         ret = intel_atomic_check_async(state);
9598                         if (ret)
9599                                 goto fail;
9600                 }
9601
9602                 if (!intel_crtc_needs_modeset(new_crtc_state) &&
9603                     !new_crtc_state->update_pipe)
9604                         continue;
9605
9606                 intel_dump_pipe_config(new_crtc_state, state,
9607                                        intel_crtc_needs_modeset(new_crtc_state) ?
9608                                        "[modeset]" : "[fastset]");
9609         }
9610
9611         return 0;
9612
9613  fail:
9614         if (ret == -EDEADLK)
9615                 return ret;
9616
9617         /*
9618          * FIXME would probably be nice to know which crtc specifically
9619          * caused the failure, in cases where we can pinpoint it.
9620          */
9621         for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state,
9622                                             new_crtc_state, i)
9623                 intel_dump_pipe_config(new_crtc_state, state, "[failed]");
9624
9625         return ret;
9626 }
9627
9628 static int intel_atomic_prepare_commit(struct intel_atomic_state *state)
9629 {
9630         struct intel_crtc_state *crtc_state;
9631         struct intel_crtc *crtc;
9632         int i, ret;
9633
9634         ret = drm_atomic_helper_prepare_planes(state->base.dev, &state->base);
9635         if (ret < 0)
9636                 return ret;
9637
9638         for_each_new_intel_crtc_in_state(state, crtc, crtc_state, i) {
9639                 bool mode_changed = intel_crtc_needs_modeset(crtc_state);
9640
9641                 if (mode_changed || crtc_state->update_pipe ||
9642                     crtc_state->uapi.color_mgmt_changed) {
9643                         intel_dsb_prepare(crtc_state);
9644                 }
9645         }
9646
9647         return 0;
9648 }
9649
9650 void intel_crtc_arm_fifo_underrun(struct intel_crtc *crtc,
9651                                   struct intel_crtc_state *crtc_state)
9652 {
9653         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
9654
9655         if (DISPLAY_VER(dev_priv) != 2 || crtc_state->active_planes)
9656                 intel_set_cpu_fifo_underrun_reporting(dev_priv, crtc->pipe, true);
9657
9658         if (crtc_state->has_pch_encoder) {
9659                 enum pipe pch_transcoder =
9660                         intel_crtc_pch_transcoder(crtc);
9661
9662                 intel_set_pch_fifo_underrun_reporting(dev_priv, pch_transcoder, true);
9663         }
9664 }
9665
9666 static void intel_pipe_fastset(const struct intel_crtc_state *old_crtc_state,
9667                                const struct intel_crtc_state *new_crtc_state)
9668 {
9669         struct intel_crtc *crtc = to_intel_crtc(new_crtc_state->uapi.crtc);
9670         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
9671
9672         /*
9673          * Update pipe size and adjust fitter if needed: the reason for this is
9674          * that in compute_mode_changes we check the native mode (not the pfit
9675          * mode) to see if we can flip rather than do a full mode set. In the
9676          * fastboot case, we'll flip, but if we don't update the pipesrc and
9677          * pfit state, we'll end up with a big fb scanned out into the wrong
9678          * sized surface.
9679          */
9680         intel_set_pipe_src_size(new_crtc_state);
9681
9682         /* on skylake this is done by detaching scalers */
9683         if (DISPLAY_VER(dev_priv) >= 9) {
9684                 if (new_crtc_state->pch_pfit.enabled)
9685                         skl_pfit_enable(new_crtc_state);
9686         } else if (HAS_PCH_SPLIT(dev_priv)) {
9687                 if (new_crtc_state->pch_pfit.enabled)
9688                         ilk_pfit_enable(new_crtc_state);
9689                 else if (old_crtc_state->pch_pfit.enabled)
9690                         ilk_pfit_disable(old_crtc_state);
9691         }
9692
9693         /*
9694          * The register is supposedly single buffered so perhaps
9695          * not 100% correct to do this here. But SKL+ calculate
9696          * this based on the adjust pixel rate so pfit changes do
9697          * affect it and so it must be updated for fastsets.
9698          * HSW/BDW only really need this here for fastboot, after
9699          * that the value should not change without a full modeset.
9700          */
9701         if (DISPLAY_VER(dev_priv) >= 9 ||
9702             IS_BROADWELL(dev_priv) || IS_HASWELL(dev_priv))
9703                 hsw_set_linetime_wm(new_crtc_state);
9704
9705         if (DISPLAY_VER(dev_priv) >= 11)
9706                 icl_set_pipe_chicken(new_crtc_state);
9707 }
9708
9709 static void commit_pipe_pre_planes(struct intel_atomic_state *state,
9710                                    struct intel_crtc *crtc)
9711 {
9712         struct drm_i915_private *dev_priv = to_i915(state->base.dev);
9713         const struct intel_crtc_state *old_crtc_state =
9714                 intel_atomic_get_old_crtc_state(state, crtc);
9715         const struct intel_crtc_state *new_crtc_state =
9716                 intel_atomic_get_new_crtc_state(state, crtc);
9717         bool modeset = intel_crtc_needs_modeset(new_crtc_state);
9718
9719         /*
9720          * During modesets pipe configuration was programmed as the
9721          * CRTC was enabled.
9722          */
9723         if (!modeset) {
9724                 if (new_crtc_state->uapi.color_mgmt_changed ||
9725                     new_crtc_state->update_pipe)
9726                         intel_color_commit(new_crtc_state);
9727
9728                 if (DISPLAY_VER(dev_priv) >= 9 || IS_BROADWELL(dev_priv))
9729                         bdw_set_pipemisc(new_crtc_state);
9730
9731                 if (new_crtc_state->update_pipe)
9732                         intel_pipe_fastset(old_crtc_state, new_crtc_state);
9733         }
9734
9735         intel_psr2_program_trans_man_trk_ctl(new_crtc_state);
9736
9737         intel_atomic_update_watermarks(state, crtc);
9738 }
9739
9740 static void commit_pipe_post_planes(struct intel_atomic_state *state,
9741                                     struct intel_crtc *crtc)
9742 {
9743         struct drm_i915_private *dev_priv = to_i915(state->base.dev);
9744         const struct intel_crtc_state *new_crtc_state =
9745                 intel_atomic_get_new_crtc_state(state, crtc);
9746
9747         /*
9748          * Disable the scaler(s) after the plane(s) so that we don't
9749          * get a catastrophic underrun even if the two operations
9750          * end up happening in two different frames.
9751          */
9752         if (DISPLAY_VER(dev_priv) >= 9 &&
9753             !intel_crtc_needs_modeset(new_crtc_state))
9754                 skl_detach_scalers(new_crtc_state);
9755 }
9756
9757 static void intel_enable_crtc(struct intel_atomic_state *state,
9758                               struct intel_crtc *crtc)
9759 {
9760         struct drm_i915_private *dev_priv = to_i915(state->base.dev);
9761         const struct intel_crtc_state *new_crtc_state =
9762                 intel_atomic_get_new_crtc_state(state, crtc);
9763
9764         if (!intel_crtc_needs_modeset(new_crtc_state))
9765                 return;
9766
9767         intel_crtc_update_active_timings(new_crtc_state);
9768
9769         dev_priv->display->crtc_enable(state, crtc);
9770
9771         if (new_crtc_state->bigjoiner_slave)
9772                 return;
9773
9774         /* vblanks work again, re-enable pipe CRC. */
9775         intel_crtc_enable_pipe_crc(crtc);
9776 }
9777
9778 static void intel_update_crtc(struct intel_atomic_state *state,
9779                               struct intel_crtc *crtc)
9780 {
9781         struct drm_i915_private *dev_priv = to_i915(state->base.dev);
9782         const struct intel_crtc_state *old_crtc_state =
9783                 intel_atomic_get_old_crtc_state(state, crtc);
9784         struct intel_crtc_state *new_crtc_state =
9785                 intel_atomic_get_new_crtc_state(state, crtc);
9786         bool modeset = intel_crtc_needs_modeset(new_crtc_state);
9787
9788         if (!modeset) {
9789                 if (new_crtc_state->preload_luts &&
9790                     (new_crtc_state->uapi.color_mgmt_changed ||
9791                      new_crtc_state->update_pipe))
9792                         intel_color_load_luts(new_crtc_state);
9793
9794                 intel_pre_plane_update(state, crtc);
9795
9796                 if (new_crtc_state->update_pipe)
9797                         intel_encoders_update_pipe(state, crtc);
9798         }
9799
9800         intel_fbc_update(state, crtc);
9801
9802         /* Perform vblank evasion around commit operation */
9803         intel_pipe_update_start(new_crtc_state);
9804
9805         commit_pipe_pre_planes(state, crtc);
9806
9807         if (DISPLAY_VER(dev_priv) >= 9)
9808                 skl_update_planes_on_crtc(state, crtc);
9809         else
9810                 i9xx_update_planes_on_crtc(state, crtc);
9811
9812         commit_pipe_post_planes(state, crtc);
9813
9814         intel_pipe_update_end(new_crtc_state);
9815
9816         /*
9817          * We usually enable FIFO underrun interrupts as part of the
9818          * CRTC enable sequence during modesets.  But when we inherit a
9819          * valid pipe configuration from the BIOS we need to take care
9820          * of enabling them on the CRTC's first fastset.
9821          */
9822         if (new_crtc_state->update_pipe && !modeset &&
9823             old_crtc_state->inherited)
9824                 intel_crtc_arm_fifo_underrun(crtc, new_crtc_state);
9825 }
9826
9827 static void intel_old_crtc_state_disables(struct intel_atomic_state *state,
9828                                           struct intel_crtc_state *old_crtc_state,
9829                                           struct intel_crtc_state *new_crtc_state,
9830                                           struct intel_crtc *crtc)
9831 {
9832         struct drm_i915_private *dev_priv = to_i915(state->base.dev);
9833
9834         drm_WARN_ON(&dev_priv->drm, old_crtc_state->bigjoiner_slave);
9835
9836         intel_encoders_pre_disable(state, crtc);
9837
9838         intel_crtc_disable_planes(state, crtc);
9839
9840         /*
9841          * We still need special handling for disabling bigjoiner master
9842          * and slaves since for slave we do not have encoder or plls
9843          * so we dont need to disable those.
9844          */
9845         if (old_crtc_state->bigjoiner) {
9846                 intel_crtc_disable_planes(state,
9847                                           old_crtc_state->bigjoiner_linked_crtc);
9848                 old_crtc_state->bigjoiner_linked_crtc->active = false;
9849         }
9850
9851         /*
9852          * We need to disable pipe CRC before disabling the pipe,
9853          * or we race against vblank off.
9854          */
9855         intel_crtc_disable_pipe_crc(crtc);
9856
9857         dev_priv->display->crtc_disable(state, crtc);
9858         crtc->active = false;
9859         intel_fbc_disable(crtc);
9860         intel_disable_shared_dpll(old_crtc_state);
9861
9862         /* FIXME unify this for all platforms */
9863         if (!new_crtc_state->hw.active &&
9864             !HAS_GMCH(dev_priv))
9865                 intel_initial_watermarks(state, crtc);
9866 }
9867
9868 static void intel_commit_modeset_disables(struct intel_atomic_state *state)
9869 {
9870         struct intel_crtc_state *new_crtc_state, *old_crtc_state;
9871         struct intel_crtc *crtc;
9872         u32 handled = 0;
9873         int i;
9874
9875         /* Only disable port sync and MST slaves */
9876         for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state,
9877                                             new_crtc_state, i) {
9878                 if (!intel_crtc_needs_modeset(new_crtc_state) || old_crtc_state->bigjoiner)
9879                         continue;
9880
9881                 if (!old_crtc_state->hw.active)
9882                         continue;
9883
9884                 /* In case of Transcoder port Sync master slave CRTCs can be
9885                  * assigned in any order and we need to make sure that
9886                  * slave CRTCs are disabled first and then master CRTC since
9887                  * Slave vblanks are masked till Master Vblanks.
9888                  */
9889                 if (!is_trans_port_sync_slave(old_crtc_state) &&
9890                     !intel_dp_mst_is_slave_trans(old_crtc_state))
9891                         continue;
9892
9893                 intel_pre_plane_update(state, crtc);
9894                 intel_old_crtc_state_disables(state, old_crtc_state,
9895                                               new_crtc_state, crtc);
9896                 handled |= BIT(crtc->pipe);
9897         }
9898
9899         /* Disable everything else left on */
9900         for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state,
9901                                             new_crtc_state, i) {
9902                 if (!intel_crtc_needs_modeset(new_crtc_state) ||
9903                     (handled & BIT(crtc->pipe)) ||
9904                     old_crtc_state->bigjoiner_slave)
9905                         continue;
9906
9907                 intel_pre_plane_update(state, crtc);
9908                 if (old_crtc_state->bigjoiner) {
9909                         struct intel_crtc *slave =
9910                                 old_crtc_state->bigjoiner_linked_crtc;
9911
9912                         intel_pre_plane_update(state, slave);
9913                 }
9914
9915                 if (old_crtc_state->hw.active)
9916                         intel_old_crtc_state_disables(state, old_crtc_state,
9917                                                       new_crtc_state, crtc);
9918         }
9919 }
9920
9921 static void intel_commit_modeset_enables(struct intel_atomic_state *state)
9922 {
9923         struct intel_crtc_state *new_crtc_state;
9924         struct intel_crtc *crtc;
9925         int i;
9926
9927         for_each_new_intel_crtc_in_state(state, crtc, new_crtc_state, i) {
9928                 if (!new_crtc_state->hw.active)
9929                         continue;
9930
9931                 intel_enable_crtc(state, crtc);
9932                 intel_update_crtc(state, crtc);
9933         }
9934 }
9935
9936 static void skl_commit_modeset_enables(struct intel_atomic_state *state)
9937 {
9938         struct drm_i915_private *dev_priv = to_i915(state->base.dev);
9939         struct intel_crtc *crtc;
9940         struct intel_crtc_state *old_crtc_state, *new_crtc_state;
9941         struct skl_ddb_entry entries[I915_MAX_PIPES] = {};
9942         u8 update_pipes = 0, modeset_pipes = 0;
9943         int i;
9944
9945         for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state, new_crtc_state, i) {
9946                 enum pipe pipe = crtc->pipe;
9947
9948                 if (!new_crtc_state->hw.active)
9949                         continue;
9950
9951                 /* ignore allocations for crtc's that have been turned off. */
9952                 if (!intel_crtc_needs_modeset(new_crtc_state)) {
9953                         entries[pipe] = old_crtc_state->wm.skl.ddb;
9954                         update_pipes |= BIT(pipe);
9955                 } else {
9956                         modeset_pipes |= BIT(pipe);
9957                 }
9958         }
9959
9960         /*
9961          * Whenever the number of active pipes changes, we need to make sure we
9962          * update the pipes in the right order so that their ddb allocations
9963          * never overlap with each other between CRTC updates. Otherwise we'll
9964          * cause pipe underruns and other bad stuff.
9965          *
9966          * So first lets enable all pipes that do not need a fullmodeset as
9967          * those don't have any external dependency.
9968          */
9969         while (update_pipes) {
9970                 for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state,
9971                                                     new_crtc_state, i) {
9972                         enum pipe pipe = crtc->pipe;
9973
9974                         if ((update_pipes & BIT(pipe)) == 0)
9975                                 continue;
9976
9977                         if (skl_ddb_allocation_overlaps(&new_crtc_state->wm.skl.ddb,
9978                                                         entries, I915_MAX_PIPES, pipe))
9979                                 continue;
9980
9981                         entries[pipe] = new_crtc_state->wm.skl.ddb;
9982                         update_pipes &= ~BIT(pipe);
9983
9984                         intel_update_crtc(state, crtc);
9985
9986                         /*
9987                          * If this is an already active pipe, it's DDB changed,
9988                          * and this isn't the last pipe that needs updating
9989                          * then we need to wait for a vblank to pass for the
9990                          * new ddb allocation to take effect.
9991                          */
9992                         if (!skl_ddb_entry_equal(&new_crtc_state->wm.skl.ddb,
9993                                                  &old_crtc_state->wm.skl.ddb) &&
9994                             (update_pipes | modeset_pipes))
9995                                 intel_wait_for_vblank(dev_priv, pipe);
9996                 }
9997         }
9998
9999         update_pipes = modeset_pipes;
10000
10001         /*
10002          * Enable all pipes that needs a modeset and do not depends on other
10003          * pipes
10004          */
10005         for_each_new_intel_crtc_in_state(state, crtc, new_crtc_state, i) {
10006                 enum pipe pipe = crtc->pipe;
10007
10008                 if ((modeset_pipes & BIT(pipe)) == 0)
10009                         continue;
10010
10011                 if (intel_dp_mst_is_slave_trans(new_crtc_state) ||
10012                     is_trans_port_sync_master(new_crtc_state) ||
10013                     (new_crtc_state->bigjoiner && !new_crtc_state->bigjoiner_slave))
10014                         continue;
10015
10016                 modeset_pipes &= ~BIT(pipe);
10017
10018                 intel_enable_crtc(state, crtc);
10019         }
10020
10021         /*
10022          * Then we enable all remaining pipes that depend on other
10023          * pipes: MST slaves and port sync masters, big joiner master
10024          */
10025         for_each_new_intel_crtc_in_state(state, crtc, new_crtc_state, i) {
10026                 enum pipe pipe = crtc->pipe;
10027
10028                 if ((modeset_pipes & BIT(pipe)) == 0)
10029                         continue;
10030
10031                 modeset_pipes &= ~BIT(pipe);
10032
10033                 intel_enable_crtc(state, crtc);
10034         }
10035
10036         /*
10037          * Finally we do the plane updates/etc. for all pipes that got enabled.
10038          */
10039         for_each_new_intel_crtc_in_state(state, crtc, new_crtc_state, i) {
10040                 enum pipe pipe = crtc->pipe;
10041
10042                 if ((update_pipes & BIT(pipe)) == 0)
10043                         continue;
10044
10045                 drm_WARN_ON(&dev_priv->drm, skl_ddb_allocation_overlaps(&new_crtc_state->wm.skl.ddb,
10046                                                                         entries, I915_MAX_PIPES, pipe));
10047
10048                 entries[pipe] = new_crtc_state->wm.skl.ddb;
10049                 update_pipes &= ~BIT(pipe);
10050
10051                 intel_update_crtc(state, crtc);
10052         }
10053
10054         drm_WARN_ON(&dev_priv->drm, modeset_pipes);
10055         drm_WARN_ON(&dev_priv->drm, update_pipes);
10056 }
10057
10058 static void intel_atomic_helper_free_state(struct drm_i915_private *dev_priv)
10059 {
10060         struct intel_atomic_state *state, *next;
10061         struct llist_node *freed;
10062
10063         freed = llist_del_all(&dev_priv->atomic_helper.free_list);
10064         llist_for_each_entry_safe(state, next, freed, freed)
10065                 drm_atomic_state_put(&state->base);
10066 }
10067
10068 static void intel_atomic_helper_free_state_worker(struct work_struct *work)
10069 {
10070         struct drm_i915_private *dev_priv =
10071                 container_of(work, typeof(*dev_priv), atomic_helper.free_work);
10072
10073         intel_atomic_helper_free_state(dev_priv);
10074 }
10075
10076 static void intel_atomic_commit_fence_wait(struct intel_atomic_state *intel_state)
10077 {
10078         struct wait_queue_entry wait_fence, wait_reset;
10079         struct drm_i915_private *dev_priv = to_i915(intel_state->base.dev);
10080
10081         init_wait_entry(&wait_fence, 0);
10082         init_wait_entry(&wait_reset, 0);
10083         for (;;) {
10084                 prepare_to_wait(&intel_state->commit_ready.wait,
10085                                 &wait_fence, TASK_UNINTERRUPTIBLE);
10086                 prepare_to_wait(bit_waitqueue(&dev_priv->gt.reset.flags,
10087                                               I915_RESET_MODESET),
10088                                 &wait_reset, TASK_UNINTERRUPTIBLE);
10089
10090
10091                 if (i915_sw_fence_done(&intel_state->commit_ready) ||
10092                     test_bit(I915_RESET_MODESET, &dev_priv->gt.reset.flags))
10093                         break;
10094
10095                 schedule();
10096         }
10097         finish_wait(&intel_state->commit_ready.wait, &wait_fence);
10098         finish_wait(bit_waitqueue(&dev_priv->gt.reset.flags,
10099                                   I915_RESET_MODESET),
10100                     &wait_reset);
10101 }
10102
10103 static void intel_cleanup_dsbs(struct intel_atomic_state *state)
10104 {
10105         struct intel_crtc_state *old_crtc_state, *new_crtc_state;
10106         struct intel_crtc *crtc;
10107         int i;
10108
10109         for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state,
10110                                             new_crtc_state, i)
10111                 intel_dsb_cleanup(old_crtc_state);
10112 }
10113
10114 static void intel_atomic_cleanup_work(struct work_struct *work)
10115 {
10116         struct intel_atomic_state *state =
10117                 container_of(work, struct intel_atomic_state, base.commit_work);
10118         struct drm_i915_private *i915 = to_i915(state->base.dev);
10119
10120         intel_cleanup_dsbs(state);
10121         drm_atomic_helper_cleanup_planes(&i915->drm, &state->base);
10122         drm_atomic_helper_commit_cleanup_done(&state->base);
10123         drm_atomic_state_put(&state->base);
10124
10125         intel_atomic_helper_free_state(i915);
10126 }
10127
10128 static void intel_atomic_prepare_plane_clear_colors(struct intel_atomic_state *state)
10129 {
10130         struct drm_i915_private *i915 = to_i915(state->base.dev);
10131         struct intel_plane *plane;
10132         struct intel_plane_state *plane_state;
10133         int i;
10134
10135         for_each_new_intel_plane_in_state(state, plane, plane_state, i) {
10136                 struct drm_framebuffer *fb = plane_state->hw.fb;
10137                 int ret;
10138
10139                 if (!fb ||
10140                     fb->modifier != I915_FORMAT_MOD_Y_TILED_GEN12_RC_CCS_CC)
10141                         continue;
10142
10143                 /*
10144                  * The layout of the fast clear color value expected by HW
10145                  * (the DRM ABI requiring this value to be located in fb at offset 0 of plane#2):
10146                  * - 4 x 4 bytes per-channel value
10147                  *   (in surface type specific float/int format provided by the fb user)
10148                  * - 8 bytes native color value used by the display
10149                  *   (converted/written by GPU during a fast clear operation using the
10150                  *    above per-channel values)
10151                  *
10152                  * The commit's FB prepare hook already ensured that FB obj is pinned and the
10153                  * caller made sure that the object is synced wrt. the related color clear value
10154                  * GPU write on it.
10155                  */
10156                 ret = i915_gem_object_read_from_page(intel_fb_obj(fb),
10157                                                      fb->offsets[2] + 16,
10158                                                      &plane_state->ccval,
10159                                                      sizeof(plane_state->ccval));
10160                 /* The above could only fail if the FB obj has an unexpected backing store type. */
10161                 drm_WARN_ON(&i915->drm, ret);
10162         }
10163 }
10164
10165 static void intel_atomic_commit_tail(struct intel_atomic_state *state)
10166 {
10167         struct drm_device *dev = state->base.dev;
10168         struct drm_i915_private *dev_priv = to_i915(dev);
10169         struct intel_crtc_state *new_crtc_state, *old_crtc_state;
10170         struct intel_crtc *crtc;
10171         u64 put_domains[I915_MAX_PIPES] = {};
10172         intel_wakeref_t wakeref = 0;
10173         int i;
10174
10175         intel_atomic_commit_fence_wait(state);
10176
10177         drm_atomic_helper_wait_for_dependencies(&state->base);
10178
10179         if (state->modeset)
10180                 wakeref = intel_display_power_get(dev_priv, POWER_DOMAIN_MODESET);
10181
10182         intel_atomic_prepare_plane_clear_colors(state);
10183
10184         for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state,
10185                                             new_crtc_state, i) {
10186                 if (intel_crtc_needs_modeset(new_crtc_state) ||
10187                     new_crtc_state->update_pipe) {
10188
10189                         put_domains[crtc->pipe] =
10190                                 modeset_get_crtc_power_domains(new_crtc_state);
10191                 }
10192         }
10193
10194         intel_commit_modeset_disables(state);
10195
10196         /* FIXME: Eventually get rid of our crtc->config pointer */
10197         for_each_new_intel_crtc_in_state(state, crtc, new_crtc_state, i)
10198                 crtc->config = new_crtc_state;
10199
10200         if (state->modeset) {
10201                 drm_atomic_helper_update_legacy_modeset_state(dev, &state->base);
10202
10203                 intel_set_cdclk_pre_plane_update(state);
10204
10205                 intel_modeset_verify_disabled(dev_priv, state);
10206         }
10207
10208         intel_sagv_pre_plane_update(state);
10209
10210         /* Complete the events for pipes that have now been disabled */
10211         for_each_new_intel_crtc_in_state(state, crtc, new_crtc_state, i) {
10212                 bool modeset = intel_crtc_needs_modeset(new_crtc_state);
10213
10214                 /* Complete events for now disable pipes here. */
10215                 if (modeset && !new_crtc_state->hw.active && new_crtc_state->uapi.event) {
10216                         spin_lock_irq(&dev->event_lock);
10217                         drm_crtc_send_vblank_event(&crtc->base,
10218                                                    new_crtc_state->uapi.event);
10219                         spin_unlock_irq(&dev->event_lock);
10220
10221                         new_crtc_state->uapi.event = NULL;
10222                 }
10223         }
10224
10225         if (state->modeset)
10226                 intel_encoders_update_prepare(state);
10227
10228         intel_dbuf_pre_plane_update(state);
10229         intel_psr_pre_plane_update(state);
10230
10231         for_each_new_intel_crtc_in_state(state, crtc, new_crtc_state, i) {
10232                 if (new_crtc_state->uapi.async_flip)
10233                         intel_crtc_enable_flip_done(state, crtc);
10234         }
10235
10236         /* Now enable the clocks, plane, pipe, and connectors that we set up. */
10237         dev_priv->display->commit_modeset_enables(state);
10238
10239         if (state->modeset) {
10240                 intel_encoders_update_complete(state);
10241
10242                 intel_set_cdclk_post_plane_update(state);
10243         }
10244
10245         /* FIXME: We should call drm_atomic_helper_commit_hw_done() here
10246          * already, but still need the state for the delayed optimization. To
10247          * fix this:
10248          * - wrap the optimization/post_plane_update stuff into a per-crtc work.
10249          * - schedule that vblank worker _before_ calling hw_done
10250          * - at the start of commit_tail, cancel it _synchrously
10251          * - switch over to the vblank wait helper in the core after that since
10252          *   we don't need out special handling any more.
10253          */
10254         drm_atomic_helper_wait_for_flip_done(dev, &state->base);
10255
10256         for_each_new_intel_crtc_in_state(state, crtc, new_crtc_state, i) {
10257                 if (new_crtc_state->uapi.async_flip)
10258                         intel_crtc_disable_flip_done(state, crtc);
10259
10260                 if (new_crtc_state->hw.active &&
10261                     !intel_crtc_needs_modeset(new_crtc_state) &&
10262                     !new_crtc_state->preload_luts &&
10263                     (new_crtc_state->uapi.color_mgmt_changed ||
10264                      new_crtc_state->update_pipe))
10265                         intel_color_load_luts(new_crtc_state);
10266         }
10267
10268         /*
10269          * Now that the vblank has passed, we can go ahead and program the
10270          * optimal watermarks on platforms that need two-step watermark
10271          * programming.
10272          *
10273          * TODO: Move this (and other cleanup) to an async worker eventually.
10274          */
10275         for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state,
10276                                             new_crtc_state, i) {
10277                 /*
10278                  * Gen2 reports pipe underruns whenever all planes are disabled.
10279                  * So re-enable underrun reporting after some planes get enabled.
10280                  *
10281                  * We do this before .optimize_watermarks() so that we have a
10282                  * chance of catching underruns with the intermediate watermarks
10283                  * vs. the new plane configuration.
10284                  */
10285                 if (DISPLAY_VER(dev_priv) == 2 && planes_enabling(old_crtc_state, new_crtc_state))
10286                         intel_set_cpu_fifo_underrun_reporting(dev_priv, crtc->pipe, true);
10287
10288                 intel_optimize_watermarks(state, crtc);
10289         }
10290
10291         intel_dbuf_post_plane_update(state);
10292         intel_psr_post_plane_update(state);
10293
10294         for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state, new_crtc_state, i) {
10295                 intel_post_plane_update(state, crtc);
10296
10297                 modeset_put_crtc_power_domains(crtc, put_domains[crtc->pipe]);
10298
10299                 intel_modeset_verify_crtc(crtc, state, old_crtc_state, new_crtc_state);
10300
10301                 /*
10302                  * DSB cleanup is done in cleanup_work aligning with framebuffer
10303                  * cleanup. So copy and reset the dsb structure to sync with
10304                  * commit_done and later do dsb cleanup in cleanup_work.
10305                  */
10306                 old_crtc_state->dsb = fetch_and_zero(&new_crtc_state->dsb);
10307         }
10308
10309         /* Underruns don't always raise interrupts, so check manually */
10310         intel_check_cpu_fifo_underruns(dev_priv);
10311         intel_check_pch_fifo_underruns(dev_priv);
10312
10313         if (state->modeset)
10314                 intel_verify_planes(state);
10315
10316         intel_sagv_post_plane_update(state);
10317
10318         drm_atomic_helper_commit_hw_done(&state->base);
10319
10320         if (state->modeset) {
10321                 /* As one of the primary mmio accessors, KMS has a high
10322                  * likelihood of triggering bugs in unclaimed access. After we
10323                  * finish modesetting, see if an error has been flagged, and if
10324                  * so enable debugging for the next modeset - and hope we catch
10325                  * the culprit.
10326                  */
10327                 intel_uncore_arm_unclaimed_mmio_detection(&dev_priv->uncore);
10328                 intel_display_power_put(dev_priv, POWER_DOMAIN_MODESET, wakeref);
10329         }
10330         intel_runtime_pm_put(&dev_priv->runtime_pm, state->wakeref);
10331
10332         /*
10333          * Defer the cleanup of the old state to a separate worker to not
10334          * impede the current task (userspace for blocking modesets) that
10335          * are executed inline. For out-of-line asynchronous modesets/flips,
10336          * deferring to a new worker seems overkill, but we would place a
10337          * schedule point (cond_resched()) here anyway to keep latencies
10338          * down.
10339          */
10340         INIT_WORK(&state->base.commit_work, intel_atomic_cleanup_work);
10341         queue_work(system_highpri_wq, &state->base.commit_work);
10342 }
10343
10344 static void intel_atomic_commit_work(struct work_struct *work)
10345 {
10346         struct intel_atomic_state *state =
10347                 container_of(work, struct intel_atomic_state, base.commit_work);
10348
10349         intel_atomic_commit_tail(state);
10350 }
10351
10352 static int __i915_sw_fence_call
10353 intel_atomic_commit_ready(struct i915_sw_fence *fence,
10354                           enum i915_sw_fence_notify notify)
10355 {
10356         struct intel_atomic_state *state =
10357                 container_of(fence, struct intel_atomic_state, commit_ready);
10358
10359         switch (notify) {
10360         case FENCE_COMPLETE:
10361                 /* we do blocking waits in the worker, nothing to do here */
10362                 break;
10363         case FENCE_FREE:
10364                 {
10365                         struct intel_atomic_helper *helper =
10366                                 &to_i915(state->base.dev)->atomic_helper;
10367
10368                         if (llist_add(&state->freed, &helper->free_list))
10369                                 schedule_work(&helper->free_work);
10370                         break;
10371                 }
10372         }
10373
10374         return NOTIFY_DONE;
10375 }
10376
10377 static void intel_atomic_track_fbs(struct intel_atomic_state *state)
10378 {
10379         struct intel_plane_state *old_plane_state, *new_plane_state;
10380         struct intel_plane *plane;
10381         int i;
10382
10383         for_each_oldnew_intel_plane_in_state(state, plane, old_plane_state,
10384                                              new_plane_state, i)
10385                 intel_frontbuffer_track(to_intel_frontbuffer(old_plane_state->hw.fb),
10386                                         to_intel_frontbuffer(new_plane_state->hw.fb),
10387                                         plane->frontbuffer_bit);
10388 }
10389
10390 static int intel_atomic_commit(struct drm_device *dev,
10391                                struct drm_atomic_state *_state,
10392                                bool nonblock)
10393 {
10394         struct intel_atomic_state *state = to_intel_atomic_state(_state);
10395         struct drm_i915_private *dev_priv = to_i915(dev);
10396         int ret = 0;
10397
10398         state->wakeref = intel_runtime_pm_get(&dev_priv->runtime_pm);
10399
10400         drm_atomic_state_get(&state->base);
10401         i915_sw_fence_init(&state->commit_ready,
10402                            intel_atomic_commit_ready);
10403
10404         /*
10405          * The intel_legacy_cursor_update() fast path takes care
10406          * of avoiding the vblank waits for simple cursor
10407          * movement and flips. For cursor on/off and size changes,
10408          * we want to perform the vblank waits so that watermark
10409          * updates happen during the correct frames. Gen9+ have
10410          * double buffered watermarks and so shouldn't need this.
10411          *
10412          * Unset state->legacy_cursor_update before the call to
10413          * drm_atomic_helper_setup_commit() because otherwise
10414          * drm_atomic_helper_wait_for_flip_done() is a noop and
10415          * we get FIFO underruns because we didn't wait
10416          * for vblank.
10417          *
10418          * FIXME doing watermarks and fb cleanup from a vblank worker
10419          * (assuming we had any) would solve these problems.
10420          */
10421         if (DISPLAY_VER(dev_priv) < 9 && state->base.legacy_cursor_update) {
10422                 struct intel_crtc_state *new_crtc_state;
10423                 struct intel_crtc *crtc;
10424                 int i;
10425
10426                 for_each_new_intel_crtc_in_state(state, crtc, new_crtc_state, i)
10427                         if (new_crtc_state->wm.need_postvbl_update ||
10428                             new_crtc_state->update_wm_post)
10429                                 state->base.legacy_cursor_update = false;
10430         }
10431
10432         ret = intel_atomic_prepare_commit(state);
10433         if (ret) {
10434                 drm_dbg_atomic(&dev_priv->drm,
10435                                "Preparing state failed with %i\n", ret);
10436                 i915_sw_fence_commit(&state->commit_ready);
10437                 intel_runtime_pm_put(&dev_priv->runtime_pm, state->wakeref);
10438                 return ret;
10439         }
10440
10441         ret = drm_atomic_helper_setup_commit(&state->base, nonblock);
10442         if (!ret)
10443                 ret = drm_atomic_helper_swap_state(&state->base, true);
10444         if (!ret)
10445                 intel_atomic_swap_global_state(state);
10446
10447         if (ret) {
10448                 struct intel_crtc_state *new_crtc_state;
10449                 struct intel_crtc *crtc;
10450                 int i;
10451
10452                 i915_sw_fence_commit(&state->commit_ready);
10453
10454                 for_each_new_intel_crtc_in_state(state, crtc, new_crtc_state, i)
10455                         intel_dsb_cleanup(new_crtc_state);
10456
10457                 drm_atomic_helper_cleanup_planes(dev, &state->base);
10458                 intel_runtime_pm_put(&dev_priv->runtime_pm, state->wakeref);
10459                 return ret;
10460         }
10461         intel_shared_dpll_swap_state(state);
10462         intel_atomic_track_fbs(state);
10463
10464         drm_atomic_state_get(&state->base);
10465         INIT_WORK(&state->base.commit_work, intel_atomic_commit_work);
10466
10467         i915_sw_fence_commit(&state->commit_ready);
10468         if (nonblock && state->modeset) {
10469                 queue_work(dev_priv->modeset_wq, &state->base.commit_work);
10470         } else if (nonblock) {
10471                 queue_work(dev_priv->flip_wq, &state->base.commit_work);
10472         } else {
10473                 if (state->modeset)
10474                         flush_workqueue(dev_priv->modeset_wq);
10475                 intel_atomic_commit_tail(state);
10476         }
10477
10478         return 0;
10479 }
10480
10481 struct wait_rps_boost {
10482         struct wait_queue_entry wait;
10483
10484         struct drm_crtc *crtc;
10485         struct i915_request *request;
10486 };
10487
10488 static int do_rps_boost(struct wait_queue_entry *_wait,
10489                         unsigned mode, int sync, void *key)
10490 {
10491         struct wait_rps_boost *wait = container_of(_wait, typeof(*wait), wait);
10492         struct i915_request *rq = wait->request;
10493
10494         /*
10495          * If we missed the vblank, but the request is already running it
10496          * is reasonable to assume that it will complete before the next
10497          * vblank without our intervention, so leave RPS alone.
10498          */
10499         if (!i915_request_started(rq))
10500                 intel_rps_boost(rq);
10501         i915_request_put(rq);
10502
10503         drm_crtc_vblank_put(wait->crtc);
10504
10505         list_del(&wait->wait.entry);
10506         kfree(wait);
10507         return 1;
10508 }
10509
10510 static void add_rps_boost_after_vblank(struct drm_crtc *crtc,
10511                                        struct dma_fence *fence)
10512 {
10513         struct wait_rps_boost *wait;
10514
10515         if (!dma_fence_is_i915(fence))
10516                 return;
10517
10518         if (DISPLAY_VER(to_i915(crtc->dev)) < 6)
10519                 return;
10520
10521         if (drm_crtc_vblank_get(crtc))
10522                 return;
10523
10524         wait = kmalloc(sizeof(*wait), GFP_KERNEL);
10525         if (!wait) {
10526                 drm_crtc_vblank_put(crtc);
10527                 return;
10528         }
10529
10530         wait->request = to_request(dma_fence_get(fence));
10531         wait->crtc = crtc;
10532
10533         wait->wait.func = do_rps_boost;
10534         wait->wait.flags = 0;
10535
10536         add_wait_queue(drm_crtc_vblank_waitqueue(crtc), &wait->wait);
10537 }
10538
10539 int intel_plane_pin_fb(struct intel_plane_state *plane_state)
10540 {
10541         struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
10542         struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
10543         struct drm_framebuffer *fb = plane_state->hw.fb;
10544         struct i915_vma *vma;
10545         bool phys_cursor =
10546                 plane->id == PLANE_CURSOR &&
10547                 INTEL_INFO(dev_priv)->display.cursor_needs_physical;
10548
10549         if (!intel_fb_uses_dpt(fb)) {
10550                 vma = intel_pin_and_fence_fb_obj(fb, phys_cursor,
10551                                                  &plane_state->view.gtt,
10552                                                  intel_plane_uses_fence(plane_state),
10553                                                  &plane_state->flags);
10554                 if (IS_ERR(vma))
10555                         return PTR_ERR(vma);
10556
10557                 plane_state->ggtt_vma = vma;
10558         } else {
10559                 struct intel_framebuffer *intel_fb = to_intel_framebuffer(fb);
10560
10561                 vma = intel_dpt_pin(intel_fb->dpt_vm);
10562                 if (IS_ERR(vma))
10563                         return PTR_ERR(vma);
10564
10565                 plane_state->ggtt_vma = vma;
10566
10567                 vma = intel_pin_fb_obj_dpt(fb, &plane_state->view.gtt, false,
10568                                            &plane_state->flags, intel_fb->dpt_vm);
10569                 if (IS_ERR(vma)) {
10570                         intel_dpt_unpin(intel_fb->dpt_vm);
10571                         plane_state->ggtt_vma = NULL;
10572                         return PTR_ERR(vma);
10573                 }
10574
10575                 plane_state->dpt_vma = vma;
10576
10577                 WARN_ON(plane_state->ggtt_vma == plane_state->dpt_vma);
10578         }
10579
10580         return 0;
10581 }
10582
10583 void intel_plane_unpin_fb(struct intel_plane_state *old_plane_state)
10584 {
10585         struct drm_framebuffer *fb = old_plane_state->hw.fb;
10586         struct i915_vma *vma;
10587
10588         if (!intel_fb_uses_dpt(fb)) {
10589                 vma = fetch_and_zero(&old_plane_state->ggtt_vma);
10590                 if (vma)
10591                         intel_unpin_fb_vma(vma, old_plane_state->flags);
10592         } else {
10593                 struct intel_framebuffer *intel_fb = to_intel_framebuffer(fb);
10594
10595                 vma = fetch_and_zero(&old_plane_state->dpt_vma);
10596                 if (vma)
10597                         intel_unpin_fb_vma(vma, old_plane_state->flags);
10598
10599                 vma = fetch_and_zero(&old_plane_state->ggtt_vma);
10600                 if (vma)
10601                         intel_dpt_unpin(intel_fb->dpt_vm);
10602         }
10603 }
10604
10605 /**
10606  * intel_prepare_plane_fb - Prepare fb for usage on plane
10607  * @_plane: drm plane to prepare for
10608  * @_new_plane_state: the plane state being prepared
10609  *
10610  * Prepares a framebuffer for usage on a display plane.  Generally this
10611  * involves pinning the underlying object and updating the frontbuffer tracking
10612  * bits.  Some older platforms need special physical address handling for
10613  * cursor planes.
10614  *
10615  * Returns 0 on success, negative error code on failure.
10616  */
10617 int
10618 intel_prepare_plane_fb(struct drm_plane *_plane,
10619                        struct drm_plane_state *_new_plane_state)
10620 {
10621         struct i915_sched_attr attr = { .priority = I915_PRIORITY_DISPLAY };
10622         struct intel_plane *plane = to_intel_plane(_plane);
10623         struct intel_plane_state *new_plane_state =
10624                 to_intel_plane_state(_new_plane_state);
10625         struct intel_atomic_state *state =
10626                 to_intel_atomic_state(new_plane_state->uapi.state);
10627         struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
10628         const struct intel_plane_state *old_plane_state =
10629                 intel_atomic_get_old_plane_state(state, plane);
10630         struct drm_i915_gem_object *obj = intel_fb_obj(new_plane_state->hw.fb);
10631         struct drm_i915_gem_object *old_obj = intel_fb_obj(old_plane_state->hw.fb);
10632         int ret;
10633
10634         if (old_obj) {
10635                 const struct intel_crtc_state *crtc_state =
10636                         intel_atomic_get_new_crtc_state(state,
10637                                                         to_intel_crtc(old_plane_state->hw.crtc));
10638
10639                 /* Big Hammer, we also need to ensure that any pending
10640                  * MI_WAIT_FOR_EVENT inside a user batch buffer on the
10641                  * current scanout is retired before unpinning the old
10642                  * framebuffer. Note that we rely on userspace rendering
10643                  * into the buffer attached to the pipe they are waiting
10644                  * on. If not, userspace generates a GPU hang with IPEHR
10645                  * point to the MI_WAIT_FOR_EVENT.
10646                  *
10647                  * This should only fail upon a hung GPU, in which case we
10648                  * can safely continue.
10649                  */
10650                 if (intel_crtc_needs_modeset(crtc_state)) {
10651                         ret = i915_sw_fence_await_reservation(&state->commit_ready,
10652                                                               old_obj->base.resv, NULL,
10653                                                               false, 0,
10654                                                               GFP_KERNEL);
10655                         if (ret < 0)
10656                                 return ret;
10657                 }
10658         }
10659
10660         if (new_plane_state->uapi.fence) { /* explicit fencing */
10661                 i915_gem_fence_wait_priority(new_plane_state->uapi.fence,
10662                                              &attr);
10663                 ret = i915_sw_fence_await_dma_fence(&state->commit_ready,
10664                                                     new_plane_state->uapi.fence,
10665                                                     i915_fence_timeout(dev_priv),
10666                                                     GFP_KERNEL);
10667                 if (ret < 0)
10668                         return ret;
10669         }
10670
10671         if (!obj)
10672                 return 0;
10673
10674
10675         ret = intel_plane_pin_fb(new_plane_state);
10676         if (ret)
10677                 return ret;
10678
10679         i915_gem_object_wait_priority(obj, 0, &attr);
10680
10681         if (!new_plane_state->uapi.fence) { /* implicit fencing */
10682                 struct dma_fence *fence;
10683
10684                 ret = i915_sw_fence_await_reservation(&state->commit_ready,
10685                                                       obj->base.resv, NULL,
10686                                                       false,
10687                                                       i915_fence_timeout(dev_priv),
10688                                                       GFP_KERNEL);
10689                 if (ret < 0)
10690                         goto unpin_fb;
10691
10692                 fence = dma_resv_get_excl_unlocked(obj->base.resv);
10693                 if (fence) {
10694                         add_rps_boost_after_vblank(new_plane_state->hw.crtc,
10695                                                    fence);
10696                         dma_fence_put(fence);
10697                 }
10698         } else {
10699                 add_rps_boost_after_vblank(new_plane_state->hw.crtc,
10700                                            new_plane_state->uapi.fence);
10701         }
10702
10703         /*
10704          * We declare pageflips to be interactive and so merit a small bias
10705          * towards upclocking to deliver the frame on time. By only changing
10706          * the RPS thresholds to sample more regularly and aim for higher
10707          * clocks we can hopefully deliver low power workloads (like kodi)
10708          * that are not quite steady state without resorting to forcing
10709          * maximum clocks following a vblank miss (see do_rps_boost()).
10710          */
10711         if (!state->rps_interactive) {
10712                 intel_rps_mark_interactive(&dev_priv->gt.rps, true);
10713                 state->rps_interactive = true;
10714         }
10715
10716         return 0;
10717
10718 unpin_fb:
10719         intel_plane_unpin_fb(new_plane_state);
10720
10721         return ret;
10722 }
10723
10724 /**
10725  * intel_cleanup_plane_fb - Cleans up an fb after plane use
10726  * @plane: drm plane to clean up for
10727  * @_old_plane_state: the state from the previous modeset
10728  *
10729  * Cleans up a framebuffer that has just been removed from a plane.
10730  */
10731 void
10732 intel_cleanup_plane_fb(struct drm_plane *plane,
10733                        struct drm_plane_state *_old_plane_state)
10734 {
10735         struct intel_plane_state *old_plane_state =
10736                 to_intel_plane_state(_old_plane_state);
10737         struct intel_atomic_state *state =
10738                 to_intel_atomic_state(old_plane_state->uapi.state);
10739         struct drm_i915_private *dev_priv = to_i915(plane->dev);
10740         struct drm_i915_gem_object *obj = intel_fb_obj(old_plane_state->hw.fb);
10741
10742         if (!obj)
10743                 return;
10744
10745         if (state->rps_interactive) {
10746                 intel_rps_mark_interactive(&dev_priv->gt.rps, false);
10747                 state->rps_interactive = false;
10748         }
10749
10750         /* Should only be called after a successful intel_prepare_plane_fb()! */
10751         intel_plane_unpin_fb(old_plane_state);
10752 }
10753
10754 /**
10755  * intel_plane_destroy - destroy a plane
10756  * @plane: plane to destroy
10757  *
10758  * Common destruction function for all types of planes (primary, cursor,
10759  * sprite).
10760  */
10761 void intel_plane_destroy(struct drm_plane *plane)
10762 {
10763         drm_plane_cleanup(plane);
10764         kfree(to_intel_plane(plane));
10765 }
10766
10767 static void intel_plane_possible_crtcs_init(struct drm_i915_private *dev_priv)
10768 {
10769         struct intel_plane *plane;
10770
10771         for_each_intel_plane(&dev_priv->drm, plane) {
10772                 struct intel_crtc *crtc = intel_get_crtc_for_pipe(dev_priv,
10773                                                                   plane->pipe);
10774
10775                 plane->base.possible_crtcs = drm_crtc_mask(&crtc->base);
10776         }
10777 }
10778
10779
10780 int intel_get_pipe_from_crtc_id_ioctl(struct drm_device *dev, void *data,
10781                                       struct drm_file *file)
10782 {
10783         struct drm_i915_get_pipe_from_crtc_id *pipe_from_crtc_id = data;
10784         struct drm_crtc *drmmode_crtc;
10785         struct intel_crtc *crtc;
10786
10787         drmmode_crtc = drm_crtc_find(dev, file, pipe_from_crtc_id->crtc_id);
10788         if (!drmmode_crtc)
10789                 return -ENOENT;
10790
10791         crtc = to_intel_crtc(drmmode_crtc);
10792         pipe_from_crtc_id->pipe = crtc->pipe;
10793
10794         return 0;
10795 }
10796
10797 static u32 intel_encoder_possible_clones(struct intel_encoder *encoder)
10798 {
10799         struct drm_device *dev = encoder->base.dev;
10800         struct intel_encoder *source_encoder;
10801         u32 possible_clones = 0;
10802
10803         for_each_intel_encoder(dev, source_encoder) {
10804                 if (encoders_cloneable(encoder, source_encoder))
10805                         possible_clones |= drm_encoder_mask(&source_encoder->base);
10806         }
10807
10808         return possible_clones;
10809 }
10810
10811 static u32 intel_encoder_possible_crtcs(struct intel_encoder *encoder)
10812 {
10813         struct drm_device *dev = encoder->base.dev;
10814         struct intel_crtc *crtc;
10815         u32 possible_crtcs = 0;
10816
10817         for_each_intel_crtc(dev, crtc) {
10818                 if (encoder->pipe_mask & BIT(crtc->pipe))
10819                         possible_crtcs |= drm_crtc_mask(&crtc->base);
10820         }
10821
10822         return possible_crtcs;
10823 }
10824
10825 static bool ilk_has_edp_a(struct drm_i915_private *dev_priv)
10826 {
10827         if (!IS_MOBILE(dev_priv))
10828                 return false;
10829
10830         if ((intel_de_read(dev_priv, DP_A) & DP_DETECTED) == 0)
10831                 return false;
10832
10833         if (IS_IRONLAKE(dev_priv) && (intel_de_read(dev_priv, FUSE_STRAP) & ILK_eDP_A_DISABLE))
10834                 return false;
10835
10836         return true;
10837 }
10838
10839 static bool intel_ddi_crt_present(struct drm_i915_private *dev_priv)
10840 {
10841         if (DISPLAY_VER(dev_priv) >= 9)
10842                 return false;
10843
10844         if (IS_HSW_ULT(dev_priv) || IS_BDW_ULT(dev_priv))
10845                 return false;
10846
10847         if (HAS_PCH_LPT_H(dev_priv) &&
10848             intel_de_read(dev_priv, SFUSE_STRAP) & SFUSE_STRAP_CRT_DISABLED)
10849                 return false;
10850
10851         /* DDI E can't be used if DDI A requires 4 lanes */
10852         if (intel_de_read(dev_priv, DDI_BUF_CTL(PORT_A)) & DDI_A_4_LANES)
10853                 return false;
10854
10855         if (!dev_priv->vbt.int_crt_support)
10856                 return false;
10857
10858         return true;
10859 }
10860
10861 static void intel_setup_outputs(struct drm_i915_private *dev_priv)
10862 {
10863         struct intel_encoder *encoder;
10864         bool dpd_is_edp = false;
10865
10866         intel_pps_unlock_regs_wa(dev_priv);
10867
10868         if (!HAS_DISPLAY(dev_priv))
10869                 return;
10870
10871         if (IS_DG2(dev_priv)) {
10872                 intel_ddi_init(dev_priv, PORT_A);
10873                 intel_ddi_init(dev_priv, PORT_B);
10874                 intel_ddi_init(dev_priv, PORT_C);
10875                 intel_ddi_init(dev_priv, PORT_D_XELPD);
10876         } else if (IS_ALDERLAKE_P(dev_priv)) {
10877                 intel_ddi_init(dev_priv, PORT_A);
10878                 intel_ddi_init(dev_priv, PORT_B);
10879                 intel_ddi_init(dev_priv, PORT_TC1);
10880                 intel_ddi_init(dev_priv, PORT_TC2);
10881                 intel_ddi_init(dev_priv, PORT_TC3);
10882                 intel_ddi_init(dev_priv, PORT_TC4);
10883                 icl_dsi_init(dev_priv);
10884         } else if (IS_ALDERLAKE_S(dev_priv)) {
10885                 intel_ddi_init(dev_priv, PORT_A);
10886                 intel_ddi_init(dev_priv, PORT_TC1);
10887                 intel_ddi_init(dev_priv, PORT_TC2);
10888                 intel_ddi_init(dev_priv, PORT_TC3);
10889                 intel_ddi_init(dev_priv, PORT_TC4);
10890         } else if (IS_DG1(dev_priv) || IS_ROCKETLAKE(dev_priv)) {
10891                 intel_ddi_init(dev_priv, PORT_A);
10892                 intel_ddi_init(dev_priv, PORT_B);
10893                 intel_ddi_init(dev_priv, PORT_TC1);
10894                 intel_ddi_init(dev_priv, PORT_TC2);
10895         } else if (DISPLAY_VER(dev_priv) >= 12) {
10896                 intel_ddi_init(dev_priv, PORT_A);
10897                 intel_ddi_init(dev_priv, PORT_B);
10898                 intel_ddi_init(dev_priv, PORT_TC1);
10899                 intel_ddi_init(dev_priv, PORT_TC2);
10900                 intel_ddi_init(dev_priv, PORT_TC3);
10901                 intel_ddi_init(dev_priv, PORT_TC4);
10902                 intel_ddi_init(dev_priv, PORT_TC5);
10903                 intel_ddi_init(dev_priv, PORT_TC6);
10904                 icl_dsi_init(dev_priv);
10905         } else if (IS_JSL_EHL(dev_priv)) {
10906                 intel_ddi_init(dev_priv, PORT_A);
10907                 intel_ddi_init(dev_priv, PORT_B);
10908                 intel_ddi_init(dev_priv, PORT_C);
10909                 intel_ddi_init(dev_priv, PORT_D);
10910                 icl_dsi_init(dev_priv);
10911         } else if (DISPLAY_VER(dev_priv) == 11) {
10912                 intel_ddi_init(dev_priv, PORT_A);
10913                 intel_ddi_init(dev_priv, PORT_B);
10914                 intel_ddi_init(dev_priv, PORT_C);
10915                 intel_ddi_init(dev_priv, PORT_D);
10916                 intel_ddi_init(dev_priv, PORT_E);
10917                 intel_ddi_init(dev_priv, PORT_F);
10918                 icl_dsi_init(dev_priv);
10919         } else if (IS_GEMINILAKE(dev_priv) || IS_BROXTON(dev_priv)) {
10920                 intel_ddi_init(dev_priv, PORT_A);
10921                 intel_ddi_init(dev_priv, PORT_B);
10922                 intel_ddi_init(dev_priv, PORT_C);
10923                 vlv_dsi_init(dev_priv);
10924         } else if (DISPLAY_VER(dev_priv) >= 9) {
10925                 intel_ddi_init(dev_priv, PORT_A);
10926                 intel_ddi_init(dev_priv, PORT_B);
10927                 intel_ddi_init(dev_priv, PORT_C);
10928                 intel_ddi_init(dev_priv, PORT_D);
10929                 intel_ddi_init(dev_priv, PORT_E);
10930         } else if (HAS_DDI(dev_priv)) {
10931                 u32 found;
10932
10933                 if (intel_ddi_crt_present(dev_priv))
10934                         intel_crt_init(dev_priv);
10935
10936                 /* Haswell uses DDI functions to detect digital outputs. */
10937                 found = intel_de_read(dev_priv, DDI_BUF_CTL(PORT_A)) & DDI_INIT_DISPLAY_DETECTED;
10938                 if (found)
10939                         intel_ddi_init(dev_priv, PORT_A);
10940
10941                 found = intel_de_read(dev_priv, SFUSE_STRAP);
10942                 if (found & SFUSE_STRAP_DDIB_DETECTED)
10943                         intel_ddi_init(dev_priv, PORT_B);
10944                 if (found & SFUSE_STRAP_DDIC_DETECTED)
10945                         intel_ddi_init(dev_priv, PORT_C);
10946                 if (found & SFUSE_STRAP_DDID_DETECTED)
10947                         intel_ddi_init(dev_priv, PORT_D);
10948                 if (found & SFUSE_STRAP_DDIF_DETECTED)
10949                         intel_ddi_init(dev_priv, PORT_F);
10950         } else if (HAS_PCH_SPLIT(dev_priv)) {
10951                 int found;
10952
10953                 /*
10954                  * intel_edp_init_connector() depends on this completing first,
10955                  * to prevent the registration of both eDP and LVDS and the
10956                  * incorrect sharing of the PPS.
10957                  */
10958                 intel_lvds_init(dev_priv);
10959                 intel_crt_init(dev_priv);
10960
10961                 dpd_is_edp = intel_dp_is_port_edp(dev_priv, PORT_D);
10962
10963                 if (ilk_has_edp_a(dev_priv))
10964                         g4x_dp_init(dev_priv, DP_A, PORT_A);
10965
10966                 if (intel_de_read(dev_priv, PCH_HDMIB) & SDVO_DETECTED) {
10967                         /* PCH SDVOB multiplex with HDMIB */
10968                         found = intel_sdvo_init(dev_priv, PCH_SDVOB, PORT_B);
10969                         if (!found)
10970                                 g4x_hdmi_init(dev_priv, PCH_HDMIB, PORT_B);
10971                         if (!found && (intel_de_read(dev_priv, PCH_DP_B) & DP_DETECTED))
10972                                 g4x_dp_init(dev_priv, PCH_DP_B, PORT_B);
10973                 }
10974
10975                 if (intel_de_read(dev_priv, PCH_HDMIC) & SDVO_DETECTED)
10976                         g4x_hdmi_init(dev_priv, PCH_HDMIC, PORT_C);
10977
10978                 if (!dpd_is_edp && intel_de_read(dev_priv, PCH_HDMID) & SDVO_DETECTED)
10979                         g4x_hdmi_init(dev_priv, PCH_HDMID, PORT_D);
10980
10981                 if (intel_de_read(dev_priv, PCH_DP_C) & DP_DETECTED)
10982                         g4x_dp_init(dev_priv, PCH_DP_C, PORT_C);
10983
10984                 if (intel_de_read(dev_priv, PCH_DP_D) & DP_DETECTED)
10985                         g4x_dp_init(dev_priv, PCH_DP_D, PORT_D);
10986         } else if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) {
10987                 bool has_edp, has_port;
10988
10989                 if (IS_VALLEYVIEW(dev_priv) && dev_priv->vbt.int_crt_support)
10990                         intel_crt_init(dev_priv);
10991
10992                 /*
10993                  * The DP_DETECTED bit is the latched state of the DDC
10994                  * SDA pin at boot. However since eDP doesn't require DDC
10995                  * (no way to plug in a DP->HDMI dongle) the DDC pins for
10996                  * eDP ports may have been muxed to an alternate function.
10997                  * Thus we can't rely on the DP_DETECTED bit alone to detect
10998                  * eDP ports. Consult the VBT as well as DP_DETECTED to
10999                  * detect eDP ports.
11000                  *
11001                  * Sadly the straps seem to be missing sometimes even for HDMI
11002                  * ports (eg. on Voyo V3 - CHT x7-Z8700), so check both strap
11003                  * and VBT for the presence of the port. Additionally we can't
11004                  * trust the port type the VBT declares as we've seen at least
11005                  * HDMI ports that the VBT claim are DP or eDP.
11006                  */
11007                 has_edp = intel_dp_is_port_edp(dev_priv, PORT_B);
11008                 has_port = intel_bios_is_port_present(dev_priv, PORT_B);
11009                 if (intel_de_read(dev_priv, VLV_DP_B) & DP_DETECTED || has_port)
11010                         has_edp &= g4x_dp_init(dev_priv, VLV_DP_B, PORT_B);
11011                 if ((intel_de_read(dev_priv, VLV_HDMIB) & SDVO_DETECTED || has_port) && !has_edp)
11012                         g4x_hdmi_init(dev_priv, VLV_HDMIB, PORT_B);
11013
11014                 has_edp = intel_dp_is_port_edp(dev_priv, PORT_C);
11015                 has_port = intel_bios_is_port_present(dev_priv, PORT_C);
11016                 if (intel_de_read(dev_priv, VLV_DP_C) & DP_DETECTED || has_port)
11017                         has_edp &= g4x_dp_init(dev_priv, VLV_DP_C, PORT_C);
11018                 if ((intel_de_read(dev_priv, VLV_HDMIC) & SDVO_DETECTED || has_port) && !has_edp)
11019                         g4x_hdmi_init(dev_priv, VLV_HDMIC, PORT_C);
11020
11021                 if (IS_CHERRYVIEW(dev_priv)) {
11022                         /*
11023                          * eDP not supported on port D,
11024                          * so no need to worry about it
11025                          */
11026                         has_port = intel_bios_is_port_present(dev_priv, PORT_D);
11027                         if (intel_de_read(dev_priv, CHV_DP_D) & DP_DETECTED || has_port)
11028                                 g4x_dp_init(dev_priv, CHV_DP_D, PORT_D);
11029                         if (intel_de_read(dev_priv, CHV_HDMID) & SDVO_DETECTED || has_port)
11030                                 g4x_hdmi_init(dev_priv, CHV_HDMID, PORT_D);
11031                 }
11032
11033                 vlv_dsi_init(dev_priv);
11034         } else if (IS_PINEVIEW(dev_priv)) {
11035                 intel_lvds_init(dev_priv);
11036                 intel_crt_init(dev_priv);
11037         } else if (IS_DISPLAY_VER(dev_priv, 3, 4)) {
11038                 bool found = false;
11039
11040                 if (IS_MOBILE(dev_priv))
11041                         intel_lvds_init(dev_priv);
11042
11043                 intel_crt_init(dev_priv);
11044
11045                 if (intel_de_read(dev_priv, GEN3_SDVOB) & SDVO_DETECTED) {
11046                         drm_dbg_kms(&dev_priv->drm, "probing SDVOB\n");
11047                         found = intel_sdvo_init(dev_priv, GEN3_SDVOB, PORT_B);
11048                         if (!found && IS_G4X(dev_priv)) {
11049                                 drm_dbg_kms(&dev_priv->drm,
11050                                             "probing HDMI on SDVOB\n");
11051                                 g4x_hdmi_init(dev_priv, GEN4_HDMIB, PORT_B);
11052                         }
11053
11054                         if (!found && IS_G4X(dev_priv))
11055                                 g4x_dp_init(dev_priv, DP_B, PORT_B);
11056                 }
11057
11058                 /* Before G4X SDVOC doesn't have its own detect register */
11059
11060                 if (intel_de_read(dev_priv, GEN3_SDVOB) & SDVO_DETECTED) {
11061                         drm_dbg_kms(&dev_priv->drm, "probing SDVOC\n");
11062                         found = intel_sdvo_init(dev_priv, GEN3_SDVOC, PORT_C);
11063                 }
11064
11065                 if (!found && (intel_de_read(dev_priv, GEN3_SDVOC) & SDVO_DETECTED)) {
11066
11067                         if (IS_G4X(dev_priv)) {
11068                                 drm_dbg_kms(&dev_priv->drm,
11069                                             "probing HDMI on SDVOC\n");
11070                                 g4x_hdmi_init(dev_priv, GEN4_HDMIC, PORT_C);
11071                         }
11072                         if (IS_G4X(dev_priv))
11073                                 g4x_dp_init(dev_priv, DP_C, PORT_C);
11074                 }
11075
11076                 if (IS_G4X(dev_priv) && (intel_de_read(dev_priv, DP_D) & DP_DETECTED))
11077                         g4x_dp_init(dev_priv, DP_D, PORT_D);
11078
11079                 if (SUPPORTS_TV(dev_priv))
11080                         intel_tv_init(dev_priv);
11081         } else if (DISPLAY_VER(dev_priv) == 2) {
11082                 if (IS_I85X(dev_priv))
11083                         intel_lvds_init(dev_priv);
11084
11085                 intel_crt_init(dev_priv);
11086                 intel_dvo_init(dev_priv);
11087         }
11088
11089         for_each_intel_encoder(&dev_priv->drm, encoder) {
11090                 encoder->base.possible_crtcs =
11091                         intel_encoder_possible_crtcs(encoder);
11092                 encoder->base.possible_clones =
11093                         intel_encoder_possible_clones(encoder);
11094         }
11095
11096         intel_init_pch_refclk(dev_priv);
11097
11098         drm_helper_move_panel_connectors_to_head(&dev_priv->drm);
11099 }
11100
11101 static enum drm_mode_status
11102 intel_mode_valid(struct drm_device *dev,
11103                  const struct drm_display_mode *mode)
11104 {
11105         struct drm_i915_private *dev_priv = to_i915(dev);
11106         int hdisplay_max, htotal_max;
11107         int vdisplay_max, vtotal_max;
11108
11109         /*
11110          * Can't reject DBLSCAN here because Xorg ddxen can add piles
11111          * of DBLSCAN modes to the output's mode list when they detect
11112          * the scaling mode property on the connector. And they don't
11113          * ask the kernel to validate those modes in any way until
11114          * modeset time at which point the client gets a protocol error.
11115          * So in order to not upset those clients we silently ignore the
11116          * DBLSCAN flag on such connectors. For other connectors we will
11117          * reject modes with the DBLSCAN flag in encoder->compute_config().
11118          * And we always reject DBLSCAN modes in connector->mode_valid()
11119          * as we never want such modes on the connector's mode list.
11120          */
11121
11122         if (mode->vscan > 1)
11123                 return MODE_NO_VSCAN;
11124
11125         if (mode->flags & DRM_MODE_FLAG_HSKEW)
11126                 return MODE_H_ILLEGAL;
11127
11128         if (mode->flags & (DRM_MODE_FLAG_CSYNC |
11129                            DRM_MODE_FLAG_NCSYNC |
11130                            DRM_MODE_FLAG_PCSYNC))
11131                 return MODE_HSYNC;
11132
11133         if (mode->flags & (DRM_MODE_FLAG_BCAST |
11134                            DRM_MODE_FLAG_PIXMUX |
11135                            DRM_MODE_FLAG_CLKDIV2))
11136                 return MODE_BAD;
11137
11138         /* Transcoder timing limits */
11139         if (DISPLAY_VER(dev_priv) >= 11) {
11140                 hdisplay_max = 16384;
11141                 vdisplay_max = 8192;
11142                 htotal_max = 16384;
11143                 vtotal_max = 8192;
11144         } else if (DISPLAY_VER(dev_priv) >= 9 ||
11145                    IS_BROADWELL(dev_priv) || IS_HASWELL(dev_priv)) {
11146                 hdisplay_max = 8192; /* FDI max 4096 handled elsewhere */
11147                 vdisplay_max = 4096;
11148                 htotal_max = 8192;
11149                 vtotal_max = 8192;
11150         } else if (DISPLAY_VER(dev_priv) >= 3) {
11151                 hdisplay_max = 4096;
11152                 vdisplay_max = 4096;
11153                 htotal_max = 8192;
11154                 vtotal_max = 8192;
11155         } else {
11156                 hdisplay_max = 2048;
11157                 vdisplay_max = 2048;
11158                 htotal_max = 4096;
11159                 vtotal_max = 4096;
11160         }
11161
11162         if (mode->hdisplay > hdisplay_max ||
11163             mode->hsync_start > htotal_max ||
11164             mode->hsync_end > htotal_max ||
11165             mode->htotal > htotal_max)
11166                 return MODE_H_ILLEGAL;
11167
11168         if (mode->vdisplay > vdisplay_max ||
11169             mode->vsync_start > vtotal_max ||
11170             mode->vsync_end > vtotal_max ||
11171             mode->vtotal > vtotal_max)
11172                 return MODE_V_ILLEGAL;
11173
11174         if (DISPLAY_VER(dev_priv) >= 5) {
11175                 if (mode->hdisplay < 64 ||
11176                     mode->htotal - mode->hdisplay < 32)
11177                         return MODE_H_ILLEGAL;
11178
11179                 if (mode->vtotal - mode->vdisplay < 5)
11180                         return MODE_V_ILLEGAL;
11181         } else {
11182                 if (mode->htotal - mode->hdisplay < 32)
11183                         return MODE_H_ILLEGAL;
11184
11185                 if (mode->vtotal - mode->vdisplay < 3)
11186                         return MODE_V_ILLEGAL;
11187         }
11188
11189         /*
11190          * Cantiga+ cannot handle modes with a hsync front porch of 0.
11191          * WaPruneModeWithIncorrectHsyncOffset:ctg,elk,ilk,snb,ivb,vlv,hsw.
11192          */
11193         if ((DISPLAY_VER(dev_priv) > 4 || IS_G4X(dev_priv)) &&
11194             mode->hsync_start == mode->hdisplay)
11195                 return MODE_H_ILLEGAL;
11196
11197         return MODE_OK;
11198 }
11199
11200 enum drm_mode_status
11201 intel_mode_valid_max_plane_size(struct drm_i915_private *dev_priv,
11202                                 const struct drm_display_mode *mode,
11203                                 bool bigjoiner)
11204 {
11205         int plane_width_max, plane_height_max;
11206
11207         /*
11208          * intel_mode_valid() should be
11209          * sufficient on older platforms.
11210          */
11211         if (DISPLAY_VER(dev_priv) < 9)
11212                 return MODE_OK;
11213
11214         /*
11215          * Most people will probably want a fullscreen
11216          * plane so let's not advertize modes that are
11217          * too big for that.
11218          */
11219         if (DISPLAY_VER(dev_priv) >= 11) {
11220                 plane_width_max = 5120 << bigjoiner;
11221                 plane_height_max = 4320;
11222         } else {
11223                 plane_width_max = 5120;
11224                 plane_height_max = 4096;
11225         }
11226
11227         if (mode->hdisplay > plane_width_max)
11228                 return MODE_H_ILLEGAL;
11229
11230         if (mode->vdisplay > plane_height_max)
11231                 return MODE_V_ILLEGAL;
11232
11233         return MODE_OK;
11234 }
11235
11236 static const struct drm_mode_config_funcs intel_mode_funcs = {
11237         .fb_create = intel_user_framebuffer_create,
11238         .get_format_info = intel_get_format_info,
11239         .output_poll_changed = intel_fbdev_output_poll_changed,
11240         .mode_valid = intel_mode_valid,
11241         .atomic_check = intel_atomic_check,
11242         .atomic_commit = intel_atomic_commit,
11243         .atomic_state_alloc = intel_atomic_state_alloc,
11244         .atomic_state_clear = intel_atomic_state_clear,
11245         .atomic_state_free = intel_atomic_state_free,
11246 };
11247
11248 static const struct drm_i915_display_funcs skl_display_funcs = {
11249         .get_pipe_config = hsw_get_pipe_config,
11250         .crtc_enable = hsw_crtc_enable,
11251         .crtc_disable = hsw_crtc_disable,
11252         .commit_modeset_enables = skl_commit_modeset_enables,
11253         .get_initial_plane_config = skl_get_initial_plane_config,
11254 };
11255
11256 static const struct drm_i915_display_funcs ddi_display_funcs = {
11257         .get_pipe_config = hsw_get_pipe_config,
11258         .crtc_enable = hsw_crtc_enable,
11259         .crtc_disable = hsw_crtc_disable,
11260         .commit_modeset_enables = intel_commit_modeset_enables,
11261         .get_initial_plane_config = i9xx_get_initial_plane_config,
11262 };
11263
11264 static const struct drm_i915_display_funcs pch_split_display_funcs = {
11265         .get_pipe_config = ilk_get_pipe_config,
11266         .crtc_enable = ilk_crtc_enable,
11267         .crtc_disable = ilk_crtc_disable,
11268         .commit_modeset_enables = intel_commit_modeset_enables,
11269         .get_initial_plane_config = i9xx_get_initial_plane_config,
11270 };
11271
11272 static const struct drm_i915_display_funcs vlv_display_funcs = {
11273         .get_pipe_config = i9xx_get_pipe_config,
11274         .crtc_enable = valleyview_crtc_enable,
11275         .crtc_disable = i9xx_crtc_disable,
11276         .commit_modeset_enables = intel_commit_modeset_enables,
11277         .get_initial_plane_config = i9xx_get_initial_plane_config,
11278 };
11279
11280 static const struct drm_i915_display_funcs i9xx_display_funcs = {
11281         .get_pipe_config = i9xx_get_pipe_config,
11282         .crtc_enable = i9xx_crtc_enable,
11283         .crtc_disable = i9xx_crtc_disable,
11284         .commit_modeset_enables = intel_commit_modeset_enables,
11285         .get_initial_plane_config = i9xx_get_initial_plane_config,
11286 };
11287
11288 /**
11289  * intel_init_display_hooks - initialize the display modesetting hooks
11290  * @dev_priv: device private
11291  */
11292 void intel_init_display_hooks(struct drm_i915_private *dev_priv)
11293 {
11294         if (!HAS_DISPLAY(dev_priv))
11295                 return;
11296
11297         intel_init_cdclk_hooks(dev_priv);
11298         intel_init_audio_hooks(dev_priv);
11299
11300         intel_dpll_init_clock_hook(dev_priv);
11301
11302         if (DISPLAY_VER(dev_priv) >= 9) {
11303                 dev_priv->display = &skl_display_funcs;
11304         } else if (HAS_DDI(dev_priv)) {
11305                 dev_priv->display = &ddi_display_funcs;
11306         } else if (HAS_PCH_SPLIT(dev_priv)) {
11307                 dev_priv->display = &pch_split_display_funcs;
11308         } else if (IS_CHERRYVIEW(dev_priv) ||
11309                    IS_VALLEYVIEW(dev_priv)) {
11310                 dev_priv->display = &vlv_display_funcs;
11311         } else {
11312                 dev_priv->display = &i9xx_display_funcs;
11313         }
11314
11315         intel_fdi_init_hook(dev_priv);
11316 }
11317
11318 void intel_modeset_init_hw(struct drm_i915_private *i915)
11319 {
11320         struct intel_cdclk_state *cdclk_state;
11321
11322         if (!HAS_DISPLAY(i915))
11323                 return;
11324
11325         cdclk_state = to_intel_cdclk_state(i915->cdclk.obj.state);
11326
11327         intel_update_cdclk(i915);
11328         intel_dump_cdclk_config(&i915->cdclk.hw, "Current CDCLK");
11329         cdclk_state->logical = cdclk_state->actual = i915->cdclk.hw;
11330 }
11331
11332 static int sanitize_watermarks_add_affected(struct drm_atomic_state *state)
11333 {
11334         struct drm_plane *plane;
11335         struct intel_crtc *crtc;
11336
11337         for_each_intel_crtc(state->dev, crtc) {
11338                 struct intel_crtc_state *crtc_state;
11339
11340                 crtc_state = intel_atomic_get_crtc_state(state, crtc);
11341                 if (IS_ERR(crtc_state))
11342                         return PTR_ERR(crtc_state);
11343
11344                 if (crtc_state->hw.active) {
11345                         /*
11346                          * Preserve the inherited flag to avoid
11347                          * taking the full modeset path.
11348                          */
11349                         crtc_state->inherited = true;
11350                 }
11351         }
11352
11353         drm_for_each_plane(plane, state->dev) {
11354                 struct drm_plane_state *plane_state;
11355
11356                 plane_state = drm_atomic_get_plane_state(state, plane);
11357                 if (IS_ERR(plane_state))
11358                         return PTR_ERR(plane_state);
11359         }
11360
11361         return 0;
11362 }
11363
11364 /*
11365  * Calculate what we think the watermarks should be for the state we've read
11366  * out of the hardware and then immediately program those watermarks so that
11367  * we ensure the hardware settings match our internal state.
11368  *
11369  * We can calculate what we think WM's should be by creating a duplicate of the
11370  * current state (which was constructed during hardware readout) and running it
11371  * through the atomic check code to calculate new watermark values in the
11372  * state object.
11373  */
11374 static void sanitize_watermarks(struct drm_i915_private *dev_priv)
11375 {
11376         struct drm_atomic_state *state;
11377         struct intel_atomic_state *intel_state;
11378         struct intel_crtc *crtc;
11379         struct intel_crtc_state *crtc_state;
11380         struct drm_modeset_acquire_ctx ctx;
11381         int ret;
11382         int i;
11383
11384         /* Only supported on platforms that use atomic watermark design */
11385         if (!dev_priv->wm_disp->optimize_watermarks)
11386                 return;
11387
11388         state = drm_atomic_state_alloc(&dev_priv->drm);
11389         if (drm_WARN_ON(&dev_priv->drm, !state))
11390                 return;
11391
11392         intel_state = to_intel_atomic_state(state);
11393
11394         drm_modeset_acquire_init(&ctx, 0);
11395
11396 retry:
11397         state->acquire_ctx = &ctx;
11398
11399         /*
11400          * Hardware readout is the only time we don't want to calculate
11401          * intermediate watermarks (since we don't trust the current
11402          * watermarks).
11403          */
11404         if (!HAS_GMCH(dev_priv))
11405                 intel_state->skip_intermediate_wm = true;
11406
11407         ret = sanitize_watermarks_add_affected(state);
11408         if (ret)
11409                 goto fail;
11410
11411         ret = intel_atomic_check(&dev_priv->drm, state);
11412         if (ret)
11413                 goto fail;
11414
11415         /* Write calculated watermark values back */
11416         for_each_new_intel_crtc_in_state(intel_state, crtc, crtc_state, i) {
11417                 crtc_state->wm.need_postvbl_update = true;
11418                 intel_optimize_watermarks(intel_state, crtc);
11419
11420                 to_intel_crtc_state(crtc->base.state)->wm = crtc_state->wm;
11421         }
11422
11423 fail:
11424         if (ret == -EDEADLK) {
11425                 drm_atomic_state_clear(state);
11426                 drm_modeset_backoff(&ctx);
11427                 goto retry;
11428         }
11429
11430         /*
11431          * If we fail here, it means that the hardware appears to be
11432          * programmed in a way that shouldn't be possible, given our
11433          * understanding of watermark requirements.  This might mean a
11434          * mistake in the hardware readout code or a mistake in the
11435          * watermark calculations for a given platform.  Raise a WARN
11436          * so that this is noticeable.
11437          *
11438          * If this actually happens, we'll have to just leave the
11439          * BIOS-programmed watermarks untouched and hope for the best.
11440          */
11441         drm_WARN(&dev_priv->drm, ret,
11442                  "Could not determine valid watermarks for inherited state\n");
11443
11444         drm_atomic_state_put(state);
11445
11446         drm_modeset_drop_locks(&ctx);
11447         drm_modeset_acquire_fini(&ctx);
11448 }
11449
11450 static int intel_initial_commit(struct drm_device *dev)
11451 {
11452         struct drm_atomic_state *state = NULL;
11453         struct drm_modeset_acquire_ctx ctx;
11454         struct intel_crtc *crtc;
11455         int ret = 0;
11456
11457         state = drm_atomic_state_alloc(dev);
11458         if (!state)
11459                 return -ENOMEM;
11460
11461         drm_modeset_acquire_init(&ctx, 0);
11462
11463 retry:
11464         state->acquire_ctx = &ctx;
11465
11466         for_each_intel_crtc(dev, crtc) {
11467                 struct intel_crtc_state *crtc_state =
11468                         intel_atomic_get_crtc_state(state, crtc);
11469
11470                 if (IS_ERR(crtc_state)) {
11471                         ret = PTR_ERR(crtc_state);
11472                         goto out;
11473                 }
11474
11475                 if (crtc_state->hw.active) {
11476                         struct intel_encoder *encoder;
11477
11478                         /*
11479                          * We've not yet detected sink capabilities
11480                          * (audio,infoframes,etc.) and thus we don't want to
11481                          * force a full state recomputation yet. We want that to
11482                          * happen only for the first real commit from userspace.
11483                          * So preserve the inherited flag for the time being.
11484                          */
11485                         crtc_state->inherited = true;
11486
11487                         ret = drm_atomic_add_affected_planes(state, &crtc->base);
11488                         if (ret)
11489                                 goto out;
11490
11491                         /*
11492                          * FIXME hack to force a LUT update to avoid the
11493                          * plane update forcing the pipe gamma on without
11494                          * having a proper LUT loaded. Remove once we
11495                          * have readout for pipe gamma enable.
11496                          */
11497                         crtc_state->uapi.color_mgmt_changed = true;
11498
11499                         for_each_intel_encoder_mask(dev, encoder,
11500                                                     crtc_state->uapi.encoder_mask) {
11501                                 if (encoder->initial_fastset_check &&
11502                                     !encoder->initial_fastset_check(encoder, crtc_state)) {
11503                                         ret = drm_atomic_add_affected_connectors(state,
11504                                                                                  &crtc->base);
11505                                         if (ret)
11506                                                 goto out;
11507                                 }
11508                         }
11509                 }
11510         }
11511
11512         ret = drm_atomic_commit(state);
11513
11514 out:
11515         if (ret == -EDEADLK) {
11516                 drm_atomic_state_clear(state);
11517                 drm_modeset_backoff(&ctx);
11518                 goto retry;
11519         }
11520
11521         drm_atomic_state_put(state);
11522
11523         drm_modeset_drop_locks(&ctx);
11524         drm_modeset_acquire_fini(&ctx);
11525
11526         return ret;
11527 }
11528
11529 static void intel_mode_config_init(struct drm_i915_private *i915)
11530 {
11531         struct drm_mode_config *mode_config = &i915->drm.mode_config;
11532
11533         drm_mode_config_init(&i915->drm);
11534         INIT_LIST_HEAD(&i915->global_obj_list);
11535
11536         mode_config->min_width = 0;
11537         mode_config->min_height = 0;
11538
11539         mode_config->preferred_depth = 24;
11540         mode_config->prefer_shadow = 1;
11541
11542         mode_config->funcs = &intel_mode_funcs;
11543
11544         mode_config->async_page_flip = HAS_ASYNC_FLIPS(i915);
11545
11546         /*
11547          * Maximum framebuffer dimensions, chosen to match
11548          * the maximum render engine surface size on gen4+.
11549          */
11550         if (DISPLAY_VER(i915) >= 7) {
11551                 mode_config->max_width = 16384;
11552                 mode_config->max_height = 16384;
11553         } else if (DISPLAY_VER(i915) >= 4) {
11554                 mode_config->max_width = 8192;
11555                 mode_config->max_height = 8192;
11556         } else if (DISPLAY_VER(i915) == 3) {
11557                 mode_config->max_width = 4096;
11558                 mode_config->max_height = 4096;
11559         } else {
11560                 mode_config->max_width = 2048;
11561                 mode_config->max_height = 2048;
11562         }
11563
11564         if (IS_I845G(i915) || IS_I865G(i915)) {
11565                 mode_config->cursor_width = IS_I845G(i915) ? 64 : 512;
11566                 mode_config->cursor_height = 1023;
11567         } else if (IS_I830(i915) || IS_I85X(i915) ||
11568                    IS_I915G(i915) || IS_I915GM(i915)) {
11569                 mode_config->cursor_width = 64;
11570                 mode_config->cursor_height = 64;
11571         } else {
11572                 mode_config->cursor_width = 256;
11573                 mode_config->cursor_height = 256;
11574         }
11575 }
11576
11577 static void intel_mode_config_cleanup(struct drm_i915_private *i915)
11578 {
11579         intel_atomic_global_obj_cleanup(i915);
11580         drm_mode_config_cleanup(&i915->drm);
11581 }
11582
11583 static void plane_config_fini(struct intel_initial_plane_config *plane_config)
11584 {
11585         if (plane_config->fb) {
11586                 struct drm_framebuffer *fb = &plane_config->fb->base;
11587
11588                 /* We may only have the stub and not a full framebuffer */
11589                 if (drm_framebuffer_read_refcount(fb))
11590                         drm_framebuffer_put(fb);
11591                 else
11592                         kfree(fb);
11593         }
11594
11595         if (plane_config->vma)
11596                 i915_vma_put(plane_config->vma);
11597 }
11598
11599 /* part #1: call before irq install */
11600 int intel_modeset_init_noirq(struct drm_i915_private *i915)
11601 {
11602         int ret;
11603
11604         if (i915_inject_probe_failure(i915))
11605                 return -ENODEV;
11606
11607         if (HAS_DISPLAY(i915)) {
11608                 ret = drm_vblank_init(&i915->drm,
11609                                       INTEL_NUM_PIPES(i915));
11610                 if (ret)
11611                         return ret;
11612         }
11613
11614         intel_bios_init(i915);
11615
11616         ret = intel_vga_register(i915);
11617         if (ret)
11618                 goto cleanup_bios;
11619
11620         /* FIXME: completely on the wrong abstraction layer */
11621         intel_power_domains_init_hw(i915, false);
11622
11623         if (!HAS_DISPLAY(i915))
11624                 return 0;
11625
11626         intel_dmc_ucode_init(i915);
11627
11628         i915->modeset_wq = alloc_ordered_workqueue("i915_modeset", 0);
11629         i915->flip_wq = alloc_workqueue("i915_flip", WQ_HIGHPRI |
11630                                         WQ_UNBOUND, WQ_UNBOUND_MAX_ACTIVE);
11631
11632         i915->framestart_delay = 1; /* 1-4 */
11633
11634         i915->window2_delay = 0; /* No DSB so no window2 delay */
11635
11636         intel_mode_config_init(i915);
11637
11638         ret = intel_cdclk_init(i915);
11639         if (ret)
11640                 goto cleanup_vga_client_pw_domain_dmc;
11641
11642         ret = intel_dbuf_init(i915);
11643         if (ret)
11644                 goto cleanup_vga_client_pw_domain_dmc;
11645
11646         ret = intel_bw_init(i915);
11647         if (ret)
11648                 goto cleanup_vga_client_pw_domain_dmc;
11649
11650         init_llist_head(&i915->atomic_helper.free_list);
11651         INIT_WORK(&i915->atomic_helper.free_work,
11652                   intel_atomic_helper_free_state_worker);
11653
11654         intel_init_quirks(i915);
11655
11656         intel_fbc_init(i915);
11657
11658         return 0;
11659
11660 cleanup_vga_client_pw_domain_dmc:
11661         intel_dmc_ucode_fini(i915);
11662         intel_power_domains_driver_remove(i915);
11663         intel_vga_unregister(i915);
11664 cleanup_bios:
11665         intel_bios_driver_remove(i915);
11666
11667         return ret;
11668 }
11669
11670 /* part #2: call after irq install, but before gem init */
11671 int intel_modeset_init_nogem(struct drm_i915_private *i915)
11672 {
11673         struct drm_device *dev = &i915->drm;
11674         enum pipe pipe;
11675         struct intel_crtc *crtc;
11676         int ret;
11677
11678         if (!HAS_DISPLAY(i915))
11679                 return 0;
11680
11681         intel_init_pm(i915);
11682
11683         intel_panel_sanitize_ssc(i915);
11684
11685         intel_pps_setup(i915);
11686
11687         intel_gmbus_setup(i915);
11688
11689         drm_dbg_kms(&i915->drm, "%d display pipe%s available.\n",
11690                     INTEL_NUM_PIPES(i915),
11691                     INTEL_NUM_PIPES(i915) > 1 ? "s" : "");
11692
11693         for_each_pipe(i915, pipe) {
11694                 ret = intel_crtc_init(i915, pipe);
11695                 if (ret) {
11696                         intel_mode_config_cleanup(i915);
11697                         return ret;
11698                 }
11699         }
11700
11701         intel_plane_possible_crtcs_init(i915);
11702         intel_shared_dpll_init(dev);
11703         intel_fdi_pll_freq_update(i915);
11704
11705         intel_update_czclk(i915);
11706         intel_modeset_init_hw(i915);
11707         intel_dpll_update_ref_clks(i915);
11708
11709         intel_hdcp_component_init(i915);
11710
11711         if (i915->max_cdclk_freq == 0)
11712                 intel_update_max_cdclk(i915);
11713
11714         /*
11715          * If the platform has HTI, we need to find out whether it has reserved
11716          * any display resources before we create our display outputs.
11717          */
11718         if (INTEL_INFO(i915)->display.has_hti)
11719                 i915->hti_state = intel_de_read(i915, HDPORT_STATE);
11720
11721         /* Just disable it once at startup */
11722         intel_vga_disable(i915);
11723         intel_setup_outputs(i915);
11724
11725         drm_modeset_lock_all(dev);
11726         intel_modeset_setup_hw_state(dev, dev->mode_config.acquire_ctx);
11727         intel_acpi_assign_connector_fwnodes(i915);
11728         drm_modeset_unlock_all(dev);
11729
11730         for_each_intel_crtc(dev, crtc) {
11731                 struct intel_initial_plane_config plane_config = {};
11732
11733                 if (!to_intel_crtc_state(crtc->base.state)->uapi.active)
11734                         continue;
11735
11736                 /*
11737                  * Note that reserving the BIOS fb up front prevents us
11738                  * from stuffing other stolen allocations like the ring
11739                  * on top.  This prevents some ugliness at boot time, and
11740                  * can even allow for smooth boot transitions if the BIOS
11741                  * fb is large enough for the active pipe configuration.
11742                  */
11743                 i915->display->get_initial_plane_config(crtc, &plane_config);
11744
11745                 /*
11746                  * If the fb is shared between multiple heads, we'll
11747                  * just get the first one.
11748                  */
11749                 intel_find_initial_plane_obj(crtc, &plane_config);
11750
11751                 plane_config_fini(&plane_config);
11752         }
11753
11754         /*
11755          * Make sure hardware watermarks really match the state we read out.
11756          * Note that we need to do this after reconstructing the BIOS fb's
11757          * since the watermark calculation done here will use pstate->fb.
11758          */
11759         if (!HAS_GMCH(i915))
11760                 sanitize_watermarks(i915);
11761
11762         return 0;
11763 }
11764
11765 /* part #3: call after gem init */
11766 int intel_modeset_init(struct drm_i915_private *i915)
11767 {
11768         int ret;
11769
11770         if (!HAS_DISPLAY(i915))
11771                 return 0;
11772
11773         /*
11774          * Force all active planes to recompute their states. So that on
11775          * mode_setcrtc after probe, all the intel_plane_state variables
11776          * are already calculated and there is no assert_plane warnings
11777          * during bootup.
11778          */
11779         ret = intel_initial_commit(&i915->drm);
11780         if (ret)
11781                 drm_dbg_kms(&i915->drm, "Initial modeset failed, %d\n", ret);
11782
11783         intel_overlay_setup(i915);
11784
11785         ret = intel_fbdev_init(&i915->drm);
11786         if (ret)
11787                 return ret;
11788
11789         /* Only enable hotplug handling once the fbdev is fully set up. */
11790         intel_hpd_init(i915);
11791         intel_hpd_poll_disable(i915);
11792
11793         intel_init_ipc(i915);
11794
11795         return 0;
11796 }
11797
11798 void i830_enable_pipe(struct drm_i915_private *dev_priv, enum pipe pipe)
11799 {
11800         struct intel_crtc *crtc = intel_get_crtc_for_pipe(dev_priv, pipe);
11801         /* 640x480@60Hz, ~25175 kHz */
11802         struct dpll clock = {
11803                 .m1 = 18,
11804                 .m2 = 7,
11805                 .p1 = 13,
11806                 .p2 = 4,
11807                 .n = 2,
11808         };
11809         u32 dpll, fp;
11810         int i;
11811
11812         drm_WARN_ON(&dev_priv->drm,
11813                     i9xx_calc_dpll_params(48000, &clock) != 25154);
11814
11815         drm_dbg_kms(&dev_priv->drm,
11816                     "enabling pipe %c due to force quirk (vco=%d dot=%d)\n",
11817                     pipe_name(pipe), clock.vco, clock.dot);
11818
11819         fp = i9xx_dpll_compute_fp(&clock);
11820         dpll = DPLL_DVO_2X_MODE |
11821                 DPLL_VGA_MODE_DIS |
11822                 ((clock.p1 - 2) << DPLL_FPA01_P1_POST_DIV_SHIFT) |
11823                 PLL_P2_DIVIDE_BY_4 |
11824                 PLL_REF_INPUT_DREFCLK |
11825                 DPLL_VCO_ENABLE;
11826
11827         intel_de_write(dev_priv, FP0(pipe), fp);
11828         intel_de_write(dev_priv, FP1(pipe), fp);
11829
11830         intel_de_write(dev_priv, HTOTAL(pipe), (640 - 1) | ((800 - 1) << 16));
11831         intel_de_write(dev_priv, HBLANK(pipe), (640 - 1) | ((800 - 1) << 16));
11832         intel_de_write(dev_priv, HSYNC(pipe), (656 - 1) | ((752 - 1) << 16));
11833         intel_de_write(dev_priv, VTOTAL(pipe), (480 - 1) | ((525 - 1) << 16));
11834         intel_de_write(dev_priv, VBLANK(pipe), (480 - 1) | ((525 - 1) << 16));
11835         intel_de_write(dev_priv, VSYNC(pipe), (490 - 1) | ((492 - 1) << 16));
11836         intel_de_write(dev_priv, PIPESRC(pipe), ((640 - 1) << 16) | (480 - 1));
11837
11838         /*
11839          * Apparently we need to have VGA mode enabled prior to changing
11840          * the P1/P2 dividers. Otherwise the DPLL will keep using the old
11841          * dividers, even though the register value does change.
11842          */
11843         intel_de_write(dev_priv, DPLL(pipe), dpll & ~DPLL_VGA_MODE_DIS);
11844         intel_de_write(dev_priv, DPLL(pipe), dpll);
11845
11846         /* Wait for the clocks to stabilize. */
11847         intel_de_posting_read(dev_priv, DPLL(pipe));
11848         udelay(150);
11849
11850         /* The pixel multiplier can only be updated once the
11851          * DPLL is enabled and the clocks are stable.
11852          *
11853          * So write it again.
11854          */
11855         intel_de_write(dev_priv, DPLL(pipe), dpll);
11856
11857         /* We do this three times for luck */
11858         for (i = 0; i < 3 ; i++) {
11859                 intel_de_write(dev_priv, DPLL(pipe), dpll);
11860                 intel_de_posting_read(dev_priv, DPLL(pipe));
11861                 udelay(150); /* wait for warmup */
11862         }
11863
11864         intel_de_write(dev_priv, PIPECONF(pipe),
11865                        PIPECONF_ENABLE | PIPECONF_PROGRESSIVE);
11866         intel_de_posting_read(dev_priv, PIPECONF(pipe));
11867
11868         intel_wait_for_pipe_scanline_moving(crtc);
11869 }
11870
11871 void i830_disable_pipe(struct drm_i915_private *dev_priv, enum pipe pipe)
11872 {
11873         struct intel_crtc *crtc = intel_get_crtc_for_pipe(dev_priv, pipe);
11874
11875         drm_dbg_kms(&dev_priv->drm, "disabling pipe %c due to force quirk\n",
11876                     pipe_name(pipe));
11877
11878         drm_WARN_ON(&dev_priv->drm,
11879                     intel_de_read(dev_priv, DSPCNTR(PLANE_A)) &
11880                     DISPLAY_PLANE_ENABLE);
11881         drm_WARN_ON(&dev_priv->drm,
11882                     intel_de_read(dev_priv, DSPCNTR(PLANE_B)) &
11883                     DISPLAY_PLANE_ENABLE);
11884         drm_WARN_ON(&dev_priv->drm,
11885                     intel_de_read(dev_priv, DSPCNTR(PLANE_C)) &
11886                     DISPLAY_PLANE_ENABLE);
11887         drm_WARN_ON(&dev_priv->drm,
11888                     intel_de_read(dev_priv, CURCNTR(PIPE_A)) & MCURSOR_MODE);
11889         drm_WARN_ON(&dev_priv->drm,
11890                     intel_de_read(dev_priv, CURCNTR(PIPE_B)) & MCURSOR_MODE);
11891
11892         intel_de_write(dev_priv, PIPECONF(pipe), 0);
11893         intel_de_posting_read(dev_priv, PIPECONF(pipe));
11894
11895         intel_wait_for_pipe_scanline_stopped(crtc);
11896
11897         intel_de_write(dev_priv, DPLL(pipe), DPLL_VGA_MODE_DIS);
11898         intel_de_posting_read(dev_priv, DPLL(pipe));
11899 }
11900
11901 static void
11902 intel_sanitize_plane_mapping(struct drm_i915_private *dev_priv)
11903 {
11904         struct intel_crtc *crtc;
11905
11906         if (DISPLAY_VER(dev_priv) >= 4)
11907                 return;
11908
11909         for_each_intel_crtc(&dev_priv->drm, crtc) {
11910                 struct intel_plane *plane =
11911                         to_intel_plane(crtc->base.primary);
11912                 struct intel_crtc *plane_crtc;
11913                 enum pipe pipe;
11914
11915                 if (!plane->get_hw_state(plane, &pipe))
11916                         continue;
11917
11918                 if (pipe == crtc->pipe)
11919                         continue;
11920
11921                 drm_dbg_kms(&dev_priv->drm,
11922                             "[PLANE:%d:%s] attached to the wrong pipe, disabling plane\n",
11923                             plane->base.base.id, plane->base.name);
11924
11925                 plane_crtc = intel_get_crtc_for_pipe(dev_priv, pipe);
11926                 intel_plane_disable_noatomic(plane_crtc, plane);
11927         }
11928 }
11929
11930 static bool intel_crtc_has_encoders(struct intel_crtc *crtc)
11931 {
11932         struct drm_device *dev = crtc->base.dev;
11933         struct intel_encoder *encoder;
11934
11935         for_each_encoder_on_crtc(dev, &crtc->base, encoder)
11936                 return true;
11937
11938         return false;
11939 }
11940
11941 static struct intel_connector *intel_encoder_find_connector(struct intel_encoder *encoder)
11942 {
11943         struct drm_device *dev = encoder->base.dev;
11944         struct intel_connector *connector;
11945
11946         for_each_connector_on_encoder(dev, &encoder->base, connector)
11947                 return connector;
11948
11949         return NULL;
11950 }
11951
11952 static bool has_pch_trancoder(struct drm_i915_private *dev_priv,
11953                               enum pipe pch_transcoder)
11954 {
11955         return HAS_PCH_IBX(dev_priv) || HAS_PCH_CPT(dev_priv) ||
11956                 (HAS_PCH_LPT_H(dev_priv) && pch_transcoder == PIPE_A);
11957 }
11958
11959 static void intel_sanitize_frame_start_delay(const struct intel_crtc_state *crtc_state)
11960 {
11961         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
11962         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
11963         enum transcoder cpu_transcoder = crtc_state->cpu_transcoder;
11964
11965         if (DISPLAY_VER(dev_priv) >= 9 ||
11966             IS_BROADWELL(dev_priv) || IS_HASWELL(dev_priv)) {
11967                 i915_reg_t reg = CHICKEN_TRANS(cpu_transcoder);
11968                 u32 val;
11969
11970                 if (transcoder_is_dsi(cpu_transcoder))
11971                         return;
11972
11973                 val = intel_de_read(dev_priv, reg);
11974                 val &= ~HSW_FRAME_START_DELAY_MASK;
11975                 val |= HSW_FRAME_START_DELAY(dev_priv->framestart_delay - 1);
11976                 intel_de_write(dev_priv, reg, val);
11977         } else {
11978                 i915_reg_t reg = PIPECONF(cpu_transcoder);
11979                 u32 val;
11980
11981                 val = intel_de_read(dev_priv, reg);
11982                 val &= ~PIPECONF_FRAME_START_DELAY_MASK;
11983                 val |= PIPECONF_FRAME_START_DELAY(dev_priv->framestart_delay - 1);
11984                 intel_de_write(dev_priv, reg, val);
11985         }
11986
11987         if (!crtc_state->has_pch_encoder)
11988                 return;
11989
11990         if (HAS_PCH_IBX(dev_priv)) {
11991                 i915_reg_t reg = PCH_TRANSCONF(crtc->pipe);
11992                 u32 val;
11993
11994                 val = intel_de_read(dev_priv, reg);
11995                 val &= ~TRANS_FRAME_START_DELAY_MASK;
11996                 val |= TRANS_FRAME_START_DELAY(dev_priv->framestart_delay - 1);
11997                 intel_de_write(dev_priv, reg, val);
11998         } else {
11999                 enum pipe pch_transcoder = intel_crtc_pch_transcoder(crtc);
12000                 i915_reg_t reg = TRANS_CHICKEN2(pch_transcoder);
12001                 u32 val;
12002
12003                 val = intel_de_read(dev_priv, reg);
12004                 val &= ~TRANS_CHICKEN2_FRAME_START_DELAY_MASK;
12005                 val |= TRANS_CHICKEN2_FRAME_START_DELAY(dev_priv->framestart_delay - 1);
12006                 intel_de_write(dev_priv, reg, val);
12007         }
12008 }
12009
12010 static void intel_sanitize_crtc(struct intel_crtc *crtc,
12011                                 struct drm_modeset_acquire_ctx *ctx)
12012 {
12013         struct drm_device *dev = crtc->base.dev;
12014         struct drm_i915_private *dev_priv = to_i915(dev);
12015         struct intel_crtc_state *crtc_state = to_intel_crtc_state(crtc->base.state);
12016
12017         if (crtc_state->hw.active) {
12018                 struct intel_plane *plane;
12019
12020                 /* Clear any frame start delays used for debugging left by the BIOS */
12021                 intel_sanitize_frame_start_delay(crtc_state);
12022
12023                 /* Disable everything but the primary plane */
12024                 for_each_intel_plane_on_crtc(dev, crtc, plane) {
12025                         const struct intel_plane_state *plane_state =
12026                                 to_intel_plane_state(plane->base.state);
12027
12028                         if (plane_state->uapi.visible &&
12029                             plane->base.type != DRM_PLANE_TYPE_PRIMARY)
12030                                 intel_plane_disable_noatomic(crtc, plane);
12031                 }
12032
12033                 /* Disable any background color/etc. set by the BIOS */
12034                 intel_color_commit(crtc_state);
12035         }
12036
12037         /* Adjust the state of the output pipe according to whether we
12038          * have active connectors/encoders. */
12039         if (crtc_state->hw.active && !intel_crtc_has_encoders(crtc) &&
12040             !crtc_state->bigjoiner_slave)
12041                 intel_crtc_disable_noatomic(crtc, ctx);
12042
12043         if (crtc_state->hw.active || HAS_GMCH(dev_priv)) {
12044                 /*
12045                  * We start out with underrun reporting disabled to avoid races.
12046                  * For correct bookkeeping mark this on active crtcs.
12047                  *
12048                  * Also on gmch platforms we dont have any hardware bits to
12049                  * disable the underrun reporting. Which means we need to start
12050                  * out with underrun reporting disabled also on inactive pipes,
12051                  * since otherwise we'll complain about the garbage we read when
12052                  * e.g. coming up after runtime pm.
12053                  *
12054                  * No protection against concurrent access is required - at
12055                  * worst a fifo underrun happens which also sets this to false.
12056                  */
12057                 crtc->cpu_fifo_underrun_disabled = true;
12058                 /*
12059                  * We track the PCH trancoder underrun reporting state
12060                  * within the crtc. With crtc for pipe A housing the underrun
12061                  * reporting state for PCH transcoder A, crtc for pipe B housing
12062                  * it for PCH transcoder B, etc. LPT-H has only PCH transcoder A,
12063                  * and marking underrun reporting as disabled for the non-existing
12064                  * PCH transcoders B and C would prevent enabling the south
12065                  * error interrupt (see cpt_can_enable_serr_int()).
12066                  */
12067                 if (has_pch_trancoder(dev_priv, crtc->pipe))
12068                         crtc->pch_fifo_underrun_disabled = true;
12069         }
12070 }
12071
12072 static bool has_bogus_dpll_config(const struct intel_crtc_state *crtc_state)
12073 {
12074         struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
12075
12076         /*
12077          * Some SNB BIOSen (eg. ASUS K53SV) are known to misprogram
12078          * the hardware when a high res displays plugged in. DPLL P
12079          * divider is zero, and the pipe timings are bonkers. We'll
12080          * try to disable everything in that case.
12081          *
12082          * FIXME would be nice to be able to sanitize this state
12083          * without several WARNs, but for now let's take the easy
12084          * road.
12085          */
12086         return IS_SANDYBRIDGE(dev_priv) &&
12087                 crtc_state->hw.active &&
12088                 crtc_state->shared_dpll &&
12089                 crtc_state->port_clock == 0;
12090 }
12091
12092 static void intel_sanitize_encoder(struct intel_encoder *encoder)
12093 {
12094         struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
12095         struct intel_connector *connector;
12096         struct intel_crtc *crtc = to_intel_crtc(encoder->base.crtc);
12097         struct intel_crtc_state *crtc_state = crtc ?
12098                 to_intel_crtc_state(crtc->base.state) : NULL;
12099
12100         /* We need to check both for a crtc link (meaning that the
12101          * encoder is active and trying to read from a pipe) and the
12102          * pipe itself being active. */
12103         bool has_active_crtc = crtc_state &&
12104                 crtc_state->hw.active;
12105
12106         if (crtc_state && has_bogus_dpll_config(crtc_state)) {
12107                 drm_dbg_kms(&dev_priv->drm,
12108                             "BIOS has misprogrammed the hardware. Disabling pipe %c\n",
12109                             pipe_name(crtc->pipe));
12110                 has_active_crtc = false;
12111         }
12112
12113         connector = intel_encoder_find_connector(encoder);
12114         if (connector && !has_active_crtc) {
12115                 drm_dbg_kms(&dev_priv->drm,
12116                             "[ENCODER:%d:%s] has active connectors but no active pipe!\n",
12117                             encoder->base.base.id,
12118                             encoder->base.name);
12119
12120                 /* Connector is active, but has no active pipe. This is
12121                  * fallout from our resume register restoring. Disable
12122                  * the encoder manually again. */
12123                 if (crtc_state) {
12124                         struct drm_encoder *best_encoder;
12125
12126                         drm_dbg_kms(&dev_priv->drm,
12127                                     "[ENCODER:%d:%s] manually disabled\n",
12128                                     encoder->base.base.id,
12129                                     encoder->base.name);
12130
12131                         /* avoid oopsing in case the hooks consult best_encoder */
12132                         best_encoder = connector->base.state->best_encoder;
12133                         connector->base.state->best_encoder = &encoder->base;
12134
12135                         /* FIXME NULL atomic state passed! */
12136                         if (encoder->disable)
12137                                 encoder->disable(NULL, encoder, crtc_state,
12138                                                  connector->base.state);
12139                         if (encoder->post_disable)
12140                                 encoder->post_disable(NULL, encoder, crtc_state,
12141                                                       connector->base.state);
12142
12143                         connector->base.state->best_encoder = best_encoder;
12144                 }
12145                 encoder->base.crtc = NULL;
12146
12147                 /* Inconsistent output/port/pipe state happens presumably due to
12148                  * a bug in one of the get_hw_state functions. Or someplace else
12149                  * in our code, like the register restore mess on resume. Clamp
12150                  * things to off as a safer default. */
12151
12152                 connector->base.dpms = DRM_MODE_DPMS_OFF;
12153                 connector->base.encoder = NULL;
12154         }
12155
12156         /* notify opregion of the sanitized encoder state */
12157         intel_opregion_notify_encoder(encoder, connector && has_active_crtc);
12158
12159         if (HAS_DDI(dev_priv))
12160                 intel_ddi_sanitize_encoder_pll_mapping(encoder);
12161 }
12162
12163 /* FIXME read out full plane state for all planes */
12164 static void readout_plane_state(struct drm_i915_private *dev_priv)
12165 {
12166         struct intel_plane *plane;
12167         struct intel_crtc *crtc;
12168
12169         for_each_intel_plane(&dev_priv->drm, plane) {
12170                 struct intel_plane_state *plane_state =
12171                         to_intel_plane_state(plane->base.state);
12172                 struct intel_crtc_state *crtc_state;
12173                 enum pipe pipe = PIPE_A;
12174                 bool visible;
12175
12176                 visible = plane->get_hw_state(plane, &pipe);
12177
12178                 crtc = intel_get_crtc_for_pipe(dev_priv, pipe);
12179                 crtc_state = to_intel_crtc_state(crtc->base.state);
12180
12181                 intel_set_plane_visible(crtc_state, plane_state, visible);
12182
12183                 drm_dbg_kms(&dev_priv->drm,
12184                             "[PLANE:%d:%s] hw state readout: %s, pipe %c\n",
12185                             plane->base.base.id, plane->base.name,
12186                             enableddisabled(visible), pipe_name(pipe));
12187         }
12188
12189         for_each_intel_crtc(&dev_priv->drm, crtc) {
12190                 struct intel_crtc_state *crtc_state =
12191                         to_intel_crtc_state(crtc->base.state);
12192
12193                 fixup_plane_bitmasks(crtc_state);
12194         }
12195 }
12196
12197 static void intel_modeset_readout_hw_state(struct drm_device *dev)
12198 {
12199         struct drm_i915_private *dev_priv = to_i915(dev);
12200         struct intel_cdclk_state *cdclk_state =
12201                 to_intel_cdclk_state(dev_priv->cdclk.obj.state);
12202         struct intel_dbuf_state *dbuf_state =
12203                 to_intel_dbuf_state(dev_priv->dbuf.obj.state);
12204         enum pipe pipe;
12205         struct intel_crtc *crtc;
12206         struct intel_encoder *encoder;
12207         struct intel_connector *connector;
12208         struct drm_connector_list_iter conn_iter;
12209         u8 active_pipes = 0;
12210
12211         for_each_intel_crtc(dev, crtc) {
12212                 struct intel_crtc_state *crtc_state =
12213                         to_intel_crtc_state(crtc->base.state);
12214
12215                 __drm_atomic_helper_crtc_destroy_state(&crtc_state->uapi);
12216                 intel_crtc_free_hw_state(crtc_state);
12217                 intel_crtc_state_reset(crtc_state, crtc);
12218
12219                 intel_crtc_get_pipe_config(crtc_state);
12220
12221                 crtc_state->hw.enable = crtc_state->hw.active;
12222
12223                 crtc->base.enabled = crtc_state->hw.enable;
12224                 crtc->active = crtc_state->hw.active;
12225
12226                 if (crtc_state->hw.active)
12227                         active_pipes |= BIT(crtc->pipe);
12228
12229                 drm_dbg_kms(&dev_priv->drm,
12230                             "[CRTC:%d:%s] hw state readout: %s\n",
12231                             crtc->base.base.id, crtc->base.name,
12232                             enableddisabled(crtc_state->hw.active));
12233         }
12234
12235         cdclk_state->active_pipes = dbuf_state->active_pipes = active_pipes;
12236
12237         readout_plane_state(dev_priv);
12238
12239         for_each_intel_encoder(dev, encoder) {
12240                 struct intel_crtc_state *crtc_state = NULL;
12241
12242                 pipe = 0;
12243
12244                 if (encoder->get_hw_state(encoder, &pipe)) {
12245                         crtc = intel_get_crtc_for_pipe(dev_priv, pipe);
12246                         crtc_state = to_intel_crtc_state(crtc->base.state);
12247
12248                         encoder->base.crtc = &crtc->base;
12249                         intel_encoder_get_config(encoder, crtc_state);
12250
12251                         /* read out to slave crtc as well for bigjoiner */
12252                         if (crtc_state->bigjoiner) {
12253                                 /* encoder should read be linked to bigjoiner master */
12254                                 WARN_ON(crtc_state->bigjoiner_slave);
12255
12256                                 crtc = crtc_state->bigjoiner_linked_crtc;
12257                                 crtc_state = to_intel_crtc_state(crtc->base.state);
12258                                 intel_encoder_get_config(encoder, crtc_state);
12259                         }
12260                 } else {
12261                         encoder->base.crtc = NULL;
12262                 }
12263
12264                 if (encoder->sync_state)
12265                         encoder->sync_state(encoder, crtc_state);
12266
12267                 drm_dbg_kms(&dev_priv->drm,
12268                             "[ENCODER:%d:%s] hw state readout: %s, pipe %c\n",
12269                             encoder->base.base.id, encoder->base.name,
12270                             enableddisabled(encoder->base.crtc),
12271                             pipe_name(pipe));
12272         }
12273
12274         intel_dpll_readout_hw_state(dev_priv);
12275
12276         drm_connector_list_iter_begin(dev, &conn_iter);
12277         for_each_intel_connector_iter(connector, &conn_iter) {
12278                 if (connector->get_hw_state(connector)) {
12279                         struct intel_crtc_state *crtc_state;
12280                         struct intel_crtc *crtc;
12281
12282                         connector->base.dpms = DRM_MODE_DPMS_ON;
12283
12284                         encoder = intel_attached_encoder(connector);
12285                         connector->base.encoder = &encoder->base;
12286
12287                         crtc = to_intel_crtc(encoder->base.crtc);
12288                         crtc_state = crtc ? to_intel_crtc_state(crtc->base.state) : NULL;
12289
12290                         if (crtc_state && crtc_state->hw.active) {
12291                                 /*
12292                                  * This has to be done during hardware readout
12293                                  * because anything calling .crtc_disable may
12294                                  * rely on the connector_mask being accurate.
12295                                  */
12296                                 crtc_state->uapi.connector_mask |=
12297                                         drm_connector_mask(&connector->base);
12298                                 crtc_state->uapi.encoder_mask |=
12299                                         drm_encoder_mask(&encoder->base);
12300                         }
12301                 } else {
12302                         connector->base.dpms = DRM_MODE_DPMS_OFF;
12303                         connector->base.encoder = NULL;
12304                 }
12305                 drm_dbg_kms(&dev_priv->drm,
12306                             "[CONNECTOR:%d:%s] hw state readout: %s\n",
12307                             connector->base.base.id, connector->base.name,
12308                             enableddisabled(connector->base.encoder));
12309         }
12310         drm_connector_list_iter_end(&conn_iter);
12311
12312         for_each_intel_crtc(dev, crtc) {
12313                 struct intel_bw_state *bw_state =
12314                         to_intel_bw_state(dev_priv->bw_obj.state);
12315                 struct intel_crtc_state *crtc_state =
12316                         to_intel_crtc_state(crtc->base.state);
12317                 struct intel_plane *plane;
12318                 int min_cdclk = 0;
12319
12320                 if (crtc_state->bigjoiner_slave)
12321                         continue;
12322
12323                 if (crtc_state->hw.active) {
12324                         /*
12325                          * The initial mode needs to be set in order to keep
12326                          * the atomic core happy. It wants a valid mode if the
12327                          * crtc's enabled, so we do the above call.
12328                          *
12329                          * But we don't set all the derived state fully, hence
12330                          * set a flag to indicate that a full recalculation is
12331                          * needed on the next commit.
12332                          */
12333                         crtc_state->inherited = true;
12334
12335                         intel_crtc_update_active_timings(crtc_state);
12336
12337                         intel_crtc_copy_hw_to_uapi_state(crtc_state);
12338                 }
12339
12340                 for_each_intel_plane_on_crtc(&dev_priv->drm, crtc, plane) {
12341                         const struct intel_plane_state *plane_state =
12342                                 to_intel_plane_state(plane->base.state);
12343
12344                         /*
12345                          * FIXME don't have the fb yet, so can't
12346                          * use intel_plane_data_rate() :(
12347                          */
12348                         if (plane_state->uapi.visible)
12349                                 crtc_state->data_rate[plane->id] =
12350                                         4 * crtc_state->pixel_rate;
12351                         /*
12352                          * FIXME don't have the fb yet, so can't
12353                          * use plane->min_cdclk() :(
12354                          */
12355                         if (plane_state->uapi.visible && plane->min_cdclk) {
12356                                 if (crtc_state->double_wide || DISPLAY_VER(dev_priv) >= 10)
12357                                         crtc_state->min_cdclk[plane->id] =
12358                                                 DIV_ROUND_UP(crtc_state->pixel_rate, 2);
12359                                 else
12360                                         crtc_state->min_cdclk[plane->id] =
12361                                                 crtc_state->pixel_rate;
12362                         }
12363                         drm_dbg_kms(&dev_priv->drm,
12364                                     "[PLANE:%d:%s] min_cdclk %d kHz\n",
12365                                     plane->base.base.id, plane->base.name,
12366                                     crtc_state->min_cdclk[plane->id]);
12367                 }
12368
12369                 if (crtc_state->hw.active) {
12370                         min_cdclk = intel_crtc_compute_min_cdclk(crtc_state);
12371                         if (drm_WARN_ON(dev, min_cdclk < 0))
12372                                 min_cdclk = 0;
12373                 }
12374
12375                 cdclk_state->min_cdclk[crtc->pipe] = min_cdclk;
12376                 cdclk_state->min_voltage_level[crtc->pipe] =
12377                         crtc_state->min_voltage_level;
12378
12379                 intel_bw_crtc_update(bw_state, crtc_state);
12380
12381                 intel_pipe_config_sanity_check(dev_priv, crtc_state);
12382
12383                 /* discard our incomplete slave state, copy it from master */
12384                 if (crtc_state->bigjoiner && crtc_state->hw.active) {
12385                         struct intel_crtc *slave = crtc_state->bigjoiner_linked_crtc;
12386                         struct intel_crtc_state *slave_crtc_state =
12387                                 to_intel_crtc_state(slave->base.state);
12388
12389                         copy_bigjoiner_crtc_state(slave_crtc_state, crtc_state);
12390                         slave->base.mode = crtc->base.mode;
12391
12392                         cdclk_state->min_cdclk[slave->pipe] = min_cdclk;
12393                         cdclk_state->min_voltage_level[slave->pipe] =
12394                                 crtc_state->min_voltage_level;
12395
12396                         for_each_intel_plane_on_crtc(&dev_priv->drm, slave, plane) {
12397                                 const struct intel_plane_state *plane_state =
12398                                         to_intel_plane_state(plane->base.state);
12399
12400                                 /*
12401                                  * FIXME don't have the fb yet, so can't
12402                                  * use intel_plane_data_rate() :(
12403                                  */
12404                                 if (plane_state->uapi.visible)
12405                                         crtc_state->data_rate[plane->id] =
12406                                                 4 * crtc_state->pixel_rate;
12407                                 else
12408                                         crtc_state->data_rate[plane->id] = 0;
12409                         }
12410
12411                         intel_bw_crtc_update(bw_state, slave_crtc_state);
12412                         drm_calc_timestamping_constants(&slave->base,
12413                                                         &slave_crtc_state->hw.adjusted_mode);
12414                 }
12415         }
12416 }
12417
12418 static void
12419 get_encoder_power_domains(struct drm_i915_private *dev_priv)
12420 {
12421         struct intel_encoder *encoder;
12422
12423         for_each_intel_encoder(&dev_priv->drm, encoder) {
12424                 struct intel_crtc_state *crtc_state;
12425
12426                 if (!encoder->get_power_domains)
12427                         continue;
12428
12429                 /*
12430                  * MST-primary and inactive encoders don't have a crtc state
12431                  * and neither of these require any power domain references.
12432                  */
12433                 if (!encoder->base.crtc)
12434                         continue;
12435
12436                 crtc_state = to_intel_crtc_state(encoder->base.crtc->state);
12437                 encoder->get_power_domains(encoder, crtc_state);
12438         }
12439 }
12440
12441 static void intel_early_display_was(struct drm_i915_private *dev_priv)
12442 {
12443         /*
12444          * Display WA #1185 WaDisableDARBFClkGating:glk,icl,ehl,tgl
12445          * Also known as Wa_14010480278.
12446          */
12447         if (IS_DISPLAY_VER(dev_priv, 10, 12))
12448                 intel_de_write(dev_priv, GEN9_CLKGATE_DIS_0,
12449                                intel_de_read(dev_priv, GEN9_CLKGATE_DIS_0) | DARBF_GATING_DIS);
12450
12451         if (IS_HASWELL(dev_priv)) {
12452                 /*
12453                  * WaRsPkgCStateDisplayPMReq:hsw
12454                  * System hang if this isn't done before disabling all planes!
12455                  */
12456                 intel_de_write(dev_priv, CHICKEN_PAR1_1,
12457                                intel_de_read(dev_priv, CHICKEN_PAR1_1) | FORCE_ARB_IDLE_PLANES);
12458         }
12459
12460         if (IS_KABYLAKE(dev_priv) || IS_COFFEELAKE(dev_priv) || IS_COMETLAKE(dev_priv)) {
12461                 /* Display WA #1142:kbl,cfl,cml */
12462                 intel_de_rmw(dev_priv, CHICKEN_PAR1_1,
12463                              KBL_ARB_FILL_SPARE_22, KBL_ARB_FILL_SPARE_22);
12464                 intel_de_rmw(dev_priv, CHICKEN_MISC_2,
12465                              KBL_ARB_FILL_SPARE_13 | KBL_ARB_FILL_SPARE_14,
12466                              KBL_ARB_FILL_SPARE_14);
12467         }
12468 }
12469
12470 static void ibx_sanitize_pch_hdmi_port(struct drm_i915_private *dev_priv,
12471                                        enum port port, i915_reg_t hdmi_reg)
12472 {
12473         u32 val = intel_de_read(dev_priv, hdmi_reg);
12474
12475         if (val & SDVO_ENABLE ||
12476             (val & SDVO_PIPE_SEL_MASK) == SDVO_PIPE_SEL(PIPE_A))
12477                 return;
12478
12479         drm_dbg_kms(&dev_priv->drm,
12480                     "Sanitizing transcoder select for HDMI %c\n",
12481                     port_name(port));
12482
12483         val &= ~SDVO_PIPE_SEL_MASK;
12484         val |= SDVO_PIPE_SEL(PIPE_A);
12485
12486         intel_de_write(dev_priv, hdmi_reg, val);
12487 }
12488
12489 static void ibx_sanitize_pch_dp_port(struct drm_i915_private *dev_priv,
12490                                      enum port port, i915_reg_t dp_reg)
12491 {
12492         u32 val = intel_de_read(dev_priv, dp_reg);
12493
12494         if (val & DP_PORT_EN ||
12495             (val & DP_PIPE_SEL_MASK) == DP_PIPE_SEL(PIPE_A))
12496                 return;
12497
12498         drm_dbg_kms(&dev_priv->drm,
12499                     "Sanitizing transcoder select for DP %c\n",
12500                     port_name(port));
12501
12502         val &= ~DP_PIPE_SEL_MASK;
12503         val |= DP_PIPE_SEL(PIPE_A);
12504
12505         intel_de_write(dev_priv, dp_reg, val);
12506 }
12507
12508 static void ibx_sanitize_pch_ports(struct drm_i915_private *dev_priv)
12509 {
12510         /*
12511          * The BIOS may select transcoder B on some of the PCH
12512          * ports even it doesn't enable the port. This would trip
12513          * assert_pch_dp_disabled() and assert_pch_hdmi_disabled().
12514          * Sanitize the transcoder select bits to prevent that. We
12515          * assume that the BIOS never actually enabled the port,
12516          * because if it did we'd actually have to toggle the port
12517          * on and back off to make the transcoder A select stick
12518          * (see. intel_dp_link_down(), intel_disable_hdmi(),
12519          * intel_disable_sdvo()).
12520          */
12521         ibx_sanitize_pch_dp_port(dev_priv, PORT_B, PCH_DP_B);
12522         ibx_sanitize_pch_dp_port(dev_priv, PORT_C, PCH_DP_C);
12523         ibx_sanitize_pch_dp_port(dev_priv, PORT_D, PCH_DP_D);
12524
12525         /* PCH SDVOB multiplex with HDMIB */
12526         ibx_sanitize_pch_hdmi_port(dev_priv, PORT_B, PCH_HDMIB);
12527         ibx_sanitize_pch_hdmi_port(dev_priv, PORT_C, PCH_HDMIC);
12528         ibx_sanitize_pch_hdmi_port(dev_priv, PORT_D, PCH_HDMID);
12529 }
12530
12531 /* Scan out the current hw modeset state,
12532  * and sanitizes it to the current state
12533  */
12534 static void
12535 intel_modeset_setup_hw_state(struct drm_device *dev,
12536                              struct drm_modeset_acquire_ctx *ctx)
12537 {
12538         struct drm_i915_private *dev_priv = to_i915(dev);
12539         struct intel_encoder *encoder;
12540         struct intel_crtc *crtc;
12541         intel_wakeref_t wakeref;
12542
12543         wakeref = intel_display_power_get(dev_priv, POWER_DOMAIN_INIT);
12544
12545         intel_early_display_was(dev_priv);
12546         intel_modeset_readout_hw_state(dev);
12547
12548         /* HW state is read out, now we need to sanitize this mess. */
12549         get_encoder_power_domains(dev_priv);
12550
12551         if (HAS_PCH_IBX(dev_priv))
12552                 ibx_sanitize_pch_ports(dev_priv);
12553
12554         /*
12555          * intel_sanitize_plane_mapping() may need to do vblank
12556          * waits, so we need vblank interrupts restored beforehand.
12557          */
12558         for_each_intel_crtc(&dev_priv->drm, crtc) {
12559                 struct intel_crtc_state *crtc_state =
12560                         to_intel_crtc_state(crtc->base.state);
12561
12562                 drm_crtc_vblank_reset(&crtc->base);
12563
12564                 if (crtc_state->hw.active)
12565                         intel_crtc_vblank_on(crtc_state);
12566         }
12567
12568         intel_sanitize_plane_mapping(dev_priv);
12569
12570         for_each_intel_encoder(dev, encoder)
12571                 intel_sanitize_encoder(encoder);
12572
12573         for_each_intel_crtc(&dev_priv->drm, crtc) {
12574                 struct intel_crtc_state *crtc_state =
12575                         to_intel_crtc_state(crtc->base.state);
12576
12577                 intel_sanitize_crtc(crtc, ctx);
12578                 intel_dump_pipe_config(crtc_state, NULL, "[setup_hw_state]");
12579         }
12580
12581         intel_modeset_update_connector_atomic_state(dev);
12582
12583         intel_dpll_sanitize_state(dev_priv);
12584
12585         if (IS_G4X(dev_priv)) {
12586                 g4x_wm_get_hw_state(dev_priv);
12587                 g4x_wm_sanitize(dev_priv);
12588         } else if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) {
12589                 vlv_wm_get_hw_state(dev_priv);
12590                 vlv_wm_sanitize(dev_priv);
12591         } else if (DISPLAY_VER(dev_priv) >= 9) {
12592                 skl_wm_get_hw_state(dev_priv);
12593         } else if (HAS_PCH_SPLIT(dev_priv)) {
12594                 ilk_wm_get_hw_state(dev_priv);
12595         }
12596
12597         for_each_intel_crtc(dev, crtc) {
12598                 struct intel_crtc_state *crtc_state =
12599                         to_intel_crtc_state(crtc->base.state);
12600                 u64 put_domains;
12601
12602                 put_domains = modeset_get_crtc_power_domains(crtc_state);
12603                 if (drm_WARN_ON(dev, put_domains))
12604                         modeset_put_crtc_power_domains(crtc, put_domains);
12605         }
12606
12607         intel_display_power_put(dev_priv, POWER_DOMAIN_INIT, wakeref);
12608 }
12609
12610 void intel_display_resume(struct drm_device *dev)
12611 {
12612         struct drm_i915_private *dev_priv = to_i915(dev);
12613         struct drm_atomic_state *state = dev_priv->modeset_restore_state;
12614         struct drm_modeset_acquire_ctx ctx;
12615         int ret;
12616
12617         if (!HAS_DISPLAY(dev_priv))
12618                 return;
12619
12620         dev_priv->modeset_restore_state = NULL;
12621         if (state)
12622                 state->acquire_ctx = &ctx;
12623
12624         drm_modeset_acquire_init(&ctx, 0);
12625
12626         while (1) {
12627                 ret = drm_modeset_lock_all_ctx(dev, &ctx);
12628                 if (ret != -EDEADLK)
12629                         break;
12630
12631                 drm_modeset_backoff(&ctx);
12632         }
12633
12634         if (!ret)
12635                 ret = __intel_display_resume(dev, state, &ctx);
12636
12637         intel_enable_ipc(dev_priv);
12638         drm_modeset_drop_locks(&ctx);
12639         drm_modeset_acquire_fini(&ctx);
12640
12641         if (ret)
12642                 drm_err(&dev_priv->drm,
12643                         "Restoring old state failed with %i\n", ret);
12644         if (state)
12645                 drm_atomic_state_put(state);
12646 }
12647
12648 static void intel_hpd_poll_fini(struct drm_i915_private *i915)
12649 {
12650         struct intel_connector *connector;
12651         struct drm_connector_list_iter conn_iter;
12652
12653         /* Kill all the work that may have been queued by hpd. */
12654         drm_connector_list_iter_begin(&i915->drm, &conn_iter);
12655         for_each_intel_connector_iter(connector, &conn_iter) {
12656                 if (connector->modeset_retry_work.func)
12657                         cancel_work_sync(&connector->modeset_retry_work);
12658                 if (connector->hdcp.shim) {
12659                         cancel_delayed_work_sync(&connector->hdcp.check_work);
12660                         cancel_work_sync(&connector->hdcp.prop_work);
12661                 }
12662         }
12663         drm_connector_list_iter_end(&conn_iter);
12664 }
12665
12666 /* part #1: call before irq uninstall */
12667 void intel_modeset_driver_remove(struct drm_i915_private *i915)
12668 {
12669         if (!HAS_DISPLAY(i915))
12670                 return;
12671
12672         flush_workqueue(i915->flip_wq);
12673         flush_workqueue(i915->modeset_wq);
12674
12675         flush_work(&i915->atomic_helper.free_work);
12676         drm_WARN_ON(&i915->drm, !llist_empty(&i915->atomic_helper.free_list));
12677 }
12678
12679 /* part #2: call after irq uninstall */
12680 void intel_modeset_driver_remove_noirq(struct drm_i915_private *i915)
12681 {
12682         if (!HAS_DISPLAY(i915))
12683                 return;
12684
12685         /*
12686          * Due to the hpd irq storm handling the hotplug work can re-arm the
12687          * poll handlers. Hence disable polling after hpd handling is shut down.
12688          */
12689         intel_hpd_poll_fini(i915);
12690
12691         /*
12692          * MST topology needs to be suspended so we don't have any calls to
12693          * fbdev after it's finalized. MST will be destroyed later as part of
12694          * drm_mode_config_cleanup()
12695          */
12696         intel_dp_mst_suspend(i915);
12697
12698         /* poll work can call into fbdev, hence clean that up afterwards */
12699         intel_fbdev_fini(i915);
12700
12701         intel_unregister_dsm_handler();
12702
12703         intel_fbc_global_disable(i915);
12704
12705         /* flush any delayed tasks or pending work */
12706         flush_scheduled_work();
12707
12708         intel_hdcp_component_fini(i915);
12709
12710         intel_mode_config_cleanup(i915);
12711
12712         intel_overlay_cleanup(i915);
12713
12714         intel_gmbus_teardown(i915);
12715
12716         destroy_workqueue(i915->flip_wq);
12717         destroy_workqueue(i915->modeset_wq);
12718
12719         intel_fbc_cleanup_cfb(i915);
12720 }
12721
12722 /* part #3: call after gem init */
12723 void intel_modeset_driver_remove_nogem(struct drm_i915_private *i915)
12724 {
12725         intel_dmc_ucode_fini(i915);
12726
12727         intel_power_domains_driver_remove(i915);
12728
12729         intel_vga_unregister(i915);
12730
12731         intel_bios_driver_remove(i915);
12732 }
12733
12734 void intel_display_driver_register(struct drm_i915_private *i915)
12735 {
12736         if (!HAS_DISPLAY(i915))
12737                 return;
12738
12739         intel_display_debugfs_register(i915);
12740
12741         /* Must be done after probing outputs */
12742         intel_opregion_register(i915);
12743         acpi_video_register();
12744
12745         intel_audio_init(i915);
12746
12747         /*
12748          * Some ports require correctly set-up hpd registers for
12749          * detection to work properly (leading to ghost connected
12750          * connector status), e.g. VGA on gm45.  Hence we can only set
12751          * up the initial fbdev config after hpd irqs are fully
12752          * enabled. We do it last so that the async config cannot run
12753          * before the connectors are registered.
12754          */
12755         intel_fbdev_initial_config_async(&i915->drm);
12756
12757         /*
12758          * We need to coordinate the hotplugs with the asynchronous
12759          * fbdev configuration, for which we use the
12760          * fbdev->async_cookie.
12761          */
12762         drm_kms_helper_poll_init(&i915->drm);
12763 }
12764
12765 void intel_display_driver_unregister(struct drm_i915_private *i915)
12766 {
12767         if (!HAS_DISPLAY(i915))
12768                 return;
12769
12770         intel_fbdev_unregister(i915);
12771         intel_audio_deinit(i915);
12772
12773         /*
12774          * After flushing the fbdev (incl. a late async config which
12775          * will have delayed queuing of a hotplug event), then flush
12776          * the hotplug events.
12777          */
12778         drm_kms_helper_poll_fini(&i915->drm);
12779         drm_atomic_helper_shutdown(&i915->drm);
12780
12781         acpi_video_unregister();
12782         intel_opregion_unregister(i915);
12783 }