drivers/gpu/drm/i915/display/intel_modeset_setup.c

   1 // SPDX-License-Identifier: MIT
   2 /*
   3  * Copyright © 2022 Intel Corporation
   4  *
   5  * Read out the current hardware modeset state, and sanitize it to the current
   6  * state.
   7  */
   8
   9 #include <drm/drm_atomic_uapi.h>
  10 #include <drm/drm_atomic_state_helper.h>
  11
  12 #include "i915_drv.h"
  13 #include "i915_reg.h"
  14 #include "i9xx_wm.h"
  15 #include "intel_atomic.h"
  16 #include "intel_bw.h"
  17 #include "intel_color.h"
  18 #include "intel_crtc.h"
  19 #include "intel_crtc_state_dump.h"
  20 #include "intel_ddi.h"
  21 #include "intel_de.h"
  22 #include "intel_display.h"
  23 #include "intel_display_power.h"
  24 #include "intel_display_types.h"
  25 #include "intel_dmc.h"
  26 #include "intel_fifo_underrun.h"
  27 #include "intel_modeset_setup.h"
  28 #include "intel_pch_display.h"
  29 #include "intel_pmdemand.h"
  30 #include "intel_tc.h"
  31 #include "intel_vblank.h"
  32 #include "intel_wm.h"
  33 #include "skl_watermark.h"
  34
  35 static void intel_crtc_disable_noatomic_begin(struct intel_crtc *crtc,
  36                                               struct drm_modeset_acquire_ctx *ctx)
  37 {
  38         struct drm_i915_private *i915 = to_i915(crtc->base.dev);
  39         struct intel_crtc_state *crtc_state =
  40                 to_intel_crtc_state(crtc->base.state);
  41         struct intel_plane *plane;
  42         struct drm_atomic_state *state;
  43         struct intel_crtc *temp_crtc;
  44         enum pipe pipe = crtc->pipe;
  45
  46         if (!crtc_state->hw.active)
  47                 return;
  48
  49         for_each_intel_plane_on_crtc(&i915->drm, crtc, plane) {
  50                 const struct intel_plane_state *plane_state =
  51                         to_intel_plane_state(plane->base.state);
  52
  53                 if (plane_state->uapi.visible)
  54                         intel_plane_disable_noatomic(crtc, plane);
  55         }
  56
  57         state = drm_atomic_state_alloc(&i915->drm);
  58         if (!state) {
  59                 drm_dbg_kms(&i915->drm,
  60                             "failed to disable [CRTC:%d:%s], out of memory",
  61                             crtc->base.base.id, crtc->base.name);
  62                 return;
  63         }
  64
  65         state->acquire_ctx = ctx;
  66         to_intel_atomic_state(state)->internal = true;
  67
  68         /* Everything's already locked, -EDEADLK can't happen. */
  69         for_each_intel_crtc_in_pipe_mask(&i915->drm, temp_crtc,
  70                                          BIT(pipe) |
  71                                          intel_crtc_bigjoiner_slave_pipes(crtc_state)) {
  72                 struct intel_crtc_state *temp_crtc_state =
  73                         intel_atomic_get_crtc_state(state, temp_crtc);
  74                 int ret;
  75
  76                 ret = drm_atomic_add_affected_connectors(state, &temp_crtc->base);
  77
  78                 drm_WARN_ON(&i915->drm, IS_ERR(temp_crtc_state) || ret);
  79         }
  80
  81         i915->display.funcs.display->crtc_disable(to_intel_atomic_state(state), crtc);
  82
  83         drm_atomic_state_put(state);
  84
  85         drm_dbg_kms(&i915->drm,
  86                     "[CRTC:%d:%s] hw state adjusted, was enabled, now disabled\n",
  87                     crtc->base.base.id, crtc->base.name);
  88
  89         crtc->active = false;
  90         crtc->base.enabled = false;
  91
  92         if (crtc_state->shared_dpll)
  93                 intel_unreference_shared_dpll_crtc(crtc,
  94                                                    crtc_state->shared_dpll,
  95                                                    &crtc_state->shared_dpll->state);
  96 }
  97
  98 static void set_encoder_for_connector(struct intel_connector *connector,
  99                                       struct intel_encoder *encoder)
 100 {
 101         struct drm_connector_state *conn_state = connector->base.state;
 102
 103         if (conn_state->crtc)
 104                 drm_connector_put(&connector->base);
 105
 106         if (encoder) {
 107                 conn_state->best_encoder = &encoder->base;
 108                 conn_state->crtc = encoder->base.crtc;
 109                 drm_connector_get(&connector->base);
 110         } else {
 111                 conn_state->best_encoder = NULL;
 112                 conn_state->crtc = NULL;
 113         }
 114 }
 115
 116 static void reset_encoder_connector_state(struct intel_encoder *encoder)
 117 {
 118         struct drm_i915_private *i915 = to_i915(encoder->base.dev);
 119         struct intel_pmdemand_state *pmdemand_state =
 120                 to_intel_pmdemand_state(i915->display.pmdemand.obj.state);
 121         struct intel_connector *connector;
 122         struct drm_connector_list_iter conn_iter;
 123
 124         drm_connector_list_iter_begin(&i915->drm, &conn_iter);
 125         for_each_intel_connector_iter(connector, &conn_iter) {
 126                 if (connector->base.encoder != &encoder->base)
 127                         continue;
 128
 129                 /* Clear the corresponding bit in pmdemand active phys mask */
 130                 intel_pmdemand_update_phys_mask(i915, encoder,
 131                                                 pmdemand_state, false);
 132
 133                 set_encoder_for_connector(connector, NULL);
 134
 135                 connector->base.dpms = DRM_MODE_DPMS_OFF;
 136                 connector->base.encoder = NULL;
 137         }
 138         drm_connector_list_iter_end(&conn_iter);
 139 }
 140
 141 static void reset_crtc_encoder_state(struct intel_crtc *crtc)
 142 {
 143         struct drm_i915_private *i915 = to_i915(crtc->base.dev);
 144         struct intel_encoder *encoder;
 145
 146         for_each_encoder_on_crtc(&i915->drm, &crtc->base, encoder) {
 147                 reset_encoder_connector_state(encoder);
 148                 encoder->base.crtc = NULL;
 149         }
 150 }
 151
 152 static void intel_crtc_disable_noatomic_complete(struct intel_crtc *crtc)
 153 {
 154         struct drm_i915_private *i915 = to_i915(crtc->base.dev);
 155         struct intel_bw_state *bw_state =
 156                 to_intel_bw_state(i915->display.bw.obj.state);
 157         struct intel_cdclk_state *cdclk_state =
 158                 to_intel_cdclk_state(i915->display.cdclk.obj.state);
 159         struct intel_dbuf_state *dbuf_state =
 160                 to_intel_dbuf_state(i915->display.dbuf.obj.state);
 161         struct intel_pmdemand_state *pmdemand_state =
 162                 to_intel_pmdemand_state(i915->display.pmdemand.obj.state);
 163         struct intel_crtc_state *crtc_state =
 164                 to_intel_crtc_state(crtc->base.state);
 165         enum pipe pipe = crtc->pipe;
 166
 167         __drm_atomic_helper_crtc_destroy_state(&crtc_state->uapi);
 168         intel_crtc_free_hw_state(crtc_state);
 169         intel_crtc_state_reset(crtc_state, crtc);
 170
 171         reset_crtc_encoder_state(crtc);
 172
 173         intel_fbc_disable(crtc);
 174         intel_update_watermarks(i915);
 175
 176         intel_display_power_put_all_in_set(i915, &crtc->enabled_power_domains);
 177
 178         cdclk_state->min_cdclk[pipe] = 0;
 179         cdclk_state->min_voltage_level[pipe] = 0;
 180         cdclk_state->active_pipes &= ~BIT(pipe);
 181
 182         dbuf_state->active_pipes &= ~BIT(pipe);
 183
 184         bw_state->data_rate[pipe] = 0;
 185         bw_state->num_active_planes[pipe] = 0;
 186
 187         intel_pmdemand_update_port_clock(i915, pmdemand_state, pipe, 0);
 188 }
 189
 190 /*
 191  * Return all the pipes using a transcoder in @transcoder_mask.
 192  * For bigjoiner configs return only the bigjoiner master.
 193  */
 194 static u8 get_transcoder_pipes(struct drm_i915_private *i915,
 195                                u8 transcoder_mask)
 196 {
 197         struct intel_crtc *temp_crtc;
 198         u8 pipes = 0;
 199
 200         for_each_intel_crtc(&i915->drm, temp_crtc) {
 201                 struct intel_crtc_state *temp_crtc_state =
 202                         to_intel_crtc_state(temp_crtc->base.state);
 203
 204                 if (temp_crtc_state->cpu_transcoder == INVALID_TRANSCODER)
 205                         continue;
 206
 207                 if (intel_crtc_is_bigjoiner_slave(temp_crtc_state))
 208                         continue;
 209
 210                 if (transcoder_mask & BIT(temp_crtc_state->cpu_transcoder))
 211                         pipes |= BIT(temp_crtc->pipe);
 212         }
 213
 214         return pipes;
 215 }
 216
 217 /*
 218  * Return the port sync master and slave pipes linked to @crtc.
 219  * For bigjoiner configs return only the bigjoiner master pipes.
 220  */
 221 static void get_portsync_pipes(struct intel_crtc *crtc,
 222                                u8 *master_pipe_mask, u8 *slave_pipes_mask)
 223 {
 224         struct drm_i915_private *i915 = to_i915(crtc->base.dev);
 225         struct intel_crtc_state *crtc_state =
 226                 to_intel_crtc_state(crtc->base.state);
 227         struct intel_crtc *master_crtc;
 228         struct intel_crtc_state *master_crtc_state;
 229         enum transcoder master_transcoder;
 230
 231         if (!is_trans_port_sync_mode(crtc_state)) {
 232                 *master_pipe_mask = BIT(crtc->pipe);
 233                 *slave_pipes_mask = 0;
 234
 235                 return;
 236         }
 237
 238         if (is_trans_port_sync_master(crtc_state))
 239                 master_transcoder = crtc_state->cpu_transcoder;
 240         else
 241                 master_transcoder = crtc_state->master_transcoder;
 242
 243         *master_pipe_mask = get_transcoder_pipes(i915, BIT(master_transcoder));
 244         drm_WARN_ON(&i915->drm, !is_power_of_2(*master_pipe_mask));
 245
 246         master_crtc = intel_crtc_for_pipe(i915, ffs(*master_pipe_mask) - 1);
 247         master_crtc_state = to_intel_crtc_state(master_crtc->base.state);
 248         *slave_pipes_mask = get_transcoder_pipes(i915, master_crtc_state->sync_mode_slaves_mask);
 249 }
 250
 251 static u8 get_bigjoiner_slave_pipes(struct drm_i915_private *i915, u8 master_pipes_mask)
 252 {
 253         struct intel_crtc *master_crtc;
 254         u8 pipes = 0;
 255
 256         for_each_intel_crtc_in_pipe_mask(&i915->drm, master_crtc, master_pipes_mask) {
 257                 struct intel_crtc_state *master_crtc_state =
 258                         to_intel_crtc_state(master_crtc->base.state);
 259
 260                 pipes |= intel_crtc_bigjoiner_slave_pipes(master_crtc_state);
 261         }
 262
 263         return pipes;
 264 }
 265
 266 static void intel_crtc_disable_noatomic(struct intel_crtc *crtc,
 267                                         struct drm_modeset_acquire_ctx *ctx)
 268 {
 269         struct drm_i915_private *i915 = to_i915(crtc->base.dev);
 270         u8 portsync_master_mask;
 271         u8 portsync_slaves_mask;
 272         u8 bigjoiner_slaves_mask;
 273         struct intel_crtc *temp_crtc;
 274
 275         /* TODO: Add support for MST */
 276         get_portsync_pipes(crtc, &portsync_master_mask, &portsync_slaves_mask);
 277         bigjoiner_slaves_mask = get_bigjoiner_slave_pipes(i915,
 278                                                           portsync_master_mask |
 279                                                           portsync_slaves_mask);
 280
 281         drm_WARN_ON(&i915->drm,
 282                     portsync_master_mask & portsync_slaves_mask ||
 283                     portsync_master_mask & bigjoiner_slaves_mask ||
 284                     portsync_slaves_mask & bigjoiner_slaves_mask);
 285
 286         for_each_intel_crtc_in_pipe_mask(&i915->drm, temp_crtc, bigjoiner_slaves_mask)
 287                 intel_crtc_disable_noatomic_begin(temp_crtc, ctx);
 288
 289         for_each_intel_crtc_in_pipe_mask(&i915->drm, temp_crtc, portsync_slaves_mask)
 290                 intel_crtc_disable_noatomic_begin(temp_crtc, ctx);
 291
 292         for_each_intel_crtc_in_pipe_mask(&i915->drm, temp_crtc, portsync_master_mask)
 293                 intel_crtc_disable_noatomic_begin(temp_crtc, ctx);
 294
 295         for_each_intel_crtc_in_pipe_mask(&i915->drm, temp_crtc,
 296                                          bigjoiner_slaves_mask |
 297                                          portsync_slaves_mask |
 298                                          portsync_master_mask)
 299                 intel_crtc_disable_noatomic_complete(temp_crtc);
 300 }
 301
 302 static void intel_modeset_update_connector_atomic_state(struct drm_i915_private *i915)
 303 {
 304         struct intel_connector *connector;
 305         struct drm_connector_list_iter conn_iter;
 306
 307         drm_connector_list_iter_begin(&i915->drm, &conn_iter);
 308         for_each_intel_connector_iter(connector, &conn_iter) {
 309                 struct drm_connector_state *conn_state = connector->base.state;
 310                 struct intel_encoder *encoder =
 311                         to_intel_encoder(connector->base.encoder);
 312
 313                 set_encoder_for_connector(connector, encoder);
 314
 315                 if (encoder) {
 316                         struct intel_crtc *crtc =
 317                                 to_intel_crtc(encoder->base.crtc);
 318                         const struct intel_crtc_state *crtc_state =
 319                                 to_intel_crtc_state(crtc->base.state);
 320
 321                         conn_state->max_bpc = (crtc_state->pipe_bpp ?: 24) / 3;
 322                 }
 323         }
 324         drm_connector_list_iter_end(&conn_iter);
 325 }
 326
 327 static void intel_crtc_copy_hw_to_uapi_state(struct intel_crtc_state *crtc_state)
 328 {
 329         if (intel_crtc_is_bigjoiner_slave(crtc_state))
 330                 return;
 331
 332         crtc_state->uapi.enable = crtc_state->hw.enable;
 333         crtc_state->uapi.active = crtc_state->hw.active;
 334         drm_WARN_ON(crtc_state->uapi.crtc->dev,
 335                     drm_atomic_set_mode_for_crtc(&crtc_state->uapi, &crtc_state->hw.mode) < 0);
 336
 337         crtc_state->uapi.adjusted_mode = crtc_state->hw.adjusted_mode;
 338         crtc_state->uapi.scaling_filter = crtc_state->hw.scaling_filter;
 339
 340         /* assume 1:1 mapping */
 341         drm_property_replace_blob(&crtc_state->hw.degamma_lut,
 342                                   crtc_state->pre_csc_lut);
 343         drm_property_replace_blob(&crtc_state->hw.gamma_lut,
 344                                   crtc_state->post_csc_lut);
 345
 346         drm_property_replace_blob(&crtc_state->uapi.degamma_lut,
 347                                   crtc_state->hw.degamma_lut);
 348         drm_property_replace_blob(&crtc_state->uapi.gamma_lut,
 349                                   crtc_state->hw.gamma_lut);
 350         drm_property_replace_blob(&crtc_state->uapi.ctm,
 351                                   crtc_state->hw.ctm);
 352 }
 353
 354 static void
 355 intel_sanitize_plane_mapping(struct drm_i915_private *i915)
 356 {
 357         struct intel_crtc *crtc;
 358
 359         if (DISPLAY_VER(i915) >= 4)
 360                 return;
 361
 362         for_each_intel_crtc(&i915->drm, crtc) {
 363                 struct intel_plane *plane =
 364                         to_intel_plane(crtc->base.primary);
 365                 struct intel_crtc *plane_crtc;
 366                 enum pipe pipe;
 367
 368                 if (!plane->get_hw_state(plane, &pipe))
 369                         continue;
 370
 371                 if (pipe == crtc->pipe)
 372                         continue;
 373
 374                 drm_dbg_kms(&i915->drm,
 375                             "[PLANE:%d:%s] attached to the wrong pipe, disabling plane\n",
 376                             plane->base.base.id, plane->base.name);
 377
 378                 plane_crtc = intel_crtc_for_pipe(i915, pipe);
 379                 intel_plane_disable_noatomic(plane_crtc, plane);
 380         }
 381 }
 382
 383 static bool intel_crtc_has_encoders(struct intel_crtc *crtc)
 384 {
 385         struct drm_device *dev = crtc->base.dev;
 386         struct intel_encoder *encoder;
 387
 388         for_each_encoder_on_crtc(dev, &crtc->base, encoder)
 389                 return true;
 390
 391         return false;
 392 }
 393
 394 static bool intel_crtc_needs_link_reset(struct intel_crtc *crtc)
 395 {
 396         struct drm_device *dev = crtc->base.dev;
 397         struct intel_encoder *encoder;
 398
 399         for_each_encoder_on_crtc(dev, &crtc->base, encoder) {
 400                 struct intel_digital_port *dig_port = enc_to_dig_port(encoder);
 401
 402                 if (dig_port && intel_tc_port_link_needs_reset(dig_port))
 403                         return true;
 404         }
 405
 406         return false;
 407 }
 408
 409 static struct intel_connector *intel_encoder_find_connector(struct intel_encoder *encoder)
 410 {
 411         struct drm_i915_private *i915 = to_i915(encoder->base.dev);
 412         struct drm_connector_list_iter conn_iter;
 413         struct intel_connector *connector;
 414         struct intel_connector *found_connector = NULL;
 415
 416         drm_connector_list_iter_begin(&i915->drm, &conn_iter);
 417         for_each_intel_connector_iter(connector, &conn_iter) {
 418                 if (&encoder->base == connector->base.encoder) {
 419                         found_connector = connector;
 420                         break;
 421                 }
 422         }
 423         drm_connector_list_iter_end(&conn_iter);
 424
 425         return found_connector;
 426 }
 427
 428 static void intel_sanitize_fifo_underrun_reporting(const struct intel_crtc_state *crtc_state)
 429 {
 430         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
 431         struct drm_i915_private *i915 = to_i915(crtc->base.dev);
 432
 433         /*
 434          * We start out with underrun reporting disabled on active
 435          * pipes to avoid races.
 436          *
 437          * Also on gmch platforms we dont have any hardware bits to
 438          * disable the underrun reporting. Which means we need to start
 439          * out with underrun reporting disabled also on inactive pipes,
 440          * since otherwise we'll complain about the garbage we read when
 441          * e.g. coming up after runtime pm.
 442          *
 443          * No protection against concurrent access is required - at
 444          * worst a fifo underrun happens which also sets this to false.
 445          */
 446         intel_init_fifo_underrun_reporting(i915, crtc,
 447                                            !crtc_state->hw.active &&
 448                                            !HAS_GMCH(i915));
 449 }
 450
 451 static bool intel_sanitize_crtc(struct intel_crtc *crtc,
 452                                 struct drm_modeset_acquire_ctx *ctx)
 453 {
 454         struct drm_i915_private *i915 = to_i915(crtc->base.dev);
 455         struct intel_crtc_state *crtc_state = to_intel_crtc_state(crtc->base.state);
 456         bool needs_link_reset;
 457
 458         if (crtc_state->hw.active) {
 459                 struct intel_plane *plane;
 460
 461                 /* Disable everything but the primary plane */
 462                 for_each_intel_plane_on_crtc(&i915->drm, crtc, plane) {
 463                         const struct intel_plane_state *plane_state =
 464                                 to_intel_plane_state(plane->base.state);
 465
 466                         if (plane_state->uapi.visible &&
 467                             plane->base.type != DRM_PLANE_TYPE_PRIMARY)
 468                                 intel_plane_disable_noatomic(crtc, plane);
 469                 }
 470
 471                 /* Disable any background color/etc. set by the BIOS */
 472                 intel_color_commit_noarm(crtc_state);
 473                 intel_color_commit_arm(crtc_state);
 474         }
 475
 476         if (!crtc_state->hw.active ||
 477             intel_crtc_is_bigjoiner_slave(crtc_state))
 478                 return false;
 479
 480         needs_link_reset = intel_crtc_needs_link_reset(crtc);
 481
 482         /*
 483          * Adjust the state of the output pipe according to whether we have
 484          * active connectors/encoders.
 485          */
 486         if (!needs_link_reset && intel_crtc_has_encoders(crtc))
 487                 return false;
 488
 489         intel_crtc_disable_noatomic(crtc, ctx);
 490
 491         /*
 492          * The HPD state on other active/disconnected TC ports may be stuck in
 493          * the connected state until this port is disabled and a ~10ms delay has
 494          * passed, wait here for that so that sanitizing other CRTCs will see the
 495          * up-to-date HPD state.
 496          */
 497         if (needs_link_reset)
 498                 msleep(20);
 499
 500         return true;
 501 }
 502
 503 static void intel_sanitize_all_crtcs(struct drm_i915_private *i915,
 504                                      struct drm_modeset_acquire_ctx *ctx)
 505 {
 506         struct intel_crtc *crtc;
 507         u32 crtcs_forced_off = 0;
 508
 509         /*
 510          * An active and disconnected TypeC port prevents the HPD live state
 511          * to get updated on other active/disconnected TypeC ports, so after
 512          * a port gets disabled the CRTCs using other TypeC ports must be
 513          * rechecked wrt. their link status.
 514          */
 515         for (;;) {
 516                 u32 old_mask = crtcs_forced_off;
 517
 518                 for_each_intel_crtc(&i915->drm, crtc) {
 519                         u32 crtc_mask = drm_crtc_mask(&crtc->base);
 520
 521                         if (crtcs_forced_off & crtc_mask)
 522                                 continue;
 523
 524                         if (intel_sanitize_crtc(crtc, ctx))
 525                                 crtcs_forced_off |= crtc_mask;
 526                 }
 527                 if (crtcs_forced_off == old_mask)
 528                         break;
 529         }
 530
 531         for_each_intel_crtc(&i915->drm, crtc) {
 532                 struct intel_crtc_state *crtc_state =
 533                         to_intel_crtc_state(crtc->base.state);
 534
 535                 intel_crtc_state_dump(crtc_state, NULL, "setup_hw_state");
 536         }
 537 }
 538
 539 static bool has_bogus_dpll_config(const struct intel_crtc_state *crtc_state)
 540 {
 541         struct drm_i915_private *i915 = to_i915(crtc_state->uapi.crtc->dev);
 542
 543         /*
 544          * Some SNB BIOSen (eg. ASUS K53SV) are known to misprogram
 545          * the hardware when a high res displays plugged in. DPLL P
 546          * divider is zero, and the pipe timings are bonkers. We'll
 547          * try to disable everything in that case.
 548          *
 549          * FIXME would be nice to be able to sanitize this state
 550          * without several WARNs, but for now let's take the easy
 551          * road.
 552          */
 553         return IS_SANDYBRIDGE(i915) &&
 554                 crtc_state->hw.active &&
 555                 crtc_state->shared_dpll &&
 556                 crtc_state->port_clock == 0;
 557 }
 558
 559 static void intel_sanitize_encoder(struct intel_encoder *encoder)
 560 {
 561         struct drm_i915_private *i915 = to_i915(encoder->base.dev);
 562         struct intel_connector *connector;
 563         struct intel_crtc *crtc = to_intel_crtc(encoder->base.crtc);
 564         struct intel_crtc_state *crtc_state = crtc ?
 565                 to_intel_crtc_state(crtc->base.state) : NULL;
 566         struct intel_pmdemand_state *pmdemand_state =
 567                 to_intel_pmdemand_state(i915->display.pmdemand.obj.state);
 568
 569         /*
 570          * We need to check both for a crtc link (meaning that the encoder is
 571          * active and trying to read from a pipe) and the pipe itself being
 572          * active.
 573          */
 574         bool has_active_crtc = crtc_state &&
 575                 crtc_state->hw.active;
 576
 577         if (crtc_state && has_bogus_dpll_config(crtc_state)) {
 578                 drm_dbg_kms(&i915->drm,
 579                             "BIOS has misprogrammed the hardware. Disabling pipe %c\n",
 580                             pipe_name(crtc->pipe));
 581                 has_active_crtc = false;
 582         }
 583
 584         connector = intel_encoder_find_connector(encoder);
 585         if (connector && !has_active_crtc) {
 586                 drm_dbg_kms(&i915->drm,
 587                             "[ENCODER:%d:%s] has active connectors but no active pipe!\n",
 588                             encoder->base.base.id,
 589                             encoder->base.name);
 590
 591                 /* Clear the corresponding bit in pmdemand active phys mask */
 592                 intel_pmdemand_update_phys_mask(i915, encoder,
 593                                                 pmdemand_state, false);
 594
 595                 /*
 596                  * Connector is active, but has no active pipe. This is fallout
 597                  * from our resume register restoring. Disable the encoder
 598                  * manually again.
 599                  */
 600                 if (crtc_state) {
 601                         struct drm_encoder *best_encoder;
 602
 603                         drm_dbg_kms(&i915->drm,
 604                                     "[ENCODER:%d:%s] manually disabled\n",
 605                                     encoder->base.base.id,
 606                                     encoder->base.name);
 607
 608                         /* avoid oopsing in case the hooks consult best_encoder */
 609                         best_encoder = connector->base.state->best_encoder;
 610                         connector->base.state->best_encoder = &encoder->base;
 611
 612                         /* FIXME NULL atomic state passed! */
 613                         if (encoder->disable)
 614                                 encoder->disable(NULL, encoder, crtc_state,
 615                                                  connector->base.state);
 616                         if (encoder->post_disable)
 617                                 encoder->post_disable(NULL, encoder, crtc_state,
 618                                                       connector->base.state);
 619
 620                         connector->base.state->best_encoder = best_encoder;
 621                 }
 622                 encoder->base.crtc = NULL;
 623
 624                 /*
 625                  * Inconsistent output/port/pipe state happens presumably due to
 626                  * a bug in one of the get_hw_state functions. Or someplace else
 627                  * in our code, like the register restore mess on resume. Clamp
 628                  * things to off as a safer default.
 629                  */
 630                 connector->base.dpms = DRM_MODE_DPMS_OFF;
 631                 connector->base.encoder = NULL;
 632         }
 633
 634         /* notify opregion of the sanitized encoder state */
 635         intel_opregion_notify_encoder(encoder, connector && has_active_crtc);
 636
 637         if (HAS_DDI(i915))
 638                 intel_ddi_sanitize_encoder_pll_mapping(encoder);
 639 }
 640
 641 /* FIXME read out full plane state for all planes */
 642 static void readout_plane_state(struct drm_i915_private *i915)
 643 {
 644         struct intel_plane *plane;
 645         struct intel_crtc *crtc;
 646
 647         for_each_intel_plane(&i915->drm, plane) {
 648                 struct intel_plane_state *plane_state =
 649                         to_intel_plane_state(plane->base.state);
 650                 struct intel_crtc_state *crtc_state;
 651                 enum pipe pipe = PIPE_A;
 652                 bool visible;
 653
 654                 visible = plane->get_hw_state(plane, &pipe);
 655
 656                 crtc = intel_crtc_for_pipe(i915, pipe);
 657                 crtc_state = to_intel_crtc_state(crtc->base.state);
 658
 659                 intel_set_plane_visible(crtc_state, plane_state, visible);
 660
 661                 drm_dbg_kms(&i915->drm,
 662                             "[PLANE:%d:%s] hw state readout: %s, pipe %c\n",
 663                             plane->base.base.id, plane->base.name,
 664                             str_enabled_disabled(visible), pipe_name(pipe));
 665         }
 666
 667         for_each_intel_crtc(&i915->drm, crtc) {
 668                 struct intel_crtc_state *crtc_state =
 669                         to_intel_crtc_state(crtc->base.state);
 670
 671                 intel_plane_fixup_bitmasks(crtc_state);
 672         }
 673 }
 674
 675 static void intel_modeset_readout_hw_state(struct drm_i915_private *i915)
 676 {
 677         struct intel_cdclk_state *cdclk_state =
 678                 to_intel_cdclk_state(i915->display.cdclk.obj.state);
 679         struct intel_dbuf_state *dbuf_state =
 680                 to_intel_dbuf_state(i915->display.dbuf.obj.state);
 681         struct intel_pmdemand_state *pmdemand_state =
 682                 to_intel_pmdemand_state(i915->display.pmdemand.obj.state);
 683         enum pipe pipe;
 684         struct intel_crtc *crtc;
 685         struct intel_encoder *encoder;
 686         struct intel_connector *connector;
 687         struct drm_connector_list_iter conn_iter;
 688         u8 active_pipes = 0;
 689
 690         for_each_intel_crtc(&i915->drm, crtc) {
 691                 struct intel_crtc_state *crtc_state =
 692                         to_intel_crtc_state(crtc->base.state);
 693
 694                 __drm_atomic_helper_crtc_destroy_state(&crtc_state->uapi);
 695                 intel_crtc_free_hw_state(crtc_state);
 696                 intel_crtc_state_reset(crtc_state, crtc);
 697
 698                 intel_crtc_get_pipe_config(crtc_state);
 699
 700                 crtc_state->hw.enable = crtc_state->hw.active;
 701
 702                 crtc->base.enabled = crtc_state->hw.enable;
 703                 crtc->active = crtc_state->hw.active;
 704
 705                 if (crtc_state->hw.active)
 706                         active_pipes |= BIT(crtc->pipe);
 707
 708                 drm_dbg_kms(&i915->drm,
 709                             "[CRTC:%d:%s] hw state readout: %s\n",
 710                             crtc->base.base.id, crtc->base.name,
 711                             str_enabled_disabled(crtc_state->hw.active));
 712         }
 713
 714         cdclk_state->active_pipes = active_pipes;
 715         dbuf_state->active_pipes = active_pipes;
 716
 717         readout_plane_state(i915);
 718
 719         for_each_intel_encoder(&i915->drm, encoder) {
 720                 struct intel_crtc_state *crtc_state = NULL;
 721
 722                 pipe = 0;
 723
 724                 if (encoder->get_hw_state(encoder, &pipe)) {
 725                         crtc = intel_crtc_for_pipe(i915, pipe);
 726                         crtc_state = to_intel_crtc_state(crtc->base.state);
 727
 728                         encoder->base.crtc = &crtc->base;
 729                         intel_encoder_get_config(encoder, crtc_state);
 730
 731                         /* read out to slave crtc as well for bigjoiner */
 732                         if (crtc_state->bigjoiner_pipes) {
 733                                 struct intel_crtc *slave_crtc;
 734
 735                                 /* encoder should read be linked to bigjoiner master */
 736                                 WARN_ON(intel_crtc_is_bigjoiner_slave(crtc_state));
 737
 738                                 for_each_intel_crtc_in_pipe_mask(&i915->drm, slave_crtc,
 739                                                                  intel_crtc_bigjoiner_slave_pipes(crtc_state)) {
 740                                         struct intel_crtc_state *slave_crtc_state;
 741
 742                                         slave_crtc_state = to_intel_crtc_state(slave_crtc->base.state);
 743                                         intel_encoder_get_config(encoder, slave_crtc_state);
 744                                 }
 745                         }
 746
 747                         intel_pmdemand_update_phys_mask(i915, encoder,
 748                                                         pmdemand_state,
 749                                                         true);
 750                 } else {
 751                         intel_pmdemand_update_phys_mask(i915, encoder,
 752                                                         pmdemand_state,
 753                                                         false);
 754
 755                         encoder->base.crtc = NULL;
 756                 }
 757
 758                 if (encoder->sync_state)
 759                         encoder->sync_state(encoder, crtc_state);
 760
 761                 drm_dbg_kms(&i915->drm,
 762                             "[ENCODER:%d:%s] hw state readout: %s, pipe %c\n",
 763                             encoder->base.base.id, encoder->base.name,
 764                             str_enabled_disabled(encoder->base.crtc),
 765                             pipe_name(pipe));
 766         }
 767
 768         intel_dpll_readout_hw_state(i915);
 769
 770         drm_connector_list_iter_begin(&i915->drm, &conn_iter);
 771         for_each_intel_connector_iter(connector, &conn_iter) {
 772                 struct intel_crtc_state *crtc_state = NULL;
 773
 774                 if (connector->get_hw_state(connector)) {
 775                         struct intel_crtc *crtc;
 776
 777                         connector->base.dpms = DRM_MODE_DPMS_ON;
 778
 779                         encoder = intel_attached_encoder(connector);
 780                         connector->base.encoder = &encoder->base;
 781
 782                         crtc = to_intel_crtc(encoder->base.crtc);
 783                         crtc_state = crtc ? to_intel_crtc_state(crtc->base.state) : NULL;
 784
 785                         if (crtc_state && crtc_state->hw.active) {
 786                                 /*
 787                                  * This has to be done during hardware readout
 788                                  * because anything calling .crtc_disable may
 789                                  * rely on the connector_mask being accurate.
 790                                  */
 791                                 crtc_state->uapi.connector_mask |=
 792                                         drm_connector_mask(&connector->base);
 793                                 crtc_state->uapi.encoder_mask |=
 794                                         drm_encoder_mask(&encoder->base);
 795                         }
 796                 } else {
 797                         connector->base.dpms = DRM_MODE_DPMS_OFF;
 798                         connector->base.encoder = NULL;
 799                 }
 800
 801                 if (connector->sync_state)
 802                         connector->sync_state(connector, crtc_state);
 803
 804                 drm_dbg_kms(&i915->drm,
 805                             "[CONNECTOR:%d:%s] hw state readout: %s\n",
 806                             connector->base.base.id, connector->base.name,
 807                             str_enabled_disabled(connector->base.encoder));
 808         }
 809         drm_connector_list_iter_end(&conn_iter);
 810
 811         for_each_intel_crtc(&i915->drm, crtc) {
 812                 struct intel_bw_state *bw_state =
 813                         to_intel_bw_state(i915->display.bw.obj.state);
 814                 struct intel_crtc_state *crtc_state =
 815                         to_intel_crtc_state(crtc->base.state);
 816                 struct intel_plane *plane;
 817                 int min_cdclk = 0;
 818
 819                 if (crtc_state->hw.active) {
 820                         /*
 821                          * The initial mode needs to be set in order to keep
 822                          * the atomic core happy. It wants a valid mode if the
 823                          * crtc's enabled, so we do the above call.
 824                          *
 825                          * But we don't set all the derived state fully, hence
 826                          * set a flag to indicate that a full recalculation is
 827                          * needed on the next commit.
 828                          */
 829                         crtc_state->inherited = true;
 830
 831                         intel_crtc_update_active_timings(crtc_state,
 832                                                          crtc_state->vrr.enable);
 833
 834                         intel_crtc_copy_hw_to_uapi_state(crtc_state);
 835                 }
 836
 837                 for_each_intel_plane_on_crtc(&i915->drm, crtc, plane) {
 838                         const struct intel_plane_state *plane_state =
 839                                 to_intel_plane_state(plane->base.state);
 840
 841                         /*
 842                          * FIXME don't have the fb yet, so can't
 843                          * use intel_plane_data_rate() :(
 844                          */
 845                         if (plane_state->uapi.visible)
 846                                 crtc_state->data_rate[plane->id] =
 847                                         4 * crtc_state->pixel_rate;
 848                         /*
 849                          * FIXME don't have the fb yet, so can't
 850                          * use plane->min_cdclk() :(
 851                          */
 852                         if (plane_state->uapi.visible && plane->min_cdclk) {
 853                                 if (crtc_state->double_wide || DISPLAY_VER(i915) >= 10)
 854                                         crtc_state->min_cdclk[plane->id] =
 855                                                 DIV_ROUND_UP(crtc_state->pixel_rate, 2);
 856                                 else
 857                                         crtc_state->min_cdclk[plane->id] =
 858                                                 crtc_state->pixel_rate;
 859                         }
 860                         drm_dbg_kms(&i915->drm,
 861                                     "[PLANE:%d:%s] min_cdclk %d kHz\n",
 862                                     plane->base.base.id, plane->base.name,
 863                                     crtc_state->min_cdclk[plane->id]);
 864                 }
 865
 866                 if (crtc_state->hw.active) {
 867                         min_cdclk = intel_crtc_compute_min_cdclk(crtc_state);
 868                         if (drm_WARN_ON(&i915->drm, min_cdclk < 0))
 869                                 min_cdclk = 0;
 870                 }
 871
 872                 cdclk_state->min_cdclk[crtc->pipe] = min_cdclk;
 873                 cdclk_state->min_voltage_level[crtc->pipe] =
 874                         crtc_state->min_voltage_level;
 875
 876                 intel_pmdemand_update_port_clock(i915, pmdemand_state, pipe,
 877                                                  crtc_state->port_clock);
 878
 879                 intel_bw_crtc_update(bw_state, crtc_state);
 880         }
 881
 882         intel_pmdemand_init_pmdemand_params(i915, pmdemand_state);
 883 }
 884
 885 static void
 886 get_encoder_power_domains(struct drm_i915_private *i915)
 887 {
 888         struct intel_encoder *encoder;
 889
 890         for_each_intel_encoder(&i915->drm, encoder) {
 891                 struct intel_crtc_state *crtc_state;
 892
 893                 if (!encoder->get_power_domains)
 894                         continue;
 895
 896                 /*
 897                  * MST-primary and inactive encoders don't have a crtc state
 898                  * and neither of these require any power domain references.
 899                  */
 900                 if (!encoder->base.crtc)
 901                         continue;
 902
 903                 crtc_state = to_intel_crtc_state(encoder->base.crtc->state);
 904                 encoder->get_power_domains(encoder, crtc_state);
 905         }
 906 }
 907
 908 static void intel_early_display_was(struct drm_i915_private *i915)
 909 {
 910         /*
 911          * Display WA #1185 WaDisableDARBFClkGating:glk,icl,ehl,tgl
 912          * Also known as Wa_14010480278.
 913          */
 914         if (IS_DISPLAY_VER(i915, 10, 12))
 915                 intel_de_rmw(i915, GEN9_CLKGATE_DIS_0, 0, DARBF_GATING_DIS);
 916
 917         /*
 918          * WaRsPkgCStateDisplayPMReq:hsw
 919          * System hang if this isn't done before disabling all planes!
 920          */
 921         if (IS_HASWELL(i915))
 922                 intel_de_rmw(i915, CHICKEN_PAR1_1, 0, FORCE_ARB_IDLE_PLANES);
 923
 924         if (IS_KABYLAKE(i915) || IS_COFFEELAKE(i915) || IS_COMETLAKE(i915)) {
 925                 /* Display WA #1142:kbl,cfl,cml */
 926                 intel_de_rmw(i915, CHICKEN_PAR1_1,
 927                              KBL_ARB_FILL_SPARE_22, KBL_ARB_FILL_SPARE_22);
 928                 intel_de_rmw(i915, CHICKEN_MISC_2,
 929                              KBL_ARB_FILL_SPARE_13 | KBL_ARB_FILL_SPARE_14,
 930                              KBL_ARB_FILL_SPARE_14);
 931         }
 932 }
 933
 934 void intel_modeset_setup_hw_state(struct drm_i915_private *i915,
 935                                   struct drm_modeset_acquire_ctx *ctx)
 936 {
 937         struct intel_encoder *encoder;
 938         struct intel_crtc *crtc;
 939         intel_wakeref_t wakeref;
 940
 941         wakeref = intel_display_power_get(i915, POWER_DOMAIN_INIT);
 942
 943         intel_early_display_was(i915);
 944         intel_modeset_readout_hw_state(i915);
 945
 946         /* HW state is read out, now we need to sanitize this mess. */
 947         get_encoder_power_domains(i915);
 948
 949         intel_pch_sanitize(i915);
 950
 951         /*
 952          * intel_sanitize_plane_mapping() may need to do vblank
 953          * waits, so we need vblank interrupts restored beforehand.
 954          */
 955         for_each_intel_crtc(&i915->drm, crtc) {
 956                 struct intel_crtc_state *crtc_state =
 957                         to_intel_crtc_state(crtc->base.state);
 958
 959                 intel_sanitize_fifo_underrun_reporting(crtc_state);
 960
 961                 drm_crtc_vblank_reset(&crtc->base);
 962
 963                 if (crtc_state->hw.active) {
 964                         intel_dmc_enable_pipe(i915, crtc->pipe);
 965                         intel_crtc_vblank_on(crtc_state);
 966                 }
 967         }
 968
 969         intel_fbc_sanitize(i915);
 970
 971         intel_sanitize_plane_mapping(i915);
 972
 973         for_each_intel_encoder(&i915->drm, encoder)
 974                 intel_sanitize_encoder(encoder);
 975
 976         /*
 977          * Sanitizing CRTCs needs their connector atomic state to be
 978          * up-to-date, so ensure that already here.
 979          */
 980         intel_modeset_update_connector_atomic_state(i915);
 981
 982         intel_sanitize_all_crtcs(i915, ctx);
 983
 984         intel_dpll_sanitize_state(i915);
 985
 986         intel_wm_get_hw_state(i915);
 987
 988         for_each_intel_crtc(&i915->drm, crtc) {
 989                 struct intel_crtc_state *crtc_state =
 990                         to_intel_crtc_state(crtc->base.state);
 991                 struct intel_power_domain_mask put_domains;
 992
 993                 intel_modeset_get_crtc_power_domains(crtc_state, &put_domains);
 994                 if (drm_WARN_ON(&i915->drm, !bitmap_empty(put_domains.bits, POWER_DOMAIN_NUM)))
 995                         intel_modeset_put_crtc_power_domains(crtc, &put_domains);
 996         }
 997
 998         intel_display_power_put(i915, POWER_DOMAIN_INIT, wakeref);
 999
1000         intel_power_domains_sanitize_state(i915);
1001 }