drivers/gpu/drm/i915/display/intel_dsb.c

   1 // SPDX-License-Identifier: MIT
   2 /*
   3  * Copyright © 2019 Intel Corporation
   4  *
   5  */
   6
   7 #include "i915_drv.h"
   8 #include "i915_irq.h"
   9 #include "i915_reg.h"
  10 #include "intel_crtc.h"
  11 #include "intel_de.h"
  12 #include "intel_display_types.h"
  13 #include "intel_dsb.h"
  14 #include "intel_dsb_buffer.h"
  15 #include "intel_dsb_regs.h"
  16 #include "intel_vblank.h"
  17 #include "intel_vrr.h"
  18 #include "skl_watermark.h"
  19
  20 #define CACHELINE_BYTES 64
  21
  22 enum dsb_id {
  23         INVALID_DSB = -1,
  24         DSB1,
  25         DSB2,
  26         DSB3,
  27         MAX_DSB_PER_PIPE
  28 };
  29
  30 struct intel_dsb {
  31         enum dsb_id id;
  32
  33         struct intel_dsb_buffer dsb_buf;
  34         struct intel_crtc *crtc;
  35
  36         /*
  37          * maximum number of dwords the buffer will hold.
  38          */
  39         unsigned int size;
  40
  41         /*
  42          * free_pos will point the first free dword and
  43          * help in calculating tail of command buffer.
  44          */
  45         unsigned int free_pos;
  46
  47         /*
  48          * ins_start_offset will help to store start dword of the dsb
  49          * instuction and help in identifying the batch of auto-increment
  50          * register.
  51          */
  52         unsigned int ins_start_offset;
  53
  54         int dewake_scanline;
  55 };
  56
  57 /**
  58  * DOC: DSB
  59  *
  60  * A DSB (Display State Buffer) is a queue of MMIO instructions in the memory
  61  * which can be offloaded to DSB HW in Display Controller. DSB HW is a DMA
  62  * engine that can be programmed to download the DSB from memory.
  63  * It allows driver to batch submit display HW programming. This helps to
  64  * reduce loading time and CPU activity, thereby making the context switch
  65  * faster. DSB Support added from Gen12 Intel graphics based platform.
  66  *
  67  * DSB's can access only the pipe, plane, and transcoder Data Island Packet
  68  * registers.
  69  *
  70  * DSB HW can support only register writes (both indexed and direct MMIO
  71  * writes). There are no registers reads possible with DSB HW engine.
  72  */
  73
  74 /* DSB opcodes. */
  75 #define DSB_OPCODE_SHIFT                24
  76 #define DSB_OPCODE_NOOP                 0x0
  77 #define DSB_OPCODE_MMIO_WRITE           0x1
  78 #define   DSB_BYTE_EN                   0xf
  79 #define   DSB_BYTE_EN_SHIFT             20
  80 #define   DSB_REG_VALUE_MASK            0xfffff
  81 #define DSB_OPCODE_WAIT_USEC            0x2
  82 #define DSB_OPCODE_WAIT_SCANLINE        0x3
  83 #define DSB_OPCODE_WAIT_VBLANKS         0x4
  84 #define DSB_OPCODE_WAIT_DSL_IN          0x5
  85 #define DSB_OPCODE_WAIT_DSL_OUT         0x6
  86 #define   DSB_SCANLINE_UPPER_SHIFT      20
  87 #define   DSB_SCANLINE_LOWER_SHIFT      0
  88 #define DSB_OPCODE_INTERRUPT            0x7
  89 #define DSB_OPCODE_INDEXED_WRITE        0x9
  90 /* see DSB_REG_VALUE_MASK */
  91 #define DSB_OPCODE_POLL                 0xA
  92 /* see DSB_REG_VALUE_MASK */
  93
  94 static bool assert_dsb_has_room(struct intel_dsb *dsb)
  95 {
  96         struct intel_crtc *crtc = dsb->crtc;
  97         struct drm_i915_private *i915 = to_i915(crtc->base.dev);
  98
  99         /* each instruction is 2 dwords */
 100         return !drm_WARN(&i915->drm, dsb->free_pos > dsb->size - 2,
 101                          "[CRTC:%d:%s] DSB %d buffer overflow\n",
 102                          crtc->base.base.id, crtc->base.name, dsb->id);
 103 }
 104
 105 static void intel_dsb_dump(struct intel_dsb *dsb)
 106 {
 107         struct intel_crtc *crtc = dsb->crtc;
 108         struct drm_i915_private *i915 = to_i915(crtc->base.dev);
 109         int i;
 110
 111         drm_dbg_kms(&i915->drm, "[CRTC:%d:%s] DSB %d commands {\n",
 112                     crtc->base.base.id, crtc->base.name, dsb->id);
 113         for (i = 0; i < ALIGN(dsb->free_pos, 64 / 4); i += 4)
 114                 drm_dbg_kms(&i915->drm,
 115                             " 0x%08x: 0x%08x 0x%08x 0x%08x 0x%08x\n", i * 4,
 116                             intel_dsb_buffer_read(&dsb->dsb_buf, i),
 117                             intel_dsb_buffer_read(&dsb->dsb_buf, i + 1),
 118                             intel_dsb_buffer_read(&dsb->dsb_buf, i + 2),
 119                             intel_dsb_buffer_read(&dsb->dsb_buf, i + 3));
 120         drm_dbg_kms(&i915->drm, "}\n");
 121 }
 122
 123 static bool is_dsb_busy(struct drm_i915_private *i915, enum pipe pipe,
 124                         enum dsb_id id)
 125 {
 126         return intel_de_read_fw(i915, DSB_CTRL(pipe, id)) & DSB_STATUS_BUSY;
 127 }
 128
 129 static void intel_dsb_emit(struct intel_dsb *dsb, u32 ldw, u32 udw)
 130 {
 131         if (!assert_dsb_has_room(dsb))
 132                 return;
 133
 134         /* Every instruction should be 8 byte aligned. */
 135         dsb->free_pos = ALIGN(dsb->free_pos, 2);
 136
 137         dsb->ins_start_offset = dsb->free_pos;
 138
 139         intel_dsb_buffer_write(&dsb->dsb_buf, dsb->free_pos++, ldw);
 140         intel_dsb_buffer_write(&dsb->dsb_buf, dsb->free_pos++, udw);
 141 }
 142
 143 static bool intel_dsb_prev_ins_is_write(struct intel_dsb *dsb,
 144                                         u32 opcode, i915_reg_t reg)
 145 {
 146         u32 prev_opcode, prev_reg;
 147
 148         /*
 149          * Nothing emitted yet? Must check before looking
 150          * at the actual data since i915_gem_object_create_internal()
 151          * does *not* give you zeroed memory!
 152          */
 153         if (dsb->free_pos == 0)
 154                 return false;
 155
 156         prev_opcode = intel_dsb_buffer_read(&dsb->dsb_buf,
 157                                             dsb->ins_start_offset + 1) & ~DSB_REG_VALUE_MASK;
 158         prev_reg =  intel_dsb_buffer_read(&dsb->dsb_buf,
 159                                           dsb->ins_start_offset + 1) & DSB_REG_VALUE_MASK;
 160
 161         return prev_opcode == opcode && prev_reg == i915_mmio_reg_offset(reg);
 162 }
 163
 164 static bool intel_dsb_prev_ins_is_mmio_write(struct intel_dsb *dsb, i915_reg_t reg)
 165 {
 166         /* only full byte-enables can be converted to indexed writes */
 167         return intel_dsb_prev_ins_is_write(dsb,
 168                                            DSB_OPCODE_MMIO_WRITE << DSB_OPCODE_SHIFT |
 169                                            DSB_BYTE_EN << DSB_BYTE_EN_SHIFT,
 170                                            reg);
 171 }
 172
 173 static bool intel_dsb_prev_ins_is_indexed_write(struct intel_dsb *dsb, i915_reg_t reg)
 174 {
 175         return intel_dsb_prev_ins_is_write(dsb,
 176                                            DSB_OPCODE_INDEXED_WRITE << DSB_OPCODE_SHIFT,
 177                                            reg);
 178 }
 179
 180 /**
 181  * intel_dsb_reg_write() - Emit register wriite to the DSB context
 182  * @dsb: DSB context
 183  * @reg: register address.
 184  * @val: value.
 185  *
 186  * This function is used for writing register-value pair in command
 187  * buffer of DSB.
 188  */
 189 void intel_dsb_reg_write(struct intel_dsb *dsb,
 190                          i915_reg_t reg, u32 val)
 191 {
 192         u32 old_val;
 193
 194         /*
 195          * For example the buffer will look like below for 3 dwords for auto
 196          * increment register:
 197          * +--------------------------------------------------------+
 198          * | size = 3 | offset &| value1 | value2 | value3 | zero   |
 199          * |          | opcode  |        |        |        |        |
 200          * +--------------------------------------------------------+
 201          * +          +         +        +        +        +        +
 202          * 0          4         8        12       16       20       24
 203          * Byte
 204          *
 205          * As every instruction is 8 byte aligned the index of dsb instruction
 206          * will start always from even number while dealing with u32 array. If
 207          * we are writing odd no of dwords, Zeros will be added in the end for
 208          * padding.
 209          */
 210         if (!intel_dsb_prev_ins_is_mmio_write(dsb, reg) &&
 211             !intel_dsb_prev_ins_is_indexed_write(dsb, reg)) {
 212                 intel_dsb_emit(dsb, val,
 213                                (DSB_OPCODE_MMIO_WRITE << DSB_OPCODE_SHIFT) |
 214                                (DSB_BYTE_EN << DSB_BYTE_EN_SHIFT) |
 215                                i915_mmio_reg_offset(reg));
 216         } else {
 217                 if (!assert_dsb_has_room(dsb))
 218                         return;
 219
 220                 /* convert to indexed write? */
 221                 if (intel_dsb_prev_ins_is_mmio_write(dsb, reg)) {
 222                         u32 prev_val = intel_dsb_buffer_read(&dsb->dsb_buf,
 223                                                              dsb->ins_start_offset + 0);
 224
 225                         intel_dsb_buffer_write(&dsb->dsb_buf,
 226                                                dsb->ins_start_offset + 0, 1); /* count */
 227                         intel_dsb_buffer_write(&dsb->dsb_buf, dsb->ins_start_offset + 1,
 228                                                (DSB_OPCODE_INDEXED_WRITE << DSB_OPCODE_SHIFT) |
 229                                                i915_mmio_reg_offset(reg));
 230                         intel_dsb_buffer_write(&dsb->dsb_buf, dsb->ins_start_offset + 2, prev_val);
 231
 232                         dsb->free_pos++;
 233                 }
 234
 235                 intel_dsb_buffer_write(&dsb->dsb_buf, dsb->free_pos++, val);
 236                 /* Update the count */
 237                 old_val = intel_dsb_buffer_read(&dsb->dsb_buf, dsb->ins_start_offset);
 238                 intel_dsb_buffer_write(&dsb->dsb_buf, dsb->ins_start_offset, old_val + 1);
 239
 240                 /* if number of data words is odd, then the last dword should be 0.*/
 241                 if (dsb->free_pos & 0x1)
 242                         intel_dsb_buffer_write(&dsb->dsb_buf, dsb->free_pos, 0);
 243         }
 244 }
 245
 246 static u32 intel_dsb_mask_to_byte_en(u32 mask)
 247 {
 248         return (!!(mask & 0xff000000) << 3 |
 249                 !!(mask & 0x00ff0000) << 2 |
 250                 !!(mask & 0x0000ff00) << 1 |
 251                 !!(mask & 0x000000ff) << 0);
 252 }
 253
 254 /* Note: mask implemented via byte enables! */
 255 void intel_dsb_reg_write_masked(struct intel_dsb *dsb,
 256                                 i915_reg_t reg, u32 mask, u32 val)
 257 {
 258         intel_dsb_emit(dsb, val,
 259                        (DSB_OPCODE_MMIO_WRITE << DSB_OPCODE_SHIFT) |
 260                        (intel_dsb_mask_to_byte_en(mask) << DSB_BYTE_EN_SHIFT) |
 261                        i915_mmio_reg_offset(reg));
 262 }
 263
 264 void intel_dsb_noop(struct intel_dsb *dsb, int count)
 265 {
 266         int i;
 267
 268         for (i = 0; i < count; i++)
 269                 intel_dsb_emit(dsb, 0,
 270                                DSB_OPCODE_NOOP << DSB_OPCODE_SHIFT);
 271 }
 272
 273 void intel_dsb_nonpost_start(struct intel_dsb *dsb)
 274 {
 275         struct intel_crtc *crtc = dsb->crtc;
 276         enum pipe pipe = crtc->pipe;
 277
 278         intel_dsb_reg_write_masked(dsb, DSB_CTRL(pipe, dsb->id),
 279                                    DSB_NON_POSTED, DSB_NON_POSTED);
 280         intel_dsb_noop(dsb, 4);
 281 }
 282
 283 void intel_dsb_nonpost_end(struct intel_dsb *dsb)
 284 {
 285         struct intel_crtc *crtc = dsb->crtc;
 286         enum pipe pipe = crtc->pipe;
 287
 288         intel_dsb_reg_write_masked(dsb, DSB_CTRL(pipe, dsb->id),
 289                                    DSB_NON_POSTED, 0);
 290         intel_dsb_noop(dsb, 4);
 291 }
 292
 293 static void intel_dsb_align_tail(struct intel_dsb *dsb)
 294 {
 295         u32 aligned_tail, tail;
 296
 297         tail = dsb->free_pos * 4;
 298         aligned_tail = ALIGN(tail, CACHELINE_BYTES);
 299
 300         if (aligned_tail > tail)
 301                 intel_dsb_buffer_memset(&dsb->dsb_buf, dsb->free_pos, 0,
 302                                         aligned_tail - tail);
 303
 304         dsb->free_pos = aligned_tail / 4;
 305 }
 306
 307 void intel_dsb_finish(struct intel_dsb *dsb)
 308 {
 309         struct intel_crtc *crtc = dsb->crtc;
 310
 311         /*
 312          * DSB_FORCE_DEWAKE remains active even after DSB is
 313          * disabled, so make sure to clear it (if set during
 314          * intel_dsb_commit()).
 315          */
 316         intel_dsb_reg_write_masked(dsb, DSB_PMCTRL_2(crtc->pipe, dsb->id),
 317                                    DSB_FORCE_DEWAKE, 0);
 318
 319         intel_dsb_align_tail(dsb);
 320
 321         intel_dsb_buffer_flush_map(&dsb->dsb_buf);
 322 }
 323
 324 static int intel_dsb_dewake_scanline(const struct intel_crtc_state *crtc_state)
 325 {
 326         struct drm_i915_private *i915 = to_i915(crtc_state->uapi.crtc->dev);
 327         const struct drm_display_mode *adjusted_mode = &crtc_state->hw.adjusted_mode;
 328         unsigned int latency = skl_watermark_max_latency(i915, 0);
 329         int vblank_start;
 330
 331         if (crtc_state->vrr.enable) {
 332                 vblank_start = intel_vrr_vmin_vblank_start(crtc_state);
 333         } else {
 334                 vblank_start = adjusted_mode->crtc_vblank_start;
 335
 336                 if (adjusted_mode->flags & DRM_MODE_FLAG_INTERLACE)
 337                         vblank_start = DIV_ROUND_UP(vblank_start, 2);
 338         }
 339
 340         return max(0, vblank_start - intel_usecs_to_scanlines(adjusted_mode, latency));
 341 }
 342
 343 static u32 dsb_chicken(struct intel_crtc *crtc)
 344 {
 345         if (crtc->mode_flags & I915_MODE_FLAG_VRR)
 346                 return DSB_SKIP_WAITS_EN |
 347                         DSB_CTRL_WAIT_SAFE_WINDOW |
 348                         DSB_CTRL_NO_WAIT_VBLANK |
 349                         DSB_INST_WAIT_SAFE_WINDOW |
 350                         DSB_INST_NO_WAIT_VBLANK;
 351         else
 352                 return DSB_SKIP_WAITS_EN;
 353 }
 354
 355 static void _intel_dsb_commit(struct intel_dsb *dsb, u32 ctrl,
 356                               int dewake_scanline)
 357 {
 358         struct intel_crtc *crtc = dsb->crtc;
 359         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
 360         enum pipe pipe = crtc->pipe;
 361         u32 tail;
 362
 363         tail = dsb->free_pos * 4;
 364         if (drm_WARN_ON(&dev_priv->drm, !IS_ALIGNED(tail, CACHELINE_BYTES)))
 365                 return;
 366
 367         if (is_dsb_busy(dev_priv, pipe, dsb->id)) {
 368                 drm_err(&dev_priv->drm, "[CRTC:%d:%s] DSB %d is busy\n",
 369                         crtc->base.base.id, crtc->base.name, dsb->id);
 370                 return;
 371         }
 372
 373         intel_de_write_fw(dev_priv, DSB_CTRL(pipe, dsb->id),
 374                           ctrl | DSB_ENABLE);
 375
 376         intel_de_write_fw(dev_priv, DSB_CHICKEN(pipe, dsb->id),
 377                           dsb_chicken(crtc));
 378
 379         intel_de_write_fw(dev_priv, DSB_HEAD(pipe, dsb->id),
 380                           intel_dsb_buffer_ggtt_offset(&dsb->dsb_buf));
 381
 382         if (dewake_scanline >= 0) {
 383                 int diff, hw_dewake_scanline;
 384
 385                 hw_dewake_scanline = intel_crtc_scanline_to_hw(crtc, dewake_scanline);
 386
 387                 intel_de_write_fw(dev_priv, DSB_PMCTRL(pipe, dsb->id),
 388                                   DSB_ENABLE_DEWAKE |
 389                                   DSB_SCANLINE_FOR_DEWAKE(hw_dewake_scanline));
 390
 391                 /*
 392                  * Force DEwake immediately if we're already past
 393                  * or close to racing past the target scanline.
 394                  */
 395                 diff = dewake_scanline - intel_get_crtc_scanline(crtc);
 396                 intel_de_write_fw(dev_priv, DSB_PMCTRL_2(pipe, dsb->id),
 397                                   (diff >= 0 && diff < 5 ? DSB_FORCE_DEWAKE : 0) |
 398                                   DSB_BLOCK_DEWAKE_EXTENSION);
 399         }
 400
 401         intel_de_write_fw(dev_priv, DSB_TAIL(pipe, dsb->id),
 402                           intel_dsb_buffer_ggtt_offset(&dsb->dsb_buf) + tail);
 403 }
 404
 405 /**
 406  * intel_dsb_commit() - Trigger workload execution of DSB.
 407  * @dsb: DSB context
 408  * @wait_for_vblank: wait for vblank before executing
 409  *
 410  * This function is used to do actual write to hardware using DSB.
 411  */
 412 void intel_dsb_commit(struct intel_dsb *dsb,
 413                       bool wait_for_vblank)
 414 {
 415         _intel_dsb_commit(dsb,
 416                           wait_for_vblank ? DSB_WAIT_FOR_VBLANK : 0,
 417                           wait_for_vblank ? dsb->dewake_scanline : -1);
 418 }
 419
 420 void intel_dsb_wait(struct intel_dsb *dsb)
 421 {
 422         struct intel_crtc *crtc = dsb->crtc;
 423         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
 424         enum pipe pipe = crtc->pipe;
 425
 426         if (wait_for(!is_dsb_busy(dev_priv, pipe, dsb->id), 1)) {
 427                 u32 offset = intel_dsb_buffer_ggtt_offset(&dsb->dsb_buf);
 428
 429                 intel_de_write_fw(dev_priv, DSB_CTRL(pipe, dsb->id),
 430                                   DSB_ENABLE | DSB_HALT);
 431
 432                 drm_err(&dev_priv->drm,
 433                         "[CRTC:%d:%s] DSB %d timed out waiting for idle (current head=0x%x, head=0x%x, tail=0x%x)\n",
 434                         crtc->base.base.id, crtc->base.name, dsb->id,
 435                         intel_de_read_fw(dev_priv, DSB_CURRENT_HEAD(pipe, dsb->id)) - offset,
 436                         intel_de_read_fw(dev_priv, DSB_HEAD(pipe, dsb->id)) - offset,
 437                         intel_de_read_fw(dev_priv, DSB_TAIL(pipe, dsb->id)) - offset);
 438
 439                 intel_dsb_dump(dsb);
 440         }
 441
 442         /* Attempt to reset it */
 443         dsb->free_pos = 0;
 444         dsb->ins_start_offset = 0;
 445         intel_de_write_fw(dev_priv, DSB_CTRL(pipe, dsb->id), 0);
 446 }
 447
 448 /**
 449  * intel_dsb_prepare() - Allocate, pin and map the DSB command buffer.
 450  * @crtc_state: the CRTC state
 451  * @max_cmds: number of commands we need to fit into command buffer
 452  *
 453  * This function prepare the command buffer which is used to store dsb
 454  * instructions with data.
 455  *
 456  * Returns:
 457  * DSB context, NULL on failure
 458  */
 459 struct intel_dsb *intel_dsb_prepare(const struct intel_crtc_state *crtc_state,
 460                                     unsigned int max_cmds)
 461 {
 462         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
 463         struct drm_i915_private *i915 = to_i915(crtc->base.dev);
 464         intel_wakeref_t wakeref;
 465         struct intel_dsb *dsb;
 466         unsigned int size;
 467
 468         if (!HAS_DSB(i915))
 469                 return NULL;
 470
 471         /* TODO: DSB is broken in Xe KMD, so disabling it until fixed */
 472         if (!IS_ENABLED(I915))
 473                 return NULL;
 474
 475         dsb = kzalloc(sizeof(*dsb), GFP_KERNEL);
 476         if (!dsb)
 477                 goto out;
 478
 479         wakeref = intel_runtime_pm_get(&i915->runtime_pm);
 480
 481         /* ~1 qword per instruction, full cachelines */
 482         size = ALIGN(max_cmds * 8, CACHELINE_BYTES);
 483
 484         if (!intel_dsb_buffer_create(crtc, &dsb->dsb_buf, size))
 485                 goto out_put_rpm;
 486
 487         intel_runtime_pm_put(&i915->runtime_pm, wakeref);
 488
 489         dsb->id = DSB1;
 490         dsb->crtc = crtc;
 491         dsb->size = size / 4; /* in dwords */
 492         dsb->free_pos = 0;
 493         dsb->ins_start_offset = 0;
 494         dsb->dewake_scanline = intel_dsb_dewake_scanline(crtc_state);
 495
 496         return dsb;
 497
 498 out_put_rpm:
 499         intel_runtime_pm_put(&i915->runtime_pm, wakeref);
 500         kfree(dsb);
 501 out:
 502         drm_info_once(&i915->drm,
 503                       "[CRTC:%d:%s] DSB %d queue setup failed, will fallback to MMIO for display HW programming\n",
 504                       crtc->base.base.id, crtc->base.name, DSB1);
 505
 506         return NULL;
 507 }
 508
 509 /**
 510  * intel_dsb_cleanup() - To cleanup DSB context.
 511  * @dsb: DSB context
 512  *
 513  * This function cleanup the DSB context by unpinning and releasing
 514  * the VMA object associated with it.
 515  */
 516 void intel_dsb_cleanup(struct intel_dsb *dsb)
 517 {
 518         intel_dsb_buffer_cleanup(&dsb->dsb_buf);
 519         kfree(dsb);
 520 }