drivers/gpu/drm/vc4/vc4_crtc.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * Copyright (C) 2015 Broadcom
   4  */
   5
   6 /**
   7  * DOC: VC4 CRTC module
   8  *
   9  * In VC4, the Pixel Valve is what most closely corresponds to the
  10  * DRM's concept of a CRTC.  The PV generates video timings from the
  11  * encoder's clock plus its configuration.  It pulls scaled pixels from
  12  * the HVS at that timing, and feeds it to the encoder.
  13  *
  14  * However, the DRM CRTC also collects the configuration of all the
  15  * DRM planes attached to it.  As a result, the CRTC is also
  16  * responsible for writing the display list for the HVS channel that
  17  * the CRTC will use.
  18  *
  19  * The 2835 has 3 different pixel valves.  pv0 in the audio power
  20  * domain feeds DSI0 or DPI, while pv1 feeds DS1 or SMI.  pv2 in the
  21  * image domain can feed either HDMI or the SDTV controller.  The
  22  * pixel valve chooses from the CPRMAN clocks (HSM for HDMI, VEC for
  23  * SDTV, etc.) according to which output type is chosen in the mux.
  24  *
  25  * For power management, the pixel valve's registers are all clocked
  26  * by the AXI clock, while the timings and FIFOs make use of the
  27  * output-specific clock.  Since the encoders also directly consume
  28  * the CPRMAN clocks, and know what timings they need, they are the
  29  * ones that set the clock.
  30  */
  31
  32 #include <linux/clk.h>
  33 #include <linux/component.h>
  34 #include <linux/of.h>
  35 #include <linux/platform_device.h>
  36 #include <linux/pm_runtime.h>
  37
  38 #include <drm/drm_atomic.h>
  39 #include <drm/drm_atomic_helper.h>
  40 #include <drm/drm_atomic_uapi.h>
  41 #include <drm/drm_fb_dma_helper.h>
  42 #include <drm/drm_framebuffer.h>
  43 #include <drm/drm_drv.h>
  44 #include <drm/drm_print.h>
  45 #include <drm/drm_probe_helper.h>
  46 #include <drm/drm_vblank.h>
  47
  48 #include "vc4_drv.h"
  49 #include "vc4_hdmi.h"
  50 #include "vc4_regs.h"
  51
  52 #define HVS_FIFO_LATENCY_PIX    6
  53
  54 #define CRTC_WRITE(offset, val)                                                         \
  55         do {                                                                            \
  56                 kunit_fail_current_test("Accessing a register in a unit test!\n");      \
  57                 writel(val, vc4_crtc->regs + (offset));                                 \
  58         } while (0)
  59
  60 #define CRTC_READ(offset)                                                               \
  61         ({                                                                              \
  62                 kunit_fail_current_test("Accessing a register in a unit test!\n");      \
  63                 readl(vc4_crtc->regs + (offset));                                       \
  64         })
  65
  66 static const struct debugfs_reg32 crtc_regs[] = {
  67         VC4_REG32(PV_CONTROL),
  68         VC4_REG32(PV_V_CONTROL),
  69         VC4_REG32(PV_VSYNCD_EVEN),
  70         VC4_REG32(PV_HORZA),
  71         VC4_REG32(PV_HORZB),
  72         VC4_REG32(PV_VERTA),
  73         VC4_REG32(PV_VERTB),
  74         VC4_REG32(PV_VERTA_EVEN),
  75         VC4_REG32(PV_VERTB_EVEN),
  76         VC4_REG32(PV_INTEN),
  77         VC4_REG32(PV_INTSTAT),
  78         VC4_REG32(PV_STAT),
  79         VC4_REG32(PV_HACT_ACT),
  80 };
  81
  82 static unsigned int
  83 vc4_crtc_get_cob_allocation(struct vc4_dev *vc4, unsigned int channel)
  84 {
  85         struct vc4_hvs *hvs = vc4->hvs;
  86         u32 dispbase, top, base;
  87
  88         /* Top/base are supposed to be 4-pixel aligned, but the
  89          * Raspberry Pi firmware fills the low bits (which are
  90          * presumably ignored).
  91          */
  92
  93         if (vc4->gen >= VC4_GEN_6_C) {
  94                 dispbase = HVS_READ(SCALER6_DISPX_COB(channel));
  95                 top = VC4_GET_FIELD(dispbase, SCALER6_DISPX_COB_TOP) & ~3;
  96                 base = VC4_GET_FIELD(dispbase, SCALER6_DISPX_COB_BASE) & ~3;
  97         } else {
  98                 dispbase = HVS_READ(SCALER_DISPBASEX(channel));
  99                 top = VC4_GET_FIELD(dispbase, SCALER_DISPBASEX_TOP) & ~3;
 100                 base = VC4_GET_FIELD(dispbase, SCALER_DISPBASEX_BASE) & ~3;
 101         }
 102
 103         return top - base + 4;
 104 }
 105
 106 static bool vc4_crtc_get_scanout_position(struct drm_crtc *crtc,
 107                                           bool in_vblank_irq,
 108                                           int *vpos, int *hpos,
 109                                           ktime_t *stime, ktime_t *etime,
 110                                           const struct drm_display_mode *mode)
 111 {
 112         struct drm_device *dev = crtc->dev;
 113         struct vc4_dev *vc4 = to_vc4_dev(dev);
 114         struct vc4_hvs *hvs = vc4->hvs;
 115         struct vc4_crtc *vc4_crtc = to_vc4_crtc(crtc);
 116         struct vc4_crtc_state *vc4_crtc_state = to_vc4_crtc_state(crtc->state);
 117         unsigned int channel = vc4_crtc_state->assigned_channel;
 118         unsigned int cob_size;
 119         u32 val;
 120         int fifo_lines;
 121         int vblank_lines;
 122         bool ret = false;
 123
 124         /* preempt_disable_rt() should go right here in PREEMPT_RT patchset. */
 125
 126         /* Get optional system timestamp before query. */
 127         if (stime)
 128                 *stime = ktime_get();
 129
 130         /*
 131          * Read vertical scanline which is currently composed for our
 132          * pixelvalve by the HVS, and also the scaler status.
 133          */
 134         if (vc4->gen >= VC4_GEN_6_C)
 135                 val = HVS_READ(SCALER6_DISPX_STATUS(channel));
 136         else
 137                 val = HVS_READ(SCALER_DISPSTATX(channel));
 138
 139         /* Get optional system timestamp after query. */
 140         if (etime)
 141                 *etime = ktime_get();
 142
 143         /* preempt_enable_rt() should go right here in PREEMPT_RT patchset. */
 144
 145         /* Vertical position of hvs composed scanline. */
 146
 147         if (vc4->gen >= VC4_GEN_6_C)
 148                 *vpos = VC4_GET_FIELD(val, SCALER6_DISPX_STATUS_YLINE);
 149         else
 150                 *vpos = VC4_GET_FIELD(val, SCALER_DISPSTATX_LINE);
 151
 152         *hpos = 0;
 153
 154         if (mode->flags & DRM_MODE_FLAG_INTERLACE) {
 155                 *vpos /= 2;
 156
 157                 /* Use hpos to correct for field offset in interlaced mode. */
 158                 if (vc4_hvs_get_fifo_frame_count(hvs, channel) % 2)
 159                         *hpos += mode->crtc_htotal / 2;
 160         }
 161
 162         cob_size = vc4_crtc_get_cob_allocation(vc4, channel);
 163         /* This is the offset we need for translating hvs -> pv scanout pos. */
 164         fifo_lines = cob_size / mode->crtc_hdisplay;
 165
 166         if (fifo_lines > 0)
 167                 ret = true;
 168
 169         /* HVS more than fifo_lines into frame for compositing? */
 170         if (*vpos > fifo_lines) {
 171                 /*
 172                  * We are in active scanout and can get some meaningful results
 173                  * from HVS. The actual PV scanout can not trail behind more
 174                  * than fifo_lines as that is the fifo's capacity. Assume that
 175                  * in active scanout the HVS and PV work in lockstep wrt. HVS
 176                  * refilling the fifo and PV consuming from the fifo, ie.
 177                  * whenever the PV consumes and frees up a scanline in the
 178                  * fifo, the HVS will immediately refill it, therefore
 179                  * incrementing vpos. Therefore we choose HVS read position -
 180                  * fifo size in scanlines as a estimate of the real scanout
 181                  * position of the PV.
 182                  */
 183                 *vpos -= fifo_lines + 1;
 184
 185                 return ret;
 186         }
 187
 188         /*
 189          * Less: This happens when we are in vblank and the HVS, after getting
 190          * the VSTART restart signal from the PV, just started refilling its
 191          * fifo with new lines from the top-most lines of the new framebuffers.
 192          * The PV does not scan out in vblank, so does not remove lines from
 193          * the fifo, so the fifo will be full quickly and the HVS has to pause.
 194          * We can't get meaningful readings wrt. scanline position of the PV
 195          * and need to make things up in a approximative but consistent way.
 196          */
 197         vblank_lines = mode->vtotal - mode->vdisplay;
 198
 199         if (in_vblank_irq) {
 200                 /*
 201                  * Assume the irq handler got called close to first
 202                  * line of vblank, so PV has about a full vblank
 203                  * scanlines to go, and as a base timestamp use the
 204                  * one taken at entry into vblank irq handler, so it
 205                  * is not affected by random delays due to lock
 206                  * contention on event_lock or vblank_time lock in
 207                  * the core.
 208                  */
 209                 *vpos = -vblank_lines;
 210
 211                 if (stime)
 212                         *stime = vc4_crtc->t_vblank;
 213                 if (etime)
 214                         *etime = vc4_crtc->t_vblank;
 215
 216                 /*
 217                  * If the HVS fifo is not yet full then we know for certain
 218                  * we are at the very beginning of vblank, as the hvs just
 219                  * started refilling, and the stime and etime timestamps
 220                  * truly correspond to start of vblank.
 221                  *
 222                  * Unfortunately there's no way to report this to upper levels
 223                  * and make it more useful.
 224                  */
 225         } else {
 226                 /*
 227                  * No clue where we are inside vblank. Return a vpos of zero,
 228                  * which will cause calling code to just return the etime
 229                  * timestamp uncorrected. At least this is no worse than the
 230                  * standard fallback.
 231                  */
 232                 *vpos = 0;
 233         }
 234
 235         return ret;
 236 }
 237
 238 static u32 vc4_get_fifo_full_level(struct vc4_crtc *vc4_crtc, u32 format)
 239 {
 240         const struct vc4_crtc_data *crtc_data = vc4_crtc_to_vc4_crtc_data(vc4_crtc);
 241         const struct vc4_pv_data *pv_data = vc4_crtc_to_vc4_pv_data(vc4_crtc);
 242         struct vc4_dev *vc4 = to_vc4_dev(vc4_crtc->base.dev);
 243
 244         /*
 245          * NOTE: Could we use register 0x68 (PV_HW_CFG1) to get the FIFO
 246          * size?
 247          */
 248         u32 fifo_len_bytes = pv_data->fifo_depth;
 249
 250         /*
 251          * Pixels are pulled from the HVS if the number of bytes is
 252          * lower than the FIFO full level.
 253          *
 254          * The latency of the pixel fetch mechanism is 6 pixels, so we
 255          * need to convert those 6 pixels in bytes, depending on the
 256          * format, and then subtract that from the length of the FIFO
 257          * to make sure we never end up in a situation where the FIFO
 258          * is full.
 259          */
 260         switch (format) {
 261         case PV_CONTROL_FORMAT_DSIV_16:
 262         case PV_CONTROL_FORMAT_DSIC_16:
 263                 return fifo_len_bytes - 2 * HVS_FIFO_LATENCY_PIX;
 264         case PV_CONTROL_FORMAT_DSIV_18:
 265                 return fifo_len_bytes - 14;
 266         case PV_CONTROL_FORMAT_24:
 267         case PV_CONTROL_FORMAT_DSIV_24:
 268         default:
 269                 /*
 270                  * For some reason, the pixelvalve4 doesn't work with
 271                  * the usual formula and will only work with 32.
 272                  */
 273                 if (crtc_data->hvs_output == 5)
 274                         return 32;
 275
 276                 /*
 277                  * It looks like in some situations, we will overflow
 278                  * the PixelValve FIFO (with the bit 10 of PV stat being
 279                  * set) and stall the HVS / PV, eventually resulting in
 280                  * a page flip timeout.
 281                  *
 282                  * Displaying the video overlay during a playback with
 283                  * Kodi on an RPi3 seems to be a great solution with a
 284                  * failure rate around 50%.
 285                  *
 286                  * Removing 1 from the FIFO full level however
 287                  * seems to completely remove that issue.
 288                  */
 289                 if (vc4->gen == VC4_GEN_4)
 290                         return fifo_len_bytes - 3 * HVS_FIFO_LATENCY_PIX - 1;
 291
 292                 return fifo_len_bytes - 3 * HVS_FIFO_LATENCY_PIX;
 293         }
 294 }
 295
 296 static u32 vc4_crtc_get_fifo_full_level_bits(struct vc4_crtc *vc4_crtc,
 297                                              u32 format)
 298 {
 299         u32 level = vc4_get_fifo_full_level(vc4_crtc, format);
 300         u32 ret = 0;
 301
 302         ret |= VC4_SET_FIELD((level >> 6),
 303                              PV5_CONTROL_FIFO_LEVEL_HIGH);
 304
 305         return ret | VC4_SET_FIELD(level & 0x3f,
 306                                    PV_CONTROL_FIFO_LEVEL);
 307 }
 308
 309 /*
 310  * Returns the encoder attached to the CRTC.
 311  *
 312  * VC4 can only scan out to one encoder at a time, while the DRM core
 313  * allows drivers to push pixels to more than one encoder from the
 314  * same CRTC.
 315  */
 316 struct drm_encoder *vc4_get_crtc_encoder(struct drm_crtc *crtc,
 317                                          struct drm_crtc_state *state)
 318 {
 319         struct drm_encoder *encoder;
 320
 321         WARN_ON(hweight32(state->encoder_mask) > 1);
 322
 323         drm_for_each_encoder_mask(encoder, crtc->dev, state->encoder_mask)
 324                 return encoder;
 325
 326         return NULL;
 327 }
 328
 329 static void vc4_crtc_pixelvalve_reset(struct drm_crtc *crtc)
 330 {
 331         struct vc4_crtc *vc4_crtc = to_vc4_crtc(crtc);
 332         struct drm_device *dev = crtc->dev;
 333         int idx;
 334
 335         if (!drm_dev_enter(dev, &idx))
 336                 return;
 337
 338         /* The PV needs to be disabled before it can be flushed */
 339         CRTC_WRITE(PV_CONTROL, CRTC_READ(PV_CONTROL) & ~PV_CONTROL_EN);
 340         CRTC_WRITE(PV_CONTROL, CRTC_READ(PV_CONTROL) | PV_CONTROL_FIFO_CLR);
 341
 342         drm_dev_exit(idx);
 343 }
 344
 345 static void vc4_crtc_config_pv(struct drm_crtc *crtc, struct drm_encoder *encoder,
 346                                struct drm_atomic_state *state)
 347 {
 348         struct drm_device *dev = crtc->dev;
 349         struct vc4_dev *vc4 = to_vc4_dev(dev);
 350         struct vc4_encoder *vc4_encoder = to_vc4_encoder(encoder);
 351         struct vc4_crtc *vc4_crtc = to_vc4_crtc(crtc);
 352         const struct vc4_pv_data *pv_data = vc4_crtc_to_vc4_pv_data(vc4_crtc);
 353         struct drm_crtc_state *crtc_state = crtc->state;
 354         struct drm_display_mode *mode = &crtc_state->adjusted_mode;
 355         bool interlace = mode->flags & DRM_MODE_FLAG_INTERLACE;
 356         bool is_hdmi = vc4_encoder->type == VC4_ENCODER_TYPE_HDMI0 ||
 357                        vc4_encoder->type == VC4_ENCODER_TYPE_HDMI1;
 358         u32 pixel_rep = ((mode->flags & DRM_MODE_FLAG_DBLCLK) && !is_hdmi) ? 2 : 1;
 359         bool is_dsi = (vc4_encoder->type == VC4_ENCODER_TYPE_DSI0 ||
 360                        vc4_encoder->type == VC4_ENCODER_TYPE_DSI1);
 361         bool is_dsi1 = vc4_encoder->type == VC4_ENCODER_TYPE_DSI1;
 362         bool is_vec = vc4_encoder->type == VC4_ENCODER_TYPE_VEC;
 363         u32 format = is_dsi1 ? PV_CONTROL_FORMAT_DSIV_24 : PV_CONTROL_FORMAT_24;
 364         u8 ppc = pv_data->pixels_per_clock;
 365
 366         u16 vert_bp = mode->crtc_vtotal - mode->crtc_vsync_end;
 367         u16 vert_sync = mode->crtc_vsync_end - mode->crtc_vsync_start;
 368         u16 vert_fp = mode->crtc_vsync_start - mode->crtc_vdisplay;
 369
 370         bool debug_dump_regs = false;
 371         int idx;
 372
 373         if (!drm_dev_enter(dev, &idx))
 374                 return;
 375
 376         if (debug_dump_regs) {
 377                 struct drm_printer p = drm_info_printer(&vc4_crtc->pdev->dev);
 378                 dev_info(&vc4_crtc->pdev->dev, "CRTC %d regs before:\n",
 379                          drm_crtc_index(crtc));
 380                 drm_print_regset32(&p, &vc4_crtc->regset);
 381         }
 382
 383         vc4_crtc_pixelvalve_reset(crtc);
 384
 385         CRTC_WRITE(PV_HORZA,
 386                    VC4_SET_FIELD((mode->htotal - mode->hsync_end) * pixel_rep / ppc,
 387                                  PV_HORZA_HBP) |
 388                    VC4_SET_FIELD((mode->hsync_end - mode->hsync_start) * pixel_rep / ppc,
 389                                  PV_HORZA_HSYNC));
 390
 391         CRTC_WRITE(PV_HORZB,
 392                    VC4_SET_FIELD((mode->hsync_start - mode->hdisplay) * pixel_rep / ppc,
 393                                  PV_HORZB_HFP) |
 394                    VC4_SET_FIELD(mode->hdisplay * pixel_rep / ppc,
 395                                  PV_HORZB_HACTIVE));
 396
 397         if (interlace) {
 398                 bool odd_field_first = false;
 399                 u32 field_delay = mode->htotal * pixel_rep / (2 * ppc);
 400                 u16 vert_bp_even = vert_bp;
 401                 u16 vert_fp_even = vert_fp;
 402
 403                 if (is_vec) {
 404                         /* VEC (composite output) */
 405                         ++field_delay;
 406                         if (mode->htotal == 858) {
 407                                 /* 525-line mode (NTSC or PAL-M) */
 408                                 odd_field_first = true;
 409                         }
 410                 }
 411
 412                 if (odd_field_first)
 413                         ++vert_fp_even;
 414                 else
 415                         ++vert_bp;
 416
 417                 CRTC_WRITE(PV_VERTA_EVEN,
 418                            VC4_SET_FIELD(vert_bp_even, PV_VERTA_VBP) |
 419                            VC4_SET_FIELD(vert_sync, PV_VERTA_VSYNC));
 420                 CRTC_WRITE(PV_VERTB_EVEN,
 421                            VC4_SET_FIELD(vert_fp_even, PV_VERTB_VFP) |
 422                            VC4_SET_FIELD(mode->crtc_vdisplay, PV_VERTB_VACTIVE));
 423
 424                 /* We set up first field even mode for HDMI and VEC's PAL.
 425                  * For NTSC, we need first field odd.
 426                  */
 427                 CRTC_WRITE(PV_V_CONTROL,
 428                            PV_VCONTROL_CONTINUOUS |
 429                            (vc4->gen >= VC4_GEN_6_C ? PV_VCONTROL_ODD_TIMING : 0) |
 430                            (is_dsi ? PV_VCONTROL_DSI : 0) |
 431                            PV_VCONTROL_INTERLACE |
 432                            (odd_field_first
 433                                    ? PV_VCONTROL_ODD_FIRST
 434                                    : VC4_SET_FIELD(field_delay,
 435                                                    PV_VCONTROL_ODD_DELAY)));
 436                 CRTC_WRITE(PV_VSYNCD_EVEN,
 437                            (odd_field_first ? field_delay : 0));
 438         } else {
 439                 CRTC_WRITE(PV_V_CONTROL,
 440                            PV_VCONTROL_CONTINUOUS |
 441                            (vc4->gen >= VC4_GEN_6_C ? PV_VCONTROL_ODD_TIMING : 0) |
 442                            (is_dsi ? PV_VCONTROL_DSI : 0));
 443                 CRTC_WRITE(PV_VSYNCD_EVEN, 0);
 444         }
 445
 446         CRTC_WRITE(PV_VERTA,
 447                    VC4_SET_FIELD(vert_bp, PV_VERTA_VBP) |
 448                    VC4_SET_FIELD(vert_sync, PV_VERTA_VSYNC));
 449         CRTC_WRITE(PV_VERTB,
 450                    VC4_SET_FIELD(vert_fp, PV_VERTB_VFP) |
 451                    VC4_SET_FIELD(mode->crtc_vdisplay, PV_VERTB_VACTIVE));
 452
 453         if (is_dsi)
 454                 CRTC_WRITE(PV_HACT_ACT, mode->hdisplay * pixel_rep);
 455
 456         if (vc4->gen >= VC4_GEN_5)
 457                 CRTC_WRITE(PV_MUX_CFG,
 458                            VC4_SET_FIELD(PV_MUX_CFG_RGB_PIXEL_MUX_MODE_NO_SWAP,
 459                                          PV_MUX_CFG_RGB_PIXEL_MUX_MODE));
 460
 461         if (vc4->gen >= VC4_GEN_6_C)
 462                 CRTC_WRITE(PV_PIPE_INIT_CTRL,
 463                            VC4_SET_FIELD(1, PV_PIPE_INIT_CTRL_PV_INIT_WIDTH) |
 464                            VC4_SET_FIELD(1, PV_PIPE_INIT_CTRL_PV_INIT_IDLE) |
 465                            PV_PIPE_INIT_CTRL_PV_INIT_EN);
 466
 467         CRTC_WRITE(PV_CONTROL, PV_CONTROL_FIFO_CLR |
 468                    vc4_crtc_get_fifo_full_level_bits(vc4_crtc, format) |
 469                    VC4_SET_FIELD(format, PV_CONTROL_FORMAT) |
 470                    VC4_SET_FIELD(pixel_rep - 1, PV_CONTROL_PIXEL_REP) |
 471                    PV_CONTROL_CLR_AT_START |
 472                    PV_CONTROL_TRIGGER_UNDERFLOW |
 473                    PV_CONTROL_WAIT_HSTART |
 474                    VC4_SET_FIELD(vc4_encoder->clock_select,
 475                                  PV_CONTROL_CLK_SELECT));
 476
 477         if (debug_dump_regs) {
 478                 struct drm_printer p = drm_info_printer(&vc4_crtc->pdev->dev);
 479                 dev_info(&vc4_crtc->pdev->dev, "CRTC %d regs after:\n",
 480                          drm_crtc_index(crtc));
 481                 drm_print_regset32(&p, &vc4_crtc->regset);
 482         }
 483
 484         drm_dev_exit(idx);
 485 }
 486
 487 static void require_hvs_enabled(struct drm_device *dev)
 488 {
 489         struct vc4_dev *vc4 = to_vc4_dev(dev);
 490         struct vc4_hvs *hvs = vc4->hvs;
 491
 492         if (vc4->gen >= VC4_GEN_6_C)
 493                 WARN_ON_ONCE(!(HVS_READ(SCALER6_CONTROL) & SCALER6_CONTROL_HVS_EN));
 494         else
 495                 WARN_ON_ONCE(!(HVS_READ(SCALER_DISPCTRL) & SCALER_DISPCTRL_ENABLE));
 496 }
 497
 498 static int vc4_crtc_disable(struct drm_crtc *crtc,
 499                             struct drm_encoder *encoder,
 500                             struct drm_atomic_state *state,
 501                             unsigned int channel)
 502 {
 503         struct vc4_encoder *vc4_encoder = to_vc4_encoder(encoder);
 504         struct vc4_crtc *vc4_crtc = to_vc4_crtc(crtc);
 505         struct drm_device *dev = crtc->dev;
 506         struct vc4_dev *vc4 = to_vc4_dev(dev);
 507         int idx, ret;
 508
 509         if (!drm_dev_enter(dev, &idx))
 510                 return -ENODEV;
 511
 512         CRTC_WRITE(PV_V_CONTROL,
 513                    CRTC_READ(PV_V_CONTROL) & ~PV_VCONTROL_VIDEN);
 514         ret = wait_for(!(CRTC_READ(PV_V_CONTROL) & PV_VCONTROL_VIDEN), 1);
 515         WARN_ONCE(ret, "Timeout waiting for !PV_VCONTROL_VIDEN\n");
 516
 517         /*
 518          * This delay is needed to avoid to get a pixel stuck in an
 519          * unflushable FIFO between the pixelvalve and the HDMI
 520          * controllers on the BCM2711.
 521          *
 522          * Timing is fairly sensitive here, so mdelay is the safest
 523          * approach.
 524          *
 525          * If it was to be reworked, the stuck pixel happens on a
 526          * BCM2711 when changing mode with a good probability, so a
 527          * script that changes mode on a regular basis should trigger
 528          * the bug after less than 10 attempts. It manifests itself with
 529          * every pixels being shifted by one to the right, and thus the
 530          * last pixel of a line actually being displayed as the first
 531          * pixel on the next line.
 532          */
 533         mdelay(20);
 534
 535         if (vc4_encoder && vc4_encoder->post_crtc_disable)
 536                 vc4_encoder->post_crtc_disable(encoder, state);
 537
 538         vc4_crtc_pixelvalve_reset(crtc);
 539         vc4_hvs_stop_channel(vc4->hvs, channel);
 540
 541         if (vc4_encoder && vc4_encoder->post_crtc_powerdown)
 542                 vc4_encoder->post_crtc_powerdown(encoder, state);
 543
 544         drm_dev_exit(idx);
 545
 546         return 0;
 547 }
 548
 549 int vc4_crtc_disable_at_boot(struct drm_crtc *crtc)
 550 {
 551         struct drm_device *drm = crtc->dev;
 552         struct vc4_dev *vc4 = to_vc4_dev(drm);
 553         struct vc4_crtc *vc4_crtc = to_vc4_crtc(crtc);
 554         enum vc4_encoder_type encoder_type;
 555         const struct vc4_pv_data *pv_data;
 556         struct drm_encoder *encoder;
 557         struct vc4_hdmi *vc4_hdmi;
 558         unsigned encoder_sel;
 559         int channel;
 560         int ret;
 561
 562         if (!(of_device_is_compatible(vc4_crtc->pdev->dev.of_node,
 563                                       "brcm,bcm2711-pixelvalve2") ||
 564               of_device_is_compatible(vc4_crtc->pdev->dev.of_node,
 565                                       "brcm,bcm2711-pixelvalve4") ||
 566               of_device_is_compatible(vc4_crtc->pdev->dev.of_node,
 567                                       "brcm,bcm2712-pixelvalve0") ||
 568               of_device_is_compatible(vc4_crtc->pdev->dev.of_node,
 569                                       "brcm,bcm2712-pixelvalve1")))
 570                 return 0;
 571
 572         if (!(CRTC_READ(PV_CONTROL) & PV_CONTROL_EN))
 573                 return 0;
 574
 575         if (!(CRTC_READ(PV_V_CONTROL) & PV_VCONTROL_VIDEN))
 576                 return 0;
 577
 578         channel = vc4_hvs_get_fifo_from_output(vc4->hvs, vc4_crtc->data->hvs_output);
 579         if (channel < 0)
 580                 return 0;
 581
 582         encoder_sel = VC4_GET_FIELD(CRTC_READ(PV_CONTROL), PV_CONTROL_CLK_SELECT);
 583         if (WARN_ON(encoder_sel != 0))
 584                 return 0;
 585
 586         pv_data = vc4_crtc_to_vc4_pv_data(vc4_crtc);
 587         encoder_type = pv_data->encoder_types[encoder_sel];
 588         encoder = vc4_find_encoder_by_type(drm, encoder_type);
 589         if (WARN_ON(!encoder))
 590                 return 0;
 591
 592         vc4_hdmi = encoder_to_vc4_hdmi(encoder);
 593         ret = pm_runtime_resume_and_get(&vc4_hdmi->pdev->dev);
 594         if (ret)
 595                 return ret;
 596
 597         ret = vc4_crtc_disable(crtc, encoder, NULL, channel);
 598         if (ret)
 599                 return ret;
 600
 601         /*
 602          * post_crtc_powerdown will have called pm_runtime_put, so we
 603          * don't need it here otherwise we'll get the reference counting
 604          * wrong.
 605          */
 606
 607         return 0;
 608 }
 609
 610 void vc4_crtc_send_vblank(struct drm_crtc *crtc)
 611 {
 612         struct drm_device *dev = crtc->dev;
 613         unsigned long flags;
 614
 615         if (!crtc->state || !crtc->state->event)
 616                 return;
 617
 618         spin_lock_irqsave(&dev->event_lock, flags);
 619         drm_crtc_send_vblank_event(crtc, crtc->state->event);
 620         crtc->state->event = NULL;
 621         spin_unlock_irqrestore(&dev->event_lock, flags);
 622 }
 623
 624 static void vc4_crtc_atomic_disable(struct drm_crtc *crtc,
 625                                     struct drm_atomic_state *state)
 626 {
 627         struct drm_crtc_state *old_state = drm_atomic_get_old_crtc_state(state,
 628                                                                          crtc);
 629         struct vc4_crtc_state *old_vc4_state = to_vc4_crtc_state(old_state);
 630         struct drm_encoder *encoder = vc4_get_crtc_encoder(crtc, old_state);
 631         struct drm_device *dev = crtc->dev;
 632
 633         drm_dbg(dev, "Disabling CRTC %s (%u) connected to Encoder %s (%u)",
 634                 crtc->name, crtc->base.id, encoder->name, encoder->base.id);
 635
 636         require_hvs_enabled(dev);
 637
 638         /* Disable vblank irq handling before crtc is disabled. */
 639         drm_crtc_vblank_off(crtc);
 640
 641         vc4_crtc_disable(crtc, encoder, state, old_vc4_state->assigned_channel);
 642
 643         /*
 644          * Make sure we issue a vblank event after disabling the CRTC if
 645          * someone was waiting it.
 646          */
 647         vc4_crtc_send_vblank(crtc);
 648 }
 649
 650 static void vc4_crtc_atomic_enable(struct drm_crtc *crtc,
 651                                    struct drm_atomic_state *state)
 652 {
 653         struct drm_crtc_state *new_state = drm_atomic_get_new_crtc_state(state,
 654                                                                          crtc);
 655         struct drm_device *dev = crtc->dev;
 656         struct vc4_crtc *vc4_crtc = to_vc4_crtc(crtc);
 657         struct drm_encoder *encoder = vc4_get_crtc_encoder(crtc, new_state);
 658         struct vc4_encoder *vc4_encoder = to_vc4_encoder(encoder);
 659         int idx;
 660
 661         drm_dbg(dev, "Enabling CRTC %s (%u) connected to Encoder %s (%u)",
 662                 crtc->name, crtc->base.id, encoder->name, encoder->base.id);
 663
 664         if (!drm_dev_enter(dev, &idx))
 665                 return;
 666
 667         require_hvs_enabled(dev);
 668
 669         /* Enable vblank irq handling before crtc is started otherwise
 670          * drm_crtc_get_vblank() fails in vc4_crtc_update_dlist().
 671          */
 672         drm_crtc_vblank_on(crtc);
 673
 674         vc4_hvs_atomic_enable(crtc, state);
 675
 676         if (vc4_encoder->pre_crtc_configure)
 677                 vc4_encoder->pre_crtc_configure(encoder, state);
 678
 679         vc4_crtc_config_pv(crtc, encoder, state);
 680
 681         CRTC_WRITE(PV_CONTROL, CRTC_READ(PV_CONTROL) | PV_CONTROL_EN);
 682
 683         if (vc4_encoder->pre_crtc_enable)
 684                 vc4_encoder->pre_crtc_enable(encoder, state);
 685
 686         /* When feeding the transposer block the pixelvalve is unneeded and
 687          * should not be enabled.
 688          */
 689         CRTC_WRITE(PV_V_CONTROL,
 690                    CRTC_READ(PV_V_CONTROL) | PV_VCONTROL_VIDEN);
 691
 692         if (vc4_encoder->post_crtc_enable)
 693                 vc4_encoder->post_crtc_enable(encoder, state);
 694
 695         drm_dev_exit(idx);
 696 }
 697
 698 static enum drm_mode_status vc4_crtc_mode_valid(struct drm_crtc *crtc,
 699                                                 const struct drm_display_mode *mode)
 700 {
 701         /* Do not allow doublescan modes from user space */
 702         if (mode->flags & DRM_MODE_FLAG_DBLSCAN) {
 703                 DRM_DEBUG_KMS("[CRTC:%d] Doublescan mode rejected.\n",
 704                               crtc->base.id);
 705                 return MODE_NO_DBLESCAN;
 706         }
 707
 708         return MODE_OK;
 709 }
 710
 711 void vc4_crtc_get_margins(struct drm_crtc_state *state,
 712                           unsigned int *left, unsigned int *right,
 713                           unsigned int *top, unsigned int *bottom)
 714 {
 715         struct vc4_crtc_state *vc4_state = to_vc4_crtc_state(state);
 716         struct drm_connector_state *conn_state;
 717         struct drm_connector *conn;
 718         int i;
 719
 720         *left = vc4_state->margins.left;
 721         *right = vc4_state->margins.right;
 722         *top = vc4_state->margins.top;
 723         *bottom = vc4_state->margins.bottom;
 724
 725         /* We have to interate over all new connector states because
 726          * vc4_crtc_get_margins() might be called before
 727          * vc4_crtc_atomic_check() which means margins info in vc4_crtc_state
 728          * might be outdated.
 729          */
 730         for_each_new_connector_in_state(state->state, conn, conn_state, i) {
 731                 if (conn_state->crtc != state->crtc)
 732                         continue;
 733
 734                 *left = conn_state->tv.margins.left;
 735                 *right = conn_state->tv.margins.right;
 736                 *top = conn_state->tv.margins.top;
 737                 *bottom = conn_state->tv.margins.bottom;
 738                 break;
 739         }
 740 }
 741
 742 int vc4_crtc_atomic_check(struct drm_crtc *crtc,
 743                           struct drm_atomic_state *state)
 744 {
 745         struct drm_crtc_state *crtc_state = drm_atomic_get_new_crtc_state(state,
 746                                                                           crtc);
 747         struct vc4_crtc_state *vc4_state = to_vc4_crtc_state(crtc_state);
 748         struct drm_connector *conn;
 749         struct drm_connector_state *conn_state;
 750         struct drm_encoder *encoder;
 751         int ret, i;
 752
 753         ret = vc4_hvs_atomic_check(crtc, state);
 754         if (ret)
 755                 return ret;
 756
 757         encoder = vc4_get_crtc_encoder(crtc, crtc_state);
 758         if (encoder) {
 759                 const struct drm_display_mode *mode = &crtc_state->adjusted_mode;
 760                 struct vc4_encoder *vc4_encoder = to_vc4_encoder(encoder);
 761
 762                 if (vc4_encoder->type == VC4_ENCODER_TYPE_HDMI0) {
 763                         vc4_state->hvs_load = max(mode->clock * mode->hdisplay / mode->htotal + 8000,
 764                                                   mode->clock * 9 / 10) * 1000;
 765                 } else {
 766                         vc4_state->hvs_load = mode->clock * 1000;
 767                 }
 768         }
 769
 770         for_each_new_connector_in_state(state, conn, conn_state,
 771                                         i) {
 772                 if (conn_state->crtc != crtc)
 773                         continue;
 774
 775                 if (memcmp(&vc4_state->margins, &conn_state->tv.margins,
 776                            sizeof(vc4_state->margins))) {
 777                         memcpy(&vc4_state->margins, &conn_state->tv.margins,
 778                                sizeof(vc4_state->margins));
 779
 780                         /*
 781                          * Need to force the dlist entries for all planes to be
 782                          * updated so that the dest rectangles are changed.
 783                          */
 784                         crtc_state->zpos_changed = true;
 785                 }
 786                 break;
 787         }
 788
 789         return 0;
 790 }
 791
 792 static int vc4_enable_vblank(struct drm_crtc *crtc)
 793 {
 794         struct vc4_crtc *vc4_crtc = to_vc4_crtc(crtc);
 795         struct drm_device *dev = crtc->dev;
 796         int idx;
 797
 798         if (!drm_dev_enter(dev, &idx))
 799                 return -ENODEV;
 800
 801         CRTC_WRITE(PV_INTEN, PV_INT_VFP_START);
 802
 803         drm_dev_exit(idx);
 804
 805         return 0;
 806 }
 807
 808 static void vc4_disable_vblank(struct drm_crtc *crtc)
 809 {
 810         struct vc4_crtc *vc4_crtc = to_vc4_crtc(crtc);
 811         struct drm_device *dev = crtc->dev;
 812         int idx;
 813
 814         if (!drm_dev_enter(dev, &idx))
 815                 return;
 816
 817         CRTC_WRITE(PV_INTEN, 0);
 818
 819         drm_dev_exit(idx);
 820 }
 821
 822 static void vc4_crtc_handle_page_flip(struct vc4_crtc *vc4_crtc)
 823 {
 824         struct drm_crtc *crtc = &vc4_crtc->base;
 825         struct drm_device *dev = crtc->dev;
 826         struct vc4_dev *vc4 = to_vc4_dev(dev);
 827         struct vc4_hvs *hvs = vc4->hvs;
 828         unsigned int current_dlist;
 829         u32 chan = vc4_crtc->current_hvs_channel;
 830         unsigned long flags;
 831
 832         spin_lock_irqsave(&dev->event_lock, flags);
 833         spin_lock(&vc4_crtc->irq_lock);
 834
 835         if (vc4->gen >= VC4_GEN_6_C)
 836                 current_dlist = VC4_GET_FIELD(HVS_READ(SCALER6_DISPX_DL(chan)),
 837                                               SCALER6_DISPX_DL_LACT);
 838         else
 839                 current_dlist = HVS_READ(SCALER_DISPLACTX(chan));
 840
 841         if (vc4_crtc->event &&
 842             (vc4_crtc->current_dlist == current_dlist || vc4_crtc->feeds_txp)) {
 843                 drm_crtc_send_vblank_event(crtc, vc4_crtc->event);
 844                 vc4_crtc->event = NULL;
 845                 drm_crtc_vblank_put(crtc);
 846
 847                 /* Wait for the page flip to unmask the underrun to ensure that
 848                  * the display list was updated by the hardware. Before that
 849                  * happens, the HVS will be using the previous display list with
 850                  * the CRTC and encoder already reconfigured, leading to
 851                  * underruns. This can be seen when reconfiguring the CRTC.
 852                  */
 853                 if (vc4->gen < VC4_GEN_6_C)
 854                         vc4_hvs_unmask_underrun(hvs, chan);
 855         }
 856         spin_unlock(&vc4_crtc->irq_lock);
 857         spin_unlock_irqrestore(&dev->event_lock, flags);
 858 }
 859
 860 void vc4_crtc_handle_vblank(struct vc4_crtc *crtc)
 861 {
 862         crtc->t_vblank = ktime_get();
 863         drm_crtc_handle_vblank(&crtc->base);
 864         vc4_crtc_handle_page_flip(crtc);
 865 }
 866
 867 static irqreturn_t vc4_crtc_irq_handler(int irq, void *data)
 868 {
 869         struct vc4_crtc *vc4_crtc = data;
 870         u32 stat = CRTC_READ(PV_INTSTAT);
 871         irqreturn_t ret = IRQ_NONE;
 872
 873         if (stat & PV_INT_VFP_START) {
 874                 CRTC_WRITE(PV_INTSTAT, PV_INT_VFP_START);
 875                 vc4_crtc_handle_vblank(vc4_crtc);
 876                 ret = IRQ_HANDLED;
 877         }
 878
 879         return ret;
 880 }
 881
 882 struct vc4_async_flip_state {
 883         struct drm_crtc *crtc;
 884         struct drm_framebuffer *fb;
 885         struct drm_framebuffer *old_fb;
 886         struct drm_pending_vblank_event *event;
 887
 888         union {
 889                 struct dma_fence_cb fence;
 890                 struct vc4_seqno_cb seqno;
 891         } cb;
 892 };
 893
 894 /* Called when the V3D execution for the BO being flipped to is done, so that
 895  * we can actually update the plane's address to point to it.
 896  */
 897 static void
 898 vc4_async_page_flip_complete(struct vc4_async_flip_state *flip_state)
 899 {
 900         struct drm_crtc *crtc = flip_state->crtc;
 901         struct drm_device *dev = crtc->dev;
 902         struct drm_plane *plane = crtc->primary;
 903
 904         vc4_plane_async_set_fb(plane, flip_state->fb);
 905         if (flip_state->event) {
 906                 unsigned long flags;
 907
 908                 spin_lock_irqsave(&dev->event_lock, flags);
 909                 drm_crtc_send_vblank_event(crtc, flip_state->event);
 910                 spin_unlock_irqrestore(&dev->event_lock, flags);
 911         }
 912
 913         drm_crtc_vblank_put(crtc);
 914         drm_framebuffer_put(flip_state->fb);
 915
 916         if (flip_state->old_fb)
 917                 drm_framebuffer_put(flip_state->old_fb);
 918
 919         kfree(flip_state);
 920 }
 921
 922 static void vc4_async_page_flip_seqno_complete(struct vc4_seqno_cb *cb)
 923 {
 924         struct vc4_async_flip_state *flip_state =
 925                 container_of(cb, struct vc4_async_flip_state, cb.seqno);
 926         struct vc4_bo *bo = NULL;
 927
 928         if (flip_state->old_fb) {
 929                 struct drm_gem_dma_object *dma_bo =
 930                         drm_fb_dma_get_gem_obj(flip_state->old_fb, 0);
 931                 bo = to_vc4_bo(&dma_bo->base);
 932         }
 933
 934         vc4_async_page_flip_complete(flip_state);
 935
 936         /*
 937          * Decrement the BO usecnt in order to keep the inc/dec
 938          * calls balanced when the planes are updated through
 939          * the async update path.
 940          *
 941          * FIXME: we should move to generic async-page-flip when
 942          * it's available, so that we can get rid of this
 943          * hand-made cleanup_fb() logic.
 944          */
 945         if (bo)
 946                 vc4_bo_dec_usecnt(bo);
 947 }
 948
 949 static void vc4_async_page_flip_fence_complete(struct dma_fence *fence,
 950                                                struct dma_fence_cb *cb)
 951 {
 952         struct vc4_async_flip_state *flip_state =
 953                 container_of(cb, struct vc4_async_flip_state, cb.fence);
 954
 955         vc4_async_page_flip_complete(flip_state);
 956         dma_fence_put(fence);
 957 }
 958
 959 static int vc4_async_set_fence_cb(struct drm_device *dev,
 960                                   struct vc4_async_flip_state *flip_state)
 961 {
 962         struct drm_framebuffer *fb = flip_state->fb;
 963         struct drm_gem_dma_object *dma_bo = drm_fb_dma_get_gem_obj(fb, 0);
 964         struct vc4_dev *vc4 = to_vc4_dev(dev);
 965         struct dma_fence *fence;
 966         int ret;
 967
 968         if (vc4->gen == VC4_GEN_4) {
 969                 struct vc4_bo *bo = to_vc4_bo(&dma_bo->base);
 970
 971                 return vc4_queue_seqno_cb(dev, &flip_state->cb.seqno, bo->seqno,
 972                                           vc4_async_page_flip_seqno_complete);
 973         }
 974
 975         ret = dma_resv_get_singleton(dma_bo->base.resv, DMA_RESV_USAGE_READ, &fence);
 976         if (ret)
 977                 return ret;
 978
 979         /* If there's no fence, complete the page flip immediately */
 980         if (!fence) {
 981                 vc4_async_page_flip_fence_complete(fence, &flip_state->cb.fence);
 982                 return 0;
 983         }
 984
 985         /* If the fence has already been completed, complete the page flip */
 986         if (dma_fence_add_callback(fence, &flip_state->cb.fence,
 987                                    vc4_async_page_flip_fence_complete))
 988                 vc4_async_page_flip_fence_complete(fence, &flip_state->cb.fence);
 989
 990         return 0;
 991 }
 992
 993 static int
 994 vc4_async_page_flip_common(struct drm_crtc *crtc,
 995                            struct drm_framebuffer *fb,
 996                            struct drm_pending_vblank_event *event,
 997                            uint32_t flags)
 998 {
 999         struct drm_device *dev = crtc->dev;
1000         struct drm_plane *plane = crtc->primary;
1001         struct vc4_async_flip_state *flip_state;
1002
1003         flip_state = kzalloc(sizeof(*flip_state), GFP_KERNEL);
1004         if (!flip_state)
1005                 return -ENOMEM;
1006
1007         drm_framebuffer_get(fb);
1008         flip_state->fb = fb;
1009         flip_state->crtc = crtc;
1010         flip_state->event = event;
1011
1012         /* Save the current FB before it's replaced by the new one in
1013          * drm_atomic_set_fb_for_plane(). We'll need the old FB in
1014          * vc4_async_page_flip_complete() to decrement the BO usecnt and keep
1015          * it consistent.
1016          * FIXME: we should move to generic async-page-flip when it's
1017          * available, so that we can get rid of this hand-made cleanup_fb()
1018          * logic.
1019          */
1020         flip_state->old_fb = plane->state->fb;
1021         if (flip_state->old_fb)
1022                 drm_framebuffer_get(flip_state->old_fb);
1023
1024         WARN_ON(drm_crtc_vblank_get(crtc) != 0);
1025
1026         /* Immediately update the plane's legacy fb pointer, so that later
1027          * modeset prep sees the state that will be present when the semaphore
1028          * is released.
1029          */
1030         drm_atomic_set_fb_for_plane(plane->state, fb);
1031
1032         vc4_async_set_fence_cb(dev, flip_state);
1033
1034         /* Driver takes ownership of state on successful async commit. */
1035         return 0;
1036 }
1037
1038 /* Implements async (non-vblank-synced) page flips.
1039  *
1040  * The page flip ioctl needs to return immediately, so we grab the
1041  * modeset semaphore on the pipe, and queue the address update for
1042  * when V3D is done with the BO being flipped to.
1043  */
1044 static int vc4_async_page_flip(struct drm_crtc *crtc,
1045                                struct drm_framebuffer *fb,
1046                                struct drm_pending_vblank_event *event,
1047                                uint32_t flags)
1048 {
1049         struct drm_device *dev = crtc->dev;
1050         struct vc4_dev *vc4 = to_vc4_dev(dev);
1051         struct drm_gem_dma_object *dma_bo = drm_fb_dma_get_gem_obj(fb, 0);
1052         struct vc4_bo *bo = to_vc4_bo(&dma_bo->base);
1053         int ret;
1054
1055         if (WARN_ON_ONCE(vc4->gen > VC4_GEN_4))
1056                 return -ENODEV;
1057
1058         /*
1059          * Increment the BO usecnt here, so that we never end up with an
1060          * unbalanced number of vc4_bo_{dec,inc}_usecnt() calls when the
1061          * plane is later updated through the non-async path.
1062          *
1063          * FIXME: we should move to generic async-page-flip when
1064          * it's available, so that we can get rid of this
1065          * hand-made prepare_fb() logic.
1066          */
1067         ret = vc4_bo_inc_usecnt(bo);
1068         if (ret)
1069                 return ret;
1070
1071         ret = vc4_async_page_flip_common(crtc, fb, event, flags);
1072         if (ret) {
1073                 vc4_bo_dec_usecnt(bo);
1074                 return ret;
1075         }
1076
1077         return 0;
1078 }
1079
1080 static int vc5_async_page_flip(struct drm_crtc *crtc,
1081                                struct drm_framebuffer *fb,
1082                                struct drm_pending_vblank_event *event,
1083                                uint32_t flags)
1084 {
1085         return vc4_async_page_flip_common(crtc, fb, event, flags);
1086 }
1087
1088 int vc4_page_flip(struct drm_crtc *crtc,
1089                   struct drm_framebuffer *fb,
1090                   struct drm_pending_vblank_event *event,
1091                   uint32_t flags,
1092                   struct drm_modeset_acquire_ctx *ctx)
1093 {
1094         if (flags & DRM_MODE_PAGE_FLIP_ASYNC) {
1095                 struct drm_device *dev = crtc->dev;
1096                 struct vc4_dev *vc4 = to_vc4_dev(dev);
1097
1098                 if (vc4->gen > VC4_GEN_4)
1099                         return vc5_async_page_flip(crtc, fb, event, flags);
1100                 else
1101                         return vc4_async_page_flip(crtc, fb, event, flags);
1102         } else {
1103                 return drm_atomic_helper_page_flip(crtc, fb, event, flags, ctx);
1104         }
1105 }
1106
1107 struct drm_crtc_state *vc4_crtc_duplicate_state(struct drm_crtc *crtc)
1108 {
1109         struct vc4_crtc_state *vc4_state, *old_vc4_state;
1110
1111         vc4_state = kzalloc(sizeof(*vc4_state), GFP_KERNEL);
1112         if (!vc4_state)
1113                 return NULL;
1114
1115         old_vc4_state = to_vc4_crtc_state(crtc->state);
1116         vc4_state->margins = old_vc4_state->margins;
1117         vc4_state->assigned_channel = old_vc4_state->assigned_channel;
1118
1119         __drm_atomic_helper_crtc_duplicate_state(crtc, &vc4_state->base);
1120         return &vc4_state->base;
1121 }
1122
1123 void vc4_crtc_destroy_state(struct drm_crtc *crtc,
1124                             struct drm_crtc_state *state)
1125 {
1126         struct vc4_dev *vc4 = to_vc4_dev(crtc->dev);
1127         struct vc4_crtc_state *vc4_state = to_vc4_crtc_state(state);
1128
1129         if (drm_mm_node_allocated(&vc4_state->mm)) {
1130                 unsigned long flags;
1131
1132                 spin_lock_irqsave(&vc4->hvs->mm_lock, flags);
1133                 drm_mm_remove_node(&vc4_state->mm);
1134                 spin_unlock_irqrestore(&vc4->hvs->mm_lock, flags);
1135
1136         }
1137
1138         drm_atomic_helper_crtc_destroy_state(crtc, state);
1139 }
1140
1141 void vc4_crtc_reset(struct drm_crtc *crtc)
1142 {
1143         struct vc4_crtc_state *vc4_crtc_state;
1144
1145         if (crtc->state)
1146                 vc4_crtc_destroy_state(crtc, crtc->state);
1147
1148         vc4_crtc_state = kzalloc(sizeof(*vc4_crtc_state), GFP_KERNEL);
1149         if (!vc4_crtc_state) {
1150                 crtc->state = NULL;
1151                 return;
1152         }
1153
1154         vc4_crtc_state->assigned_channel = VC4_HVS_CHANNEL_DISABLED;
1155         __drm_atomic_helper_crtc_reset(crtc, &vc4_crtc_state->base);
1156 }
1157
1158 int vc4_crtc_late_register(struct drm_crtc *crtc)
1159 {
1160         struct drm_device *drm = crtc->dev;
1161         struct vc4_crtc *vc4_crtc = to_vc4_crtc(crtc);
1162         const struct vc4_crtc_data *crtc_data = vc4_crtc_to_vc4_crtc_data(vc4_crtc);
1163
1164         vc4_debugfs_add_regset32(drm, crtc_data->debugfs_name,
1165                                  &vc4_crtc->regset);
1166
1167         return 0;
1168 }
1169
1170 static const struct drm_crtc_funcs vc4_crtc_funcs = {
1171         .set_config = drm_atomic_helper_set_config,
1172         .page_flip = vc4_page_flip,
1173         .set_property = NULL,
1174         .cursor_set = NULL, /* handled by drm_mode_cursor_universal */
1175         .cursor_move = NULL, /* handled by drm_mode_cursor_universal */
1176         .reset = vc4_crtc_reset,
1177         .atomic_duplicate_state = vc4_crtc_duplicate_state,
1178         .atomic_destroy_state = vc4_crtc_destroy_state,
1179         .enable_vblank = vc4_enable_vblank,
1180         .disable_vblank = vc4_disable_vblank,
1181         .get_vblank_timestamp = drm_crtc_vblank_helper_get_vblank_timestamp,
1182         .late_register = vc4_crtc_late_register,
1183 };
1184
1185 static const struct drm_crtc_helper_funcs vc4_crtc_helper_funcs = {
1186         .mode_valid = vc4_crtc_mode_valid,
1187         .atomic_check = vc4_crtc_atomic_check,
1188         .atomic_begin = vc4_hvs_atomic_begin,
1189         .atomic_flush = vc4_hvs_atomic_flush,
1190         .atomic_enable = vc4_crtc_atomic_enable,
1191         .atomic_disable = vc4_crtc_atomic_disable,
1192         .get_scanout_position = vc4_crtc_get_scanout_position,
1193 };
1194
1195 const struct vc4_pv_data bcm2835_pv0_data = {
1196         .base = {
1197                 .name = "pixelvalve-0",
1198                 .debugfs_name = "crtc0_regs",
1199                 .hvs_available_channels = BIT(0),
1200                 .hvs_output = 0,
1201         },
1202         .fifo_depth = 64,
1203         .pixels_per_clock = 1,
1204         .encoder_types = {
1205                 [PV_CONTROL_CLK_SELECT_DSI] = VC4_ENCODER_TYPE_DSI0,
1206                 [PV_CONTROL_CLK_SELECT_DPI_SMI_HDMI] = VC4_ENCODER_TYPE_DPI,
1207         },
1208 };
1209
1210 const struct vc4_pv_data bcm2835_pv1_data = {
1211         .base = {
1212                 .name = "pixelvalve-1",
1213                 .debugfs_name = "crtc1_regs",
1214                 .hvs_available_channels = BIT(2),
1215                 .hvs_output = 2,
1216         },
1217         .fifo_depth = 64,
1218         .pixels_per_clock = 1,
1219         .encoder_types = {
1220                 [PV_CONTROL_CLK_SELECT_DSI] = VC4_ENCODER_TYPE_DSI1,
1221                 [PV_CONTROL_CLK_SELECT_DPI_SMI_HDMI] = VC4_ENCODER_TYPE_SMI,
1222         },
1223 };
1224
1225 const struct vc4_pv_data bcm2835_pv2_data = {
1226         .base = {
1227                 .name = "pixelvalve-2",
1228                 .debugfs_name = "crtc2_regs",
1229                 .hvs_available_channels = BIT(1),
1230                 .hvs_output = 1,
1231         },
1232         .fifo_depth = 64,
1233         .pixels_per_clock = 1,
1234         .encoder_types = {
1235                 [PV_CONTROL_CLK_SELECT_DPI_SMI_HDMI] = VC4_ENCODER_TYPE_HDMI0,
1236                 [PV_CONTROL_CLK_SELECT_VEC] = VC4_ENCODER_TYPE_VEC,
1237         },
1238 };
1239
1240 const struct vc4_pv_data bcm2711_pv0_data = {
1241         .base = {
1242                 .name = "pixelvalve-0",
1243                 .debugfs_name = "crtc0_regs",
1244                 .hvs_available_channels = BIT(0),
1245                 .hvs_output = 0,
1246         },
1247         .fifo_depth = 64,
1248         .pixels_per_clock = 1,
1249         .encoder_types = {
1250                 [0] = VC4_ENCODER_TYPE_DSI0,
1251                 [1] = VC4_ENCODER_TYPE_DPI,
1252         },
1253 };
1254
1255 const struct vc4_pv_data bcm2711_pv1_data = {
1256         .base = {
1257                 .name = "pixelvalve-1",
1258                 .debugfs_name = "crtc1_regs",
1259                 .hvs_available_channels = BIT(0) | BIT(1) | BIT(2),
1260                 .hvs_output = 3,
1261         },
1262         .fifo_depth = 64,
1263         .pixels_per_clock = 1,
1264         .encoder_types = {
1265                 [0] = VC4_ENCODER_TYPE_DSI1,
1266                 [1] = VC4_ENCODER_TYPE_SMI,
1267         },
1268 };
1269
1270 const struct vc4_pv_data bcm2711_pv2_data = {
1271         .base = {
1272                 .name = "pixelvalve-2",
1273                 .debugfs_name = "crtc2_regs",
1274                 .hvs_available_channels = BIT(0) | BIT(1) | BIT(2),
1275                 .hvs_output = 4,
1276         },
1277         .fifo_depth = 256,
1278         .pixels_per_clock = 2,
1279         .encoder_types = {
1280                 [0] = VC4_ENCODER_TYPE_HDMI0,
1281         },
1282 };
1283
1284 const struct vc4_pv_data bcm2711_pv3_data = {
1285         .base = {
1286                 .name = "pixelvalve-3",
1287                 .debugfs_name = "crtc3_regs",
1288                 .hvs_available_channels = BIT(1),
1289                 .hvs_output = 1,
1290         },
1291         .fifo_depth = 64,
1292         .pixels_per_clock = 1,
1293         .encoder_types = {
1294                 [PV_CONTROL_CLK_SELECT_VEC] = VC4_ENCODER_TYPE_VEC,
1295         },
1296 };
1297
1298 const struct vc4_pv_data bcm2711_pv4_data = {
1299         .base = {
1300                 .name = "pixelvalve-4",
1301                 .debugfs_name = "crtc4_regs",
1302                 .hvs_available_channels = BIT(0) | BIT(1) | BIT(2),
1303                 .hvs_output = 5,
1304         },
1305         .fifo_depth = 64,
1306         .pixels_per_clock = 2,
1307         .encoder_types = {
1308                 [0] = VC4_ENCODER_TYPE_HDMI1,
1309         },
1310 };
1311
1312 const struct vc4_pv_data bcm2712_pv0_data = {
1313         .base = {
1314                 .debugfs_name = "crtc0_regs",
1315                 .hvs_available_channels = BIT(0),
1316                 .hvs_output = 0,
1317         },
1318         .fifo_depth = 64,
1319         .pixels_per_clock = 1,
1320         .encoder_types = {
1321                 [0] = VC4_ENCODER_TYPE_HDMI0,
1322         },
1323 };
1324
1325 const struct vc4_pv_data bcm2712_pv1_data = {
1326         .base = {
1327                 .debugfs_name = "crtc1_regs",
1328                 .hvs_available_channels = BIT(1),
1329                 .hvs_output = 1,
1330         },
1331         .fifo_depth = 64,
1332         .pixels_per_clock = 1,
1333         .encoder_types = {
1334                 [0] = VC4_ENCODER_TYPE_HDMI1,
1335         },
1336 };
1337
1338 static const struct of_device_id vc4_crtc_dt_match[] = {
1339         { .compatible = "brcm,bcm2835-pixelvalve0", .data = &bcm2835_pv0_data },
1340         { .compatible = "brcm,bcm2835-pixelvalve1", .data = &bcm2835_pv1_data },
1341         { .compatible = "brcm,bcm2835-pixelvalve2", .data = &bcm2835_pv2_data },
1342         { .compatible = "brcm,bcm2711-pixelvalve0", .data = &bcm2711_pv0_data },
1343         { .compatible = "brcm,bcm2711-pixelvalve1", .data = &bcm2711_pv1_data },
1344         { .compatible = "brcm,bcm2711-pixelvalve2", .data = &bcm2711_pv2_data },
1345         { .compatible = "brcm,bcm2711-pixelvalve3", .data = &bcm2711_pv3_data },
1346         { .compatible = "brcm,bcm2711-pixelvalve4", .data = &bcm2711_pv4_data },
1347         { .compatible = "brcm,bcm2712-pixelvalve0", .data = &bcm2712_pv0_data },
1348         { .compatible = "brcm,bcm2712-pixelvalve1", .data = &bcm2712_pv1_data },
1349         {}
1350 };
1351
1352 static void vc4_set_crtc_possible_masks(struct drm_device *drm,
1353                                         struct drm_crtc *crtc)
1354 {
1355         struct vc4_crtc *vc4_crtc = to_vc4_crtc(crtc);
1356         const struct vc4_pv_data *pv_data = vc4_crtc_to_vc4_pv_data(vc4_crtc);
1357         const enum vc4_encoder_type *encoder_types = pv_data->encoder_types;
1358         struct drm_encoder *encoder;
1359
1360         drm_for_each_encoder(encoder, drm) {
1361                 struct vc4_encoder *vc4_encoder;
1362                 int i;
1363
1364                 if (encoder->encoder_type == DRM_MODE_ENCODER_VIRTUAL)
1365                         continue;
1366
1367                 vc4_encoder = to_vc4_encoder(encoder);
1368                 for (i = 0; i < ARRAY_SIZE(pv_data->encoder_types); i++) {
1369                         if (vc4_encoder->type == encoder_types[i]) {
1370                                 vc4_encoder->clock_select = i;
1371                                 encoder->possible_crtcs |= drm_crtc_mask(crtc);
1372                                 break;
1373                         }
1374                 }
1375         }
1376 }
1377
1378 /**
1379  * __vc4_crtc_init - Initializes a CRTC
1380  * @drm: DRM Device
1381  * @pdev: CRTC Platform Device
1382  * @vc4_crtc: CRTC Object to Initialize
1383  * @data: Configuration data associated with this CRTC
1384  * @primary_plane: Primary plane for CRTC
1385  * @crtc_funcs: Callbacks for the new CRTC
1386  * @crtc_helper_funcs: Helper Callbacks for the new CRTC
1387  * @feeds_txp: Is this CRTC connected to the TXP?
1388  *
1389  * Initializes our private CRTC structure. This function is mostly
1390  * relevant for KUnit testing, all other users should use
1391  * vc4_crtc_init() instead.
1392  *
1393  * Returns:
1394  * 0 on success, a negative error code on failure.
1395  */
1396 int __vc4_crtc_init(struct drm_device *drm,
1397                     struct platform_device *pdev,
1398                     struct vc4_crtc *vc4_crtc,
1399                     const struct vc4_crtc_data *data,
1400                     struct drm_plane *primary_plane,
1401                     const struct drm_crtc_funcs *crtc_funcs,
1402                     const struct drm_crtc_helper_funcs *crtc_helper_funcs,
1403                     bool feeds_txp)
1404 {
1405         struct vc4_dev *vc4 = to_vc4_dev(drm);
1406         struct drm_crtc *crtc = &vc4_crtc->base;
1407         unsigned int i;
1408         int ret;
1409
1410         vc4_crtc->data = data;
1411         vc4_crtc->pdev = pdev;
1412         vc4_crtc->feeds_txp = feeds_txp;
1413         spin_lock_init(&vc4_crtc->irq_lock);
1414         ret = drmm_crtc_init_with_planes(drm, crtc, primary_plane, NULL,
1415                                          crtc_funcs, data->name);
1416         if (ret)
1417                 return ret;
1418
1419         drm_crtc_helper_add(crtc, crtc_helper_funcs);
1420
1421         if (vc4->gen == VC4_GEN_4) {
1422                 drm_mode_crtc_set_gamma_size(crtc, ARRAY_SIZE(vc4_crtc->lut_r));
1423                 drm_crtc_enable_color_mgmt(crtc, 0, false, crtc->gamma_size);
1424
1425                 /* We support CTM, but only for one CRTC at a time. It's therefore
1426                  * implemented as private driver state in vc4_kms, not here.
1427                  */
1428                 drm_crtc_enable_color_mgmt(crtc, 0, true, crtc->gamma_size);
1429         }
1430
1431         for (i = 0; i < crtc->gamma_size; i++) {
1432                 vc4_crtc->lut_r[i] = i;
1433                 vc4_crtc->lut_g[i] = i;
1434                 vc4_crtc->lut_b[i] = i;
1435         }
1436
1437         return 0;
1438 }
1439
1440 int vc4_crtc_init(struct drm_device *drm, struct platform_device *pdev,
1441                   struct vc4_crtc *vc4_crtc,
1442                   const struct vc4_crtc_data *data,
1443                   const struct drm_crtc_funcs *crtc_funcs,
1444                   const struct drm_crtc_helper_funcs *crtc_helper_funcs,
1445                   bool feeds_txp)
1446 {
1447         struct drm_plane *primary_plane;
1448
1449         /* For now, we create just the primary and the legacy cursor
1450          * planes.  We should be able to stack more planes on easily,
1451          * but to do that we would need to compute the bandwidth
1452          * requirement of the plane configuration, and reject ones
1453          * that will take too much.
1454          */
1455         primary_plane = vc4_plane_init(drm, DRM_PLANE_TYPE_PRIMARY, 0);
1456         if (IS_ERR(primary_plane)) {
1457                 dev_err(drm->dev, "failed to construct primary plane\n");
1458                 return PTR_ERR(primary_plane);
1459         }
1460
1461         return __vc4_crtc_init(drm, pdev, vc4_crtc, data, primary_plane,
1462                                crtc_funcs, crtc_helper_funcs, feeds_txp);
1463 }
1464
1465 static int vc4_crtc_bind(struct device *dev, struct device *master, void *data)
1466 {
1467         struct platform_device *pdev = to_platform_device(dev);
1468         struct drm_device *drm = dev_get_drvdata(master);
1469         const struct vc4_pv_data *pv_data;
1470         struct vc4_crtc *vc4_crtc;
1471         struct drm_crtc *crtc;
1472         int ret;
1473
1474         vc4_crtc = drmm_kzalloc(drm, sizeof(*vc4_crtc), GFP_KERNEL);
1475         if (!vc4_crtc)
1476                 return -ENOMEM;
1477         crtc = &vc4_crtc->base;
1478
1479         pv_data = of_device_get_match_data(dev);
1480         if (!pv_data)
1481                 return -ENODEV;
1482
1483         vc4_crtc->regs = vc4_ioremap_regs(pdev, 0);
1484         if (IS_ERR(vc4_crtc->regs))
1485                 return PTR_ERR(vc4_crtc->regs);
1486
1487         vc4_crtc->regset.base = vc4_crtc->regs;
1488         vc4_crtc->regset.regs = crtc_regs;
1489         vc4_crtc->regset.nregs = ARRAY_SIZE(crtc_regs);
1490
1491         ret = vc4_crtc_init(drm, pdev, vc4_crtc, &pv_data->base,
1492                             &vc4_crtc_funcs, &vc4_crtc_helper_funcs,
1493                             false);
1494         if (ret)
1495                 return ret;
1496         vc4_set_crtc_possible_masks(drm, crtc);
1497
1498         CRTC_WRITE(PV_INTEN, 0);
1499         CRTC_WRITE(PV_INTSTAT, PV_INT_VFP_START);
1500         ret = devm_request_irq(dev, platform_get_irq(pdev, 0),
1501                                vc4_crtc_irq_handler,
1502                                IRQF_SHARED,
1503                                "vc4 crtc", vc4_crtc);
1504         if (ret)
1505                 return ret;
1506
1507         platform_set_drvdata(pdev, vc4_crtc);
1508
1509         return 0;
1510 }
1511
1512 static void vc4_crtc_unbind(struct device *dev, struct device *master,
1513                             void *data)
1514 {
1515         struct platform_device *pdev = to_platform_device(dev);
1516         struct vc4_crtc *vc4_crtc = dev_get_drvdata(dev);
1517
1518         CRTC_WRITE(PV_INTEN, 0);
1519
1520         platform_set_drvdata(pdev, NULL);
1521 }
1522
1523 static const struct component_ops vc4_crtc_ops = {
1524         .bind   = vc4_crtc_bind,
1525         .unbind = vc4_crtc_unbind,
1526 };
1527
1528 static int vc4_crtc_dev_probe(struct platform_device *pdev)
1529 {
1530         return component_add(&pdev->dev, &vc4_crtc_ops);
1531 }
1532
1533 static void vc4_crtc_dev_remove(struct platform_device *pdev)
1534 {
1535         component_del(&pdev->dev, &vc4_crtc_ops);
1536 }
1537
1538 struct platform_driver vc4_crtc_driver = {
1539         .probe = vc4_crtc_dev_probe,
1540         .remove = vc4_crtc_dev_remove,
1541         .driver = {
1542                 .name = "vc4_crtc",
1543                 .of_match_table = vc4_crtc_dt_match,
1544         },
1545 };