drivers/gpu/drm/vc4/vc4_hvs.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * Copyright (C) 2015 Broadcom
   4  */
   5
   6 /**
   7  * DOC: VC4 HVS module.
   8  *
   9  * The Hardware Video Scaler (HVS) is the piece of hardware that does
  10  * translation, scaling, colorspace conversion, and compositing of
  11  * pixels stored in framebuffers into a FIFO of pixels going out to
  12  * the Pixel Valve (CRTC).  It operates at the system clock rate (the
  13  * system audio clock gate, specifically), which is much higher than
  14  * the pixel clock rate.
  15  *
  16  * There is a single global HVS, with multiple output FIFOs that can
  17  * be consumed by the PVs.  This file just manages the resources for
  18  * the HVS, while the vc4_crtc.c code actually drives HVS setup for
  19  * each CRTC.
  20  */
  21
  22 #include <linux/bitfield.h>
  23 #include <linux/clk.h>
  24 #include <linux/component.h>
  25 #include <linux/platform_device.h>
  26
  27 #include <drm/drm_atomic_helper.h>
  28 #include <drm/drm_drv.h>
  29 #include <drm/drm_vblank.h>
  30
  31 #include <soc/bcm2835/raspberrypi-firmware.h>
  32
  33 #include "vc4_drv.h"
  34 #include "vc4_regs.h"
  35
  36 static const struct debugfs_reg32 vc4_hvs_regs[] = {
  37         VC4_REG32(SCALER_DISPCTRL),
  38         VC4_REG32(SCALER_DISPSTAT),
  39         VC4_REG32(SCALER_DISPID),
  40         VC4_REG32(SCALER_DISPECTRL),
  41         VC4_REG32(SCALER_DISPPROF),
  42         VC4_REG32(SCALER_DISPDITHER),
  43         VC4_REG32(SCALER_DISPEOLN),
  44         VC4_REG32(SCALER_DISPLIST0),
  45         VC4_REG32(SCALER_DISPLIST1),
  46         VC4_REG32(SCALER_DISPLIST2),
  47         VC4_REG32(SCALER_DISPLSTAT),
  48         VC4_REG32(SCALER_DISPLACT0),
  49         VC4_REG32(SCALER_DISPLACT1),
  50         VC4_REG32(SCALER_DISPLACT2),
  51         VC4_REG32(SCALER_DISPCTRL0),
  52         VC4_REG32(SCALER_DISPBKGND0),
  53         VC4_REG32(SCALER_DISPSTAT0),
  54         VC4_REG32(SCALER_DISPBASE0),
  55         VC4_REG32(SCALER_DISPCTRL1),
  56         VC4_REG32(SCALER_DISPBKGND1),
  57         VC4_REG32(SCALER_DISPSTAT1),
  58         VC4_REG32(SCALER_DISPBASE1),
  59         VC4_REG32(SCALER_DISPCTRL2),
  60         VC4_REG32(SCALER_DISPBKGND2),
  61         VC4_REG32(SCALER_DISPSTAT2),
  62         VC4_REG32(SCALER_DISPBASE2),
  63         VC4_REG32(SCALER_DISPALPHA2),
  64         VC4_REG32(SCALER_OLEDOFFS),
  65         VC4_REG32(SCALER_OLEDCOEF0),
  66         VC4_REG32(SCALER_OLEDCOEF1),
  67         VC4_REG32(SCALER_OLEDCOEF2),
  68 };
  69
  70 static const struct debugfs_reg32 vc6_hvs_regs[] = {
  71         VC4_REG32(SCALER6_VERSION),
  72         VC4_REG32(SCALER6_CXM_SIZE),
  73         VC4_REG32(SCALER6_LBM_SIZE),
  74         VC4_REG32(SCALER6_UBM_SIZE),
  75         VC4_REG32(SCALER6_COBA_SIZE),
  76         VC4_REG32(SCALER6_COB_SIZE),
  77         VC4_REG32(SCALER6_CONTROL),
  78         VC4_REG32(SCALER6_FETCHER_STATUS),
  79         VC4_REG32(SCALER6_FETCH_STATUS),
  80         VC4_REG32(SCALER6_HANDLE_ERROR),
  81         VC4_REG32(SCALER6_DISP0_CTRL0),
  82         VC4_REG32(SCALER6_DISP0_CTRL1),
  83         VC4_REG32(SCALER6_DISP0_BGND),
  84         VC4_REG32(SCALER6_DISP0_LPTRS),
  85         VC4_REG32(SCALER6_DISP0_COB),
  86         VC4_REG32(SCALER6_DISP0_STATUS),
  87         VC4_REG32(SCALER6_DISP0_DL),
  88         VC4_REG32(SCALER6_DISP0_RUN),
  89         VC4_REG32(SCALER6_DISP1_CTRL0),
  90         VC4_REG32(SCALER6_DISP1_CTRL1),
  91         VC4_REG32(SCALER6_DISP1_BGND),
  92         VC4_REG32(SCALER6_DISP1_LPTRS),
  93         VC4_REG32(SCALER6_DISP1_COB),
  94         VC4_REG32(SCALER6_DISP1_STATUS),
  95         VC4_REG32(SCALER6_DISP1_DL),
  96         VC4_REG32(SCALER6_DISP1_RUN),
  97         VC4_REG32(SCALER6_DISP2_CTRL0),
  98         VC4_REG32(SCALER6_DISP2_CTRL1),
  99         VC4_REG32(SCALER6_DISP2_BGND),
 100         VC4_REG32(SCALER6_DISP2_LPTRS),
 101         VC4_REG32(SCALER6_DISP2_COB),
 102         VC4_REG32(SCALER6_DISP2_STATUS),
 103         VC4_REG32(SCALER6_DISP2_DL),
 104         VC4_REG32(SCALER6_DISP2_RUN),
 105         VC4_REG32(SCALER6_EOLN),
 106         VC4_REG32(SCALER6_DL_STATUS),
 107         VC4_REG32(SCALER6_BFG_MISC),
 108         VC4_REG32(SCALER6_QOS0),
 109         VC4_REG32(SCALER6_PROF0),
 110         VC4_REG32(SCALER6_QOS1),
 111         VC4_REG32(SCALER6_PROF1),
 112         VC4_REG32(SCALER6_QOS2),
 113         VC4_REG32(SCALER6_PROF2),
 114         VC4_REG32(SCALER6_PRI_MAP0),
 115         VC4_REG32(SCALER6_PRI_MAP1),
 116         VC4_REG32(SCALER6_HISTCTRL),
 117         VC4_REG32(SCALER6_HISTBIN0),
 118         VC4_REG32(SCALER6_HISTBIN1),
 119         VC4_REG32(SCALER6_HISTBIN2),
 120         VC4_REG32(SCALER6_HISTBIN3),
 121         VC4_REG32(SCALER6_HISTBIN4),
 122         VC4_REG32(SCALER6_HISTBIN5),
 123         VC4_REG32(SCALER6_HISTBIN6),
 124         VC4_REG32(SCALER6_HISTBIN7),
 125         VC4_REG32(SCALER6_HDR_CFG_REMAP),
 126         VC4_REG32(SCALER6_COL_SPACE),
 127         VC4_REG32(SCALER6_HVS_ID),
 128         VC4_REG32(SCALER6_CFC1),
 129         VC4_REG32(SCALER6_DISP_UPM_ISO0),
 130         VC4_REG32(SCALER6_DISP_UPM_ISO1),
 131         VC4_REG32(SCALER6_DISP_UPM_ISO2),
 132         VC4_REG32(SCALER6_DISP_LBM_ISO0),
 133         VC4_REG32(SCALER6_DISP_LBM_ISO1),
 134         VC4_REG32(SCALER6_DISP_LBM_ISO2),
 135         VC4_REG32(SCALER6_DISP_COB_ISO0),
 136         VC4_REG32(SCALER6_DISP_COB_ISO1),
 137         VC4_REG32(SCALER6_DISP_COB_ISO2),
 138         VC4_REG32(SCALER6_BAD_COB),
 139         VC4_REG32(SCALER6_BAD_LBM),
 140         VC4_REG32(SCALER6_BAD_UPM),
 141         VC4_REG32(SCALER6_BAD_AXI),
 142 };
 143
 144 static const struct debugfs_reg32 vc6_d_hvs_regs[] = {
 145         VC4_REG32(SCALER6D_VERSION),
 146         VC4_REG32(SCALER6D_CXM_SIZE),
 147         VC4_REG32(SCALER6D_LBM_SIZE),
 148         VC4_REG32(SCALER6D_UBM_SIZE),
 149         VC4_REG32(SCALER6D_COBA_SIZE),
 150         VC4_REG32(SCALER6D_COB_SIZE),
 151         VC4_REG32(SCALER6D_CONTROL),
 152         VC4_REG32(SCALER6D_FETCHER_STATUS),
 153         VC4_REG32(SCALER6D_FETCH_STATUS),
 154         VC4_REG32(SCALER6D_HANDLE_ERROR),
 155         VC4_REG32(SCALER6D_DISP0_CTRL0),
 156         VC4_REG32(SCALER6D_DISP0_CTRL1),
 157         VC4_REG32(SCALER6D_DISP0_BGND0),
 158         VC4_REG32(SCALER6D_DISP0_BGND1),
 159         VC4_REG32(SCALER6D_DISP0_LPTRS),
 160         VC4_REG32(SCALER6D_DISP0_COB),
 161         VC4_REG32(SCALER6D_DISP0_STATUS),
 162         VC4_REG32(SCALER6D_DISP0_DL),
 163         VC4_REG32(SCALER6D_DISP0_RUN),
 164         VC4_REG32(SCALER6D_DISP1_CTRL0),
 165         VC4_REG32(SCALER6D_DISP1_CTRL1),
 166         VC4_REG32(SCALER6D_DISP1_BGND0),
 167         VC4_REG32(SCALER6D_DISP1_BGND1),
 168         VC4_REG32(SCALER6D_DISP1_LPTRS),
 169         VC4_REG32(SCALER6D_DISP1_COB),
 170         VC4_REG32(SCALER6D_DISP1_STATUS),
 171         VC4_REG32(SCALER6D_DISP1_DL),
 172         VC4_REG32(SCALER6D_DISP1_RUN),
 173         VC4_REG32(SCALER6D_DISP2_CTRL0),
 174         VC4_REG32(SCALER6D_DISP2_CTRL1),
 175         VC4_REG32(SCALER6D_DISP2_BGND0),
 176         VC4_REG32(SCALER6D_DISP2_BGND1),
 177         VC4_REG32(SCALER6D_DISP2_LPTRS),
 178         VC4_REG32(SCALER6D_DISP2_COB),
 179         VC4_REG32(SCALER6D_DISP2_STATUS),
 180         VC4_REG32(SCALER6D_DISP2_DL),
 181         VC4_REG32(SCALER6D_DISP2_RUN),
 182         VC4_REG32(SCALER6D_EOLN),
 183         VC4_REG32(SCALER6D_DL_STATUS),
 184         VC4_REG32(SCALER6D_QOS0),
 185         VC4_REG32(SCALER6D_PROF0),
 186         VC4_REG32(SCALER6D_QOS1),
 187         VC4_REG32(SCALER6D_PROF1),
 188         VC4_REG32(SCALER6D_QOS2),
 189         VC4_REG32(SCALER6D_PROF2),
 190         VC4_REG32(SCALER6D_PRI_MAP0),
 191         VC4_REG32(SCALER6D_PRI_MAP1),
 192         VC4_REG32(SCALER6D_HISTCTRL),
 193         VC4_REG32(SCALER6D_HISTBIN0),
 194         VC4_REG32(SCALER6D_HISTBIN1),
 195         VC4_REG32(SCALER6D_HISTBIN2),
 196         VC4_REG32(SCALER6D_HISTBIN3),
 197         VC4_REG32(SCALER6D_HISTBIN4),
 198         VC4_REG32(SCALER6D_HISTBIN5),
 199         VC4_REG32(SCALER6D_HISTBIN6),
 200         VC4_REG32(SCALER6D_HISTBIN7),
 201         VC4_REG32(SCALER6D_HVS_ID),
 202 };
 203
 204 void vc4_hvs_dump_state(struct vc4_hvs *hvs)
 205 {
 206         struct drm_device *drm = &hvs->vc4->base;
 207         struct drm_printer p = drm_info_printer(&hvs->pdev->dev);
 208         int idx, i;
 209
 210         if (!drm_dev_enter(drm, &idx))
 211                 return;
 212
 213         drm_print_regset32(&p, &hvs->regset);
 214
 215         DRM_INFO("HVS ctx:\n");
 216         for (i = 0; i < 64; i += 4) {
 217                 DRM_INFO("0x%08x (%s): 0x%08x 0x%08x 0x%08x 0x%08x\n",
 218                          i * 4, i < HVS_BOOTLOADER_DLIST_END ? "B" : "D",
 219                          readl((u32 __iomem *)hvs->dlist + i + 0),
 220                          readl((u32 __iomem *)hvs->dlist + i + 1),
 221                          readl((u32 __iomem *)hvs->dlist + i + 2),
 222                          readl((u32 __iomem *)hvs->dlist + i + 3));
 223         }
 224
 225         drm_dev_exit(idx);
 226 }
 227
 228 static int vc4_hvs_debugfs_underrun(struct seq_file *m, void *data)
 229 {
 230         struct drm_debugfs_entry *entry = m->private;
 231         struct drm_device *dev = entry->dev;
 232         struct vc4_dev *vc4 = to_vc4_dev(dev);
 233         struct drm_printer p = drm_seq_file_printer(m);
 234
 235         drm_printf(&p, "%d\n", atomic_read(&vc4->underrun));
 236
 237         return 0;
 238 }
 239
 240 static int vc4_hvs_debugfs_dlist(struct seq_file *m, void *data)
 241 {
 242         struct drm_debugfs_entry *entry = m->private;
 243         struct drm_device *dev = entry->dev;
 244         struct vc4_dev *vc4 = to_vc4_dev(dev);
 245         struct vc4_hvs *hvs = vc4->hvs;
 246         struct drm_printer p = drm_seq_file_printer(m);
 247         unsigned int dlist_mem_size = hvs->dlist_mem_size;
 248         unsigned int next_entry_start;
 249         unsigned int i, j;
 250         u32 dlist_word, dispstat;
 251
 252         for (i = 0; i < SCALER_CHANNELS_COUNT; i++) {
 253                 dispstat = VC4_GET_FIELD(HVS_READ(SCALER_DISPSTATX(i)),
 254                                          SCALER_DISPSTATX_MODE);
 255                 if (dispstat == SCALER_DISPSTATX_MODE_DISABLED ||
 256                     dispstat == SCALER_DISPSTATX_MODE_EOF) {
 257                         drm_printf(&p, "HVS chan %u disabled\n", i);
 258                         continue;
 259                 }
 260
 261                 drm_printf(&p, "HVS chan %u:\n", i);
 262                 next_entry_start = 0;
 263
 264                 for (j = HVS_READ(SCALER_DISPLISTX(i)); j < dlist_mem_size; j++) {
 265                         dlist_word = readl((u32 __iomem *)vc4->hvs->dlist + j);
 266                         drm_printf(&p, "dlist: %02d: 0x%08x\n", j,
 267                                    dlist_word);
 268                         if (!next_entry_start ||
 269                             next_entry_start == j) {
 270                                 if (dlist_word & SCALER_CTL0_END)
 271                                         break;
 272                                 next_entry_start = j +
 273                                         VC4_GET_FIELD(dlist_word,
 274                                                       SCALER_CTL0_SIZE);
 275                         }
 276                 }
 277         }
 278
 279         return 0;
 280 }
 281
 282 static int vc6_hvs_debugfs_dlist(struct seq_file *m, void *data)
 283 {
 284         struct drm_info_node *node = m->private;
 285         struct drm_device *dev = node->minor->dev;
 286         struct vc4_dev *vc4 = to_vc4_dev(dev);
 287         struct vc4_hvs *hvs = vc4->hvs;
 288         struct drm_printer p = drm_seq_file_printer(m);
 289         unsigned int dlist_mem_size = hvs->dlist_mem_size;
 290         unsigned int next_entry_start;
 291         unsigned int i;
 292
 293         for (i = 0; i < SCALER_CHANNELS_COUNT; i++) {
 294                 unsigned int active_dlist, dispstat;
 295                 unsigned int j;
 296
 297                 dispstat = VC4_GET_FIELD(HVS_READ(SCALER6_DISPX_STATUS(i)),
 298                                          SCALER6_DISPX_STATUS_MODE);
 299                 if (dispstat == SCALER6_DISPX_STATUS_MODE_DISABLED ||
 300                     dispstat == SCALER6_DISPX_STATUS_MODE_EOF) {
 301                         drm_printf(&p, "HVS chan %u disabled\n", i);
 302                         continue;
 303                 }
 304
 305                 drm_printf(&p, "HVS chan %u:\n", i);
 306
 307                 active_dlist = VC4_GET_FIELD(HVS_READ(SCALER6_DISPX_DL(i)),
 308                                              SCALER6_DISPX_DL_LACT);
 309                 next_entry_start = 0;
 310
 311                 for (j = active_dlist; j < dlist_mem_size; j++) {
 312                         u32 dlist_word;
 313
 314                         dlist_word = readl((u32 __iomem *)vc4->hvs->dlist + j);
 315                         drm_printf(&p, "dlist: %02d: 0x%08x\n", j,
 316                                    dlist_word);
 317                         if (!next_entry_start ||
 318                             next_entry_start == j) {
 319                                 if (dlist_word & SCALER_CTL0_END)
 320                                         break;
 321                                 next_entry_start = j +
 322                                         VC4_GET_FIELD(dlist_word,
 323                                                       SCALER_CTL0_SIZE);
 324                         }
 325                 }
 326         }
 327
 328         return 0;
 329 }
 330
 331 static int vc6_hvs_debugfs_upm_allocs(struct seq_file *m, void *data)
 332 {
 333         struct drm_debugfs_entry *entry = m->private;
 334         struct drm_device *dev = entry->dev;
 335         struct vc4_dev *vc4 = to_vc4_dev(dev);
 336         struct vc4_hvs *hvs = vc4->hvs;
 337         struct drm_printer p = drm_seq_file_printer(m);
 338         struct vc4_upm_refcounts *refcount;
 339         unsigned int i;
 340
 341         drm_printf(&p, "UPM Handles:\n");
 342         for (i = 1; i <= VC4_NUM_UPM_HANDLES; i++) {
 343                 refcount = &hvs->upm_refcounts[i];
 344                 drm_printf(&p, "handle %u: refcount %u, size %zu [%08llx + %08llx]\n",
 345                            i, refcount_read(&refcount->refcount), refcount->size,
 346                            refcount->upm.start, refcount->upm.size);
 347         }
 348
 349         return 0;
 350 }
 351
 352 /* The filter kernel is composed of dwords each containing 3 9-bit
 353  * signed integers packed next to each other.
 354  */
 355 #define VC4_INT_TO_COEFF(coeff) (coeff & 0x1ff)
 356 #define VC4_PPF_FILTER_WORD(c0, c1, c2)                         \
 357         ((((c0) & 0x1ff) << 0) |                                \
 358          (((c1) & 0x1ff) << 9) |                                \
 359          (((c2) & 0x1ff) << 18))
 360
 361 /* The whole filter kernel is arranged as the coefficients 0-16 going
 362  * up, then a pad, then 17-31 going down and reversed within the
 363  * dwords.  This means that a linear phase kernel (where it's
 364  * symmetrical at the boundary between 15 and 16) has the last 5
 365  * dwords matching the first 5, but reversed.
 366  */
 367 #define VC4_LINEAR_PHASE_KERNEL(c0, c1, c2, c3, c4, c5, c6, c7, c8,     \
 368                                 c9, c10, c11, c12, c13, c14, c15)       \
 369         {VC4_PPF_FILTER_WORD(c0, c1, c2),                               \
 370          VC4_PPF_FILTER_WORD(c3, c4, c5),                               \
 371          VC4_PPF_FILTER_WORD(c6, c7, c8),                               \
 372          VC4_PPF_FILTER_WORD(c9, c10, c11),                             \
 373          VC4_PPF_FILTER_WORD(c12, c13, c14),                            \
 374          VC4_PPF_FILTER_WORD(c15, c15, 0)}
 375
 376 #define VC4_LINEAR_PHASE_KERNEL_DWORDS 6
 377 #define VC4_KERNEL_DWORDS (VC4_LINEAR_PHASE_KERNEL_DWORDS * 2 - 1)
 378
 379 /* Recommended B=1/3, C=1/3 filter choice from Mitchell/Netravali.
 380  * http://www.cs.utexas.edu/~fussell/courses/cs384g/lectures/mitchell/Mitchell.pdf
 381  */
 382 static const u32 mitchell_netravali_1_3_1_3_kernel[] =
 383         VC4_LINEAR_PHASE_KERNEL(0, -2, -6, -8, -10, -8, -3, 2, 18,
 384                                 50, 82, 119, 155, 187, 213, 227);
 385
 386 static int vc4_hvs_upload_linear_kernel(struct vc4_hvs *hvs,
 387                                         struct drm_mm_node *space,
 388                                         const u32 *kernel)
 389 {
 390         int ret, i;
 391         u32 __iomem *dst_kernel;
 392
 393         /*
 394          * NOTE: We don't need a call to drm_dev_enter()/drm_dev_exit()
 395          * here since that function is only called from vc4_hvs_bind().
 396          */
 397
 398         ret = drm_mm_insert_node(&hvs->dlist_mm, space, VC4_KERNEL_DWORDS);
 399         if (ret) {
 400                 drm_err(&hvs->vc4->base, "Failed to allocate space for filter kernel: %d\n",
 401                         ret);
 402                 return ret;
 403         }
 404
 405         dst_kernel = hvs->dlist + space->start;
 406
 407         for (i = 0; i < VC4_KERNEL_DWORDS; i++) {
 408                 if (i < VC4_LINEAR_PHASE_KERNEL_DWORDS)
 409                         writel(kernel[i], &dst_kernel[i]);
 410                 else {
 411                         writel(kernel[VC4_KERNEL_DWORDS - i - 1],
 412                                &dst_kernel[i]);
 413                 }
 414         }
 415
 416         return 0;
 417 }
 418
 419 static void vc4_hvs_lut_load(struct vc4_hvs *hvs,
 420                              struct vc4_crtc *vc4_crtc)
 421 {
 422         struct vc4_dev *vc4 = hvs->vc4;
 423         struct drm_device *drm = &vc4->base;
 424         struct drm_crtc *crtc = &vc4_crtc->base;
 425         struct vc4_crtc_state *vc4_state = to_vc4_crtc_state(crtc->state);
 426         int idx;
 427         u32 i;
 428
 429         WARN_ON_ONCE(vc4->gen > VC4_GEN_5);
 430
 431         if (!drm_dev_enter(drm, &idx))
 432                 return;
 433
 434         if (hvs->vc4->gen != VC4_GEN_4)
 435                 goto exit;
 436
 437         /* The LUT memory is laid out with each HVS channel in order,
 438          * each of which takes 256 writes for R, 256 for G, then 256
 439          * for B.
 440          */
 441         HVS_WRITE(SCALER_GAMADDR,
 442                   SCALER_GAMADDR_AUTOINC |
 443                   (vc4_state->assigned_channel * 3 * crtc->gamma_size));
 444
 445         for (i = 0; i < crtc->gamma_size; i++)
 446                 HVS_WRITE(SCALER_GAMDATA, vc4_crtc->lut_r[i]);
 447         for (i = 0; i < crtc->gamma_size; i++)
 448                 HVS_WRITE(SCALER_GAMDATA, vc4_crtc->lut_g[i]);
 449         for (i = 0; i < crtc->gamma_size; i++)
 450                 HVS_WRITE(SCALER_GAMDATA, vc4_crtc->lut_b[i]);
 451
 452 exit:
 453         drm_dev_exit(idx);
 454 }
 455
 456 static void vc4_hvs_update_gamma_lut(struct vc4_hvs *hvs,
 457                                      struct vc4_crtc *vc4_crtc)
 458 {
 459         struct drm_crtc_state *crtc_state = vc4_crtc->base.state;
 460         struct drm_color_lut *lut = crtc_state->gamma_lut->data;
 461         u32 length = drm_color_lut_size(crtc_state->gamma_lut);
 462         u32 i;
 463
 464         for (i = 0; i < length; i++) {
 465                 vc4_crtc->lut_r[i] = drm_color_lut_extract(lut[i].red, 8);
 466                 vc4_crtc->lut_g[i] = drm_color_lut_extract(lut[i].green, 8);
 467                 vc4_crtc->lut_b[i] = drm_color_lut_extract(lut[i].blue, 8);
 468         }
 469
 470         vc4_hvs_lut_load(hvs, vc4_crtc);
 471 }
 472
 473 u8 vc4_hvs_get_fifo_frame_count(struct vc4_hvs *hvs, unsigned int fifo)
 474 {
 475         struct vc4_dev *vc4 = hvs->vc4;
 476         struct drm_device *drm = &vc4->base;
 477         u8 field = 0;
 478         int idx;
 479
 480         WARN_ON_ONCE(vc4->gen > VC4_GEN_6_D);
 481
 482         if (!drm_dev_enter(drm, &idx))
 483                 return 0;
 484
 485         switch (vc4->gen) {
 486         case VC4_GEN_6_C:
 487         case VC4_GEN_6_D:
 488                 field = VC4_GET_FIELD(HVS_READ(SCALER6_DISPX_STATUS(fifo)),
 489                                       SCALER6_DISPX_STATUS_FRCNT);
 490                 break;
 491         case VC4_GEN_5:
 492                 switch (fifo) {
 493                 case 0:
 494                         field = VC4_GET_FIELD(HVS_READ(SCALER_DISPSTAT1),
 495                                               SCALER5_DISPSTAT1_FRCNT0);
 496                         break;
 497                 case 1:
 498                         field = VC4_GET_FIELD(HVS_READ(SCALER_DISPSTAT1),
 499                                               SCALER5_DISPSTAT1_FRCNT1);
 500                         break;
 501                 case 2:
 502                         field = VC4_GET_FIELD(HVS_READ(SCALER_DISPSTAT2),
 503                                               SCALER5_DISPSTAT2_FRCNT2);
 504                         break;
 505                 }
 506                 break;
 507         case VC4_GEN_4:
 508                 switch (fifo) {
 509                 case 0:
 510                         field = VC4_GET_FIELD(HVS_READ(SCALER_DISPSTAT1),
 511                                               SCALER_DISPSTAT1_FRCNT0);
 512                         break;
 513                 case 1:
 514                         field = VC4_GET_FIELD(HVS_READ(SCALER_DISPSTAT1),
 515                                               SCALER_DISPSTAT1_FRCNT1);
 516                         break;
 517                 case 2:
 518                         field = VC4_GET_FIELD(HVS_READ(SCALER_DISPSTAT2),
 519                                               SCALER_DISPSTAT2_FRCNT2);
 520                         break;
 521                 }
 522                 break;
 523         default:
 524                 drm_err(drm, "Unknown VC4 generation: %d", vc4->gen);
 525                 break;
 526         }
 527
 528         drm_dev_exit(idx);
 529         return field;
 530 }
 531
 532 int vc4_hvs_get_fifo_from_output(struct vc4_hvs *hvs, unsigned int output)
 533 {
 534         struct vc4_dev *vc4 = hvs->vc4;
 535         u32 reg;
 536         int ret;
 537
 538         WARN_ON_ONCE(vc4->gen > VC4_GEN_6_D);
 539
 540         switch (vc4->gen) {
 541         case VC4_GEN_4:
 542                 return output;
 543
 544         case VC4_GEN_5:
 545                 /*
 546                  * NOTE: We should probably use
 547                  * drm_dev_enter()/drm_dev_exit() here, but this
 548                  * function is only used during the DRM device
 549                  * initialization, so we should be fine.
 550                  */
 551
 552                 switch (output) {
 553                 case 0:
 554                         return 0;
 555
 556                 case 1:
 557                         return 1;
 558
 559                 case 2:
 560                         reg = HVS_READ(SCALER_DISPECTRL);
 561                         ret = FIELD_GET(SCALER_DISPECTRL_DSP2_MUX_MASK, reg);
 562                         if (ret == 0)
 563                                 return 2;
 564
 565                         return 0;
 566
 567                 case 3:
 568                         reg = HVS_READ(SCALER_DISPCTRL);
 569                         ret = FIELD_GET(SCALER_DISPCTRL_DSP3_MUX_MASK, reg);
 570                         if (ret == 3)
 571                                 return -EPIPE;
 572
 573                         return ret;
 574
 575                 case 4:
 576                         reg = HVS_READ(SCALER_DISPEOLN);
 577                         ret = FIELD_GET(SCALER_DISPEOLN_DSP4_MUX_MASK, reg);
 578                         if (ret == 3)
 579                                 return -EPIPE;
 580
 581                         return ret;
 582
 583                 case 5:
 584                         reg = HVS_READ(SCALER_DISPDITHER);
 585                         ret = FIELD_GET(SCALER_DISPDITHER_DSP5_MUX_MASK, reg);
 586                         if (ret == 3)
 587                                 return -EPIPE;
 588
 589                         return ret;
 590
 591                 default:
 592                         return -EPIPE;
 593                 }
 594
 595         case VC4_GEN_6_C:
 596         case VC4_GEN_6_D:
 597                 switch (output) {
 598                 case 0:
 599                         return 0;
 600
 601                 case 2:
 602                         return 2;
 603
 604                 case 1:
 605                 case 3:
 606                 case 4:
 607                         return 1;
 608
 609                 default:
 610                         return -EPIPE;
 611                 }
 612
 613         default:
 614                 return -EPIPE;
 615         }
 616 }
 617
 618 static int vc4_hvs_init_channel(struct vc4_hvs *hvs, struct drm_crtc *crtc,
 619                                 struct drm_display_mode *mode, bool oneshot)
 620 {
 621         struct vc4_dev *vc4 = hvs->vc4;
 622         struct drm_device *drm = &vc4->base;
 623         struct vc4_crtc *vc4_crtc = to_vc4_crtc(crtc);
 624         struct vc4_crtc_state *vc4_crtc_state = to_vc4_crtc_state(crtc->state);
 625         unsigned int chan = vc4_crtc_state->assigned_channel;
 626         bool interlace = mode->flags & DRM_MODE_FLAG_INTERLACE;
 627         u32 dispbkgndx;
 628         u32 dispctrl;
 629         int idx;
 630
 631         WARN_ON_ONCE(vc4->gen > VC4_GEN_5);
 632
 633         if (!drm_dev_enter(drm, &idx))
 634                 return -ENODEV;
 635
 636         HVS_WRITE(SCALER_DISPCTRLX(chan), 0);
 637         HVS_WRITE(SCALER_DISPCTRLX(chan), SCALER_DISPCTRLX_RESET);
 638         HVS_WRITE(SCALER_DISPCTRLX(chan), 0);
 639
 640         /* Turn on the scaler, which will wait for vstart to start
 641          * compositing.
 642          * When feeding the transposer, we should operate in oneshot
 643          * mode.
 644          */
 645         dispctrl = SCALER_DISPCTRLX_ENABLE;
 646         dispbkgndx = HVS_READ(SCALER_DISPBKGNDX(chan));
 647
 648         if (vc4->gen == VC4_GEN_4) {
 649                 dispctrl |= VC4_SET_FIELD(mode->hdisplay,
 650                                           SCALER_DISPCTRLX_WIDTH) |
 651                             VC4_SET_FIELD(mode->vdisplay,
 652                                           SCALER_DISPCTRLX_HEIGHT) |
 653                             (oneshot ? SCALER_DISPCTRLX_ONESHOT : 0);
 654                 dispbkgndx |= SCALER_DISPBKGND_AUTOHS;
 655         } else {
 656                 dispctrl |= VC4_SET_FIELD(mode->hdisplay,
 657                                           SCALER5_DISPCTRLX_WIDTH) |
 658                             VC4_SET_FIELD(mode->vdisplay,
 659                                           SCALER5_DISPCTRLX_HEIGHT) |
 660                             (oneshot ? SCALER5_DISPCTRLX_ONESHOT : 0);
 661                 dispbkgndx &= ~SCALER5_DISPBKGND_BCK2BCK;
 662         }
 663
 664         HVS_WRITE(SCALER_DISPCTRLX(chan), dispctrl);
 665
 666         dispbkgndx &= ~SCALER_DISPBKGND_GAMMA;
 667         dispbkgndx &= ~SCALER_DISPBKGND_INTERLACE;
 668
 669         HVS_WRITE(SCALER_DISPBKGNDX(chan), dispbkgndx |
 670                   ((vc4->gen == VC4_GEN_4) ? SCALER_DISPBKGND_GAMMA : 0) |
 671                   (interlace ? SCALER_DISPBKGND_INTERLACE : 0));
 672
 673         /* Reload the LUT, since the SRAMs would have been disabled if
 674          * all CRTCs had SCALER_DISPBKGND_GAMMA unset at once.
 675          */
 676         vc4_hvs_lut_load(hvs, vc4_crtc);
 677
 678         drm_dev_exit(idx);
 679
 680         return 0;
 681 }
 682
 683 static int vc6_hvs_init_channel(struct vc4_hvs *hvs, struct drm_crtc *crtc,
 684                                 struct drm_display_mode *mode, bool oneshot)
 685 {
 686         struct vc4_dev *vc4 = hvs->vc4;
 687         struct drm_device *drm = &vc4->base;
 688         struct vc4_crtc_state *vc4_crtc_state = to_vc4_crtc_state(crtc->state);
 689         unsigned int chan = vc4_crtc_state->assigned_channel;
 690         bool interlace = mode->flags & DRM_MODE_FLAG_INTERLACE;
 691         u32 disp_ctrl1;
 692         int idx;
 693
 694         WARN_ON_ONCE(vc4->gen < VC4_GEN_6_C);
 695
 696         if (!drm_dev_enter(drm, &idx))
 697                 return -ENODEV;
 698
 699         HVS_WRITE(SCALER6_DISPX_CTRL0(chan), SCALER6_DISPX_CTRL0_RESET);
 700
 701         disp_ctrl1 = HVS_READ(SCALER6_DISPX_CTRL1(chan));
 702         disp_ctrl1 &= ~SCALER6_DISPX_CTRL1_INTLACE;
 703         HVS_WRITE(SCALER6_DISPX_CTRL1(chan),
 704                   disp_ctrl1 | (interlace ? SCALER6_DISPX_CTRL1_INTLACE : 0));
 705
 706         HVS_WRITE(SCALER6_DISPX_CTRL0(chan),
 707                   SCALER6_DISPX_CTRL0_ENB |
 708                   VC4_SET_FIELD(mode->hdisplay - 1,
 709                                 SCALER6_DISPX_CTRL0_FWIDTH) |
 710                   (oneshot ? SCALER6_DISPX_CTRL0_ONESHOT : 0) |
 711                   VC4_SET_FIELD(mode->vdisplay - 1,
 712                                 SCALER6_DISPX_CTRL0_LINES));
 713
 714         drm_dev_exit(idx);
 715
 716         return 0;
 717 }
 718
 719 static void __vc4_hvs_stop_channel(struct vc4_hvs *hvs, unsigned int chan)
 720 {
 721         struct vc4_dev *vc4 = hvs->vc4;
 722         struct drm_device *drm = &vc4->base;
 723         int idx;
 724
 725         WARN_ON_ONCE(vc4->gen > VC4_GEN_5);
 726
 727         if (!drm_dev_enter(drm, &idx))
 728                 return;
 729
 730         if (!(HVS_READ(SCALER_DISPCTRLX(chan)) & SCALER_DISPCTRLX_ENABLE))
 731                 goto out;
 732
 733         HVS_WRITE(SCALER_DISPCTRLX(chan), SCALER_DISPCTRLX_RESET);
 734         HVS_WRITE(SCALER_DISPCTRLX(chan), 0);
 735
 736         /* Once we leave, the scaler should be disabled and its fifo empty. */
 737         WARN_ON_ONCE(HVS_READ(SCALER_DISPCTRLX(chan)) & SCALER_DISPCTRLX_RESET);
 738
 739         WARN_ON_ONCE(VC4_GET_FIELD(HVS_READ(SCALER_DISPSTATX(chan)),
 740                                    SCALER_DISPSTATX_MODE) !=
 741                      SCALER_DISPSTATX_MODE_DISABLED);
 742
 743         WARN_ON_ONCE((HVS_READ(SCALER_DISPSTATX(chan)) &
 744                       (SCALER_DISPSTATX_FULL | SCALER_DISPSTATX_EMPTY)) !=
 745                      SCALER_DISPSTATX_EMPTY);
 746
 747 out:
 748         drm_dev_exit(idx);
 749 }
 750
 751 static void __vc6_hvs_stop_channel(struct vc4_hvs *hvs, unsigned int chan)
 752 {
 753         struct vc4_dev *vc4 = hvs->vc4;
 754         struct drm_device *drm = &vc4->base;
 755         int idx;
 756
 757         WARN_ON_ONCE(vc4->gen < VC4_GEN_6_C);
 758
 759         if (!drm_dev_enter(drm, &idx))
 760                 return;
 761
 762         if (!(HVS_READ(SCALER6_DISPX_CTRL0(chan)) & SCALER6_DISPX_CTRL0_ENB))
 763                 goto out;
 764
 765         HVS_WRITE(SCALER6_DISPX_CTRL0(chan),
 766                   HVS_READ(SCALER6_DISPX_CTRL0(chan)) | SCALER6_DISPX_CTRL0_RESET);
 767
 768         HVS_WRITE(SCALER6_DISPX_CTRL0(chan),
 769                   HVS_READ(SCALER6_DISPX_CTRL0(chan)) & ~SCALER6_DISPX_CTRL0_ENB);
 770
 771         WARN_ON_ONCE(VC4_GET_FIELD(HVS_READ(SCALER6_DISPX_STATUS(chan)),
 772                                    SCALER6_DISPX_STATUS_MODE) !=
 773                      SCALER6_DISPX_STATUS_MODE_DISABLED);
 774
 775 out:
 776         drm_dev_exit(idx);
 777 }
 778
 779 void vc4_hvs_stop_channel(struct vc4_hvs *hvs, unsigned int chan)
 780 {
 781         struct vc4_dev *vc4 = hvs->vc4;
 782
 783         if (vc4->gen >= VC4_GEN_6_C)
 784                 __vc6_hvs_stop_channel(hvs, chan);
 785         else
 786                 __vc4_hvs_stop_channel(hvs, chan);
 787 }
 788
 789 int vc4_hvs_atomic_check(struct drm_crtc *crtc, struct drm_atomic_state *state)
 790 {
 791         struct drm_crtc_state *crtc_state = drm_atomic_get_new_crtc_state(state, crtc);
 792         struct vc4_crtc_state *vc4_state = to_vc4_crtc_state(crtc_state);
 793         struct drm_device *dev = crtc->dev;
 794         struct vc4_dev *vc4 = to_vc4_dev(dev);
 795         struct drm_plane *plane;
 796         unsigned long flags;
 797         const struct drm_plane_state *plane_state;
 798         u32 dlist_count = 0;
 799         int ret;
 800
 801         /* The pixelvalve can only feed one encoder (and encoders are
 802          * 1:1 with connectors.)
 803          */
 804         if (hweight32(crtc_state->connector_mask) > 1)
 805                 return -EINVAL;
 806
 807         drm_atomic_crtc_state_for_each_plane_state(plane, plane_state, crtc_state) {
 808                 u32 plane_dlist_count = vc4_plane_dlist_size(plane_state);
 809
 810                 drm_dbg_driver(dev, "[CRTC:%d:%s] Found [PLANE:%d:%s] with DLIST size: %u\n",
 811                                crtc->base.id, crtc->name,
 812                                plane->base.id, plane->name,
 813                                plane_dlist_count);
 814
 815                 dlist_count += plane_dlist_count;
 816         }
 817
 818         dlist_count++; /* Account for SCALER_CTL0_END. */
 819
 820         drm_dbg_driver(dev, "[CRTC:%d:%s] Allocating DLIST block with size: %u\n",
 821                        crtc->base.id, crtc->name, dlist_count);
 822         spin_lock_irqsave(&vc4->hvs->mm_lock, flags);
 823         ret = drm_mm_insert_node(&vc4->hvs->dlist_mm, &vc4_state->mm,
 824                                  dlist_count);
 825         spin_unlock_irqrestore(&vc4->hvs->mm_lock, flags);
 826         if (ret) {
 827                 drm_err(dev, "Failed to allocate DLIST entry: %d\n", ret);
 828                 return ret;
 829         }
 830
 831         return 0;
 832 }
 833
 834 static void vc4_hvs_install_dlist(struct drm_crtc *crtc)
 835 {
 836         struct drm_device *dev = crtc->dev;
 837         struct vc4_dev *vc4 = to_vc4_dev(dev);
 838         struct vc4_hvs *hvs = vc4->hvs;
 839         struct vc4_crtc_state *vc4_state = to_vc4_crtc_state(crtc->state);
 840         int idx;
 841
 842         if (!drm_dev_enter(dev, &idx))
 843                 return;
 844
 845         if (vc4->gen >= VC4_GEN_6_C)
 846                 HVS_WRITE(SCALER6_DISPX_LPTRS(vc4_state->assigned_channel),
 847                           VC4_SET_FIELD(vc4_state->mm.start,
 848                                         SCALER6_DISPX_LPTRS_HEADE));
 849         else
 850                 HVS_WRITE(SCALER_DISPLISTX(vc4_state->assigned_channel),
 851                           vc4_state->mm.start);
 852
 853         drm_dev_exit(idx);
 854 }
 855
 856 static void vc4_hvs_update_dlist(struct drm_crtc *crtc)
 857 {
 858         struct drm_device *dev = crtc->dev;
 859         struct vc4_crtc *vc4_crtc = to_vc4_crtc(crtc);
 860         struct vc4_crtc_state *vc4_state = to_vc4_crtc_state(crtc->state);
 861         unsigned long flags;
 862
 863         if (crtc->state->event) {
 864                 crtc->state->event->pipe = drm_crtc_index(crtc);
 865
 866                 WARN_ON(drm_crtc_vblank_get(crtc) != 0);
 867
 868                 spin_lock_irqsave(&dev->event_lock, flags);
 869
 870                 if (!vc4_crtc->feeds_txp || vc4_state->txp_armed) {
 871                         vc4_crtc->event = crtc->state->event;
 872                         crtc->state->event = NULL;
 873                 }
 874
 875                 spin_unlock_irqrestore(&dev->event_lock, flags);
 876         }
 877
 878         spin_lock_irqsave(&vc4_crtc->irq_lock, flags);
 879         vc4_crtc->current_dlist = vc4_state->mm.start;
 880         spin_unlock_irqrestore(&vc4_crtc->irq_lock, flags);
 881 }
 882
 883 void vc4_hvs_atomic_begin(struct drm_crtc *crtc,
 884                           struct drm_atomic_state *state)
 885 {
 886         struct vc4_crtc *vc4_crtc = to_vc4_crtc(crtc);
 887         struct vc4_crtc_state *vc4_state = to_vc4_crtc_state(crtc->state);
 888         unsigned long flags;
 889
 890         spin_lock_irqsave(&vc4_crtc->irq_lock, flags);
 891         vc4_crtc->current_hvs_channel = vc4_state->assigned_channel;
 892         spin_unlock_irqrestore(&vc4_crtc->irq_lock, flags);
 893 }
 894
 895 void vc4_hvs_atomic_enable(struct drm_crtc *crtc,
 896                            struct drm_atomic_state *state)
 897 {
 898         struct drm_device *dev = crtc->dev;
 899         struct vc4_dev *vc4 = to_vc4_dev(dev);
 900         struct drm_display_mode *mode = &crtc->state->adjusted_mode;
 901         struct vc4_crtc *vc4_crtc = to_vc4_crtc(crtc);
 902         bool oneshot = vc4_crtc->feeds_txp;
 903
 904         vc4_hvs_install_dlist(crtc);
 905         vc4_hvs_update_dlist(crtc);
 906
 907         if (vc4->gen >= VC4_GEN_6_C)
 908                 vc6_hvs_init_channel(vc4->hvs, crtc, mode, oneshot);
 909         else
 910                 vc4_hvs_init_channel(vc4->hvs, crtc, mode, oneshot);
 911 }
 912
 913 void vc4_hvs_atomic_disable(struct drm_crtc *crtc,
 914                             struct drm_atomic_state *state)
 915 {
 916         struct drm_device *dev = crtc->dev;
 917         struct vc4_dev *vc4 = to_vc4_dev(dev);
 918         struct drm_crtc_state *old_state = drm_atomic_get_old_crtc_state(state, crtc);
 919         struct vc4_crtc_state *vc4_state = to_vc4_crtc_state(old_state);
 920         unsigned int chan = vc4_state->assigned_channel;
 921
 922         vc4_hvs_stop_channel(vc4->hvs, chan);
 923 }
 924
 925 void vc4_hvs_atomic_flush(struct drm_crtc *crtc,
 926                           struct drm_atomic_state *state)
 927 {
 928         struct drm_crtc_state *old_state = drm_atomic_get_old_crtc_state(state,
 929                                                                          crtc);
 930         struct drm_device *dev = crtc->dev;
 931         struct vc4_dev *vc4 = to_vc4_dev(dev);
 932         struct vc4_hvs *hvs = vc4->hvs;
 933         struct vc4_crtc *vc4_crtc = to_vc4_crtc(crtc);
 934         struct vc4_crtc_state *vc4_state = to_vc4_crtc_state(crtc->state);
 935         unsigned int channel = vc4_state->assigned_channel;
 936         struct drm_plane *plane;
 937         struct vc4_plane_state *vc4_plane_state;
 938         bool debug_dump_regs = false;
 939         bool enable_bg_fill = true;
 940         u32 __iomem *dlist_start = vc4->hvs->dlist + vc4_state->mm.start;
 941         u32 __iomem *dlist_next = dlist_start;
 942         unsigned int zpos = 0;
 943         bool found = false;
 944         int idx;
 945
 946         WARN_ON_ONCE(vc4->gen > VC4_GEN_6_D);
 947
 948         if (!drm_dev_enter(dev, &idx)) {
 949                 vc4_crtc_send_vblank(crtc);
 950                 return;
 951         }
 952
 953         if (vc4_state->assigned_channel == VC4_HVS_CHANNEL_DISABLED)
 954                 goto exit;
 955
 956         if (debug_dump_regs) {
 957                 DRM_INFO("CRTC %d HVS before:\n", drm_crtc_index(crtc));
 958                 vc4_hvs_dump_state(hvs);
 959         }
 960
 961         /* Copy all the active planes' dlist contents to the hardware dlist. */
 962         do {
 963                 found = false;
 964
 965                 drm_atomic_crtc_for_each_plane(plane, crtc) {
 966                         if (plane->state->normalized_zpos != zpos)
 967                                 continue;
 968
 969                         /* Is this the first active plane? */
 970                         if (dlist_next == dlist_start) {
 971                                 /* We need to enable background fill when a plane
 972                                  * could be alpha blending from the background, i.e.
 973                                  * where no other plane is underneath. It suffices to
 974                                  * consider the first active plane here since we set
 975                                  * needs_bg_fill such that either the first plane
 976                                  * already needs it or all planes on top blend from
 977                                  * the first or a lower plane.
 978                                  */
 979                                 vc4_plane_state = to_vc4_plane_state(plane->state);
 980                                 enable_bg_fill = vc4_plane_state->needs_bg_fill;
 981                         }
 982
 983                         dlist_next += vc4_plane_write_dlist(plane, dlist_next);
 984
 985                         found = true;
 986                 }
 987
 988                 zpos++;
 989         } while (found);
 990
 991         writel(SCALER_CTL0_END, dlist_next);
 992         dlist_next++;
 993
 994         WARN_ON_ONCE(dlist_next - dlist_start != vc4_state->mm.size);
 995
 996         if (vc4->gen >= VC4_GEN_6_C) {
 997                 /* This sets a black background color fill, as is the case
 998                  * with other DRM drivers.
 999                  */
1000                 if (enable_bg_fill)
1001                         HVS_WRITE(SCALER6_DISPX_CTRL1(channel),
1002                                   HVS_READ(SCALER6_DISPX_CTRL1(channel)) |
1003                                   SCALER6_DISPX_CTRL1_BGENB);
1004                 else
1005                         HVS_WRITE(SCALER6_DISPX_CTRL1(channel),
1006                                   HVS_READ(SCALER6_DISPX_CTRL1(channel)) &
1007                                   ~SCALER6_DISPX_CTRL1_BGENB);
1008         } else {
1009                 /* we can actually run with a lower core clock when background
1010                  * fill is enabled on VC4_GEN_5 so leave it enabled always.
1011                  */
1012                 HVS_WRITE(SCALER_DISPBKGNDX(channel),
1013                           HVS_READ(SCALER_DISPBKGNDX(channel)) |
1014                           SCALER_DISPBKGND_FILL);
1015         }
1016
1017         /* Only update DISPLIST if the CRTC was already running and is not
1018          * being disabled.
1019          * vc4_crtc_enable() takes care of updating the dlist just after
1020          * re-enabling VBLANK interrupts and before enabling the engine.
1021          * If the CRTC is being disabled, there's no point in updating this
1022          * information.
1023          */
1024         if (crtc->state->active && old_state->active) {
1025                 vc4_hvs_install_dlist(crtc);
1026                 vc4_hvs_update_dlist(crtc);
1027         }
1028
1029         if (crtc->state->color_mgmt_changed) {
1030                 u32 dispbkgndx = HVS_READ(SCALER_DISPBKGNDX(channel));
1031
1032                 WARN_ON_ONCE(vc4->gen > VC4_GEN_5);
1033
1034                 if (crtc->state->gamma_lut) {
1035                         vc4_hvs_update_gamma_lut(hvs, vc4_crtc);
1036                         dispbkgndx |= SCALER_DISPBKGND_GAMMA;
1037                 } else {
1038                         /* Unsetting DISPBKGND_GAMMA skips the gamma lut step
1039                          * in hardware, which is the same as a linear lut that
1040                          * DRM expects us to use in absence of a user lut.
1041                          */
1042                         dispbkgndx &= ~SCALER_DISPBKGND_GAMMA;
1043                 }
1044                 HVS_WRITE(SCALER_DISPBKGNDX(channel), dispbkgndx);
1045         }
1046
1047         if (debug_dump_regs) {
1048                 DRM_INFO("CRTC %d HVS after:\n", drm_crtc_index(crtc));
1049                 vc4_hvs_dump_state(hvs);
1050         }
1051
1052 exit:
1053         drm_dev_exit(idx);
1054 }
1055
1056 void vc4_hvs_mask_underrun(struct vc4_hvs *hvs, int channel)
1057 {
1058         struct vc4_dev *vc4 = hvs->vc4;
1059         struct drm_device *drm = &vc4->base;
1060         u32 dispctrl;
1061         int idx;
1062
1063         WARN_ON(vc4->gen > VC4_GEN_5);
1064
1065         if (!drm_dev_enter(drm, &idx))
1066                 return;
1067
1068         dispctrl = HVS_READ(SCALER_DISPCTRL);
1069         dispctrl &= ~((vc4->gen == VC4_GEN_5) ?
1070                       SCALER5_DISPCTRL_DSPEISLUR(channel) :
1071                       SCALER_DISPCTRL_DSPEISLUR(channel));
1072
1073         HVS_WRITE(SCALER_DISPCTRL, dispctrl);
1074
1075         drm_dev_exit(idx);
1076 }
1077
1078 void vc4_hvs_unmask_underrun(struct vc4_hvs *hvs, int channel)
1079 {
1080         struct vc4_dev *vc4 = hvs->vc4;
1081         struct drm_device *drm = &vc4->base;
1082         u32 dispctrl;
1083         int idx;
1084
1085         WARN_ON(vc4->gen > VC4_GEN_5);
1086
1087         if (!drm_dev_enter(drm, &idx))
1088                 return;
1089
1090         dispctrl = HVS_READ(SCALER_DISPCTRL);
1091         dispctrl |= ((vc4->gen == VC4_GEN_5) ?
1092                      SCALER5_DISPCTRL_DSPEISLUR(channel) :
1093                      SCALER_DISPCTRL_DSPEISLUR(channel));
1094
1095         HVS_WRITE(SCALER_DISPSTAT,
1096                   SCALER_DISPSTAT_EUFLOW(channel));
1097         HVS_WRITE(SCALER_DISPCTRL, dispctrl);
1098
1099         drm_dev_exit(idx);
1100 }
1101
1102 static void vc4_hvs_report_underrun(struct drm_device *dev)
1103 {
1104         struct vc4_dev *vc4 = to_vc4_dev(dev);
1105
1106         atomic_inc(&vc4->underrun);
1107         DRM_DEV_ERROR(dev->dev, "HVS underrun\n");
1108 }
1109
1110 static irqreturn_t vc4_hvs_irq_handler(int irq, void *data)
1111 {
1112         struct drm_device *dev = data;
1113         struct vc4_dev *vc4 = to_vc4_dev(dev);
1114         struct vc4_hvs *hvs = vc4->hvs;
1115         irqreturn_t irqret = IRQ_NONE;
1116         int channel;
1117         u32 control;
1118         u32 status;
1119         u32 dspeislur;
1120
1121         WARN_ON(vc4->gen > VC4_GEN_5);
1122
1123         /*
1124          * NOTE: We don't need to protect the register access using
1125          * drm_dev_enter() there because the interrupt handler lifetime
1126          * is tied to the device itself, and not to the DRM device.
1127          *
1128          * So when the device will be gone, one of the first thing we
1129          * will be doing will be to unregister the interrupt handler,
1130          * and then unregister the DRM device. drm_dev_enter() would
1131          * thus always succeed if we are here.
1132          */
1133
1134         status = HVS_READ(SCALER_DISPSTAT);
1135         control = HVS_READ(SCALER_DISPCTRL);
1136
1137         for (channel = 0; channel < SCALER_CHANNELS_COUNT; channel++) {
1138                 dspeislur = (vc4->gen == VC4_GEN_5) ?
1139                         SCALER5_DISPCTRL_DSPEISLUR(channel) :
1140                         SCALER_DISPCTRL_DSPEISLUR(channel);
1141
1142                 /* Interrupt masking is not always honored, so check it here. */
1143                 if (status & SCALER_DISPSTAT_EUFLOW(channel) &&
1144                     control & dspeislur) {
1145                         vc4_hvs_mask_underrun(hvs, channel);
1146                         vc4_hvs_report_underrun(dev);
1147
1148                         irqret = IRQ_HANDLED;
1149                 }
1150         }
1151
1152         /* Clear every per-channel interrupt flag. */
1153         HVS_WRITE(SCALER_DISPSTAT, SCALER_DISPSTAT_IRQMASK(0) |
1154                                    SCALER_DISPSTAT_IRQMASK(1) |
1155                                    SCALER_DISPSTAT_IRQMASK(2));
1156
1157         return irqret;
1158 }
1159
1160 int vc4_hvs_debugfs_init(struct drm_minor *minor)
1161 {
1162         struct drm_device *drm = minor->dev;
1163         struct vc4_dev *vc4 = to_vc4_dev(drm);
1164         struct vc4_hvs *hvs = vc4->hvs;
1165
1166         if (!vc4->hvs)
1167                 return -ENODEV;
1168
1169         if (vc4->gen == VC4_GEN_4)
1170                 debugfs_create_bool("hvs_load_tracker", S_IRUGO | S_IWUSR,
1171                                     minor->debugfs_root,
1172                                     &vc4->load_tracker_enabled);
1173
1174         if (vc4->gen >= VC4_GEN_6_C) {
1175                 drm_debugfs_add_file(drm, "hvs_dlists", vc6_hvs_debugfs_dlist, NULL);
1176                 drm_debugfs_add_file(drm, "hvs_upm", vc6_hvs_debugfs_upm_allocs, NULL);
1177         } else {
1178                 drm_debugfs_add_file(drm, "hvs_dlists", vc4_hvs_debugfs_dlist, NULL);
1179         }
1180
1181         drm_debugfs_add_file(drm, "hvs_underrun", vc4_hvs_debugfs_underrun, NULL);
1182
1183         vc4_debugfs_add_regset32(drm, "hvs_regs", &hvs->regset);
1184
1185         return 0;
1186 }
1187
1188 struct vc4_hvs *__vc4_hvs_alloc(struct vc4_dev *vc4,
1189                                 void __iomem *regs,
1190                                 struct platform_device *pdev)
1191 {
1192         struct drm_device *drm = &vc4->base;
1193         struct vc4_hvs *hvs;
1194         unsigned int dlist_start;
1195         size_t dlist_size;
1196         size_t lbm_size;
1197         unsigned int i;
1198
1199         hvs = drmm_kzalloc(drm, sizeof(*hvs), GFP_KERNEL);
1200         if (!hvs)
1201                 return ERR_PTR(-ENOMEM);
1202
1203         hvs->vc4 = vc4;
1204         hvs->regs = regs;
1205         hvs->pdev = pdev;
1206
1207         spin_lock_init(&hvs->mm_lock);
1208
1209         switch (vc4->gen) {
1210         case VC4_GEN_4:
1211         case VC4_GEN_5:
1212                 /* Set up the HVS display list memory manager. We never
1213                  * overwrite the setup from the bootloader (just 128b
1214                  * out of our 16K), since we don't want to scramble the
1215                  * screen when transitioning from the firmware's boot
1216                  * setup to runtime.
1217                  */
1218                 dlist_start = HVS_BOOTLOADER_DLIST_END;
1219                 dlist_size = (SCALER_DLIST_SIZE >> 2) - HVS_BOOTLOADER_DLIST_END;
1220                 break;
1221
1222         case VC4_GEN_6_C:
1223         case VC4_GEN_6_D:
1224                 dlist_start = HVS_BOOTLOADER_DLIST_END;
1225
1226                 /*
1227                  * If we are running a test, it means that we can't
1228                  * access a register. Use a plausible size then.
1229                  */
1230                 if (!kunit_get_current_test())
1231                         dlist_size = HVS_READ(SCALER6_CXM_SIZE);
1232                 else
1233                         dlist_size = 4096;
1234
1235                 for (i = 0; i < VC4_NUM_UPM_HANDLES; i++) {
1236                         refcount_set(&hvs->upm_refcounts[i].refcount, 0);
1237                         hvs->upm_refcounts[i].hvs = hvs;
1238                 }
1239
1240                 break;
1241
1242         default:
1243                 drm_err(drm, "Unknown VC4 generation: %d", vc4->gen);
1244                 return ERR_PTR(-ENODEV);
1245         }
1246
1247         drm_mm_init(&hvs->dlist_mm, dlist_start, dlist_size);
1248
1249         hvs->dlist_mem_size = dlist_size;
1250
1251         /* Set up the HVS LBM memory manager.  We could have some more
1252          * complicated data structure that allowed reuse of LBM areas
1253          * between planes when they don't overlap on the screen, but
1254          * for now we just allocate globally.
1255          */
1256
1257         switch (vc4->gen) {
1258         case VC4_GEN_4:
1259                 /* 48k words of 2x12-bit pixels */
1260                 lbm_size = 48 * SZ_1K;
1261                 break;
1262
1263         case VC4_GEN_5:
1264                 /* 60k words of 4x12-bit pixels */
1265                 lbm_size = 60 * SZ_1K;
1266                 break;
1267
1268         case VC4_GEN_6_C:
1269         case VC4_GEN_6_D:
1270                 /*
1271                  * If we are running a test, it means that we can't
1272                  * access a register. Use a plausible size then.
1273                  */
1274                 lbm_size = 1024;
1275                 break;
1276
1277         default:
1278                 drm_err(drm, "Unknown VC4 generation: %d", vc4->gen);
1279                 return ERR_PTR(-ENODEV);
1280         }
1281
1282         drm_mm_init(&hvs->lbm_mm, 0, lbm_size);
1283
1284         if (vc4->gen >= VC4_GEN_6_C) {
1285                 ida_init(&hvs->upm_handles);
1286
1287                 /*
1288                  * NOTE: On BCM2712, the size can also be read through
1289                  * the SCALER_UBM_SIZE register. We would need to do a
1290                  * register access though, which we can't do with kunit
1291                  * that also uses this function to create its mock
1292                  * device.
1293                  */
1294                 drm_mm_init(&hvs->upm_mm, 0, 1024 * HVS_UBM_WORD_SIZE);
1295         }
1296
1297
1298         vc4->hvs = hvs;
1299
1300         return hvs;
1301 }
1302
1303 static int vc4_hvs_hw_init(struct vc4_hvs *hvs)
1304 {
1305         struct vc4_dev *vc4 = hvs->vc4;
1306         u32 dispctrl, reg;
1307
1308         dispctrl = HVS_READ(SCALER_DISPCTRL);
1309         dispctrl |= SCALER_DISPCTRL_ENABLE;
1310         HVS_WRITE(SCALER_DISPCTRL, dispctrl);
1311
1312         reg = HVS_READ(SCALER_DISPECTRL);
1313         reg &= ~SCALER_DISPECTRL_DSP2_MUX_MASK;
1314         HVS_WRITE(SCALER_DISPECTRL,
1315                   reg | VC4_SET_FIELD(0, SCALER_DISPECTRL_DSP2_MUX));
1316
1317         reg = HVS_READ(SCALER_DISPCTRL);
1318         reg &= ~SCALER_DISPCTRL_DSP3_MUX_MASK;
1319         HVS_WRITE(SCALER_DISPCTRL,
1320                   reg | VC4_SET_FIELD(3, SCALER_DISPCTRL_DSP3_MUX));
1321
1322         reg = HVS_READ(SCALER_DISPEOLN);
1323         reg &= ~SCALER_DISPEOLN_DSP4_MUX_MASK;
1324         HVS_WRITE(SCALER_DISPEOLN,
1325                   reg | VC4_SET_FIELD(3, SCALER_DISPEOLN_DSP4_MUX));
1326
1327         reg = HVS_READ(SCALER_DISPDITHER);
1328         reg &= ~SCALER_DISPDITHER_DSP5_MUX_MASK;
1329         HVS_WRITE(SCALER_DISPDITHER,
1330                   reg | VC4_SET_FIELD(3, SCALER_DISPDITHER_DSP5_MUX));
1331
1332         dispctrl = HVS_READ(SCALER_DISPCTRL);
1333         dispctrl |= SCALER_DISPCTRL_DISPEIRQ(0) |
1334                     SCALER_DISPCTRL_DISPEIRQ(1) |
1335                     SCALER_DISPCTRL_DISPEIRQ(2);
1336
1337         if (vc4->gen == VC4_GEN_4)
1338                 dispctrl &= ~(SCALER_DISPCTRL_DMAEIRQ |
1339                               SCALER_DISPCTRL_SLVWREIRQ |
1340                               SCALER_DISPCTRL_SLVRDEIRQ |
1341                               SCALER_DISPCTRL_DSPEIEOF(0) |
1342                               SCALER_DISPCTRL_DSPEIEOF(1) |
1343                               SCALER_DISPCTRL_DSPEIEOF(2) |
1344                               SCALER_DISPCTRL_DSPEIEOLN(0) |
1345                               SCALER_DISPCTRL_DSPEIEOLN(1) |
1346                               SCALER_DISPCTRL_DSPEIEOLN(2) |
1347                               SCALER_DISPCTRL_DSPEISLUR(0) |
1348                               SCALER_DISPCTRL_DSPEISLUR(1) |
1349                               SCALER_DISPCTRL_DSPEISLUR(2) |
1350                               SCALER_DISPCTRL_SCLEIRQ);
1351         else
1352                 dispctrl &= ~(SCALER_DISPCTRL_DMAEIRQ |
1353                               SCALER5_DISPCTRL_SLVEIRQ |
1354                               SCALER5_DISPCTRL_DSPEIEOF(0) |
1355                               SCALER5_DISPCTRL_DSPEIEOF(1) |
1356                               SCALER5_DISPCTRL_DSPEIEOF(2) |
1357                               SCALER5_DISPCTRL_DSPEIEOLN(0) |
1358                               SCALER5_DISPCTRL_DSPEIEOLN(1) |
1359                               SCALER5_DISPCTRL_DSPEIEOLN(2) |
1360                               SCALER5_DISPCTRL_DSPEISLUR(0) |
1361                               SCALER5_DISPCTRL_DSPEISLUR(1) |
1362                               SCALER5_DISPCTRL_DSPEISLUR(2) |
1363                               SCALER_DISPCTRL_SCLEIRQ);
1364
1365
1366         /* Set AXI panic mode.
1367          * VC4 panics when < 2 lines in FIFO.
1368          * VC5 panics when less than 1 line in the FIFO.
1369          */
1370         dispctrl &= ~(SCALER_DISPCTRL_PANIC0_MASK |
1371                       SCALER_DISPCTRL_PANIC1_MASK |
1372                       SCALER_DISPCTRL_PANIC2_MASK);
1373         dispctrl |= VC4_SET_FIELD(2, SCALER_DISPCTRL_PANIC0);
1374         dispctrl |= VC4_SET_FIELD(2, SCALER_DISPCTRL_PANIC1);
1375         dispctrl |= VC4_SET_FIELD(2, SCALER_DISPCTRL_PANIC2);
1376
1377         /* Set AXI panic mode.
1378          * VC4 panics when < 2 lines in FIFO.
1379          * VC5 panics when less than 1 line in the FIFO.
1380          */
1381         dispctrl &= ~(SCALER_DISPCTRL_PANIC0_MASK |
1382                       SCALER_DISPCTRL_PANIC1_MASK |
1383                       SCALER_DISPCTRL_PANIC2_MASK);
1384         dispctrl |= VC4_SET_FIELD(2, SCALER_DISPCTRL_PANIC0);
1385         dispctrl |= VC4_SET_FIELD(2, SCALER_DISPCTRL_PANIC1);
1386         dispctrl |= VC4_SET_FIELD(2, SCALER_DISPCTRL_PANIC2);
1387
1388         HVS_WRITE(SCALER_DISPCTRL, dispctrl);
1389
1390         return 0;
1391 }
1392
1393 #define CFC1_N_NL_CSC_CTRL(x)           (0xa000 + ((x) * 0x3000))
1394 #define CFC1_N_MA_CSC_COEFF_C00(x)      (0xa008 + ((x) * 0x3000))
1395 #define CFC1_N_MA_CSC_COEFF_C01(x)      (0xa00c + ((x) * 0x3000))
1396 #define CFC1_N_MA_CSC_COEFF_C02(x)      (0xa010 + ((x) * 0x3000))
1397 #define CFC1_N_MA_CSC_COEFF_C03(x)      (0xa014 + ((x) * 0x3000))
1398 #define CFC1_N_MA_CSC_COEFF_C04(x)      (0xa018 + ((x) * 0x3000))
1399 #define CFC1_N_MA_CSC_COEFF_C10(x)      (0xa01c + ((x) * 0x3000))
1400 #define CFC1_N_MA_CSC_COEFF_C11(x)      (0xa020 + ((x) * 0x3000))
1401 #define CFC1_N_MA_CSC_COEFF_C12(x)      (0xa024 + ((x) * 0x3000))
1402 #define CFC1_N_MA_CSC_COEFF_C13(x)      (0xa028 + ((x) * 0x3000))
1403 #define CFC1_N_MA_CSC_COEFF_C14(x)      (0xa02c + ((x) * 0x3000))
1404 #define CFC1_N_MA_CSC_COEFF_C20(x)      (0xa030 + ((x) * 0x3000))
1405 #define CFC1_N_MA_CSC_COEFF_C21(x)      (0xa034 + ((x) * 0x3000))
1406 #define CFC1_N_MA_CSC_COEFF_C22(x)      (0xa038 + ((x) * 0x3000))
1407 #define CFC1_N_MA_CSC_COEFF_C23(x)      (0xa03c + ((x) * 0x3000))
1408 #define CFC1_N_MA_CSC_COEFF_C24(x)      (0xa040 + ((x) * 0x3000))
1409
1410 #define SCALER_PI_CMP_CSC_RED0(x)               (0x200 + ((x) * 0x40))
1411 #define SCALER_PI_CMP_CSC_RED1(x)               (0x204 + ((x) * 0x40))
1412 #define SCALER_PI_CMP_CSC_RED_CLAMP(x)          (0x208 + ((x) * 0x40))
1413 #define SCALER_PI_CMP_CSC_CFG(x)                (0x20c + ((x) * 0x40))
1414 #define SCALER_PI_CMP_CSC_GREEN0(x)             (0x210 + ((x) * 0x40))
1415 #define SCALER_PI_CMP_CSC_GREEN1(x)             (0x214 + ((x) * 0x40))
1416 #define SCALER_PI_CMP_CSC_GREEN_CLAMP(x)        (0x218 + ((x) * 0x40))
1417 #define SCALER_PI_CMP_CSC_BLUE0(x)              (0x220 + ((x) * 0x40))
1418 #define SCALER_PI_CMP_CSC_BLUE1(x)              (0x224 + ((x) * 0x40))
1419 #define SCALER_PI_CMP_CSC_BLUE_CLAMP(x)         (0x228 + ((x) * 0x40))
1420
1421 /* 4 S2.22 multiplication factors, and 1 S9.15 addititive element for each of 3
1422  * output components
1423  */
1424 struct vc6_csc_coeff_entry {
1425         u32 csc[3][5];
1426 };
1427
1428 static const struct vc6_csc_coeff_entry csc_coeffs[2][3] = {
1429         [DRM_COLOR_YCBCR_LIMITED_RANGE] = {
1430                 [DRM_COLOR_YCBCR_BT601] = {
1431                         .csc = {
1432                                 { 0x004A8542, 0x0, 0x0066254A, 0x0, 0xFF908A0D },
1433                                 { 0x004A8542, 0xFFE6ED5D, 0xFFCBF856, 0x0, 0x0043C9A3 },
1434                                 { 0x004A8542, 0x00811A54, 0x0, 0x0, 0xFF759502 }
1435                         }
1436                 },
1437                 [DRM_COLOR_YCBCR_BT709] = {
1438                         .csc = {
1439                                 { 0x004A8542, 0x0, 0x0072BC44, 0x0, 0xFF83F312 },
1440                                 { 0x004A8542, 0xFFF25A22, 0xFFDDE4D0, 0x0, 0x00267064 },
1441                                 { 0x004A8542, 0x00873197, 0x0, 0x0, 0xFF6F7DC0 }
1442                         }
1443                 },
1444                 [DRM_COLOR_YCBCR_BT2020] = {
1445                         .csc = {
1446                                 { 0x004A8542, 0x0, 0x006B4A17, 0x0, 0xFF8B653F },
1447                                 { 0x004A8542, 0xFFF402D9, 0xFFDDE4D0, 0x0, 0x0024C7AE },
1448                                 { 0x004A8542, 0x008912CC, 0x0, 0x0, 0xFF6D9C8B }
1449                         }
1450                 }
1451         },
1452         [DRM_COLOR_YCBCR_FULL_RANGE] = {
1453                 [DRM_COLOR_YCBCR_BT601] = {
1454                         .csc = {
1455                                 { 0x00400000, 0x0, 0x0059BA5E, 0x0, 0xFFA645A1 },
1456                                 { 0x00400000, 0xFFE9F9AC, 0xFFD24B97, 0x0, 0x0043BABB },
1457                                 { 0x00400000, 0x00716872, 0x0, 0x0, 0xFF8E978D }
1458                         }
1459                 },
1460                 [DRM_COLOR_YCBCR_BT709] = {
1461                         .csc = {
1462                                 { 0x00400000, 0x0, 0x0064C985, 0x0, 0xFF9B367A },
1463                                 { 0x00400000, 0xFFF402E1, 0xFFE20A40, 0x0, 0x0029F2DE },
1464                                 { 0x00400000, 0x0076C226, 0x0, 0x0, 0xFF893DD9 }
1465                         }
1466                 },
1467                 [DRM_COLOR_YCBCR_BT2020] = {
1468                         .csc = {
1469                                 { 0x00400000, 0x0, 0x005E3F14, 0x0, 0xFFA1C0EB },
1470                                 { 0x00400000, 0xFFF577F6, 0xFFDB580F, 0x0, 0x002F2FFA },
1471                                 { 0x00400000, 0x007868DB, 0x0, 0x0, 0xFF879724 }
1472                         }
1473                 }
1474         }
1475 };
1476
1477 static int vc6_hvs_hw_init(struct vc4_hvs *hvs)
1478 {
1479         const struct vc6_csc_coeff_entry *coeffs;
1480         unsigned int i;
1481
1482         HVS_WRITE(SCALER6_CONTROL,
1483                   SCALER6_CONTROL_HVS_EN |
1484                   VC4_SET_FIELD(8, SCALER6_CONTROL_PF_LINES) |
1485                   VC4_SET_FIELD(15, SCALER6_CONTROL_MAX_REQS));
1486
1487         /* Set HVS arbiter priority to max */
1488         HVS_WRITE(SCALER6(PRI_MAP0), 0xffffffff);
1489         HVS_WRITE(SCALER6(PRI_MAP1), 0xffffffff);
1490
1491         if (hvs->vc4->gen == VC4_GEN_6_C) {
1492                 for (i = 0; i < 6; i++) {
1493                         coeffs = &csc_coeffs[i / 3][i % 3];
1494
1495                         HVS_WRITE(CFC1_N_MA_CSC_COEFF_C00(i), coeffs->csc[0][0]);
1496                         HVS_WRITE(CFC1_N_MA_CSC_COEFF_C01(i), coeffs->csc[0][1]);
1497                         HVS_WRITE(CFC1_N_MA_CSC_COEFF_C02(i), coeffs->csc[0][2]);
1498                         HVS_WRITE(CFC1_N_MA_CSC_COEFF_C03(i), coeffs->csc[0][3]);
1499                         HVS_WRITE(CFC1_N_MA_CSC_COEFF_C04(i), coeffs->csc[0][4]);
1500
1501                         HVS_WRITE(CFC1_N_MA_CSC_COEFF_C10(i), coeffs->csc[1][0]);
1502                         HVS_WRITE(CFC1_N_MA_CSC_COEFF_C11(i), coeffs->csc[1][1]);
1503                         HVS_WRITE(CFC1_N_MA_CSC_COEFF_C12(i), coeffs->csc[1][2]);
1504                         HVS_WRITE(CFC1_N_MA_CSC_COEFF_C13(i), coeffs->csc[1][3]);
1505                         HVS_WRITE(CFC1_N_MA_CSC_COEFF_C14(i), coeffs->csc[1][4]);
1506
1507                         HVS_WRITE(CFC1_N_MA_CSC_COEFF_C20(i), coeffs->csc[2][0]);
1508                         HVS_WRITE(CFC1_N_MA_CSC_COEFF_C21(i), coeffs->csc[2][1]);
1509                         HVS_WRITE(CFC1_N_MA_CSC_COEFF_C22(i), coeffs->csc[2][2]);
1510                         HVS_WRITE(CFC1_N_MA_CSC_COEFF_C23(i), coeffs->csc[2][3]);
1511                         HVS_WRITE(CFC1_N_MA_CSC_COEFF_C24(i), coeffs->csc[2][4]);
1512
1513                         HVS_WRITE(CFC1_N_NL_CSC_CTRL(i), BIT(15));
1514                 }
1515         } else {
1516                 for (i = 0; i < 8; i++) {
1517                         HVS_WRITE(SCALER_PI_CMP_CSC_RED0(i), 0x1f002566);
1518                         HVS_WRITE(SCALER_PI_CMP_CSC_RED1(i), 0x3994);
1519                         HVS_WRITE(SCALER_PI_CMP_CSC_RED_CLAMP(i), 0xfff00000);
1520                         HVS_WRITE(SCALER_PI_CMP_CSC_CFG(i), 0x1);
1521                         HVS_WRITE(SCALER_PI_CMP_CSC_GREEN0(i), 0x18002566);
1522                         HVS_WRITE(SCALER_PI_CMP_CSC_GREEN1(i), 0xf927eee2);
1523                         HVS_WRITE(SCALER_PI_CMP_CSC_GREEN_CLAMP(i), 0xfff00000);
1524                         HVS_WRITE(SCALER_PI_CMP_CSC_BLUE0(i), 0x18002566);
1525                         HVS_WRITE(SCALER_PI_CMP_CSC_BLUE1(i), 0x43d80000);
1526                         HVS_WRITE(SCALER_PI_CMP_CSC_BLUE_CLAMP(i), 0xfff00000);
1527                 }
1528         }
1529
1530         return 0;
1531 }
1532
1533 static int vc4_hvs_cob_init(struct vc4_hvs *hvs)
1534 {
1535         struct vc4_dev *vc4 = hvs->vc4;
1536         u32 reg, top, base;
1537
1538         /*
1539          * Recompute Composite Output Buffer (COB) allocations for the
1540          * displays
1541          */
1542         switch (vc4->gen) {
1543         case VC4_GEN_4:
1544                 /* The COB is 20736 pixels, or just over 10 lines at 2048 wide.
1545                  * The bottom 2048 pixels are full 32bpp RGBA (intended for the
1546                  * TXP composing RGBA to memory), whilst the remainder are only
1547                  * 24bpp RGB.
1548                  *
1549                  * Assign 3 lines to channels 1 & 2, and just over 4 lines to
1550                  * channel 0.
1551                  */
1552                 #define VC4_COB_SIZE            20736
1553                 #define VC4_COB_LINE_WIDTH      2048
1554                 #define VC4_COB_NUM_LINES       3
1555                 reg = 0;
1556                 top = VC4_COB_LINE_WIDTH * VC4_COB_NUM_LINES;
1557                 reg |= (top - 1) << 16;
1558                 HVS_WRITE(SCALER_DISPBASE2, reg);
1559                 reg = top;
1560                 top += VC4_COB_LINE_WIDTH * VC4_COB_NUM_LINES;
1561                 reg |= (top - 1) << 16;
1562                 HVS_WRITE(SCALER_DISPBASE1, reg);
1563                 reg = top;
1564                 top = VC4_COB_SIZE;
1565                 reg |= (top - 1) << 16;
1566                 HVS_WRITE(SCALER_DISPBASE0, reg);
1567                 break;
1568
1569         case VC4_GEN_5:
1570                 /* The COB is 44416 pixels, or 10.8 lines at 4096 wide.
1571                  * The bottom 4096 pixels are full RGBA (intended for the TXP
1572                  * composing RGBA to memory), whilst the remainder are only
1573                  * RGB. Addressing is always pixel wide.
1574                  *
1575                  * Assign 3 lines of 4096 to channels 1 & 2, and just over 4
1576                  * lines. to channel 0.
1577                  */
1578                 #define VC5_COB_SIZE            44416
1579                 #define VC5_COB_LINE_WIDTH      4096
1580                 #define VC5_COB_NUM_LINES       3
1581                 reg = 0;
1582                 top = VC5_COB_LINE_WIDTH * VC5_COB_NUM_LINES;
1583                 reg |= top << 16;
1584                 HVS_WRITE(SCALER_DISPBASE2, reg);
1585                 top += 16;
1586                 reg = top;
1587                 top += VC5_COB_LINE_WIDTH * VC5_COB_NUM_LINES;
1588                 reg |= top << 16;
1589                 HVS_WRITE(SCALER_DISPBASE1, reg);
1590                 top += 16;
1591                 reg = top;
1592                 top = VC5_COB_SIZE;
1593                 reg |= top << 16;
1594                 HVS_WRITE(SCALER_DISPBASE0, reg);
1595                 break;
1596
1597         case VC4_GEN_6_C:
1598         case VC4_GEN_6_D:
1599                 #define VC6_COB_LINE_WIDTH      3840
1600                 #define VC6_COB_NUM_LINES       4
1601                 base = 0;
1602                 top = 3840;
1603
1604                 HVS_WRITE(SCALER6_DISPX_COB(2),
1605                           VC4_SET_FIELD(top, SCALER6_DISPX_COB_TOP) |
1606                           VC4_SET_FIELD(base, SCALER6_DISPX_COB_BASE));
1607
1608                 base = top + 16;
1609                 top += VC6_COB_LINE_WIDTH * VC6_COB_NUM_LINES;
1610
1611                 HVS_WRITE(SCALER6_DISPX_COB(1),
1612                           VC4_SET_FIELD(top, SCALER6_DISPX_COB_TOP) |
1613                           VC4_SET_FIELD(base, SCALER6_DISPX_COB_BASE));
1614
1615                 base = top + 16;
1616                 top += VC6_COB_LINE_WIDTH * VC6_COB_NUM_LINES;
1617
1618                 HVS_WRITE(SCALER6_DISPX_COB(0),
1619                           VC4_SET_FIELD(top, SCALER6_DISPX_COB_TOP) |
1620                           VC4_SET_FIELD(base, SCALER6_DISPX_COB_BASE));
1621                 break;
1622
1623         default:
1624                 return -EINVAL;
1625         }
1626
1627         return 0;
1628 }
1629
1630 static int vc4_hvs_bind(struct device *dev, struct device *master, void *data)
1631 {
1632         struct platform_device *pdev = to_platform_device(dev);
1633         struct drm_device *drm = dev_get_drvdata(master);
1634         struct vc4_dev *vc4 = to_vc4_dev(drm);
1635         struct vc4_hvs *hvs = NULL;
1636         void __iomem *regs;
1637         int ret;
1638
1639         regs = vc4_ioremap_regs(pdev, 0);
1640         if (IS_ERR(regs))
1641                 return PTR_ERR(regs);
1642
1643         hvs = __vc4_hvs_alloc(vc4, regs, pdev);
1644         if (IS_ERR(hvs))
1645                 return PTR_ERR(hvs);
1646
1647         hvs->regset.base = hvs->regs;
1648
1649         if (vc4->gen == VC4_GEN_6_C) {
1650                 hvs->regset.regs = vc6_hvs_regs;
1651                 hvs->regset.nregs = ARRAY_SIZE(vc6_hvs_regs);
1652
1653                 if (VC4_GET_FIELD(HVS_READ(SCALER6_VERSION), SCALER6_VERSION) ==
1654                                                 SCALER6_VERSION_D0) {
1655                         vc4->gen = VC4_GEN_6_D;
1656                         hvs->regset.regs = vc6_d_hvs_regs;
1657                         hvs->regset.nregs = ARRAY_SIZE(vc6_d_hvs_regs);
1658                 }
1659         } else {
1660                 hvs->regset.regs = vc4_hvs_regs;
1661                 hvs->regset.nregs = ARRAY_SIZE(vc4_hvs_regs);
1662         }
1663
1664         if (vc4->gen >= VC4_GEN_5) {
1665                 struct rpi_firmware *firmware;
1666                 struct device_node *node;
1667                 unsigned int max_rate;
1668
1669                 node = rpi_firmware_find_node();
1670                 if (!node)
1671                         return -EINVAL;
1672
1673                 firmware = rpi_firmware_get(node);
1674                 of_node_put(node);
1675                 if (!firmware)
1676                         return -EPROBE_DEFER;
1677
1678                 hvs->core_clk = devm_clk_get(&pdev->dev,
1679                                              (vc4->gen >= VC4_GEN_6_C) ? "core" : NULL);
1680                 if (IS_ERR(hvs->core_clk)) {
1681                         dev_err(&pdev->dev, "Couldn't get core clock\n");
1682                         return PTR_ERR(hvs->core_clk);
1683                 }
1684
1685                 hvs->disp_clk = devm_clk_get(&pdev->dev,
1686                                              (vc4->gen >= VC4_GEN_6_C) ? "disp" : NULL);
1687                 if (IS_ERR(hvs->disp_clk)) {
1688                         dev_err(&pdev->dev, "Couldn't get disp clock\n");
1689                         return PTR_ERR(hvs->disp_clk);
1690                 }
1691
1692                 max_rate = rpi_firmware_clk_get_max_rate(firmware,
1693                                                          RPI_FIRMWARE_CORE_CLK_ID);
1694                 rpi_firmware_put(firmware);
1695                 if (max_rate >= 550000000)
1696                         hvs->vc5_hdmi_enable_hdmi_20 = true;
1697
1698                 if (max_rate >= 600000000)
1699                         hvs->vc5_hdmi_enable_4096by2160 = true;
1700
1701                 hvs->max_core_rate = max_rate;
1702
1703                 ret = clk_prepare_enable(hvs->core_clk);
1704                 if (ret) {
1705                         dev_err(&pdev->dev, "Couldn't enable the core clock\n");
1706                         return ret;
1707                 }
1708
1709                 ret = clk_prepare_enable(hvs->disp_clk);
1710                 if (ret) {
1711                         dev_err(&pdev->dev, "Couldn't enable the disp clock\n");
1712                         return ret;
1713                 }
1714         }
1715
1716         if (vc4->gen >= VC4_GEN_5)
1717                 hvs->dlist = hvs->regs + SCALER5_DLIST_START;
1718         else
1719                 hvs->dlist = hvs->regs + SCALER_DLIST_START;
1720
1721         if (vc4->gen >= VC4_GEN_6_C)
1722                 ret = vc6_hvs_hw_init(hvs);
1723         else
1724                 ret = vc4_hvs_hw_init(hvs);
1725         if (ret)
1726                 return ret;
1727
1728         /* Upload filter kernels.  We only have the one for now, so we
1729          * keep it around for the lifetime of the driver.
1730          */
1731         ret = vc4_hvs_upload_linear_kernel(hvs,
1732                                            &hvs->mitchell_netravali_filter,
1733                                            mitchell_netravali_1_3_1_3_kernel);
1734         if (ret)
1735                 return ret;
1736
1737         ret = vc4_hvs_cob_init(hvs);
1738         if (ret)
1739                 return ret;
1740
1741         if (vc4->gen < VC4_GEN_6_C) {
1742                 ret = devm_request_irq(dev, platform_get_irq(pdev, 0),
1743                                        vc4_hvs_irq_handler, 0, "vc4 hvs", drm);
1744                 if (ret)
1745                         return ret;
1746         }
1747
1748         return 0;
1749 }
1750
1751 static void vc4_hvs_unbind(struct device *dev, struct device *master,
1752                            void *data)
1753 {
1754         struct drm_device *drm = dev_get_drvdata(master);
1755         struct vc4_dev *vc4 = to_vc4_dev(drm);
1756         struct vc4_hvs *hvs = vc4->hvs;
1757         struct drm_mm_node *node, *next;
1758
1759         if (drm_mm_node_allocated(&vc4->hvs->mitchell_netravali_filter))
1760                 drm_mm_remove_node(&vc4->hvs->mitchell_netravali_filter);
1761
1762         drm_mm_for_each_node_safe(node, next, &vc4->hvs->dlist_mm)
1763                 drm_mm_remove_node(node);
1764
1765         drm_mm_takedown(&vc4->hvs->dlist_mm);
1766
1767         drm_mm_for_each_node_safe(node, next, &vc4->hvs->lbm_mm)
1768                 drm_mm_remove_node(node);
1769         drm_mm_takedown(&vc4->hvs->lbm_mm);
1770
1771         clk_disable_unprepare(hvs->disp_clk);
1772         clk_disable_unprepare(hvs->core_clk);
1773
1774         vc4->hvs = NULL;
1775 }
1776
1777 static const struct component_ops vc4_hvs_ops = {
1778         .bind   = vc4_hvs_bind,
1779         .unbind = vc4_hvs_unbind,
1780 };
1781
1782 static int vc4_hvs_dev_probe(struct platform_device *pdev)
1783 {
1784         return component_add(&pdev->dev, &vc4_hvs_ops);
1785 }
1786
1787 static void vc4_hvs_dev_remove(struct platform_device *pdev)
1788 {
1789         component_del(&pdev->dev, &vc4_hvs_ops);
1790 }
1791
1792 static const struct of_device_id vc4_hvs_dt_match[] = {
1793         { .compatible = "brcm,bcm2711-hvs" },
1794         { .compatible = "brcm,bcm2712-hvs" },
1795         { .compatible = "brcm,bcm2835-hvs" },
1796         {}
1797 };
1798
1799 struct platform_driver vc4_hvs_driver = {
1800         .probe = vc4_hvs_dev_probe,
1801         .remove = vc4_hvs_dev_remove,
1802         .driver = {
1803                 .name = "vc4_hvs",
1804                 .of_match_table = vc4_hvs_dt_match,
1805         },
1806 };