drivers/gpu/drm/i915/display/intel_fb.c

   1 // SPDX-License-Identifier: MIT
   2 /*
   3  * Copyright © 2021 Intel Corporation
   4  */
   5
   6 #include <linux/dma-fence.h>
   7 #include <linux/dma-resv.h>
   8
   9 #include <drm/drm_blend.h>
  10 #include <drm/drm_gem.h>
  11 #include <drm/drm_modeset_helper.h>
  12
  13 #include "i915_drv.h"
  14 #include "intel_atomic_plane.h"
  15 #include "intel_bo.h"
  16 #include "intel_display.h"
  17 #include "intel_display_types.h"
  18 #include "intel_dpt.h"
  19 #include "intel_fb.h"
  20 #include "intel_fb_bo.h"
  21 #include "intel_frontbuffer.h"
  22
  23 #define check_array_bounds(i915, a, i) drm_WARN_ON(&(i915)->drm, (i) >= ARRAY_SIZE(a))
  24
  25 /*
  26  * From the Sky Lake PRM:
  27  * "The Color Control Surface (CCS) contains the compression status of
  28  *  the cache-line pairs. The compression state of the cache-line pair
  29  *  is specified by 2 bits in the CCS. Each CCS cache-line represents
  30  *  an area on the main surface of 16 x16 sets of 128 byte Y-tiled
  31  *  cache-line-pairs. CCS is always Y tiled."
  32  *
  33  * Since cache line pairs refers to horizontally adjacent cache lines,
  34  * each cache line in the CCS corresponds to an area of 32x16 cache
  35  * lines on the main surface. Since each pixel is 4 bytes, this gives
  36  * us a ratio of one byte in the CCS for each 8x16 pixels in the
  37  * main surface.
  38  */
  39 static const struct drm_format_info skl_ccs_formats[] = {
  40         { .format = DRM_FORMAT_XRGB8888, .depth = 24, .num_planes = 2,
  41           .cpp = { 4, 1, }, .hsub = 8, .vsub = 16, },
  42         { .format = DRM_FORMAT_XBGR8888, .depth = 24, .num_planes = 2,
  43           .cpp = { 4, 1, }, .hsub = 8, .vsub = 16, },
  44         { .format = DRM_FORMAT_ARGB8888, .depth = 32, .num_planes = 2,
  45           .cpp = { 4, 1, }, .hsub = 8, .vsub = 16, .has_alpha = true, },
  46         { .format = DRM_FORMAT_ABGR8888, .depth = 32, .num_planes = 2,
  47           .cpp = { 4, 1, }, .hsub = 8, .vsub = 16, .has_alpha = true, },
  48         { .format = DRM_FORMAT_XRGB2101010, .depth = 30, .num_planes = 2,
  49           .cpp = { 4, 1, }, .hsub = 8, .vsub = 16, },
  50         { .format = DRM_FORMAT_XBGR2101010, .depth = 30, .num_planes = 2,
  51           .cpp = { 4, 1, }, .hsub = 8, .vsub = 16, },
  52         { .format = DRM_FORMAT_ARGB2101010, .depth = 30, .num_planes = 2,
  53           .cpp = { 4, 1, }, .hsub = 8, .vsub = 16, .has_alpha = true, },
  54         { .format = DRM_FORMAT_ABGR2101010, .depth = 30, .num_planes = 2,
  55           .cpp = { 4, 1, }, .hsub = 8, .vsub = 16, .has_alpha = true, },
  56 };
  57
  58 /*
  59  * Gen-12 compression uses 4 bits of CCS data for each cache line pair in the
  60  * main surface. And each 64B CCS cache line represents an area of 4x1 Y-tiles
  61  * in the main surface. With 4 byte pixels and each Y-tile having dimensions of
  62  * 32x32 pixels, the ratio turns out to 1B in the CCS for every 2x32 pixels in
  63  * the main surface.
  64  */
  65 static const struct drm_format_info gen12_ccs_formats[] = {
  66         { .format = DRM_FORMAT_XRGB8888, .depth = 24, .num_planes = 2,
  67           .char_per_block = { 4, 1 }, .block_w = { 1, 2 }, .block_h = { 1, 1 },
  68           .hsub = 1, .vsub = 1, },
  69         { .format = DRM_FORMAT_XBGR8888, .depth = 24, .num_planes = 2,
  70           .char_per_block = { 4, 1 }, .block_w = { 1, 2 }, .block_h = { 1, 1 },
  71           .hsub = 1, .vsub = 1, },
  72         { .format = DRM_FORMAT_ARGB8888, .depth = 32, .num_planes = 2,
  73           .char_per_block = { 4, 1 }, .block_w = { 1, 2 }, .block_h = { 1, 1 },
  74           .hsub = 1, .vsub = 1, .has_alpha = true },
  75         { .format = DRM_FORMAT_ABGR8888, .depth = 32, .num_planes = 2,
  76           .char_per_block = { 4, 1 }, .block_w = { 1, 2 }, .block_h = { 1, 1 },
  77           .hsub = 1, .vsub = 1, .has_alpha = true },
  78         { .format = DRM_FORMAT_XRGB2101010, .depth = 30, .num_planes = 2,
  79           .char_per_block = { 4, 1 }, .block_w = { 1, 2 }, .block_h = { 1, 1 },
  80           .hsub = 1, .vsub = 1, },
  81         { .format = DRM_FORMAT_XBGR2101010, .depth = 30, .num_planes = 2,
  82           .char_per_block = { 4, 1 }, .block_w = { 1, 2 }, .block_h = { 1, 1 },
  83           .hsub = 1, .vsub = 1, },
  84         { .format = DRM_FORMAT_ARGB2101010, .depth = 30, .num_planes = 2,
  85           .char_per_block = { 4, 1 }, .block_w = { 1, 2 }, .block_h = { 1, 1 },
  86           .hsub = 1, .vsub = 1, .has_alpha = true },
  87         { .format = DRM_FORMAT_ABGR2101010, .depth = 30, .num_planes = 2,
  88           .char_per_block = { 4, 1 }, .block_w = { 1, 2 }, .block_h = { 1, 1 },
  89           .hsub = 1, .vsub = 1, .has_alpha = true },
  90         { .format = DRM_FORMAT_XRGB16161616F, .depth = 0, .num_planes = 2,
  91           .char_per_block = { 8, 1 }, .block_w = { 1, 1 }, .block_h = { 1, 1 },
  92           .hsub = 1, .vsub = 1, },
  93         { .format = DRM_FORMAT_XBGR16161616F, .depth = 0, .num_planes = 2,
  94           .char_per_block = { 8, 1 }, .block_w = { 1, 1 }, .block_h = { 1, 1 },
  95           .hsub = 1, .vsub = 1, },
  96         { .format = DRM_FORMAT_ARGB16161616F, .depth = 0, .num_planes = 2,
  97           .char_per_block = { 8, 1 }, .block_w = { 1, 1 }, .block_h = { 1, 1 },
  98           .hsub = 1, .vsub = 1, .has_alpha = true },
  99         { .format = DRM_FORMAT_ABGR16161616F, .depth = 0, .num_planes = 2,
 100           .char_per_block = { 8, 1 }, .block_w = { 1, 1 }, .block_h = { 1, 1 },
 101           .hsub = 1, .vsub = 1, .has_alpha = true },
 102         { .format = DRM_FORMAT_YUYV, .num_planes = 2,
 103           .char_per_block = { 2, 1 }, .block_w = { 1, 2 }, .block_h = { 1, 1 },
 104           .hsub = 2, .vsub = 1, .is_yuv = true },
 105         { .format = DRM_FORMAT_YVYU, .num_planes = 2,
 106           .char_per_block = { 2, 1 }, .block_w = { 1, 2 }, .block_h = { 1, 1 },
 107           .hsub = 2, .vsub = 1, .is_yuv = true },
 108         { .format = DRM_FORMAT_UYVY, .num_planes = 2,
 109           .char_per_block = { 2, 1 }, .block_w = { 1, 2 }, .block_h = { 1, 1 },
 110           .hsub = 2, .vsub = 1, .is_yuv = true },
 111         { .format = DRM_FORMAT_VYUY, .num_planes = 2,
 112           .char_per_block = { 2, 1 }, .block_w = { 1, 2 }, .block_h = { 1, 1 },
 113           .hsub = 2, .vsub = 1, .is_yuv = true },
 114         { .format = DRM_FORMAT_XYUV8888, .num_planes = 2,
 115           .char_per_block = { 4, 1 }, .block_w = { 1, 2 }, .block_h = { 1, 1 },
 116           .hsub = 1, .vsub = 1, .is_yuv = true },
 117         { .format = DRM_FORMAT_NV12, .num_planes = 4,
 118           .char_per_block = { 1, 2, 1, 1 }, .block_w = { 1, 1, 4, 4 }, .block_h = { 1, 1, 1, 1 },
 119           .hsub = 2, .vsub = 2, .is_yuv = true },
 120         { .format = DRM_FORMAT_P010, .num_planes = 4,
 121           .char_per_block = { 2, 4, 1, 1 }, .block_w = { 1, 1, 2, 2 }, .block_h = { 1, 1, 1, 1 },
 122           .hsub = 2, .vsub = 2, .is_yuv = true },
 123         { .format = DRM_FORMAT_P012, .num_planes = 4,
 124           .char_per_block = { 2, 4, 1, 1 }, .block_w = { 1, 1, 2, 2 }, .block_h = { 1, 1, 1, 1 },
 125           .hsub = 2, .vsub = 2, .is_yuv = true },
 126         { .format = DRM_FORMAT_P016, .num_planes = 4,
 127           .char_per_block = { 2, 4, 1, 1 }, .block_w = { 1, 1, 2, 2 }, .block_h = { 1, 1, 1, 1 },
 128           .hsub = 2, .vsub = 2, .is_yuv = true },
 129 };
 130
 131 /*
 132  * Same as gen12_ccs_formats[] above, but with additional surface used
 133  * to pass Clear Color information in plane 2 with 64 bits of data.
 134  */
 135 static const struct drm_format_info gen12_ccs_cc_formats[] = {
 136         { .format = DRM_FORMAT_XRGB8888, .depth = 24, .num_planes = 3,
 137           .char_per_block = { 4, 1, 0 }, .block_w = { 1, 2, 0 }, .block_h = { 1, 1, 0 },
 138           .hsub = 1, .vsub = 1, },
 139         { .format = DRM_FORMAT_XBGR8888, .depth = 24, .num_planes = 3,
 140           .char_per_block = { 4, 1, 0 }, .block_w = { 1, 2, 0 }, .block_h = { 1, 1, 0 },
 141           .hsub = 1, .vsub = 1, },
 142         { .format = DRM_FORMAT_ARGB8888, .depth = 32, .num_planes = 3,
 143           .char_per_block = { 4, 1, 0 }, .block_w = { 1, 2, 0 }, .block_h = { 1, 1, 0 },
 144           .hsub = 1, .vsub = 1, .has_alpha = true },
 145         { .format = DRM_FORMAT_ABGR8888, .depth = 32, .num_planes = 3,
 146           .char_per_block = { 4, 1, 0 }, .block_w = { 1, 2, 0 }, .block_h = { 1, 1, 0 },
 147           .hsub = 1, .vsub = 1, .has_alpha = true },
 148         { .format = DRM_FORMAT_XRGB2101010, .depth = 30, .num_planes = 3,
 149           .char_per_block = { 4, 1, 0 }, .block_w = { 1, 2, 0 }, .block_h = { 1, 1, 0 },
 150           .hsub = 1, .vsub = 1, },
 151         { .format = DRM_FORMAT_XBGR2101010, .depth = 30, .num_planes = 3,
 152           .char_per_block = { 4, 1, 0 }, .block_w = { 1, 2, 0 }, .block_h = { 1, 1, 0 },
 153           .hsub = 1, .vsub = 1, },
 154         { .format = DRM_FORMAT_ARGB2101010, .depth = 30, .num_planes = 3,
 155           .char_per_block = { 4, 1, 0 }, .block_w = { 1, 2, 0 }, .block_h = { 1, 1, 0 },
 156           .hsub = 1, .vsub = 1, .has_alpha = true },
 157         { .format = DRM_FORMAT_ABGR2101010, .depth = 30, .num_planes = 3,
 158           .char_per_block = { 4, 1, 0 }, .block_w = { 1, 2, 0 }, .block_h = { 1, 1, 0 },
 159           .hsub = 1, .vsub = 1, .has_alpha = true },
 160         { .format = DRM_FORMAT_XRGB16161616F, .depth = 0, .num_planes = 3,
 161           .char_per_block = { 8, 1, 0 }, .block_w = { 1, 1, 0 }, .block_h = { 1, 1, 0 },
 162           .hsub = 1, .vsub = 1, },
 163         { .format = DRM_FORMAT_XBGR16161616F, .depth = 0, .num_planes = 3,
 164           .char_per_block = { 8, 1, 0 }, .block_w = { 1, 1, 0 }, .block_h = { 1, 1, 0 },
 165           .hsub = 1, .vsub = 1, },
 166         { .format = DRM_FORMAT_ARGB16161616F, .depth = 0, .num_planes = 3,
 167           .char_per_block = { 8, 1, 0 }, .block_w = { 1, 1, 0 }, .block_h = { 1, 1, 0 },
 168           .hsub = 1, .vsub = 1, .has_alpha = true },
 169         { .format = DRM_FORMAT_ABGR16161616F, .depth = 0, .num_planes = 3,
 170           .char_per_block = { 8, 1, 0 }, .block_w = { 1, 1, 0 }, .block_h = { 1, 1, 0 },
 171           .hsub = 1, .vsub = 1, .has_alpha = true },
 172 };
 173
 174 static const struct drm_format_info gen12_flat_ccs_cc_formats[] = {
 175         { .format = DRM_FORMAT_XRGB8888, .depth = 24, .num_planes = 2,
 176           .char_per_block = { 4, 0 }, .block_w = { 1, 0 }, .block_h = { 1, 0 },
 177           .hsub = 1, .vsub = 1, },
 178         { .format = DRM_FORMAT_XBGR8888, .depth = 24, .num_planes = 2,
 179           .char_per_block = { 4, 0 }, .block_w = { 1, 0 }, .block_h = { 1, 0 },
 180           .hsub = 1, .vsub = 1, },
 181         { .format = DRM_FORMAT_ARGB8888, .depth = 32, .num_planes = 2,
 182           .char_per_block = { 4, 0 }, .block_w = { 1, 0 }, .block_h = { 1, 0 },
 183           .hsub = 1, .vsub = 1, .has_alpha = true },
 184         { .format = DRM_FORMAT_ABGR8888, .depth = 32, .num_planes = 2,
 185           .char_per_block = { 4, 0 }, .block_w = { 1, 0 }, .block_h = { 1, 0 },
 186           .hsub = 1, .vsub = 1, .has_alpha = true },
 187         { .format = DRM_FORMAT_XRGB2101010, .depth = 30, .num_planes = 2,
 188           .char_per_block = { 4, 0 }, .block_w = { 1, 0 }, .block_h = { 1, 0 },
 189           .hsub = 1, .vsub = 1, },
 190         { .format = DRM_FORMAT_XBGR2101010, .depth = 30, .num_planes = 2,
 191           .char_per_block = { 4, 0 }, .block_w = { 1, 0 }, .block_h = { 1, 0 },
 192           .hsub = 1, .vsub = 1, },
 193         { .format = DRM_FORMAT_ARGB2101010, .depth = 30, .num_planes = 2,
 194           .char_per_block = { 4, 0 }, .block_w = { 1, 0 }, .block_h = { 1, 0 },
 195           .hsub = 1, .vsub = 1, .has_alpha = true },
 196         { .format = DRM_FORMAT_ABGR2101010, .depth = 30, .num_planes = 2,
 197           .char_per_block = { 4, 0 }, .block_w = { 1, 0 }, .block_h = { 1, 0 },
 198           .hsub = 1, .vsub = 1, .has_alpha = true },
 199         { .format = DRM_FORMAT_XRGB16161616F, .depth = 0, .num_planes = 2,
 200           .char_per_block = { 8, 0 }, .block_w = { 1, 0 }, .block_h = { 1, 0 },
 201           .hsub = 1, .vsub = 1, },
 202         { .format = DRM_FORMAT_XBGR16161616F, .depth = 0, .num_planes = 2,
 203           .char_per_block = { 8, 0 }, .block_w = { 1, 0 }, .block_h = { 1, 0 },
 204           .hsub = 1, .vsub = 1, },
 205         { .format = DRM_FORMAT_ARGB16161616F, .depth = 0, .num_planes = 2,
 206           .char_per_block = { 8, 0 }, .block_w = { 1, 0 }, .block_h = { 1, 0 },
 207           .hsub = 1, .vsub = 1, .has_alpha = true },
 208         { .format = DRM_FORMAT_ABGR16161616F, .depth = 0, .num_planes = 2,
 209           .char_per_block = { 8, 0 }, .block_w = { 1, 0 }, .block_h = { 1, 0 },
 210           .hsub = 1, .vsub = 1, .has_alpha = true },
 211 };
 212
 213 struct intel_modifier_desc {
 214         u64 modifier;
 215         struct {
 216                 u8 from;
 217                 u8 until;
 218         } display_ver;
 219 #define DISPLAY_VER_ALL         { 0, -1 }
 220
 221         const struct drm_format_info *formats;
 222         int format_count;
 223 #define FORMAT_OVERRIDE(format_list) \
 224         .formats = format_list, \
 225         .format_count = ARRAY_SIZE(format_list)
 226
 227         u8 plane_caps;
 228
 229         struct {
 230                 u8 cc_planes:3;
 231                 u8 packed_aux_planes:4;
 232                 u8 planar_aux_planes:4;
 233         } ccs;
 234 };
 235
 236 #define INTEL_PLANE_CAP_CCS_MASK        (INTEL_PLANE_CAP_CCS_RC | \
 237                                          INTEL_PLANE_CAP_CCS_RC_CC | \
 238                                          INTEL_PLANE_CAP_CCS_MC)
 239 #define INTEL_PLANE_CAP_TILING_MASK     (INTEL_PLANE_CAP_TILING_X | \
 240                                          INTEL_PLANE_CAP_TILING_Y | \
 241                                          INTEL_PLANE_CAP_TILING_Yf | \
 242                                          INTEL_PLANE_CAP_TILING_4)
 243 #define INTEL_PLANE_CAP_TILING_NONE     0
 244
 245 static const struct intel_modifier_desc intel_modifiers[] = {
 246         {
 247                 .modifier = I915_FORMAT_MOD_4_TILED_LNL_CCS,
 248                 .display_ver = { 20, -1 },
 249                 .plane_caps = INTEL_PLANE_CAP_TILING_4,
 250         }, {
 251                 .modifier = I915_FORMAT_MOD_4_TILED_BMG_CCS,
 252                 .display_ver = { 14, -1 },
 253                 .plane_caps = INTEL_PLANE_CAP_TILING_4 | INTEL_PLANE_CAP_NEED64K_PHYS,
 254         }, {
 255                 .modifier = I915_FORMAT_MOD_4_TILED_MTL_MC_CCS,
 256                 .display_ver = { 14, 14 },
 257                 .plane_caps = INTEL_PLANE_CAP_TILING_4 | INTEL_PLANE_CAP_CCS_MC,
 258
 259                 .ccs.packed_aux_planes = BIT(1),
 260                 .ccs.planar_aux_planes = BIT(2) | BIT(3),
 261
 262                 FORMAT_OVERRIDE(gen12_ccs_formats),
 263         }, {
 264                 .modifier = I915_FORMAT_MOD_4_TILED_MTL_RC_CCS,
 265                 .display_ver = { 14, 14 },
 266                 .plane_caps = INTEL_PLANE_CAP_TILING_4 | INTEL_PLANE_CAP_CCS_RC,
 267
 268                 .ccs.packed_aux_planes = BIT(1),
 269
 270                 FORMAT_OVERRIDE(gen12_ccs_formats),
 271         }, {
 272                 .modifier = I915_FORMAT_MOD_4_TILED_MTL_RC_CCS_CC,
 273                 .display_ver = { 14, 14 },
 274                 .plane_caps = INTEL_PLANE_CAP_TILING_4 | INTEL_PLANE_CAP_CCS_RC_CC,
 275
 276                 .ccs.cc_planes = BIT(2),
 277                 .ccs.packed_aux_planes = BIT(1),
 278
 279                 FORMAT_OVERRIDE(gen12_ccs_cc_formats),
 280         }, {
 281                 .modifier = I915_FORMAT_MOD_4_TILED_DG2_MC_CCS,
 282                 .display_ver = { 13, 13 },
 283                 .plane_caps = INTEL_PLANE_CAP_TILING_4 | INTEL_PLANE_CAP_CCS_MC,
 284         }, {
 285                 .modifier = I915_FORMAT_MOD_4_TILED_DG2_RC_CCS_CC,
 286                 .display_ver = { 13, 13 },
 287                 .plane_caps = INTEL_PLANE_CAP_TILING_4 | INTEL_PLANE_CAP_CCS_RC_CC,
 288
 289                 .ccs.cc_planes = BIT(1),
 290
 291                 FORMAT_OVERRIDE(gen12_flat_ccs_cc_formats),
 292         }, {
 293                 .modifier = I915_FORMAT_MOD_4_TILED_DG2_RC_CCS,
 294                 .display_ver = { 13, 13 },
 295                 .plane_caps = INTEL_PLANE_CAP_TILING_4 | INTEL_PLANE_CAP_CCS_RC,
 296         }, {
 297                 .modifier = I915_FORMAT_MOD_4_TILED,
 298                 .display_ver = { 13, -1 },
 299                 .plane_caps = INTEL_PLANE_CAP_TILING_4,
 300         }, {
 301                 .modifier = I915_FORMAT_MOD_Y_TILED_GEN12_MC_CCS,
 302                 .display_ver = { 12, 13 },
 303                 .plane_caps = INTEL_PLANE_CAP_TILING_Y | INTEL_PLANE_CAP_CCS_MC,
 304
 305                 .ccs.packed_aux_planes = BIT(1),
 306                 .ccs.planar_aux_planes = BIT(2) | BIT(3),
 307
 308                 FORMAT_OVERRIDE(gen12_ccs_formats),
 309         }, {
 310                 .modifier = I915_FORMAT_MOD_Y_TILED_GEN12_RC_CCS,
 311                 .display_ver = { 12, 13 },
 312                 .plane_caps = INTEL_PLANE_CAP_TILING_Y | INTEL_PLANE_CAP_CCS_RC,
 313
 314                 .ccs.packed_aux_planes = BIT(1),
 315
 316                 FORMAT_OVERRIDE(gen12_ccs_formats),
 317         }, {
 318                 .modifier = I915_FORMAT_MOD_Y_TILED_GEN12_RC_CCS_CC,
 319                 .display_ver = { 12, 13 },
 320                 .plane_caps = INTEL_PLANE_CAP_TILING_Y | INTEL_PLANE_CAP_CCS_RC_CC,
 321
 322                 .ccs.cc_planes = BIT(2),
 323                 .ccs.packed_aux_planes = BIT(1),
 324
 325                 FORMAT_OVERRIDE(gen12_ccs_cc_formats),
 326         }, {
 327                 .modifier = I915_FORMAT_MOD_Yf_TILED_CCS,
 328                 .display_ver = { 9, 11 },
 329                 .plane_caps = INTEL_PLANE_CAP_TILING_Yf | INTEL_PLANE_CAP_CCS_RC,
 330
 331                 .ccs.packed_aux_planes = BIT(1),
 332
 333                 FORMAT_OVERRIDE(skl_ccs_formats),
 334         }, {
 335                 .modifier = I915_FORMAT_MOD_Y_TILED_CCS,
 336                 .display_ver = { 9, 11 },
 337                 .plane_caps = INTEL_PLANE_CAP_TILING_Y | INTEL_PLANE_CAP_CCS_RC,
 338
 339                 .ccs.packed_aux_planes = BIT(1),
 340
 341                 FORMAT_OVERRIDE(skl_ccs_formats),
 342         }, {
 343                 .modifier = I915_FORMAT_MOD_Yf_TILED,
 344                 .display_ver = { 9, 11 },
 345                 .plane_caps = INTEL_PLANE_CAP_TILING_Yf,
 346         }, {
 347                 .modifier = I915_FORMAT_MOD_Y_TILED,
 348                 .display_ver = { 9, 13 },
 349                 .plane_caps = INTEL_PLANE_CAP_TILING_Y,
 350         }, {
 351                 .modifier = I915_FORMAT_MOD_X_TILED,
 352                 .display_ver = { 0, 29 },
 353                 .plane_caps = INTEL_PLANE_CAP_TILING_X,
 354         }, {
 355                 .modifier = DRM_FORMAT_MOD_LINEAR,
 356                 .display_ver = DISPLAY_VER_ALL,
 357         },
 358 };
 359
 360 static const struct intel_modifier_desc *lookup_modifier_or_null(u64 modifier)
 361 {
 362         int i;
 363
 364         for (i = 0; i < ARRAY_SIZE(intel_modifiers); i++)
 365                 if (intel_modifiers[i].modifier == modifier)
 366                         return &intel_modifiers[i];
 367
 368         return NULL;
 369 }
 370
 371 static const struct intel_modifier_desc *lookup_modifier(u64 modifier)
 372 {
 373         const struct intel_modifier_desc *md = lookup_modifier_or_null(modifier);
 374
 375         if (WARN_ON(!md))
 376                 return &intel_modifiers[0];
 377
 378         return md;
 379 }
 380
 381 static const struct drm_format_info *
 382 lookup_format_info(const struct drm_format_info formats[],
 383                    int num_formats, u32 format)
 384 {
 385         int i;
 386
 387         for (i = 0; i < num_formats; i++) {
 388                 if (formats[i].format == format)
 389                         return &formats[i];
 390         }
 391
 392         return NULL;
 393 }
 394
 395 unsigned int intel_fb_modifier_to_tiling(u64 fb_modifier)
 396 {
 397         const struct intel_modifier_desc *md;
 398         u8 tiling_caps;
 399
 400         md = lookup_modifier_or_null(fb_modifier);
 401         if (!md)
 402                 return I915_TILING_NONE;
 403
 404         tiling_caps = lookup_modifier_or_null(fb_modifier)->plane_caps &
 405                          INTEL_PLANE_CAP_TILING_MASK;
 406
 407         switch (tiling_caps) {
 408         case INTEL_PLANE_CAP_TILING_Y:
 409                 return I915_TILING_Y;
 410         case INTEL_PLANE_CAP_TILING_X:
 411                 return I915_TILING_X;
 412         case INTEL_PLANE_CAP_TILING_4:
 413         case INTEL_PLANE_CAP_TILING_Yf:
 414         case INTEL_PLANE_CAP_TILING_NONE:
 415                 return I915_TILING_NONE;
 416         default:
 417                 MISSING_CASE(tiling_caps);
 418                 return I915_TILING_NONE;
 419         }
 420 }
 421
 422 /**
 423  * intel_fb_get_format_info: Get a modifier specific format information
 424  * @cmd: FB add command structure
 425  *
 426  * Returns:
 427  * Returns the format information for @cmd->pixel_format specific to @cmd->modifier[0],
 428  * or %NULL if the modifier doesn't override the format.
 429  */
 430 const struct drm_format_info *
 431 intel_fb_get_format_info(const struct drm_mode_fb_cmd2 *cmd)
 432 {
 433         const struct intel_modifier_desc *md = lookup_modifier_or_null(cmd->modifier[0]);
 434
 435         if (!md || !md->formats)
 436                 return NULL;
 437
 438         return lookup_format_info(md->formats, md->format_count, cmd->pixel_format);
 439 }
 440
 441 static bool plane_caps_contain_any(u8 caps, u8 mask)
 442 {
 443         return caps & mask;
 444 }
 445
 446 static bool plane_caps_contain_all(u8 caps, u8 mask)
 447 {
 448         return (caps & mask) == mask;
 449 }
 450
 451 /**
 452  * intel_fb_is_tiled_modifier: Check if a modifier is a tiled modifier type
 453  * @modifier: Modifier to check
 454  *
 455  * Returns:
 456  * Returns %true if @modifier is a tiled modifier.
 457  */
 458 bool intel_fb_is_tiled_modifier(u64 modifier)
 459 {
 460         return plane_caps_contain_any(lookup_modifier(modifier)->plane_caps,
 461                                       INTEL_PLANE_CAP_TILING_MASK);
 462 }
 463
 464 /**
 465  * intel_fb_is_ccs_modifier: Check if a modifier is a CCS modifier type
 466  * @modifier: Modifier to check
 467  *
 468  * Returns:
 469  * Returns %true if @modifier is a render, render with color clear or
 470  * media compression modifier.
 471  */
 472 bool intel_fb_is_ccs_modifier(u64 modifier)
 473 {
 474         return plane_caps_contain_any(lookup_modifier(modifier)->plane_caps,
 475                                       INTEL_PLANE_CAP_CCS_MASK);
 476 }
 477
 478 /**
 479  * intel_fb_is_rc_ccs_cc_modifier: Check if a modifier is an RC CCS CC modifier type
 480  * @modifier: Modifier to check
 481  *
 482  * Returns:
 483  * Returns %true if @modifier is a render with color clear modifier.
 484  */
 485 bool intel_fb_is_rc_ccs_cc_modifier(u64 modifier)
 486 {
 487         return plane_caps_contain_any(lookup_modifier(modifier)->plane_caps,
 488                                       INTEL_PLANE_CAP_CCS_RC_CC);
 489 }
 490
 491 /**
 492  * intel_fb_is_mc_ccs_modifier: Check if a modifier is an MC CCS modifier type
 493  * @modifier: Modifier to check
 494  *
 495  * Returns:
 496  * Returns %true if @modifier is a media compression modifier.
 497  */
 498 bool intel_fb_is_mc_ccs_modifier(u64 modifier)
 499 {
 500         return plane_caps_contain_any(lookup_modifier(modifier)->plane_caps,
 501                                       INTEL_PLANE_CAP_CCS_MC);
 502 }
 503
 504 /**
 505  * intel_fb_needs_64k_phys: Check if modifier requires 64k physical placement.
 506  * @modifier: Modifier to check
 507  *
 508  * Returns:
 509  * Returns %true if @modifier requires 64k aligned physical pages.
 510  */
 511 bool intel_fb_needs_64k_phys(u64 modifier)
 512 {
 513         const struct intel_modifier_desc *md = lookup_modifier_or_null(modifier);
 514
 515         if (!md)
 516                 return false;
 517
 518         return plane_caps_contain_any(md->plane_caps,
 519                                       INTEL_PLANE_CAP_NEED64K_PHYS);
 520 }
 521
 522 /**
 523  * intel_fb_is_tile4_modifier: Check if a modifier is a tile4 modifier type
 524  * @modifier: Modifier to check
 525  *
 526  * Returns:
 527  * Returns %true if @modifier is a tile4 modifier.
 528  */
 529 bool intel_fb_is_tile4_modifier(u64 modifier)
 530 {
 531         return plane_caps_contain_any(lookup_modifier(modifier)->plane_caps,
 532                                       INTEL_PLANE_CAP_TILING_4);
 533 }
 534
 535 static bool check_modifier_display_ver_range(const struct intel_modifier_desc *md,
 536                                              u8 display_ver_from, u8 display_ver_until)
 537 {
 538         return md->display_ver.from <= display_ver_until &&
 539                 display_ver_from <= md->display_ver.until;
 540 }
 541
 542 static bool plane_has_modifier(struct drm_i915_private *i915,
 543                                u8 plane_caps,
 544                                const struct intel_modifier_desc *md)
 545 {
 546         if (!IS_DISPLAY_VER(i915, md->display_ver.from, md->display_ver.until))
 547                 return false;
 548
 549         if (!plane_caps_contain_all(plane_caps, md->plane_caps))
 550                 return false;
 551
 552         /*
 553          * Separate AuxCCS and Flat CCS modifiers to be run only on platforms
 554          * where supported.
 555          */
 556         if (intel_fb_is_ccs_modifier(md->modifier) &&
 557             HAS_FLAT_CCS(i915) != !md->ccs.packed_aux_planes)
 558                 return false;
 559
 560         if (md->modifier == I915_FORMAT_MOD_4_TILED_BMG_CCS &&
 561             (GRAPHICS_VER(i915) < 20 || !IS_DGFX(i915)))
 562                 return false;
 563
 564         if (md->modifier == I915_FORMAT_MOD_4_TILED_LNL_CCS &&
 565             (GRAPHICS_VER(i915) < 20 || IS_DGFX(i915)))
 566                 return false;
 567
 568         return true;
 569 }
 570
 571 /**
 572  * intel_fb_plane_get_modifiers: Get the modifiers for the given platform and plane capabilities
 573  * @i915: i915 device instance
 574  * @plane_caps: capabilities for the plane the modifiers are queried for
 575  *
 576  * Returns:
 577  * Returns the list of modifiers allowed by the @i915 platform and @plane_caps.
 578  * The caller must free the returned buffer.
 579  */
 580 u64 *intel_fb_plane_get_modifiers(struct drm_i915_private *i915,
 581                                   u8 plane_caps)
 582 {
 583         u64 *list, *p;
 584         int count = 1;          /* +1 for invalid modifier terminator */
 585         int i;
 586
 587         for (i = 0; i < ARRAY_SIZE(intel_modifiers); i++) {
 588                 if (plane_has_modifier(i915, plane_caps, &intel_modifiers[i]))
 589                         count++;
 590         }
 591
 592         list = kmalloc_array(count, sizeof(*list), GFP_KERNEL);
 593         if (drm_WARN_ON(&i915->drm, !list))
 594                 return NULL;
 595
 596         p = list;
 597         for (i = 0; i < ARRAY_SIZE(intel_modifiers); i++) {
 598                 if (plane_has_modifier(i915, plane_caps, &intel_modifiers[i]))
 599                         *p++ = intel_modifiers[i].modifier;
 600         }
 601         *p++ = DRM_FORMAT_MOD_INVALID;
 602
 603         return list;
 604 }
 605
 606 /**
 607  * intel_fb_plane_supports_modifier: Determine if a modifier is supported by the given plane
 608  * @plane: Plane to check the modifier support for
 609  * @modifier: The modifier to check the support for
 610  *
 611  * Returns:
 612  * %true if the @modifier is supported on @plane.
 613  */
 614 bool intel_fb_plane_supports_modifier(struct intel_plane *plane, u64 modifier)
 615 {
 616         int i;
 617
 618         for (i = 0; i < plane->base.modifier_count; i++)
 619                 if (plane->base.modifiers[i] == modifier)
 620                         return true;
 621
 622         return false;
 623 }
 624
 625 static bool format_is_yuv_semiplanar(const struct intel_modifier_desc *md,
 626                                      const struct drm_format_info *info)
 627 {
 628         if (!info->is_yuv)
 629                 return false;
 630
 631         if (hweight8(md->ccs.planar_aux_planes) == 2)
 632                 return info->num_planes == 4;
 633         else
 634                 return info->num_planes == 2;
 635 }
 636
 637 /**
 638  * intel_format_info_is_yuv_semiplanar: Check if the given format is YUV semiplanar
 639  * @info: format to check
 640  * @modifier: modifier used with the format
 641  *
 642  * Returns:
 643  * %true if @info / @modifier is YUV semiplanar.
 644  */
 645 bool intel_format_info_is_yuv_semiplanar(const struct drm_format_info *info,
 646                                          u64 modifier)
 647 {
 648         return format_is_yuv_semiplanar(lookup_modifier(modifier), info);
 649 }
 650
 651 static u8 ccs_aux_plane_mask(const struct intel_modifier_desc *md,
 652                              const struct drm_format_info *format)
 653 {
 654         if (format_is_yuv_semiplanar(md, format))
 655                 return md->ccs.planar_aux_planes;
 656         else
 657                 return md->ccs.packed_aux_planes;
 658 }
 659
 660 /**
 661  * intel_fb_is_ccs_aux_plane: Check if a framebuffer color plane is a CCS AUX plane
 662  * @fb: Framebuffer
 663  * @color_plane: color plane index to check
 664  *
 665  * Returns:
 666  * Returns %true if @fb's color plane at index @color_plane is a CCS AUX plane.
 667  */
 668 bool intel_fb_is_ccs_aux_plane(const struct drm_framebuffer *fb, int color_plane)
 669 {
 670         const struct intel_modifier_desc *md = lookup_modifier(fb->modifier);
 671
 672         return ccs_aux_plane_mask(md, fb->format) & BIT(color_plane);
 673 }
 674
 675 /**
 676  * intel_fb_is_gen12_ccs_aux_plane: Check if a framebuffer color plane is a GEN12 CCS AUX plane
 677  * @fb: Framebuffer
 678  * @color_plane: color plane index to check
 679  *
 680  * Returns:
 681  * Returns %true if @fb's color plane at index @color_plane is a GEN12 CCS AUX plane.
 682  */
 683 static bool intel_fb_is_gen12_ccs_aux_plane(const struct drm_framebuffer *fb, int color_plane)
 684 {
 685         const struct intel_modifier_desc *md = lookup_modifier(fb->modifier);
 686
 687         return check_modifier_display_ver_range(md, 12, 14) &&
 688                ccs_aux_plane_mask(md, fb->format) & BIT(color_plane);
 689 }
 690
 691 /**
 692  * intel_fb_rc_ccs_cc_plane: Get the CCS CC color plane index for a framebuffer
 693  * @fb: Framebuffer
 694  *
 695  * Returns:
 696  * Returns the index of the color clear plane for @fb, or -1 if @fb is not a
 697  * framebuffer using a render compression/color clear modifier.
 698  */
 699 int intel_fb_rc_ccs_cc_plane(const struct drm_framebuffer *fb)
 700 {
 701         const struct intel_modifier_desc *md = lookup_modifier(fb->modifier);
 702
 703         if (!md->ccs.cc_planes)
 704                 return -1;
 705
 706         drm_WARN_ON_ONCE(fb->dev, hweight8(md->ccs.cc_planes) > 1);
 707
 708         return ilog2((int)md->ccs.cc_planes);
 709 }
 710
 711 static bool is_gen12_ccs_cc_plane(const struct drm_framebuffer *fb, int color_plane)
 712 {
 713         return intel_fb_rc_ccs_cc_plane(fb) == color_plane;
 714 }
 715
 716 bool is_surface_linear(const struct drm_framebuffer *fb, int color_plane)
 717 {
 718         return fb->modifier == DRM_FORMAT_MOD_LINEAR ||
 719                intel_fb_is_gen12_ccs_aux_plane(fb, color_plane) ||
 720                is_gen12_ccs_cc_plane(fb, color_plane);
 721 }
 722
 723 int main_to_ccs_plane(const struct drm_framebuffer *fb, int main_plane)
 724 {
 725         drm_WARN_ON(fb->dev, !intel_fb_is_ccs_modifier(fb->modifier) ||
 726                     (main_plane && main_plane >= fb->format->num_planes / 2));
 727
 728         return fb->format->num_planes / 2 + main_plane;
 729 }
 730
 731 int skl_ccs_to_main_plane(const struct drm_framebuffer *fb, int ccs_plane)
 732 {
 733         drm_WARN_ON(fb->dev, !intel_fb_is_ccs_modifier(fb->modifier) ||
 734                     ccs_plane < fb->format->num_planes / 2);
 735
 736         if (is_gen12_ccs_cc_plane(fb, ccs_plane))
 737                 return 0;
 738
 739         return ccs_plane - fb->format->num_planes / 2;
 740 }
 741
 742 static unsigned int gen12_ccs_aux_stride(struct intel_framebuffer *fb, int ccs_plane)
 743 {
 744         int main_plane = skl_ccs_to_main_plane(&fb->base, ccs_plane);
 745         unsigned int main_stride = fb->base.pitches[main_plane];
 746         unsigned int main_tile_width = intel_tile_width_bytes(&fb->base, main_plane);
 747
 748         return DIV_ROUND_UP(main_stride, 4 * main_tile_width) * 64;
 749 }
 750
 751 int skl_main_to_aux_plane(const struct drm_framebuffer *fb, int main_plane)
 752 {
 753         const struct intel_modifier_desc *md = lookup_modifier(fb->modifier);
 754         struct drm_i915_private *i915 = to_i915(fb->dev);
 755
 756         if (md->ccs.packed_aux_planes | md->ccs.planar_aux_planes)
 757                 return main_to_ccs_plane(fb, main_plane);
 758         else if (DISPLAY_VER(i915) < 11 &&
 759                  format_is_yuv_semiplanar(md, fb->format))
 760                 return 1;
 761         else
 762                 return 0;
 763 }
 764
 765 unsigned int intel_tile_size(const struct drm_i915_private *i915)
 766 {
 767         return DISPLAY_VER(i915) == 2 ? 2048 : 4096;
 768 }
 769
 770 unsigned int
 771 intel_tile_width_bytes(const struct drm_framebuffer *fb, int color_plane)
 772 {
 773         struct drm_i915_private *dev_priv = to_i915(fb->dev);
 774         unsigned int cpp = fb->format->cpp[color_plane];
 775
 776         switch (fb->modifier) {
 777         case DRM_FORMAT_MOD_LINEAR:
 778                 return intel_tile_size(dev_priv);
 779         case I915_FORMAT_MOD_X_TILED:
 780                 if (DISPLAY_VER(dev_priv) == 2)
 781                         return 128;
 782                 else
 783                         return 512;
 784         case I915_FORMAT_MOD_4_TILED_BMG_CCS:
 785         case I915_FORMAT_MOD_4_TILED_LNL_CCS:
 786         case I915_FORMAT_MOD_4_TILED_DG2_RC_CCS:
 787         case I915_FORMAT_MOD_4_TILED_DG2_RC_CCS_CC:
 788         case I915_FORMAT_MOD_4_TILED_DG2_MC_CCS:
 789         case I915_FORMAT_MOD_4_TILED:
 790                 /*
 791                  * Each 4K tile consists of 64B(8*8) subtiles, with
 792                  * same shape as Y Tile(i.e 4*16B OWords)
 793                  */
 794                 return 128;
 795         case I915_FORMAT_MOD_Y_TILED_CCS:
 796                 if (intel_fb_is_ccs_aux_plane(fb, color_plane))
 797                         return 128;
 798                 fallthrough;
 799         case I915_FORMAT_MOD_4_TILED_MTL_RC_CCS:
 800         case I915_FORMAT_MOD_4_TILED_MTL_RC_CCS_CC:
 801         case I915_FORMAT_MOD_4_TILED_MTL_MC_CCS:
 802         case I915_FORMAT_MOD_Y_TILED_GEN12_RC_CCS:
 803         case I915_FORMAT_MOD_Y_TILED_GEN12_RC_CCS_CC:
 804         case I915_FORMAT_MOD_Y_TILED_GEN12_MC_CCS:
 805                 if (intel_fb_is_ccs_aux_plane(fb, color_plane) ||
 806                     is_gen12_ccs_cc_plane(fb, color_plane))
 807                         return 64;
 808                 fallthrough;
 809         case I915_FORMAT_MOD_Y_TILED:
 810                 if (DISPLAY_VER(dev_priv) == 2 || HAS_128_BYTE_Y_TILING(dev_priv))
 811                         return 128;
 812                 else
 813                         return 512;
 814         case I915_FORMAT_MOD_Yf_TILED_CCS:
 815                 if (intel_fb_is_ccs_aux_plane(fb, color_plane))
 816                         return 128;
 817                 fallthrough;
 818         case I915_FORMAT_MOD_Yf_TILED:
 819                 switch (cpp) {
 820                 case 1:
 821                         return 64;
 822                 case 2:
 823                 case 4:
 824                         return 128;
 825                 case 8:
 826                 case 16:
 827                         return 256;
 828                 default:
 829                         MISSING_CASE(cpp);
 830                         return cpp;
 831                 }
 832                 break;
 833         default:
 834                 MISSING_CASE(fb->modifier);
 835                 return cpp;
 836         }
 837 }
 838
 839 unsigned int intel_tile_height(const struct drm_framebuffer *fb, int color_plane)
 840 {
 841         return intel_tile_size(to_i915(fb->dev)) /
 842                 intel_tile_width_bytes(fb, color_plane);
 843 }
 844
 845 /*
 846  * Return the tile dimensions in pixel units, based on the (2 or 4 kbyte) GTT
 847  * page tile size.
 848  */
 849 static void intel_tile_dims(const struct drm_framebuffer *fb, int color_plane,
 850                             unsigned int *tile_width,
 851                             unsigned int *tile_height)
 852 {
 853         unsigned int tile_width_bytes = intel_tile_width_bytes(fb, color_plane);
 854         unsigned int cpp = fb->format->cpp[color_plane];
 855
 856         *tile_width = tile_width_bytes / cpp;
 857         *tile_height = intel_tile_height(fb, color_plane);
 858 }
 859
 860 /*
 861  * Return the tile dimensions in pixel units, based on the tile block size.
 862  * The block covers the full GTT page sized tile on all tiled surfaces and
 863  * it's a 64 byte portion of the tile on TGL+ CCS surfaces.
 864  */
 865 static void intel_tile_block_dims(const struct drm_framebuffer *fb, int color_plane,
 866                                   unsigned int *tile_width,
 867                                   unsigned int *tile_height)
 868 {
 869         intel_tile_dims(fb, color_plane, tile_width, tile_height);
 870
 871         if (intel_fb_is_gen12_ccs_aux_plane(fb, color_plane))
 872                 *tile_height = 1;
 873 }
 874
 875 unsigned int intel_tile_row_size(const struct drm_framebuffer *fb, int color_plane)
 876 {
 877         unsigned int tile_width, tile_height;
 878
 879         intel_tile_dims(fb, color_plane, &tile_width, &tile_height);
 880
 881         return fb->pitches[color_plane] * tile_height;
 882 }
 883
 884 unsigned int
 885 intel_fb_align_height(const struct drm_framebuffer *fb,
 886                       int color_plane, unsigned int height)
 887 {
 888         unsigned int tile_height = intel_tile_height(fb, color_plane);
 889
 890         return ALIGN(height, tile_height);
 891 }
 892
 893 bool intel_fb_modifier_uses_dpt(struct drm_i915_private *i915, u64 modifier)
 894 {
 895         return HAS_DPT(i915) && modifier != DRM_FORMAT_MOD_LINEAR;
 896 }
 897
 898 bool intel_fb_uses_dpt(const struct drm_framebuffer *fb)
 899 {
 900         return to_i915(fb->dev)->display.params.enable_dpt &&
 901                 intel_fb_modifier_uses_dpt(to_i915(fb->dev), fb->modifier);
 902 }
 903
 904 void intel_fb_plane_get_subsampling(int *hsub, int *vsub,
 905                                     const struct drm_framebuffer *fb,
 906                                     int color_plane)
 907 {
 908         int main_plane;
 909
 910         if (color_plane == 0) {
 911                 *hsub = 1;
 912                 *vsub = 1;
 913
 914                 return;
 915         }
 916
 917         /*
 918          * TODO: Deduct the subsampling from the char block for all CCS
 919          * formats and planes.
 920          */
 921         if (!intel_fb_is_gen12_ccs_aux_plane(fb, color_plane)) {
 922                 *hsub = fb->format->hsub;
 923                 *vsub = fb->format->vsub;
 924
 925                 return;
 926         }
 927
 928         main_plane = skl_ccs_to_main_plane(fb, color_plane);
 929         *hsub = drm_format_info_block_width(fb->format, color_plane) /
 930                 drm_format_info_block_width(fb->format, main_plane);
 931
 932         /*
 933          * The min stride check in the core framebuffer_check() function
 934          * assumes that format->hsub applies to every plane except for the
 935          * first plane. That's incorrect for the CCS AUX plane of the first
 936          * plane, but for the above check to pass we must define the block
 937          * width with that subsampling applied to it. Adjust the width here
 938          * accordingly, so we can calculate the actual subsampling factor.
 939          */
 940         if (main_plane == 0)
 941                 *hsub *= fb->format->hsub;
 942
 943         *vsub = 32;
 944 }
 945
 946 static void intel_fb_plane_dims(const struct intel_framebuffer *fb, int color_plane, int *w, int *h)
 947 {
 948         int main_plane = intel_fb_is_ccs_aux_plane(&fb->base, color_plane) ?
 949                          skl_ccs_to_main_plane(&fb->base, color_plane) : 0;
 950         unsigned int main_width = fb->base.width;
 951         unsigned int main_height = fb->base.height;
 952         int main_hsub, main_vsub;
 953         int hsub, vsub;
 954
 955         intel_fb_plane_get_subsampling(&main_hsub, &main_vsub, &fb->base, main_plane);
 956         intel_fb_plane_get_subsampling(&hsub, &vsub, &fb->base, color_plane);
 957
 958         *w = DIV_ROUND_UP(main_width, main_hsub * hsub);
 959         *h = DIV_ROUND_UP(main_height, main_vsub * vsub);
 960 }
 961
 962 static u32 intel_adjust_tile_offset(int *x, int *y,
 963                                     unsigned int tile_width,
 964                                     unsigned int tile_height,
 965                                     unsigned int tile_size,
 966                                     unsigned int pitch_tiles,
 967                                     u32 old_offset,
 968                                     u32 new_offset)
 969 {
 970         unsigned int pitch_pixels = pitch_tiles * tile_width;
 971         unsigned int tiles;
 972
 973         WARN_ON(old_offset & (tile_size - 1));
 974         WARN_ON(new_offset & (tile_size - 1));
 975         WARN_ON(new_offset > old_offset);
 976
 977         tiles = (old_offset - new_offset) / tile_size;
 978
 979         *y += tiles / pitch_tiles * tile_height;
 980         *x += tiles % pitch_tiles * tile_width;
 981
 982         /* minimize x in case it got needlessly big */
 983         *y += *x / pitch_pixels * tile_height;
 984         *x %= pitch_pixels;
 985
 986         return new_offset;
 987 }
 988
 989 static u32 intel_adjust_linear_offset(int *x, int *y,
 990                                       unsigned int cpp,
 991                                       unsigned int pitch,
 992                                       u32 old_offset,
 993                                       u32 new_offset)
 994 {
 995         old_offset += *y * pitch + *x * cpp;
 996
 997         *y = (old_offset - new_offset) / pitch;
 998         *x = ((old_offset - new_offset) - *y * pitch) / cpp;
 999
1000         return new_offset;
1001 }
1002
1003 static u32 intel_adjust_aligned_offset(int *x, int *y,
1004                                        const struct drm_framebuffer *fb,
1005                                        int color_plane,
1006                                        unsigned int rotation,
1007                                        unsigned int pitch,
1008                                        u32 old_offset, u32 new_offset)
1009 {
1010         struct drm_i915_private *i915 = to_i915(fb->dev);
1011         unsigned int cpp = fb->format->cpp[color_plane];
1012
1013         drm_WARN_ON(&i915->drm, new_offset > old_offset);
1014
1015         if (!is_surface_linear(fb, color_plane)) {
1016                 unsigned int tile_size, tile_width, tile_height;
1017                 unsigned int pitch_tiles;
1018
1019                 tile_size = intel_tile_size(i915);
1020                 intel_tile_dims(fb, color_plane, &tile_width, &tile_height);
1021
1022                 if (drm_rotation_90_or_270(rotation)) {
1023                         pitch_tiles = pitch / tile_height;
1024                         swap(tile_width, tile_height);
1025                 } else {
1026                         pitch_tiles = pitch / (tile_width * cpp);
1027                 }
1028
1029                 intel_adjust_tile_offset(x, y, tile_width, tile_height,
1030                                          tile_size, pitch_tiles,
1031                                          old_offset, new_offset);
1032         } else {
1033                 intel_adjust_linear_offset(x, y, cpp, pitch,
1034                                            old_offset, new_offset);
1035         }
1036
1037         return new_offset;
1038 }
1039
1040 /*
1041  * Adjust the tile offset by moving the difference into
1042  * the x/y offsets.
1043  */
1044 u32 intel_plane_adjust_aligned_offset(int *x, int *y,
1045                                       const struct intel_plane_state *state,
1046                                       int color_plane,
1047                                       u32 old_offset, u32 new_offset)
1048 {
1049         return intel_adjust_aligned_offset(x, y, state->hw.fb, color_plane,
1050                                            state->hw.rotation,
1051                                            state->view.color_plane[color_plane].mapping_stride,
1052                                            old_offset, new_offset);
1053 }
1054
1055 /*
1056  * Computes the aligned offset to the base tile and adjusts
1057  * x, y. bytes per pixel is assumed to be a power-of-two.
1058  *
1059  * In the 90/270 rotated case, x and y are assumed
1060  * to be already rotated to match the rotated GTT view, and
1061  * pitch is the tile_height aligned framebuffer height.
1062  *
1063  * This function is used when computing the derived information
1064  * under intel_framebuffer, so using any of that information
1065  * here is not allowed. Anything under drm_framebuffer can be
1066  * used. This is why the user has to pass in the pitch since it
1067  * is specified in the rotated orientation.
1068  */
1069 static u32 intel_compute_aligned_offset(struct drm_i915_private *i915,
1070                                         int *x, int *y,
1071                                         const struct drm_framebuffer *fb,
1072                                         int color_plane,
1073                                         unsigned int pitch,
1074                                         unsigned int rotation,
1075                                         unsigned int alignment)
1076 {
1077         unsigned int cpp = fb->format->cpp[color_plane];
1078         u32 offset, offset_aligned;
1079
1080         if (!is_surface_linear(fb, color_plane)) {
1081                 unsigned int tile_size, tile_width, tile_height;
1082                 unsigned int tile_rows, tiles, pitch_tiles;
1083
1084                 tile_size = intel_tile_size(i915);
1085                 intel_tile_dims(fb, color_plane, &tile_width, &tile_height);
1086
1087                 if (drm_rotation_90_or_270(rotation)) {
1088                         pitch_tiles = pitch / tile_height;
1089                         swap(tile_width, tile_height);
1090                 } else {
1091                         pitch_tiles = pitch / (tile_width * cpp);
1092                 }
1093
1094                 tile_rows = *y / tile_height;
1095                 *y %= tile_height;
1096
1097                 tiles = *x / tile_width;
1098                 *x %= tile_width;
1099
1100                 offset = (tile_rows * pitch_tiles + tiles) * tile_size;
1101
1102                 offset_aligned = offset;
1103                 if (alignment)
1104                         offset_aligned = rounddown(offset_aligned, alignment);
1105
1106                 intel_adjust_tile_offset(x, y, tile_width, tile_height,
1107                                          tile_size, pitch_tiles,
1108                                          offset, offset_aligned);
1109         } else {
1110                 offset = *y * pitch + *x * cpp;
1111                 offset_aligned = offset;
1112                 if (alignment) {
1113                         offset_aligned = rounddown(offset_aligned, alignment);
1114                         *y = (offset % alignment) / pitch;
1115                         *x = ((offset % alignment) - *y * pitch) / cpp;
1116                 } else {
1117                         *y = *x = 0;
1118                 }
1119         }
1120
1121         return offset_aligned;
1122 }
1123
1124 u32 intel_plane_compute_aligned_offset(int *x, int *y,
1125                                        const struct intel_plane_state *state,
1126                                        int color_plane)
1127 {
1128         struct intel_plane *plane = to_intel_plane(state->uapi.plane);
1129         struct drm_i915_private *i915 = to_i915(plane->base.dev);
1130         const struct drm_framebuffer *fb = state->hw.fb;
1131         unsigned int rotation = state->hw.rotation;
1132         unsigned int pitch = state->view.color_plane[color_plane].mapping_stride;
1133         unsigned int alignment = plane->min_alignment(plane, fb, color_plane);
1134
1135         return intel_compute_aligned_offset(i915, x, y, fb, color_plane,
1136                                             pitch, rotation, alignment);
1137 }
1138
1139 /* Convert the fb->offset[] into x/y offsets */
1140 static int intel_fb_offset_to_xy(int *x, int *y,
1141                                  const struct drm_framebuffer *fb,
1142                                  int color_plane)
1143 {
1144         struct drm_i915_private *i915 = to_i915(fb->dev);
1145         unsigned int height, alignment, unused;
1146
1147         if (fb->modifier != DRM_FORMAT_MOD_LINEAR)
1148                 alignment = intel_tile_size(i915);
1149         else
1150                 alignment = 0;
1151
1152         if (alignment != 0 && fb->offsets[color_plane] % alignment) {
1153                 drm_dbg_kms(&i915->drm,
1154                             "Misaligned offset 0x%08x for color plane %d\n",
1155                             fb->offsets[color_plane], color_plane);
1156                 return -EINVAL;
1157         }
1158
1159         height = drm_format_info_plane_height(fb->format, fb->height, color_plane);
1160         height = ALIGN(height, intel_tile_height(fb, color_plane));
1161
1162         /* Catch potential overflows early */
1163         if (check_add_overflow(mul_u32_u32(height, fb->pitches[color_plane]),
1164                                fb->offsets[color_plane], &unused)) {
1165                 drm_dbg_kms(&i915->drm,
1166                             "Bad offset 0x%08x or pitch %d for color plane %d\n",
1167                             fb->offsets[color_plane], fb->pitches[color_plane],
1168                             color_plane);
1169                 return -ERANGE;
1170         }
1171
1172         *x = 0;
1173         *y = 0;
1174
1175         intel_adjust_aligned_offset(x, y,
1176                                     fb, color_plane, DRM_MODE_ROTATE_0,
1177                                     fb->pitches[color_plane],
1178                                     fb->offsets[color_plane], 0);
1179
1180         return 0;
1181 }
1182
1183 static int intel_fb_check_ccs_xy(const struct drm_framebuffer *fb, int ccs_plane, int x, int y)
1184 {
1185         struct drm_i915_private *i915 = to_i915(fb->dev);
1186         const struct intel_framebuffer *intel_fb = to_intel_framebuffer(fb);
1187         int main_plane;
1188         int hsub, vsub;
1189         int tile_width, tile_height;
1190         int ccs_x, ccs_y;
1191         int main_x, main_y;
1192
1193         if (!intel_fb_is_ccs_aux_plane(fb, ccs_plane))
1194                 return 0;
1195
1196         /*
1197          * While all the tile dimensions are based on a 2k or 4k GTT page size
1198          * here the main and CCS coordinates must match only within a (64 byte
1199          * on TGL+) block inside the tile.
1200          */
1201         intel_tile_block_dims(fb, ccs_plane, &tile_width, &tile_height);
1202         intel_fb_plane_get_subsampling(&hsub, &vsub, fb, ccs_plane);
1203
1204         tile_width *= hsub;
1205         tile_height *= vsub;
1206
1207         ccs_x = (x * hsub) % tile_width;
1208         ccs_y = (y * vsub) % tile_height;
1209
1210         main_plane = skl_ccs_to_main_plane(fb, ccs_plane);
1211         main_x = intel_fb->normal_view.color_plane[main_plane].x % tile_width;
1212         main_y = intel_fb->normal_view.color_plane[main_plane].y % tile_height;
1213
1214         /*
1215          * CCS doesn't have its own x/y offset register, so the intra CCS tile
1216          * x/y offsets must match between CCS and the main surface.
1217          */
1218         if (main_x != ccs_x || main_y != ccs_y) {
1219                 drm_dbg_kms(&i915->drm,
1220                               "Bad CCS x/y (main %d,%d ccs %d,%d) full (main %d,%d ccs %d,%d)\n",
1221                               main_x, main_y,
1222                               ccs_x, ccs_y,
1223                               intel_fb->normal_view.color_plane[main_plane].x,
1224                               intel_fb->normal_view.color_plane[main_plane].y,
1225                               x, y);
1226                 return -EINVAL;
1227         }
1228
1229         return 0;
1230 }
1231
1232 static bool intel_plane_can_remap(const struct intel_plane_state *plane_state)
1233 {
1234         struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
1235         struct drm_i915_private *i915 = to_i915(plane->base.dev);
1236         const struct drm_framebuffer *fb = plane_state->hw.fb;
1237         int i;
1238
1239         /* We don't want to deal with remapping with cursors */
1240         if (plane->id == PLANE_CURSOR)
1241                 return false;
1242
1243         /*
1244          * The display engine limits already match/exceed the
1245          * render engine limits, so not much point in remapping.
1246          * Would also need to deal with the fence POT alignment
1247          * and gen2 2KiB GTT tile size.
1248          */
1249         if (DISPLAY_VER(i915) < 4)
1250                 return false;
1251
1252         /*
1253          * The new CCS hash mode isn't compatible with remapping as
1254          * the virtual address of the pages affects the compressed data.
1255          */
1256         if (intel_fb_is_ccs_modifier(fb->modifier))
1257                 return false;
1258
1259         /* Linear needs a page aligned stride for remapping */
1260         if (fb->modifier == DRM_FORMAT_MOD_LINEAR) {
1261                 unsigned int alignment = intel_tile_size(i915) - 1;
1262
1263                 for (i = 0; i < fb->format->num_planes; i++) {
1264                         if (fb->pitches[i] & alignment)
1265                                 return false;
1266                 }
1267         }
1268
1269         return true;
1270 }
1271
1272 bool intel_fb_needs_pot_stride_remap(const struct intel_framebuffer *fb)
1273 {
1274         struct drm_i915_private *i915 = to_i915(fb->base.dev);
1275
1276         return (IS_ALDERLAKE_P(i915) || DISPLAY_VER(i915) >= 14) &&
1277                 intel_fb_uses_dpt(&fb->base);
1278 }
1279
1280 static int intel_fb_pitch(const struct intel_framebuffer *fb, int color_plane, unsigned int rotation)
1281 {
1282         if (drm_rotation_90_or_270(rotation))
1283                 return fb->rotated_view.color_plane[color_plane].mapping_stride;
1284         else if (intel_fb_needs_pot_stride_remap(fb))
1285                 return fb->remapped_view.color_plane[color_plane].mapping_stride;
1286         else
1287                 return fb->normal_view.color_plane[color_plane].mapping_stride;
1288 }
1289
1290 static bool intel_plane_needs_remap(const struct intel_plane_state *plane_state)
1291 {
1292         struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
1293         const struct intel_framebuffer *fb = to_intel_framebuffer(plane_state->hw.fb);
1294         unsigned int rotation = plane_state->hw.rotation;
1295         u32 stride, max_stride;
1296
1297         /*
1298          * No remapping for invisible planes since we don't have
1299          * an actual source viewport to remap.
1300          */
1301         if (!plane_state->uapi.visible)
1302                 return false;
1303
1304         if (!intel_plane_can_remap(plane_state))
1305                 return false;
1306
1307         /*
1308          * FIXME: aux plane limits on gen9+ are
1309          * unclear in Bspec, for now no checking.
1310          */
1311         stride = intel_fb_pitch(fb, 0, rotation);
1312         max_stride = plane->max_stride(plane, fb->base.format->format,
1313                                        fb->base.modifier, rotation);
1314
1315         return stride > max_stride;
1316 }
1317
1318 static int convert_plane_offset_to_xy(const struct intel_framebuffer *fb, int color_plane,
1319                                       int plane_width, int *x, int *y)
1320 {
1321         struct drm_gem_object *obj = intel_fb_bo(&fb->base);
1322         int ret;
1323
1324         ret = intel_fb_offset_to_xy(x, y, &fb->base, color_plane);
1325         if (ret) {
1326                 drm_dbg_kms(fb->base.dev,
1327                             "bad fb plane %d offset: 0x%x\n",
1328                             color_plane, fb->base.offsets[color_plane]);
1329                 return ret;
1330         }
1331
1332         ret = intel_fb_check_ccs_xy(&fb->base, color_plane, *x, *y);
1333         if (ret)
1334                 return ret;
1335
1336         /*
1337          * The fence (if used) is aligned to the start of the object
1338          * so having the framebuffer wrap around across the edge of the
1339          * fenced region doesn't really work. We have no API to configure
1340          * the fence start offset within the object (nor could we probably
1341          * on gen2/3). So it's just easier if we just require that the
1342          * fb layout agrees with the fence layout. We already check that the
1343          * fb stride matches the fence stride elsewhere.
1344          */
1345         if (color_plane == 0 && intel_bo_is_tiled(obj) &&
1346             (*x + plane_width) * fb->base.format->cpp[color_plane] > fb->base.pitches[color_plane]) {
1347                 drm_dbg_kms(fb->base.dev,
1348                             "bad fb plane %d offset: 0x%x\n",
1349                             color_plane, fb->base.offsets[color_plane]);
1350                 return -EINVAL;
1351         }
1352
1353         return 0;
1354 }
1355
1356 static u32 calc_plane_aligned_offset(const struct intel_framebuffer *fb, int color_plane, int *x, int *y)
1357 {
1358         struct drm_i915_private *i915 = to_i915(fb->base.dev);
1359         unsigned int tile_size = intel_tile_size(i915);
1360         u32 offset;
1361
1362         offset = intel_compute_aligned_offset(i915, x, y, &fb->base, color_plane,
1363                                               fb->base.pitches[color_plane],
1364                                               DRM_MODE_ROTATE_0,
1365                                               tile_size);
1366
1367         return offset / tile_size;
1368 }
1369
1370 struct fb_plane_view_dims {
1371         unsigned int width, height;
1372         unsigned int tile_width, tile_height;
1373 };
1374
1375 static void init_plane_view_dims(const struct intel_framebuffer *fb, int color_plane,
1376                                  unsigned int width, unsigned int height,
1377                                  struct fb_plane_view_dims *dims)
1378 {
1379         dims->width = width;
1380         dims->height = height;
1381
1382         intel_tile_dims(&fb->base, color_plane, &dims->tile_width, &dims->tile_height);
1383 }
1384
1385 static unsigned int
1386 plane_view_src_stride_tiles(const struct intel_framebuffer *fb, int color_plane,
1387                             const struct fb_plane_view_dims *dims)
1388 {
1389         return DIV_ROUND_UP(fb->base.pitches[color_plane],
1390                             dims->tile_width * fb->base.format->cpp[color_plane]);
1391 }
1392
1393 static unsigned int
1394 plane_view_dst_stride_tiles(const struct intel_framebuffer *fb, int color_plane,
1395                             unsigned int pitch_tiles)
1396 {
1397         if (intel_fb_needs_pot_stride_remap(fb)) {
1398                 /*
1399                  * ADL_P, the only platform needing a POT stride has a minimum
1400                  * of 8 main surface tiles.
1401                  */
1402                 return roundup_pow_of_two(max(pitch_tiles, 8u));
1403         } else {
1404                 return pitch_tiles;
1405         }
1406 }
1407
1408 static unsigned int
1409 plane_view_scanout_stride(const struct intel_framebuffer *fb, int color_plane,
1410                           unsigned int tile_width,
1411                           unsigned int src_stride_tiles, unsigned int dst_stride_tiles)
1412 {
1413         struct drm_i915_private *i915 = to_i915(fb->base.dev);
1414         unsigned int stride_tiles;
1415
1416         if ((IS_ALDERLAKE_P(i915) || DISPLAY_VER(i915) >= 14) &&
1417             src_stride_tiles < dst_stride_tiles)
1418                 stride_tiles = src_stride_tiles;
1419         else
1420                 stride_tiles = dst_stride_tiles;
1421
1422         return stride_tiles * tile_width * fb->base.format->cpp[color_plane];
1423 }
1424
1425 static unsigned int
1426 plane_view_width_tiles(const struct intel_framebuffer *fb, int color_plane,
1427                        const struct fb_plane_view_dims *dims,
1428                        int x)
1429 {
1430         return DIV_ROUND_UP(x + dims->width, dims->tile_width);
1431 }
1432
1433 static unsigned int
1434 plane_view_height_tiles(const struct intel_framebuffer *fb, int color_plane,
1435                         const struct fb_plane_view_dims *dims,
1436                         int y)
1437 {
1438         return DIV_ROUND_UP(y + dims->height, dims->tile_height);
1439 }
1440
1441 static unsigned int
1442 plane_view_linear_tiles(const struct intel_framebuffer *fb, int color_plane,
1443                         const struct fb_plane_view_dims *dims,
1444                         int x, int y)
1445 {
1446         struct drm_i915_private *i915 = to_i915(fb->base.dev);
1447         unsigned int size;
1448
1449         size = (y + dims->height) * fb->base.pitches[color_plane] +
1450                 x * fb->base.format->cpp[color_plane];
1451
1452         return DIV_ROUND_UP(size, intel_tile_size(i915));
1453 }
1454
1455 #define assign_chk_ovf(i915, var, val) ({ \
1456         drm_WARN_ON(&(i915)->drm, overflows_type(val, var)); \
1457         (var) = (val); \
1458 })
1459
1460 #define assign_bfld_chk_ovf(i915, var, val) ({ \
1461         (var) = (val); \
1462         drm_WARN_ON(&(i915)->drm, (var) != (val)); \
1463         (var); \
1464 })
1465
1466 static u32 calc_plane_remap_info(const struct intel_framebuffer *fb, int color_plane,
1467                                  const struct fb_plane_view_dims *dims,
1468                                  u32 obj_offset, u32 gtt_offset, int x, int y,
1469                                  struct intel_fb_view *view)
1470 {
1471         struct drm_i915_private *i915 = to_i915(fb->base.dev);
1472         struct intel_remapped_plane_info *remap_info = &view->gtt.remapped.plane[color_plane];
1473         struct i915_color_plane_view *color_plane_info = &view->color_plane[color_plane];
1474         unsigned int tile_width = dims->tile_width;
1475         unsigned int tile_height = dims->tile_height;
1476         unsigned int tile_size = intel_tile_size(i915);
1477         struct drm_rect r;
1478         u32 size = 0;
1479
1480         assign_bfld_chk_ovf(i915, remap_info->offset, obj_offset);
1481
1482         if (intel_fb_is_gen12_ccs_aux_plane(&fb->base, color_plane)) {
1483                 remap_info->linear = 1;
1484
1485                 assign_chk_ovf(i915, remap_info->size,
1486                                plane_view_linear_tiles(fb, color_plane, dims, x, y));
1487         } else {
1488                 remap_info->linear = 0;
1489
1490                 assign_chk_ovf(i915, remap_info->src_stride,
1491                                plane_view_src_stride_tiles(fb, color_plane, dims));
1492                 assign_chk_ovf(i915, remap_info->width,
1493                                plane_view_width_tiles(fb, color_plane, dims, x));
1494                 assign_chk_ovf(i915, remap_info->height,
1495                                plane_view_height_tiles(fb, color_plane, dims, y));
1496         }
1497
1498         if (view->gtt.type == I915_GTT_VIEW_ROTATED) {
1499                 drm_WARN_ON(&i915->drm, remap_info->linear);
1500                 check_array_bounds(i915, view->gtt.rotated.plane, color_plane);
1501
1502                 assign_chk_ovf(i915, remap_info->dst_stride,
1503                                plane_view_dst_stride_tiles(fb, color_plane, remap_info->height));
1504
1505                 /* rotate the x/y offsets to match the GTT view */
1506                 drm_rect_init(&r, x, y, dims->width, dims->height);
1507                 drm_rect_rotate(&r,
1508                                 remap_info->width * tile_width,
1509                                 remap_info->height * tile_height,
1510                                 DRM_MODE_ROTATE_270);
1511
1512                 color_plane_info->x = r.x1;
1513                 color_plane_info->y = r.y1;
1514
1515                 color_plane_info->mapping_stride = remap_info->dst_stride * tile_height;
1516                 color_plane_info->scanout_stride = color_plane_info->mapping_stride;
1517
1518                 size += remap_info->dst_stride * remap_info->width;
1519
1520                 /* rotate the tile dimensions to match the GTT view */
1521                 swap(tile_width, tile_height);
1522         } else {
1523                 drm_WARN_ON(&i915->drm, view->gtt.type != I915_GTT_VIEW_REMAPPED);
1524
1525                 check_array_bounds(i915, view->gtt.remapped.plane, color_plane);
1526
1527                 if (view->gtt.remapped.plane_alignment) {
1528                         u32 aligned_offset = ALIGN(gtt_offset,
1529                                                    view->gtt.remapped.plane_alignment);
1530
1531                         size += aligned_offset - gtt_offset;
1532                         gtt_offset = aligned_offset;
1533                 }
1534
1535                 color_plane_info->x = x;
1536                 color_plane_info->y = y;
1537
1538                 if (remap_info->linear) {
1539                         color_plane_info->mapping_stride = fb->base.pitches[color_plane];
1540                         color_plane_info->scanout_stride = color_plane_info->mapping_stride;
1541
1542                         size += remap_info->size;
1543                 } else {
1544                         unsigned int dst_stride;
1545
1546                         /*
1547                          * The hardware automagically calculates the CCS AUX surface
1548                          * stride from the main surface stride so can't really remap a
1549                          * smaller subset (unless we'd remap in whole AUX page units).
1550                          */
1551                         if (intel_fb_needs_pot_stride_remap(fb) &&
1552                             intel_fb_is_ccs_modifier(fb->base.modifier))
1553                                 dst_stride = remap_info->src_stride;
1554                         else
1555                                 dst_stride = remap_info->width;
1556
1557                         dst_stride = plane_view_dst_stride_tiles(fb, color_plane, dst_stride);
1558
1559                         assign_chk_ovf(i915, remap_info->dst_stride, dst_stride);
1560                         color_plane_info->mapping_stride = dst_stride *
1561                                                            tile_width *
1562                                                            fb->base.format->cpp[color_plane];
1563                         color_plane_info->scanout_stride =
1564                                 plane_view_scanout_stride(fb, color_plane, tile_width,
1565                                                           remap_info->src_stride,
1566                                                           dst_stride);
1567
1568                         size += dst_stride * remap_info->height;
1569                 }
1570         }
1571
1572         /*
1573          * We only keep the x/y offsets, so push all of the gtt offset into
1574          * the x/y offsets.  x,y will hold the first pixel of the framebuffer
1575          * plane from the start of the remapped/rotated gtt mapping.
1576          */
1577         if (remap_info->linear)
1578                 intel_adjust_linear_offset(&color_plane_info->x, &color_plane_info->y,
1579                                            fb->base.format->cpp[color_plane],
1580                                            color_plane_info->mapping_stride,
1581                                            gtt_offset * tile_size, 0);
1582         else
1583                 intel_adjust_tile_offset(&color_plane_info->x, &color_plane_info->y,
1584                                          tile_width, tile_height,
1585                                          tile_size, remap_info->dst_stride,
1586                                          gtt_offset * tile_size, 0);
1587
1588         return size;
1589 }
1590
1591 #undef assign_chk_ovf
1592
1593 /* Return number of tiles @color_plane needs. */
1594 static unsigned int
1595 calc_plane_normal_size(const struct intel_framebuffer *fb, int color_plane,
1596                        const struct fb_plane_view_dims *dims,
1597                        int x, int y)
1598 {
1599         unsigned int tiles;
1600
1601         if (is_surface_linear(&fb->base, color_plane)) {
1602                 tiles = plane_view_linear_tiles(fb, color_plane, dims, x, y);
1603         } else {
1604                 tiles = plane_view_src_stride_tiles(fb, color_plane, dims) *
1605                         plane_view_height_tiles(fb, color_plane, dims, y);
1606                 /*
1607                  * If the plane isn't horizontally tile aligned,
1608                  * we need one more tile.
1609                  */
1610                 if (x != 0)
1611                         tiles++;
1612         }
1613
1614         return tiles;
1615 }
1616
1617 static void intel_fb_view_init(struct drm_i915_private *i915, struct intel_fb_view *view,
1618                                enum i915_gtt_view_type view_type)
1619 {
1620         memset(view, 0, sizeof(*view));
1621         view->gtt.type = view_type;
1622
1623         if (view_type == I915_GTT_VIEW_REMAPPED &&
1624             (IS_ALDERLAKE_P(i915) || DISPLAY_VER(i915) >= 14))
1625                 view->gtt.remapped.plane_alignment = SZ_2M / PAGE_SIZE;
1626 }
1627
1628 bool intel_fb_supports_90_270_rotation(const struct intel_framebuffer *fb)
1629 {
1630         if (DISPLAY_VER(to_i915(fb->base.dev)) >= 13)
1631                 return false;
1632
1633         return fb->base.modifier == I915_FORMAT_MOD_Y_TILED ||
1634                fb->base.modifier == I915_FORMAT_MOD_Yf_TILED;
1635 }
1636
1637 static unsigned int intel_fb_min_alignment(const struct drm_framebuffer *fb)
1638 {
1639         struct drm_i915_private *i915 = to_i915(fb->dev);
1640         struct intel_plane *plane;
1641         unsigned int min_alignment = 0;
1642
1643         for_each_intel_plane(&i915->drm, plane) {
1644                 unsigned int plane_min_alignment;
1645
1646                 if (!drm_plane_has_format(&plane->base, fb->format->format, fb->modifier))
1647                         continue;
1648
1649                 plane_min_alignment = plane->min_alignment(plane, fb, 0);
1650
1651                 drm_WARN_ON(&i915->drm, plane_min_alignment &&
1652                             !is_power_of_2(plane_min_alignment));
1653
1654                 if (intel_plane_needs_physical(plane))
1655                         continue;
1656
1657                 min_alignment = max(min_alignment, plane_min_alignment);
1658         }
1659
1660         return min_alignment;
1661 }
1662
1663 int intel_fill_fb_info(struct drm_i915_private *i915, struct intel_framebuffer *fb)
1664 {
1665         struct drm_gem_object *obj = intel_fb_bo(&fb->base);
1666         u32 gtt_offset_rotated = 0;
1667         u32 gtt_offset_remapped = 0;
1668         unsigned int max_size = 0;
1669         int i, num_planes = fb->base.format->num_planes;
1670         unsigned int tile_size = intel_tile_size(i915);
1671
1672         intel_fb_view_init(i915, &fb->normal_view, I915_GTT_VIEW_NORMAL);
1673
1674         drm_WARN_ON(&i915->drm,
1675                     intel_fb_supports_90_270_rotation(fb) &&
1676                     intel_fb_needs_pot_stride_remap(fb));
1677
1678         if (intel_fb_supports_90_270_rotation(fb))
1679                 intel_fb_view_init(i915, &fb->rotated_view, I915_GTT_VIEW_ROTATED);
1680         if (intel_fb_needs_pot_stride_remap(fb))
1681                 intel_fb_view_init(i915, &fb->remapped_view, I915_GTT_VIEW_REMAPPED);
1682
1683         for (i = 0; i < num_planes; i++) {
1684                 struct fb_plane_view_dims view_dims;
1685                 unsigned int width, height;
1686                 unsigned int size;
1687                 u32 offset;
1688                 int x, y;
1689                 int ret;
1690
1691                 /*
1692                  * Plane 2 of Render Compression with Clear Color fb modifier
1693                  * is consumed by the driver and not passed to DE. Skip the
1694                  * arithmetic related to alignment and offset calculation.
1695                  */
1696                 if (is_gen12_ccs_cc_plane(&fb->base, i)) {
1697                         if (IS_ALIGNED(fb->base.offsets[i], PAGE_SIZE))
1698                                 continue;
1699                         else
1700                                 return -EINVAL;
1701                 }
1702
1703                 intel_fb_plane_dims(fb, i, &width, &height);
1704
1705                 ret = convert_plane_offset_to_xy(fb, i, width, &x, &y);
1706                 if (ret)
1707                         return ret;
1708
1709                 init_plane_view_dims(fb, i, width, height, &view_dims);
1710
1711                 /*
1712                  * First pixel of the framebuffer from
1713                  * the start of the normal gtt mapping.
1714                  */
1715                 fb->normal_view.color_plane[i].x = x;
1716                 fb->normal_view.color_plane[i].y = y;
1717                 fb->normal_view.color_plane[i].mapping_stride = fb->base.pitches[i];
1718                 fb->normal_view.color_plane[i].scanout_stride =
1719                         fb->normal_view.color_plane[i].mapping_stride;
1720
1721                 offset = calc_plane_aligned_offset(fb, i, &x, &y);
1722
1723                 if (intel_fb_supports_90_270_rotation(fb))
1724                         gtt_offset_rotated += calc_plane_remap_info(fb, i, &view_dims,
1725                                                                     offset, gtt_offset_rotated, x, y,
1726                                                                     &fb->rotated_view);
1727
1728                 if (intel_fb_needs_pot_stride_remap(fb))
1729                         gtt_offset_remapped += calc_plane_remap_info(fb, i, &view_dims,
1730                                                                      offset, gtt_offset_remapped, x, y,
1731                                                                      &fb->remapped_view);
1732
1733                 size = calc_plane_normal_size(fb, i, &view_dims, x, y);
1734                 /* how many tiles in total needed in the bo */
1735                 max_size = max(max_size, offset + size);
1736         }
1737
1738         if (mul_u32_u32(max_size, tile_size) > obj->size) {
1739                 drm_dbg_kms(&i915->drm,
1740                             "fb too big for bo (need %llu bytes, have %zu bytes)\n",
1741                             mul_u32_u32(max_size, tile_size), obj->size);
1742                 return -EINVAL;
1743         }
1744
1745         fb->min_alignment = intel_fb_min_alignment(&fb->base);
1746
1747         return 0;
1748 }
1749
1750 static void intel_plane_remap_gtt(struct intel_plane_state *plane_state)
1751 {
1752         struct drm_i915_private *i915 =
1753                 to_i915(plane_state->uapi.plane->dev);
1754         struct drm_framebuffer *fb = plane_state->hw.fb;
1755         struct intel_framebuffer *intel_fb = to_intel_framebuffer(fb);
1756         unsigned int rotation = plane_state->hw.rotation;
1757         int i, num_planes = fb->format->num_planes;
1758         unsigned int src_x, src_y;
1759         unsigned int src_w, src_h;
1760         u32 gtt_offset = 0;
1761
1762         intel_fb_view_init(i915, &plane_state->view,
1763                            drm_rotation_90_or_270(rotation) ? I915_GTT_VIEW_ROTATED :
1764                                                               I915_GTT_VIEW_REMAPPED);
1765
1766         src_x = plane_state->uapi.src.x1 >> 16;
1767         src_y = plane_state->uapi.src.y1 >> 16;
1768         src_w = drm_rect_width(&plane_state->uapi.src) >> 16;
1769         src_h = drm_rect_height(&plane_state->uapi.src) >> 16;
1770
1771         drm_WARN_ON(&i915->drm, intel_fb_is_ccs_modifier(fb->modifier));
1772
1773         /* Make src coordinates relative to the viewport */
1774         drm_rect_translate(&plane_state->uapi.src,
1775                            -(src_x << 16), -(src_y << 16));
1776
1777         /* Rotate src coordinates to match rotated GTT view */
1778         if (drm_rotation_90_or_270(rotation))
1779                 drm_rect_rotate(&plane_state->uapi.src,
1780                                 src_w << 16, src_h << 16,
1781                                 DRM_MODE_ROTATE_270);
1782
1783         for (i = 0; i < num_planes; i++) {
1784                 unsigned int hsub = i ? fb->format->hsub : 1;
1785                 unsigned int vsub = i ? fb->format->vsub : 1;
1786                 struct fb_plane_view_dims view_dims;
1787                 unsigned int width, height;
1788                 unsigned int x, y;
1789                 u32 offset;
1790
1791                 x = src_x / hsub;
1792                 y = src_y / vsub;
1793                 width = src_w / hsub;
1794                 height = src_h / vsub;
1795
1796                 init_plane_view_dims(intel_fb, i, width, height, &view_dims);
1797
1798                 /*
1799                  * First pixel of the src viewport from the
1800                  * start of the normal gtt mapping.
1801                  */
1802                 x += intel_fb->normal_view.color_plane[i].x;
1803                 y += intel_fb->normal_view.color_plane[i].y;
1804
1805                 offset = calc_plane_aligned_offset(intel_fb, i, &x, &y);
1806
1807                 gtt_offset += calc_plane_remap_info(intel_fb, i, &view_dims,
1808                                                     offset, gtt_offset, x, y,
1809                                                     &plane_state->view);
1810         }
1811 }
1812
1813 void intel_fb_fill_view(const struct intel_framebuffer *fb, unsigned int rotation,
1814                         struct intel_fb_view *view)
1815 {
1816         if (drm_rotation_90_or_270(rotation))
1817                 *view = fb->rotated_view;
1818         else if (intel_fb_needs_pot_stride_remap(fb))
1819                 *view = fb->remapped_view;
1820         else
1821                 *view = fb->normal_view;
1822 }
1823
1824 static
1825 u32 intel_fb_max_stride(struct drm_i915_private *dev_priv,
1826                         u32 pixel_format, u64 modifier)
1827 {
1828         /*
1829          * Arbitrary limit for gen4+ chosen to match the
1830          * render engine max stride.
1831          *
1832          * The new CCS hash mode makes remapping impossible
1833          */
1834         if (DISPLAY_VER(dev_priv) < 4 || intel_fb_is_ccs_modifier(modifier) ||
1835             intel_fb_modifier_uses_dpt(dev_priv, modifier))
1836                 return intel_plane_fb_max_stride(dev_priv, pixel_format, modifier);
1837         else if (DISPLAY_VER(dev_priv) >= 7)
1838                 return 256 * 1024;
1839         else
1840                 return 128 * 1024;
1841 }
1842
1843 static unsigned int
1844 intel_fb_stride_alignment(const struct drm_framebuffer *fb, int color_plane)
1845 {
1846         struct drm_i915_private *dev_priv = to_i915(fb->dev);
1847         unsigned int tile_width;
1848
1849         if (is_surface_linear(fb, color_plane)) {
1850                 unsigned int max_stride = intel_plane_fb_max_stride(dev_priv,
1851                                                                     fb->format->format,
1852                                                                     fb->modifier);
1853
1854                 /*
1855                  * To make remapping with linear generally feasible
1856                  * we need the stride to be page aligned.
1857                  */
1858                 if (fb->pitches[color_plane] > max_stride &&
1859                     !intel_fb_is_ccs_modifier(fb->modifier))
1860                         return intel_tile_size(dev_priv);
1861                 else
1862                         return 64;
1863         }
1864
1865         tile_width = intel_tile_width_bytes(fb, color_plane);
1866         if (intel_fb_is_ccs_modifier(fb->modifier)) {
1867                 /*
1868                  * On TGL the surface stride must be 4 tile aligned, mapped by
1869                  * one 64 byte cacheline on the CCS AUX surface.
1870                  */
1871                 if (DISPLAY_VER(dev_priv) >= 12)
1872                         tile_width *= 4;
1873                 /*
1874                  * Display WA #0531: skl,bxt,kbl,glk
1875                  *
1876                  * Render decompression and plane width > 3840
1877                  * combined with horizontal panning requires the
1878                  * plane stride to be a multiple of 4. We'll just
1879                  * require the entire fb to accommodate that to avoid
1880                  * potential runtime errors at plane configuration time.
1881                  */
1882                 else if ((DISPLAY_VER(dev_priv) == 9 || IS_GEMINILAKE(dev_priv)) &&
1883                          color_plane == 0 && fb->width > 3840)
1884                         tile_width *= 4;
1885         }
1886         return tile_width;
1887 }
1888
1889 static int intel_plane_check_stride(const struct intel_plane_state *plane_state)
1890 {
1891         struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
1892         const struct drm_framebuffer *fb = plane_state->hw.fb;
1893         unsigned int rotation = plane_state->hw.rotation;
1894         u32 stride, max_stride;
1895
1896         /*
1897          * We ignore stride for all invisible planes that
1898          * can be remapped. Otherwise we could end up
1899          * with a false positive when the remapping didn't
1900          * kick in due the plane being invisible.
1901          */
1902         if (intel_plane_can_remap(plane_state) &&
1903             !plane_state->uapi.visible)
1904                 return 0;
1905
1906         /* FIXME other color planes? */
1907         stride = plane_state->view.color_plane[0].mapping_stride;
1908         max_stride = plane->max_stride(plane, fb->format->format,
1909                                        fb->modifier, rotation);
1910
1911         if (stride > max_stride) {
1912                 drm_dbg_kms(plane->base.dev,
1913                             "[FB:%d] stride (%d) exceeds [PLANE:%d:%s] max stride (%d)\n",
1914                             fb->base.id, stride,
1915                             plane->base.base.id, plane->base.name, max_stride);
1916                 return -EINVAL;
1917         }
1918
1919         return 0;
1920 }
1921
1922 int intel_plane_compute_gtt(struct intel_plane_state *plane_state)
1923 {
1924         const struct intel_framebuffer *fb =
1925                 to_intel_framebuffer(plane_state->hw.fb);
1926         unsigned int rotation = plane_state->hw.rotation;
1927
1928         if (!fb)
1929                 return 0;
1930
1931         if (intel_plane_needs_remap(plane_state)) {
1932                 intel_plane_remap_gtt(plane_state);
1933
1934                 /*
1935                  * Sometimes even remapping can't overcome
1936                  * the stride limitations :( Can happen with
1937                  * big plane sizes and suitably misaligned
1938                  * offsets.
1939                  */
1940                 return intel_plane_check_stride(plane_state);
1941         }
1942
1943         intel_fb_fill_view(fb, rotation, &plane_state->view);
1944
1945         /* Rotate src coordinates to match rotated GTT view */
1946         if (drm_rotation_90_or_270(rotation))
1947                 drm_rect_rotate(&plane_state->uapi.src,
1948                                 fb->base.width << 16, fb->base.height << 16,
1949                                 DRM_MODE_ROTATE_270);
1950
1951         return intel_plane_check_stride(plane_state);
1952 }
1953
1954 static void intel_user_framebuffer_destroy(struct drm_framebuffer *fb)
1955 {
1956         struct intel_framebuffer *intel_fb = to_intel_framebuffer(fb);
1957
1958         drm_framebuffer_cleanup(fb);
1959
1960         if (intel_fb_uses_dpt(fb))
1961                 intel_dpt_destroy(intel_fb->dpt_vm);
1962
1963         intel_frontbuffer_put(intel_fb->frontbuffer);
1964
1965         intel_fb_bo_framebuffer_fini(intel_fb_bo(fb));
1966
1967         kfree(intel_fb);
1968 }
1969
1970 static int intel_user_framebuffer_create_handle(struct drm_framebuffer *fb,
1971                                                 struct drm_file *file,
1972                                                 unsigned int *handle)
1973 {
1974         struct drm_gem_object *obj = intel_fb_bo(fb);
1975         struct intel_display *display = to_intel_display(obj->dev);
1976
1977         if (intel_bo_is_userptr(obj)) {
1978                 drm_dbg(display->drm,
1979                         "attempting to use a userptr for a framebuffer, denied\n");
1980                 return -EINVAL;
1981         }
1982
1983         return drm_gem_handle_create(file, obj, handle);
1984 }
1985
1986 struct frontbuffer_fence_cb {
1987         struct dma_fence_cb base;
1988         struct intel_frontbuffer *front;
1989 };
1990
1991 static void intel_user_framebuffer_fence_wake(struct dma_fence *dma,
1992                                               struct dma_fence_cb *data)
1993 {
1994         struct frontbuffer_fence_cb *cb = container_of(data, typeof(*cb), base);
1995
1996         intel_frontbuffer_queue_flush(cb->front);
1997         kfree(cb);
1998         dma_fence_put(dma);
1999 }
2000
2001 static int intel_user_framebuffer_dirty(struct drm_framebuffer *fb,
2002                                         struct drm_file *file,
2003                                         unsigned int flags, unsigned int color,
2004                                         struct drm_clip_rect *clips,
2005                                         unsigned int num_clips)
2006 {
2007         struct drm_gem_object *obj = intel_fb_bo(fb);
2008         struct intel_frontbuffer *front = to_intel_frontbuffer(fb);
2009         struct dma_fence *fence;
2010         struct frontbuffer_fence_cb *cb;
2011         int ret = 0;
2012
2013         if (!atomic_read(&front->bits))
2014                 return 0;
2015
2016         if (dma_resv_test_signaled(obj->resv, dma_resv_usage_rw(false)))
2017                 goto flush;
2018
2019         ret = dma_resv_get_singleton(obj->resv, dma_resv_usage_rw(false),
2020                                      &fence);
2021         if (ret || !fence)
2022                 goto flush;
2023
2024         cb = kmalloc(sizeof(*cb), GFP_KERNEL);
2025         if (!cb) {
2026                 dma_fence_put(fence);
2027                 ret = -ENOMEM;
2028                 goto flush;
2029         }
2030
2031         cb->front = front;
2032
2033         intel_frontbuffer_invalidate(front, ORIGIN_DIRTYFB);
2034
2035         ret = dma_fence_add_callback(fence, &cb->base,
2036                                      intel_user_framebuffer_fence_wake);
2037         if (ret) {
2038                 intel_user_framebuffer_fence_wake(fence, &cb->base);
2039                 if (ret == -ENOENT)
2040                         ret = 0;
2041         }
2042
2043         return ret;
2044
2045 flush:
2046         intel_bo_flush_if_display(obj);
2047         intel_frontbuffer_flush(front, ORIGIN_DIRTYFB);
2048         return ret;
2049 }
2050
2051 static const struct drm_framebuffer_funcs intel_fb_funcs = {
2052         .destroy = intel_user_framebuffer_destroy,
2053         .create_handle = intel_user_framebuffer_create_handle,
2054         .dirty = intel_user_framebuffer_dirty,
2055 };
2056
2057 int intel_framebuffer_init(struct intel_framebuffer *intel_fb,
2058                            struct drm_gem_object *obj,
2059                            struct drm_mode_fb_cmd2 *mode_cmd)
2060 {
2061         struct drm_i915_private *dev_priv = to_i915(obj->dev);
2062         struct drm_framebuffer *fb = &intel_fb->base;
2063         u32 max_stride;
2064         int ret = -EINVAL;
2065         int i;
2066
2067         ret = intel_fb_bo_framebuffer_init(intel_fb, obj, mode_cmd);
2068         if (ret)
2069                 return ret;
2070
2071         intel_fb->frontbuffer = intel_frontbuffer_get(obj);
2072         if (!intel_fb->frontbuffer) {
2073                 ret = -ENOMEM;
2074                 goto err;
2075         }
2076
2077         ret = -EINVAL;
2078         if (!drm_any_plane_has_format(&dev_priv->drm,
2079                                       mode_cmd->pixel_format,
2080                                       mode_cmd->modifier[0])) {
2081                 drm_dbg_kms(&dev_priv->drm,
2082                             "unsupported pixel format %p4cc / modifier 0x%llx\n",
2083                             &mode_cmd->pixel_format, mode_cmd->modifier[0]);
2084                 goto err_frontbuffer_put;
2085         }
2086
2087         max_stride = intel_fb_max_stride(dev_priv, mode_cmd->pixel_format,
2088                                          mode_cmd->modifier[0]);
2089         if (mode_cmd->pitches[0] > max_stride) {
2090                 drm_dbg_kms(&dev_priv->drm,
2091                             "%s pitch (%u) must be at most %d\n",
2092                             mode_cmd->modifier[0] != DRM_FORMAT_MOD_LINEAR ?
2093                             "tiled" : "linear",
2094                             mode_cmd->pitches[0], max_stride);
2095                 goto err_frontbuffer_put;
2096         }
2097
2098         /* FIXME need to adjust LINOFF/TILEOFF accordingly. */
2099         if (mode_cmd->offsets[0] != 0) {
2100                 drm_dbg_kms(&dev_priv->drm,
2101                             "plane 0 offset (0x%08x) must be 0\n",
2102                             mode_cmd->offsets[0]);
2103                 goto err_frontbuffer_put;
2104         }
2105
2106         drm_helper_mode_fill_fb_struct(&dev_priv->drm, fb, mode_cmd);
2107
2108         for (i = 0; i < fb->format->num_planes; i++) {
2109                 unsigned int stride_alignment;
2110
2111                 if (mode_cmd->handles[i] != mode_cmd->handles[0]) {
2112                         drm_dbg_kms(&dev_priv->drm, "bad plane %d handle\n",
2113                                     i);
2114                         goto err_frontbuffer_put;
2115                 }
2116
2117                 stride_alignment = intel_fb_stride_alignment(fb, i);
2118                 if (fb->pitches[i] & (stride_alignment - 1)) {
2119                         drm_dbg_kms(&dev_priv->drm,
2120                                     "plane %d pitch (%d) must be at least %u byte aligned\n",
2121                                     i, fb->pitches[i], stride_alignment);
2122                         goto err_frontbuffer_put;
2123                 }
2124
2125                 if (intel_fb_is_gen12_ccs_aux_plane(fb, i)) {
2126                         unsigned int ccs_aux_stride = gen12_ccs_aux_stride(intel_fb, i);
2127
2128                         if (fb->pitches[i] != ccs_aux_stride) {
2129                                 drm_dbg_kms(&dev_priv->drm,
2130                                             "ccs aux plane %d pitch (%d) must be %d\n",
2131                                             i,
2132                                             fb->pitches[i], ccs_aux_stride);
2133                                 goto err_frontbuffer_put;
2134                         }
2135                 }
2136
2137                 fb->obj[i] = obj;
2138         }
2139
2140         ret = intel_fill_fb_info(dev_priv, intel_fb);
2141         if (ret)
2142                 goto err_frontbuffer_put;
2143
2144         if (intel_fb_uses_dpt(fb)) {
2145                 struct i915_address_space *vm;
2146
2147                 vm = intel_dpt_create(intel_fb);
2148                 if (IS_ERR(vm)) {
2149                         drm_dbg_kms(&dev_priv->drm, "failed to create DPT\n");
2150                         ret = PTR_ERR(vm);
2151                         goto err_frontbuffer_put;
2152                 }
2153
2154                 intel_fb->dpt_vm = vm;
2155         }
2156
2157         ret = drm_framebuffer_init(&dev_priv->drm, fb, &intel_fb_funcs);
2158         if (ret) {
2159                 drm_err(&dev_priv->drm, "framebuffer init failed %d\n", ret);
2160                 goto err_free_dpt;
2161         }
2162
2163         return 0;
2164
2165 err_free_dpt:
2166         if (intel_fb_uses_dpt(fb))
2167                 intel_dpt_destroy(intel_fb->dpt_vm);
2168 err_frontbuffer_put:
2169         intel_frontbuffer_put(intel_fb->frontbuffer);
2170 err:
2171         intel_fb_bo_framebuffer_fini(obj);
2172         return ret;
2173 }
2174
2175 struct drm_framebuffer *
2176 intel_user_framebuffer_create(struct drm_device *dev,
2177                               struct drm_file *filp,
2178                               const struct drm_mode_fb_cmd2 *user_mode_cmd)
2179 {
2180         struct drm_framebuffer *fb;
2181         struct drm_gem_object *obj;
2182         struct drm_mode_fb_cmd2 mode_cmd = *user_mode_cmd;
2183         struct drm_i915_private *i915 = to_i915(dev);
2184
2185         obj = intel_fb_bo_lookup_valid_bo(i915, filp, &mode_cmd);
2186         if (IS_ERR(obj))
2187                 return ERR_CAST(obj);
2188
2189         fb = intel_framebuffer_create(obj, &mode_cmd);
2190         drm_gem_object_put(obj);
2191
2192         return fb;
2193 }
2194
2195 struct drm_framebuffer *
2196 intel_framebuffer_create(struct drm_gem_object *obj,
2197                          struct drm_mode_fb_cmd2 *mode_cmd)
2198 {
2199         struct intel_framebuffer *intel_fb;
2200         int ret;
2201
2202         intel_fb = kzalloc(sizeof(*intel_fb), GFP_KERNEL);
2203         if (!intel_fb)
2204                 return ERR_PTR(-ENOMEM);
2205
2206         ret = intel_framebuffer_init(intel_fb, obj, mode_cmd);
2207         if (ret)
2208                 goto err;
2209
2210         return &intel_fb->base;
2211
2212 err:
2213         kfree(intel_fb);
2214         return ERR_PTR(ret);
2215 }
2216
2217 struct drm_gem_object *intel_fb_bo(const struct drm_framebuffer *fb)
2218 {
2219         return fb ? fb->obj[0] : NULL;
2220 }