drivers/gpu/drm/i915/display/intel_dpt.c

   1 // SPDX-License-Identifier: MIT
   2 /*
   3  * Copyright © 2021 Intel Corporation
   4  */
   5
   6 #include "gem/i915_gem_domain.h"
   7 #include "gem/i915_gem_internal.h"
   8 #include "gem/i915_gem_lmem.h"
   9 #include "gt/gen8_ppgtt.h"
  10
  11 #include "i915_drv.h"
  12 #include "i915_reg.h"
  13 #include "intel_de.h"
  14 #include "intel_display_types.h"
  15 #include "intel_dpt.h"
  16 #include "intel_fb.h"
  17
  18 struct i915_dpt {
  19         struct i915_address_space vm;
  20
  21         struct drm_i915_gem_object *obj;
  22         struct i915_vma *vma;
  23         void __iomem *iomem;
  24 };
  25
  26 #define i915_is_dpt(vm) ((vm)->is_dpt)
  27
  28 static inline struct i915_dpt *
  29 i915_vm_to_dpt(struct i915_address_space *vm)
  30 {
  31         BUILD_BUG_ON(offsetof(struct i915_dpt, vm));
  32         drm_WARN_ON(&vm->i915->drm, !i915_is_dpt(vm));
  33         return container_of(vm, struct i915_dpt, vm);
  34 }
  35
  36 #define dpt_total_entries(dpt) ((dpt)->vm.total >> PAGE_SHIFT)
  37
  38 static void gen8_set_pte(void __iomem *addr, gen8_pte_t pte)
  39 {
  40         writeq(pte, addr);
  41 }
  42
  43 static void dpt_insert_page(struct i915_address_space *vm,
  44                             dma_addr_t addr,
  45                             u64 offset,
  46                             unsigned int pat_index,
  47                             u32 flags)
  48 {
  49         struct i915_dpt *dpt = i915_vm_to_dpt(vm);
  50         gen8_pte_t __iomem *base = dpt->iomem;
  51
  52         gen8_set_pte(base + offset / I915_GTT_PAGE_SIZE,
  53                      vm->pte_encode(addr, pat_index, flags));
  54 }
  55
  56 static void dpt_insert_entries(struct i915_address_space *vm,
  57                                struct i915_vma_resource *vma_res,
  58                                unsigned int pat_index,
  59                                u32 flags)
  60 {
  61         struct i915_dpt *dpt = i915_vm_to_dpt(vm);
  62         gen8_pte_t __iomem *base = dpt->iomem;
  63         const gen8_pte_t pte_encode = vm->pte_encode(0, pat_index, flags);
  64         struct sgt_iter sgt_iter;
  65         dma_addr_t addr;
  66         int i;
  67
  68         /*
  69          * Note that we ignore PTE_READ_ONLY here. The caller must be careful
  70          * not to allow the user to override access to a read only page.
  71          */
  72
  73         i = vma_res->start / I915_GTT_PAGE_SIZE;
  74         for_each_sgt_daddr(addr, sgt_iter, vma_res->bi.pages)
  75                 gen8_set_pte(&base[i++], pte_encode | addr);
  76 }
  77
  78 static void dpt_clear_range(struct i915_address_space *vm,
  79                             u64 start, u64 length)
  80 {
  81 }
  82
  83 static void dpt_bind_vma(struct i915_address_space *vm,
  84                          struct i915_vm_pt_stash *stash,
  85                          struct i915_vma_resource *vma_res,
  86                          unsigned int pat_index,
  87                          u32 flags)
  88 {
  89         u32 pte_flags;
  90
  91         if (vma_res->bound_flags)
  92                 return;
  93
  94         /* Applicable to VLV (gen8+ do not support RO in the GGTT) */
  95         pte_flags = 0;
  96         if (vm->has_read_only && vma_res->bi.readonly)
  97                 pte_flags |= PTE_READ_ONLY;
  98         if (vma_res->bi.lmem)
  99                 pte_flags |= PTE_LM;
 100
 101         vm->insert_entries(vm, vma_res, pat_index, pte_flags);
 102
 103         vma_res->page_sizes_gtt = I915_GTT_PAGE_SIZE;
 104
 105         /*
 106          * Without aliasing PPGTT there's no difference between
 107          * GLOBAL/LOCAL_BIND, it's all the same ptes. Hence unconditionally
 108          * upgrade to both bound if we bind either to avoid double-binding.
 109          */
 110         vma_res->bound_flags = I915_VMA_GLOBAL_BIND | I915_VMA_LOCAL_BIND;
 111 }
 112
 113 static void dpt_unbind_vma(struct i915_address_space *vm,
 114                            struct i915_vma_resource *vma_res)
 115 {
 116         vm->clear_range(vm, vma_res->start, vma_res->vma_size);
 117 }
 118
 119 static void dpt_cleanup(struct i915_address_space *vm)
 120 {
 121         struct i915_dpt *dpt = i915_vm_to_dpt(vm);
 122
 123         i915_gem_object_put(dpt->obj);
 124 }
 125
 126 struct i915_vma *intel_dpt_pin(struct i915_address_space *vm)
 127 {
 128         struct drm_i915_private *i915 = vm->i915;
 129         struct i915_dpt *dpt = i915_vm_to_dpt(vm);
 130         intel_wakeref_t wakeref;
 131         struct i915_vma *vma;
 132         void __iomem *iomem;
 133         struct i915_gem_ww_ctx ww;
 134         u64 pin_flags = 0;
 135         int err;
 136
 137         if (i915_gem_object_is_stolen(dpt->obj))
 138                 pin_flags |= PIN_MAPPABLE;
 139
 140         wakeref = intel_runtime_pm_get(&i915->runtime_pm);
 141         atomic_inc(&i915->gpu_error.pending_fb_pin);
 142
 143         for_i915_gem_ww(&ww, err, true) {
 144                 err = i915_gem_object_lock(dpt->obj, &ww);
 145                 if (err)
 146                         continue;
 147
 148                 vma = i915_gem_object_ggtt_pin_ww(dpt->obj, &ww, NULL, 0, 4096,
 149                                                   pin_flags);
 150                 if (IS_ERR(vma)) {
 151                         err = PTR_ERR(vma);
 152                         continue;
 153                 }
 154
 155                 iomem = i915_vma_pin_iomap(vma);
 156                 i915_vma_unpin(vma);
 157
 158                 if (IS_ERR(iomem)) {
 159                         err = PTR_ERR(iomem);
 160                         continue;
 161                 }
 162
 163                 dpt->vma = vma;
 164                 dpt->iomem = iomem;
 165
 166                 i915_vma_get(vma);
 167         }
 168
 169         dpt->obj->mm.dirty = true;
 170
 171         atomic_dec(&i915->gpu_error.pending_fb_pin);
 172         intel_runtime_pm_put(&i915->runtime_pm, wakeref);
 173
 174         return err ? ERR_PTR(err) : vma;
 175 }
 176
 177 void intel_dpt_unpin(struct i915_address_space *vm)
 178 {
 179         struct i915_dpt *dpt = i915_vm_to_dpt(vm);
 180
 181         i915_vma_unpin_iomap(dpt->vma);
 182         i915_vma_put(dpt->vma);
 183 }
 184
 185 /**
 186  * intel_dpt_resume - restore the memory mapping for all DPT FBs during system resume
 187  * @i915: device instance
 188  *
 189  * Restore the memory mapping during system resume for all framebuffers which
 190  * are mapped to HW via a GGTT->DPT page table. The content of these page
 191  * tables are not stored in the hibernation image during S4 and S3RST->S4
 192  * transitions, so here we reprogram the PTE entries in those tables.
 193  *
 194  * This function must be called after the mappings in GGTT have been restored calling
 195  * i915_ggtt_resume().
 196  */
 197 void intel_dpt_resume(struct drm_i915_private *i915)
 198 {
 199         struct drm_framebuffer *drm_fb;
 200
 201         if (!HAS_DISPLAY(i915))
 202                 return;
 203
 204         mutex_lock(&i915->drm.mode_config.fb_lock);
 205         drm_for_each_fb(drm_fb, &i915->drm) {
 206                 struct intel_framebuffer *fb = to_intel_framebuffer(drm_fb);
 207
 208                 if (fb->dpt_vm)
 209                         i915_ggtt_resume_vm(fb->dpt_vm);
 210         }
 211         mutex_unlock(&i915->drm.mode_config.fb_lock);
 212 }
 213
 214 /**
 215  * intel_dpt_suspend - suspend the memory mapping for all DPT FBs during system suspend
 216  * @i915: device instance
 217  *
 218  * Suspend the memory mapping during system suspend for all framebuffers which
 219  * are mapped to HW via a GGTT->DPT page table.
 220  *
 221  * This function must be called before the mappings in GGTT are suspended calling
 222  * i915_ggtt_suspend().
 223  */
 224 void intel_dpt_suspend(struct drm_i915_private *i915)
 225 {
 226         struct drm_framebuffer *drm_fb;
 227
 228         if (!HAS_DISPLAY(i915))
 229                 return;
 230
 231         mutex_lock(&i915->drm.mode_config.fb_lock);
 232
 233         drm_for_each_fb(drm_fb, &i915->drm) {
 234                 struct intel_framebuffer *fb = to_intel_framebuffer(drm_fb);
 235
 236                 if (fb->dpt_vm)
 237                         i915_ggtt_suspend_vm(fb->dpt_vm);
 238         }
 239
 240         mutex_unlock(&i915->drm.mode_config.fb_lock);
 241 }
 242
 243 struct i915_address_space *
 244 intel_dpt_create(struct intel_framebuffer *fb)
 245 {
 246         struct drm_gem_object *obj = &intel_fb_obj(&fb->base)->base;
 247         struct drm_i915_private *i915 = to_i915(obj->dev);
 248         struct drm_i915_gem_object *dpt_obj;
 249         struct i915_address_space *vm;
 250         struct i915_dpt *dpt;
 251         size_t size;
 252         int ret;
 253
 254         if (intel_fb_needs_pot_stride_remap(fb))
 255                 size = intel_remapped_info_size(&fb->remapped_view.gtt.remapped);
 256         else
 257                 size = DIV_ROUND_UP_ULL(obj->size, I915_GTT_PAGE_SIZE);
 258
 259         size = round_up(size * sizeof(gen8_pte_t), I915_GTT_PAGE_SIZE);
 260
 261         dpt_obj = i915_gem_object_create_lmem(i915, size, I915_BO_ALLOC_CONTIGUOUS);
 262         if (IS_ERR(dpt_obj) && i915_ggtt_has_aperture(to_gt(i915)->ggtt))
 263                 dpt_obj = i915_gem_object_create_stolen(i915, size);
 264         if (IS_ERR(dpt_obj) && !HAS_LMEM(i915)) {
 265                 drm_dbg_kms(&i915->drm, "Allocating dpt from smem\n");
 266                 dpt_obj = i915_gem_object_create_shmem(i915, size);
 267         }
 268         if (IS_ERR(dpt_obj))
 269                 return ERR_CAST(dpt_obj);
 270
 271         ret = i915_gem_object_lock_interruptible(dpt_obj, NULL);
 272         if (!ret) {
 273                 ret = i915_gem_object_set_cache_level(dpt_obj, I915_CACHE_NONE);
 274                 i915_gem_object_unlock(dpt_obj);
 275         }
 276         if (ret) {
 277                 i915_gem_object_put(dpt_obj);
 278                 return ERR_PTR(ret);
 279         }
 280
 281         dpt = kzalloc(sizeof(*dpt), GFP_KERNEL);
 282         if (!dpt) {
 283                 i915_gem_object_put(dpt_obj);
 284                 return ERR_PTR(-ENOMEM);
 285         }
 286
 287         vm = &dpt->vm;
 288
 289         vm->gt = to_gt(i915);
 290         vm->i915 = i915;
 291         vm->dma = i915->drm.dev;
 292         vm->total = (size / sizeof(gen8_pte_t)) * I915_GTT_PAGE_SIZE;
 293         vm->is_dpt = true;
 294
 295         i915_address_space_init(vm, VM_CLASS_DPT);
 296
 297         vm->insert_page = dpt_insert_page;
 298         vm->clear_range = dpt_clear_range;
 299         vm->insert_entries = dpt_insert_entries;
 300         vm->cleanup = dpt_cleanup;
 301
 302         vm->vma_ops.bind_vma    = dpt_bind_vma;
 303         vm->vma_ops.unbind_vma  = dpt_unbind_vma;
 304
 305         vm->pte_encode = vm->gt->ggtt->vm.pte_encode;
 306
 307         dpt->obj = dpt_obj;
 308         dpt->obj->is_dpt = true;
 309
 310         return &dpt->vm;
 311 }
 312
 313 void intel_dpt_destroy(struct i915_address_space *vm)
 314 {
 315         struct i915_dpt *dpt = i915_vm_to_dpt(vm);
 316
 317         dpt->obj->is_dpt = false;
 318         i915_vm_put(&dpt->vm);
 319 }
 320
 321 void intel_dpt_configure(struct intel_crtc *crtc)
 322 {
 323         struct drm_i915_private *i915 = to_i915(crtc->base.dev);
 324
 325         if (DISPLAY_VER(i915) == 14) {
 326                 enum pipe pipe = crtc->pipe;
 327                 enum plane_id plane_id;
 328
 329                 for_each_plane_id_on_crtc(crtc, plane_id) {
 330                         if (plane_id == PLANE_CURSOR)
 331                                 continue;
 332
 333                         intel_de_rmw(i915, PLANE_CHICKEN(pipe, plane_id),
 334                                      PLANE_CHICKEN_DISABLE_DPT,
 335                                      i915->params.enable_dpt ? 0 : PLANE_CHICKEN_DISABLE_DPT);
 336                 }
 337         } else if (DISPLAY_VER(i915) == 13) {
 338                 intel_de_rmw(i915, CHICKEN_MISC_2,
 339                              CHICKEN_MISC_DISABLE_DPT,
 340                              i915->params.enable_dpt ? 0 : CHICKEN_MISC_DISABLE_DPT);
 341         }
 342 }