drivers/gpu/drm/i915/gt/gen6_ppgtt.c

   1 // SPDX-License-Identifier: MIT
   2 /*
   3  * Copyright © 2020 Intel Corporation
   4  */
   5
   6 #include <linux/log2.h>
   7
   8 #include "gem/i915_gem_internal.h"
   9
  10 #include "gen6_ppgtt.h"
  11 #include "i915_scatterlist.h"
  12 #include "i915_trace.h"
  13 #include "i915_vgpu.h"
  14 #include "intel_gt_regs.h"
  15 #include "intel_engine_regs.h"
  16 #include "intel_gt.h"
  17
  18 /* Write pde (index) from the page directory @pd to the page table @pt */
  19 static void gen6_write_pde(const struct gen6_ppgtt *ppgtt,
  20                            const unsigned int pde,
  21                            const struct i915_page_table *pt)
  22 {
  23         dma_addr_t addr = pt ? px_dma(pt) : px_dma(ppgtt->base.vm.scratch[1]);
  24
  25         /* Caller needs to make sure the write completes if necessary */
  26         iowrite32(GEN6_PDE_ADDR_ENCODE(addr) | GEN6_PDE_VALID,
  27                   ppgtt->pd_addr + pde);
  28 }
  29
  30 void gen7_ppgtt_enable(struct intel_gt *gt)
  31 {
  32         struct drm_i915_private *i915 = gt->i915;
  33         struct intel_uncore *uncore = gt->uncore;
  34         u32 ecochk;
  35
  36         intel_uncore_rmw(uncore, GAC_ECO_BITS, 0, ECOBITS_PPGTT_CACHE64B);
  37
  38         ecochk = intel_uncore_read(uncore, GAM_ECOCHK);
  39         if (IS_HASWELL(i915)) {
  40                 ecochk |= ECOCHK_PPGTT_WB_HSW;
  41         } else {
  42                 ecochk |= ECOCHK_PPGTT_LLC_IVB;
  43                 ecochk &= ~ECOCHK_PPGTT_GFDT_IVB;
  44         }
  45         intel_uncore_write(uncore, GAM_ECOCHK, ecochk);
  46 }
  47
  48 void gen6_ppgtt_enable(struct intel_gt *gt)
  49 {
  50         struct intel_uncore *uncore = gt->uncore;
  51
  52         intel_uncore_rmw(uncore,
  53                          GAC_ECO_BITS,
  54                          0,
  55                          ECOBITS_SNB_BIT | ECOBITS_PPGTT_CACHE64B);
  56
  57         intel_uncore_rmw(uncore,
  58                          GAB_CTL,
  59                          0,
  60                          GAB_CTL_CONT_AFTER_PAGEFAULT);
  61
  62         intel_uncore_rmw(uncore,
  63                          GAM_ECOCHK,
  64                          0,
  65                          ECOCHK_SNB_BIT | ECOCHK_PPGTT_CACHE64B);
  66
  67         if (HAS_PPGTT(uncore->i915)) /* may be disabled for VT-d */
  68                 intel_uncore_write(uncore,
  69                                    GFX_MODE,
  70                                    _MASKED_BIT_ENABLE(GFX_PPGTT_ENABLE));
  71 }
  72
  73 /* PPGTT support for Sandybdrige/Gen6 and later */
  74 static void gen6_ppgtt_clear_range(struct i915_address_space *vm,
  75                                    u64 start, u64 length)
  76 {
  77         struct gen6_ppgtt * const ppgtt = to_gen6_ppgtt(i915_vm_to_ppgtt(vm));
  78         const unsigned int first_entry = start / I915_GTT_PAGE_SIZE;
  79         const gen6_pte_t scratch_pte = vm->scratch[0]->encode;
  80         unsigned int pde = first_entry / GEN6_PTES;
  81         unsigned int pte = first_entry % GEN6_PTES;
  82         unsigned int num_entries = length / I915_GTT_PAGE_SIZE;
  83
  84         while (num_entries) {
  85                 struct i915_page_table * const pt =
  86                         i915_pt_entry(ppgtt->base.pd, pde++);
  87                 const unsigned int count = min(num_entries, GEN6_PTES - pte);
  88                 gen6_pte_t *vaddr;
  89
  90                 num_entries -= count;
  91
  92                 GEM_BUG_ON(count > atomic_read(&pt->used));
  93                 if (!atomic_sub_return(count, &pt->used))
  94                         ppgtt->scan_for_unused_pt = true;
  95
  96                 /*
  97                  * Note that the hw doesn't support removing PDE on the fly
  98                  * (they are cached inside the context with no means to
  99                  * invalidate the cache), so we can only reset the PTE
 100                  * entries back to scratch.
 101                  */
 102
 103                 vaddr = px_vaddr(pt);
 104                 memset32(vaddr + pte, scratch_pte, count);
 105
 106                 pte = 0;
 107         }
 108 }
 109
 110 static void gen6_ppgtt_insert_entries(struct i915_address_space *vm,
 111                                       struct i915_vma_resource *vma_res,
 112                                       unsigned int pat_index,
 113                                       u32 flags)
 114 {
 115         struct i915_ppgtt *ppgtt = i915_vm_to_ppgtt(vm);
 116         struct i915_page_directory * const pd = ppgtt->pd;
 117         unsigned int first_entry = vma_res->start / I915_GTT_PAGE_SIZE;
 118         unsigned int act_pt = first_entry / GEN6_PTES;
 119         unsigned int act_pte = first_entry % GEN6_PTES;
 120         const u32 pte_encode = vm->pte_encode(0, pat_index, flags);
 121         struct sgt_dma iter = sgt_dma(vma_res);
 122         gen6_pte_t *vaddr;
 123
 124         GEM_BUG_ON(!pd->entry[act_pt]);
 125
 126         vaddr = px_vaddr(i915_pt_entry(pd, act_pt));
 127         do {
 128                 GEM_BUG_ON(sg_dma_len(iter.sg) < I915_GTT_PAGE_SIZE);
 129                 vaddr[act_pte] = pte_encode | GEN6_PTE_ADDR_ENCODE(iter.dma);
 130
 131                 iter.dma += I915_GTT_PAGE_SIZE;
 132                 if (iter.dma == iter.max) {
 133                         iter.sg = __sg_next(iter.sg);
 134                         if (!iter.sg || sg_dma_len(iter.sg) == 0)
 135                                 break;
 136
 137                         iter.dma = sg_dma_address(iter.sg);
 138                         iter.max = iter.dma + sg_dma_len(iter.sg);
 139                 }
 140
 141                 if (++act_pte == GEN6_PTES) {
 142                         vaddr = px_vaddr(i915_pt_entry(pd, ++act_pt));
 143                         act_pte = 0;
 144                 }
 145         } while (1);
 146
 147         vma_res->page_sizes_gtt = I915_GTT_PAGE_SIZE;
 148 }
 149
 150 static void gen6_flush_pd(struct gen6_ppgtt *ppgtt, u64 start, u64 end)
 151 {
 152         struct i915_page_directory * const pd = ppgtt->base.pd;
 153         struct i915_page_table *pt;
 154         unsigned int pde;
 155
 156         start = round_down(start, SZ_64K);
 157         end = round_up(end, SZ_64K) - start;
 158
 159         mutex_lock(&ppgtt->flush);
 160
 161         gen6_for_each_pde(pt, pd, start, end, pde)
 162                 gen6_write_pde(ppgtt, pde, pt);
 163
 164         mb();
 165         ioread32(ppgtt->pd_addr + pde - 1);
 166         gen6_ggtt_invalidate(ppgtt->base.vm.gt->ggtt);
 167         mb();
 168
 169         mutex_unlock(&ppgtt->flush);
 170 }
 171
 172 static void gen6_alloc_va_range(struct i915_address_space *vm,
 173                                 struct i915_vm_pt_stash *stash,
 174                                 u64 start, u64 length)
 175 {
 176         struct gen6_ppgtt *ppgtt = to_gen6_ppgtt(i915_vm_to_ppgtt(vm));
 177         struct i915_page_directory * const pd = ppgtt->base.pd;
 178         struct i915_page_table *pt;
 179         bool flush = false;
 180         u64 from = start;
 181         unsigned int pde;
 182
 183         spin_lock(&pd->lock);
 184         gen6_for_each_pde(pt, pd, start, length, pde) {
 185                 const unsigned int count = gen6_pte_count(start, length);
 186
 187                 if (!pt) {
 188                         spin_unlock(&pd->lock);
 189
 190                         pt = stash->pt[0];
 191                         __i915_gem_object_pin_pages(pt->base);
 192
 193                         fill32_px(pt, vm->scratch[0]->encode);
 194
 195                         spin_lock(&pd->lock);
 196                         if (!pd->entry[pde]) {
 197                                 stash->pt[0] = pt->stash;
 198                                 atomic_set(&pt->used, 0);
 199                                 pd->entry[pde] = pt;
 200                         } else {
 201                                 pt = pd->entry[pde];
 202                         }
 203
 204                         flush = true;
 205                 }
 206
 207                 atomic_add(count, &pt->used);
 208         }
 209         spin_unlock(&pd->lock);
 210
 211         if (flush && i915_vma_is_bound(ppgtt->vma, I915_VMA_GLOBAL_BIND)) {
 212                 intel_wakeref_t wakeref;
 213
 214                 with_intel_runtime_pm(&vm->i915->runtime_pm, wakeref)
 215                         gen6_flush_pd(ppgtt, from, start);
 216         }
 217 }
 218
 219 static int gen6_ppgtt_init_scratch(struct gen6_ppgtt *ppgtt)
 220 {
 221         struct i915_address_space * const vm = &ppgtt->base.vm;
 222         int ret;
 223
 224         ret = setup_scratch_page(vm);
 225         if (ret)
 226                 return ret;
 227
 228         vm->scratch[0]->encode =
 229                 vm->pte_encode(px_dma(vm->scratch[0]),
 230                                i915_gem_get_pat_index(vm->i915,
 231                                                       I915_CACHE_NONE),
 232                                PTE_READ_ONLY);
 233
 234         vm->scratch[1] = vm->alloc_pt_dma(vm, I915_GTT_PAGE_SIZE_4K);
 235         if (IS_ERR(vm->scratch[1])) {
 236                 ret = PTR_ERR(vm->scratch[1]);
 237                 goto err_scratch0;
 238         }
 239
 240         ret = map_pt_dma(vm, vm->scratch[1]);
 241         if (ret)
 242                 goto err_scratch1;
 243
 244         fill32_px(vm->scratch[1], vm->scratch[0]->encode);
 245
 246         return 0;
 247
 248 err_scratch1:
 249         i915_gem_object_put(vm->scratch[1]);
 250 err_scratch0:
 251         i915_gem_object_put(vm->scratch[0]);
 252         vm->scratch[0] = NULL;
 253         return ret;
 254 }
 255
 256 static void gen6_ppgtt_free_pd(struct gen6_ppgtt *ppgtt)
 257 {
 258         struct i915_page_directory * const pd = ppgtt->base.pd;
 259         struct i915_page_table *pt;
 260         u32 pde;
 261
 262         gen6_for_all_pdes(pt, pd, pde)
 263                 if (pt)
 264                         free_pt(&ppgtt->base.vm, pt);
 265 }
 266
 267 static void gen6_ppgtt_cleanup(struct i915_address_space *vm)
 268 {
 269         struct gen6_ppgtt *ppgtt = to_gen6_ppgtt(i915_vm_to_ppgtt(vm));
 270
 271         gen6_ppgtt_free_pd(ppgtt);
 272         free_scratch(vm);
 273
 274         if (ppgtt->base.pd)
 275                 free_pd(&ppgtt->base.vm, ppgtt->base.pd);
 276
 277         mutex_destroy(&ppgtt->flush);
 278 }
 279
 280 static void pd_vma_bind(struct i915_address_space *vm,
 281                         struct i915_vm_pt_stash *stash,
 282                         struct i915_vma_resource *vma_res,
 283                         unsigned int pat_index,
 284                         u32 unused)
 285 {
 286         struct i915_ggtt *ggtt = i915_vm_to_ggtt(vm);
 287         struct gen6_ppgtt *ppgtt = vma_res->private;
 288         u32 ggtt_offset = vma_res->start / I915_GTT_PAGE_SIZE;
 289
 290         ppgtt->pp_dir = ggtt_offset * sizeof(gen6_pte_t) << 10;
 291         ppgtt->pd_addr = (gen6_pte_t __iomem *)ggtt->gsm + ggtt_offset;
 292
 293         gen6_flush_pd(ppgtt, 0, ppgtt->base.vm.total);
 294 }
 295
 296 static void pd_vma_unbind(struct i915_address_space *vm,
 297                           struct i915_vma_resource *vma_res)
 298 {
 299         struct gen6_ppgtt *ppgtt = vma_res->private;
 300         struct i915_page_directory * const pd = ppgtt->base.pd;
 301         struct i915_page_table *pt;
 302         unsigned int pde;
 303
 304         if (!ppgtt->scan_for_unused_pt)
 305                 return;
 306
 307         /* Free all no longer used page tables */
 308         gen6_for_all_pdes(pt, ppgtt->base.pd, pde) {
 309                 if (!pt || atomic_read(&pt->used))
 310                         continue;
 311
 312                 free_pt(&ppgtt->base.vm, pt);
 313                 pd->entry[pde] = NULL;
 314         }
 315
 316         ppgtt->scan_for_unused_pt = false;
 317 }
 318
 319 static const struct i915_vma_ops pd_vma_ops = {
 320         .bind_vma = pd_vma_bind,
 321         .unbind_vma = pd_vma_unbind,
 322 };
 323
 324 int gen6_ppgtt_pin(struct i915_ppgtt *base, struct i915_gem_ww_ctx *ww)
 325 {
 326         struct gen6_ppgtt *ppgtt = to_gen6_ppgtt(base);
 327         int err;
 328
 329         GEM_BUG_ON(!kref_read(&ppgtt->base.vm.ref));
 330
 331         /*
 332          * Workaround the limited maximum vma->pin_count and the aliasing_ppgtt
 333          * which will be pinned into every active context.
 334          * (When vma->pin_count becomes atomic, I expect we will naturally
 335          * need a larger, unpacked, type and kill this redundancy.)
 336          */
 337         if (atomic_add_unless(&ppgtt->pin_count, 1, 0))
 338                 return 0;
 339
 340         /* grab the ppgtt resv to pin the object */
 341         err = i915_vm_lock_objects(&ppgtt->base.vm, ww);
 342         if (err)
 343                 return err;
 344
 345         /*
 346          * PPGTT PDEs reside in the GGTT and consists of 512 entries. The
 347          * allocator works in address space sizes, so it's multiplied by page
 348          * size. We allocate at the top of the GTT to avoid fragmentation.
 349          */
 350         if (!atomic_read(&ppgtt->pin_count)) {
 351                 err = i915_ggtt_pin(ppgtt->vma, ww, GEN6_PD_ALIGN, PIN_HIGH);
 352
 353                 GEM_BUG_ON(ppgtt->vma->fence);
 354                 clear_bit(I915_VMA_CAN_FENCE_BIT, __i915_vma_flags(ppgtt->vma));
 355         }
 356         if (!err)
 357                 atomic_inc(&ppgtt->pin_count);
 358
 359         return err;
 360 }
 361
 362 static int pd_dummy_obj_get_pages(struct drm_i915_gem_object *obj)
 363 {
 364         obj->mm.pages = ZERO_SIZE_PTR;
 365         return 0;
 366 }
 367
 368 static void pd_dummy_obj_put_pages(struct drm_i915_gem_object *obj,
 369                                    struct sg_table *pages)
 370 {
 371 }
 372
 373 static const struct drm_i915_gem_object_ops pd_dummy_obj_ops = {
 374         .name = "pd_dummy_obj",
 375         .get_pages = pd_dummy_obj_get_pages,
 376         .put_pages = pd_dummy_obj_put_pages,
 377 };
 378
 379 static struct i915_page_directory *
 380 gen6_alloc_top_pd(struct gen6_ppgtt *ppgtt)
 381 {
 382         struct i915_ggtt * const ggtt = ppgtt->base.vm.gt->ggtt;
 383         struct i915_page_directory *pd;
 384         int err;
 385
 386         pd = __alloc_pd(I915_PDES);
 387         if (unlikely(!pd))
 388                 return ERR_PTR(-ENOMEM);
 389
 390         pd->pt.base = __i915_gem_object_create_internal(ppgtt->base.vm.gt->i915,
 391                                                         &pd_dummy_obj_ops,
 392                                                         I915_PDES * SZ_4K);
 393         if (IS_ERR(pd->pt.base)) {
 394                 err = PTR_ERR(pd->pt.base);
 395                 pd->pt.base = NULL;
 396                 goto err_pd;
 397         }
 398
 399         pd->pt.base->base.resv = i915_vm_resv_get(&ppgtt->base.vm);
 400         pd->pt.base->shares_resv_from = &ppgtt->base.vm;
 401
 402         ppgtt->vma = i915_vma_instance(pd->pt.base, &ggtt->vm, NULL);
 403         if (IS_ERR(ppgtt->vma)) {
 404                 err = PTR_ERR(ppgtt->vma);
 405                 ppgtt->vma = NULL;
 406                 goto err_pd;
 407         }
 408
 409         /* The dummy object we create is special, override ops.. */
 410         ppgtt->vma->ops = &pd_vma_ops;
 411         ppgtt->vma->private = ppgtt;
 412         return pd;
 413
 414 err_pd:
 415         free_pd(&ppgtt->base.vm, pd);
 416         return ERR_PTR(err);
 417 }
 418
 419 void gen6_ppgtt_unpin(struct i915_ppgtt *base)
 420 {
 421         struct gen6_ppgtt *ppgtt = to_gen6_ppgtt(base);
 422
 423         GEM_BUG_ON(!atomic_read(&ppgtt->pin_count));
 424         if (atomic_dec_and_test(&ppgtt->pin_count))
 425                 i915_vma_unpin(ppgtt->vma);
 426 }
 427
 428 struct i915_ppgtt *gen6_ppgtt_create(struct intel_gt *gt)
 429 {
 430         struct i915_ggtt * const ggtt = gt->ggtt;
 431         struct gen6_ppgtt *ppgtt;
 432         int err;
 433
 434         ppgtt = kzalloc(sizeof(*ppgtt), GFP_KERNEL);
 435         if (!ppgtt)
 436                 return ERR_PTR(-ENOMEM);
 437
 438         mutex_init(&ppgtt->flush);
 439
 440         ppgtt_init(&ppgtt->base, gt, 0);
 441         ppgtt->base.vm.pd_shift = ilog2(SZ_4K * SZ_4K / sizeof(gen6_pte_t));
 442         ppgtt->base.vm.top = 1;
 443
 444         ppgtt->base.vm.bind_async_flags = I915_VMA_LOCAL_BIND;
 445         ppgtt->base.vm.allocate_va_range = gen6_alloc_va_range;
 446         ppgtt->base.vm.clear_range = gen6_ppgtt_clear_range;
 447         ppgtt->base.vm.insert_entries = gen6_ppgtt_insert_entries;
 448         ppgtt->base.vm.cleanup = gen6_ppgtt_cleanup;
 449
 450         ppgtt->base.vm.alloc_pt_dma = alloc_pt_dma;
 451         ppgtt->base.vm.alloc_scratch_dma = alloc_pt_dma;
 452         ppgtt->base.vm.pte_encode = ggtt->vm.pte_encode;
 453
 454         err = gen6_ppgtt_init_scratch(ppgtt);
 455         if (err)
 456                 goto err_put;
 457
 458         ppgtt->base.pd = gen6_alloc_top_pd(ppgtt);
 459         if (IS_ERR(ppgtt->base.pd)) {
 460                 err = PTR_ERR(ppgtt->base.pd);
 461                 goto err_put;
 462         }
 463
 464         return &ppgtt->base;
 465
 466 err_put:
 467         i915_vm_put(&ppgtt->base.vm);
 468         return ERR_PTR(err);
 469 }