drivers/gpu/drm/i915/gt/intel_ggtt.c

   1 // SPDX-License-Identifier: MIT
   2 /*
   3  * Copyright © 2020 Intel Corporation
   4  */
   5
   6 #include <linux/stop_machine.h>
   7
   8 #include <asm/set_memory.h>
   9 #include <asm/smp.h>
  10
  11 #include <drm/i915_drm.h>
  12
  13 #include "intel_gt.h"
  14 #include "i915_drv.h"
  15 #include "i915_scatterlist.h"
  16 #include "i915_vgpu.h"
  17
  18 #include "intel_gtt.h"
  19
  20 static int
  21 i915_get_ggtt_vma_pages(struct i915_vma *vma);
  22
  23 static void i915_ggtt_color_adjust(const struct drm_mm_node *node,
  24                                    unsigned long color,
  25                                    u64 *start,
  26                                    u64 *end)
  27 {
  28         if (i915_node_color_differs(node, color))
  29                 *start += I915_GTT_PAGE_SIZE;
  30
  31         /*
  32          * Also leave a space between the unallocated reserved node after the
  33          * GTT and any objects within the GTT, i.e. we use the color adjustment
  34          * to insert a guard page to prevent prefetches crossing over the
  35          * GTT boundary.
  36          */
  37         node = list_next_entry(node, node_list);
  38         if (node->color != color)
  39                 *end -= I915_GTT_PAGE_SIZE;
  40 }
  41
  42 static int ggtt_init_hw(struct i915_ggtt *ggtt)
  43 {
  44         struct drm_i915_private *i915 = ggtt->vm.i915;
  45
  46         i915_address_space_init(&ggtt->vm, VM_CLASS_GGTT);
  47
  48         ggtt->vm.is_ggtt = true;
  49
  50         /* Only VLV supports read-only GGTT mappings */
  51         ggtt->vm.has_read_only = IS_VALLEYVIEW(i915);
  52
  53         if (!HAS_LLC(i915) && !HAS_PPGTT(i915))
  54                 ggtt->vm.mm.color_adjust = i915_ggtt_color_adjust;
  55
  56         if (ggtt->mappable_end) {
  57                 if (!io_mapping_init_wc(&ggtt->iomap,
  58                                         ggtt->gmadr.start,
  59                                         ggtt->mappable_end)) {
  60                         ggtt->vm.cleanup(&ggtt->vm);
  61                         return -EIO;
  62                 }
  63
  64                 ggtt->mtrr = arch_phys_wc_add(ggtt->gmadr.start,
  65                                               ggtt->mappable_end);
  66         }
  67
  68         intel_ggtt_init_fences(ggtt);
  69
  70         return 0;
  71 }
  72
  73 /**
  74  * i915_ggtt_init_hw - Initialize GGTT hardware
  75  * @i915: i915 device
  76  */
  77 int i915_ggtt_init_hw(struct drm_i915_private *i915)
  78 {
  79         int ret;
  80
  81         /*
  82          * Note that we use page colouring to enforce a guard page at the
  83          * end of the address space. This is required as the CS may prefetch
  84          * beyond the end of the batch buffer, across the page boundary,
  85          * and beyond the end of the GTT if we do not provide a guard.
  86          */
  87         ret = ggtt_init_hw(&i915->ggtt);
  88         if (ret)
  89                 return ret;
  90
  91         return 0;
  92 }
  93
  94 /*
  95  * Certain Gen5 chipsets require require idling the GPU before
  96  * unmapping anything from the GTT when VT-d is enabled.
  97  */
  98 static bool needs_idle_maps(struct drm_i915_private *i915)
  99 {
 100         /*
 101          * Query intel_iommu to see if we need the workaround. Presumably that
 102          * was loaded first.
 103          */
 104         return IS_GEN(i915, 5) && IS_MOBILE(i915) && intel_vtd_active();
 105 }
 106
 107 void i915_ggtt_suspend(struct i915_ggtt *ggtt)
 108 {
 109         struct i915_vma *vma, *vn;
 110         int open;
 111
 112         mutex_lock(&ggtt->vm.mutex);
 113
 114         /* Skip rewriting PTE on VMA unbind. */
 115         open = atomic_xchg(&ggtt->vm.open, 0);
 116
 117         list_for_each_entry_safe(vma, vn, &ggtt->vm.bound_list, vm_link) {
 118                 GEM_BUG_ON(!drm_mm_node_allocated(&vma->node));
 119                 i915_vma_wait_for_bind(vma);
 120
 121                 if (i915_vma_is_pinned(vma))
 122                         continue;
 123
 124                 if (!i915_vma_is_bound(vma, I915_VMA_GLOBAL_BIND)) {
 125                         __i915_vma_evict(vma);
 126                         drm_mm_remove_node(&vma->node);
 127                 }
 128         }
 129
 130         ggtt->vm.clear_range(&ggtt->vm, 0, ggtt->vm.total);
 131         ggtt->invalidate(ggtt);
 132         atomic_set(&ggtt->vm.open, open);
 133
 134         mutex_unlock(&ggtt->vm.mutex);
 135
 136         intel_gt_check_and_clear_faults(ggtt->vm.gt);
 137 }
 138
 139 void gen6_ggtt_invalidate(struct i915_ggtt *ggtt)
 140 {
 141         struct intel_uncore *uncore = ggtt->vm.gt->uncore;
 142
 143         spin_lock_irq(&uncore->lock);
 144         intel_uncore_write_fw(uncore, GFX_FLSH_CNTL_GEN6, GFX_FLSH_CNTL_EN);
 145         intel_uncore_read_fw(uncore, GFX_FLSH_CNTL_GEN6);
 146         spin_unlock_irq(&uncore->lock);
 147 }
 148
 149 static void gen8_ggtt_invalidate(struct i915_ggtt *ggtt)
 150 {
 151         struct intel_uncore *uncore = ggtt->vm.gt->uncore;
 152
 153         /*
 154          * Note that as an uncached mmio write, this will flush the
 155          * WCB of the writes into the GGTT before it triggers the invalidate.
 156          */
 157         intel_uncore_write_fw(uncore, GFX_FLSH_CNTL_GEN6, GFX_FLSH_CNTL_EN);
 158 }
 159
 160 static void guc_ggtt_invalidate(struct i915_ggtt *ggtt)
 161 {
 162         struct intel_uncore *uncore = ggtt->vm.gt->uncore;
 163         struct drm_i915_private *i915 = ggtt->vm.i915;
 164
 165         gen8_ggtt_invalidate(ggtt);
 166
 167         if (INTEL_GEN(i915) >= 12)
 168                 intel_uncore_write_fw(uncore, GEN12_GUC_TLB_INV_CR,
 169                                       GEN12_GUC_TLB_INV_CR_INVALIDATE);
 170         else
 171                 intel_uncore_write_fw(uncore, GEN8_GTCR, GEN8_GTCR_INVALIDATE);
 172 }
 173
 174 static void gmch_ggtt_invalidate(struct i915_ggtt *ggtt)
 175 {
 176         intel_gtt_chipset_flush();
 177 }
 178
 179 static u64 gen8_ggtt_pte_encode(dma_addr_t addr,
 180                                 enum i915_cache_level level,
 181                                 u32 flags)
 182 {
 183         return addr | _PAGE_PRESENT;
 184 }
 185
 186 static void gen8_set_pte(void __iomem *addr, gen8_pte_t pte)
 187 {
 188         writeq(pte, addr);
 189 }
 190
 191 static void gen8_ggtt_insert_page(struct i915_address_space *vm,
 192                                   dma_addr_t addr,
 193                                   u64 offset,
 194                                   enum i915_cache_level level,
 195                                   u32 unused)
 196 {
 197         struct i915_ggtt *ggtt = i915_vm_to_ggtt(vm);
 198         gen8_pte_t __iomem *pte =
 199                 (gen8_pte_t __iomem *)ggtt->gsm + offset / I915_GTT_PAGE_SIZE;
 200
 201         gen8_set_pte(pte, gen8_ggtt_pte_encode(addr, level, 0));
 202
 203         ggtt->invalidate(ggtt);
 204 }
 205
 206 static void gen8_ggtt_insert_entries(struct i915_address_space *vm,
 207                                      struct i915_vma *vma,
 208                                      enum i915_cache_level level,
 209                                      u32 flags)
 210 {
 211         const gen8_pte_t pte_encode = gen8_ggtt_pte_encode(0, level, 0);
 212         struct i915_ggtt *ggtt = i915_vm_to_ggtt(vm);
 213         gen8_pte_t __iomem *gte;
 214         gen8_pte_t __iomem *end;
 215         struct sgt_iter iter;
 216         dma_addr_t addr;
 217
 218         /*
 219          * Note that we ignore PTE_READ_ONLY here. The caller must be careful
 220          * not to allow the user to override access to a read only page.
 221          */
 222
 223         gte = (gen8_pte_t __iomem *)ggtt->gsm;
 224         gte += vma->node.start / I915_GTT_PAGE_SIZE;
 225         end = gte + vma->node.size / I915_GTT_PAGE_SIZE;
 226
 227         for_each_sgt_daddr(addr, iter, vma->pages)
 228                 gen8_set_pte(gte++, pte_encode | addr);
 229         GEM_BUG_ON(gte > end);
 230
 231         /* Fill the allocated but "unused" space beyond the end of the buffer */
 232         while (gte < end)
 233                 gen8_set_pte(gte++, vm->scratch[0]->encode);
 234
 235         /*
 236          * We want to flush the TLBs only after we're certain all the PTE
 237          * updates have finished.
 238          */
 239         ggtt->invalidate(ggtt);
 240 }
 241
 242 static void gen6_ggtt_insert_page(struct i915_address_space *vm,
 243                                   dma_addr_t addr,
 244                                   u64 offset,
 245                                   enum i915_cache_level level,
 246                                   u32 flags)
 247 {
 248         struct i915_ggtt *ggtt = i915_vm_to_ggtt(vm);
 249         gen6_pte_t __iomem *pte =
 250                 (gen6_pte_t __iomem *)ggtt->gsm + offset / I915_GTT_PAGE_SIZE;
 251
 252         iowrite32(vm->pte_encode(addr, level, flags), pte);
 253
 254         ggtt->invalidate(ggtt);
 255 }
 256
 257 /*
 258  * Binds an object into the global gtt with the specified cache level.
 259  * The object will be accessible to the GPU via commands whose operands
 260  * reference offsets within the global GTT as well as accessible by the GPU
 261  * through the GMADR mapped BAR (i915->mm.gtt->gtt).
 262  */
 263 static void gen6_ggtt_insert_entries(struct i915_address_space *vm,
 264                                      struct i915_vma *vma,
 265                                      enum i915_cache_level level,
 266                                      u32 flags)
 267 {
 268         struct i915_ggtt *ggtt = i915_vm_to_ggtt(vm);
 269         gen6_pte_t __iomem *gte;
 270         gen6_pte_t __iomem *end;
 271         struct sgt_iter iter;
 272         dma_addr_t addr;
 273
 274         gte = (gen6_pte_t __iomem *)ggtt->gsm;
 275         gte += vma->node.start / I915_GTT_PAGE_SIZE;
 276         end = gte + vma->node.size / I915_GTT_PAGE_SIZE;
 277
 278         for_each_sgt_daddr(addr, iter, vma->pages)
 279                 iowrite32(vm->pte_encode(addr, level, flags), gte++);
 280         GEM_BUG_ON(gte > end);
 281
 282         /* Fill the allocated but "unused" space beyond the end of the buffer */
 283         while (gte < end)
 284                 iowrite32(vm->scratch[0]->encode, gte++);
 285
 286         /*
 287          * We want to flush the TLBs only after we're certain all the PTE
 288          * updates have finished.
 289          */
 290         ggtt->invalidate(ggtt);
 291 }
 292
 293 static void nop_clear_range(struct i915_address_space *vm,
 294                             u64 start, u64 length)
 295 {
 296 }
 297
 298 static void gen8_ggtt_clear_range(struct i915_address_space *vm,
 299                                   u64 start, u64 length)
 300 {
 301         struct i915_ggtt *ggtt = i915_vm_to_ggtt(vm);
 302         unsigned int first_entry = start / I915_GTT_PAGE_SIZE;
 303         unsigned int num_entries = length / I915_GTT_PAGE_SIZE;
 304         const gen8_pte_t scratch_pte = vm->scratch[0]->encode;
 305         gen8_pte_t __iomem *gtt_base =
 306                 (gen8_pte_t __iomem *)ggtt->gsm + first_entry;
 307         const int max_entries = ggtt_total_entries(ggtt) - first_entry;
 308         int i;
 309
 310         if (WARN(num_entries > max_entries,
 311                  "First entry = %d; Num entries = %d (max=%d)\n",
 312                  first_entry, num_entries, max_entries))
 313                 num_entries = max_entries;
 314
 315         for (i = 0; i < num_entries; i++)
 316                 gen8_set_pte(&gtt_base[i], scratch_pte);
 317 }
 318
 319 static void bxt_vtd_ggtt_wa(struct i915_address_space *vm)
 320 {
 321         /*
 322          * Make sure the internal GAM fifo has been cleared of all GTT
 323          * writes before exiting stop_machine(). This guarantees that
 324          * any aperture accesses waiting to start in another process
 325          * cannot back up behind the GTT writes causing a hang.
 326          * The register can be any arbitrary GAM register.
 327          */
 328         intel_uncore_posting_read_fw(vm->gt->uncore, GFX_FLSH_CNTL_GEN6);
 329 }
 330
 331 struct insert_page {
 332         struct i915_address_space *vm;
 333         dma_addr_t addr;
 334         u64 offset;
 335         enum i915_cache_level level;
 336 };
 337
 338 static int bxt_vtd_ggtt_insert_page__cb(void *_arg)
 339 {
 340         struct insert_page *arg = _arg;
 341
 342         gen8_ggtt_insert_page(arg->vm, arg->addr, arg->offset, arg->level, 0);
 343         bxt_vtd_ggtt_wa(arg->vm);
 344
 345         return 0;
 346 }
 347
 348 static void bxt_vtd_ggtt_insert_page__BKL(struct i915_address_space *vm,
 349                                           dma_addr_t addr,
 350                                           u64 offset,
 351                                           enum i915_cache_level level,
 352                                           u32 unused)
 353 {
 354         struct insert_page arg = { vm, addr, offset, level };
 355
 356         stop_machine(bxt_vtd_ggtt_insert_page__cb, &arg, NULL);
 357 }
 358
 359 struct insert_entries {
 360         struct i915_address_space *vm;
 361         struct i915_vma *vma;
 362         enum i915_cache_level level;
 363         u32 flags;
 364 };
 365
 366 static int bxt_vtd_ggtt_insert_entries__cb(void *_arg)
 367 {
 368         struct insert_entries *arg = _arg;
 369
 370         gen8_ggtt_insert_entries(arg->vm, arg->vma, arg->level, arg->flags);
 371         bxt_vtd_ggtt_wa(arg->vm);
 372
 373         return 0;
 374 }
 375
 376 static void bxt_vtd_ggtt_insert_entries__BKL(struct i915_address_space *vm,
 377                                              struct i915_vma *vma,
 378                                              enum i915_cache_level level,
 379                                              u32 flags)
 380 {
 381         struct insert_entries arg = { vm, vma, level, flags };
 382
 383         stop_machine(bxt_vtd_ggtt_insert_entries__cb, &arg, NULL);
 384 }
 385
 386 static void gen6_ggtt_clear_range(struct i915_address_space *vm,
 387                                   u64 start, u64 length)
 388 {
 389         struct i915_ggtt *ggtt = i915_vm_to_ggtt(vm);
 390         unsigned int first_entry = start / I915_GTT_PAGE_SIZE;
 391         unsigned int num_entries = length / I915_GTT_PAGE_SIZE;
 392         gen6_pte_t scratch_pte, __iomem *gtt_base =
 393                 (gen6_pte_t __iomem *)ggtt->gsm + first_entry;
 394         const int max_entries = ggtt_total_entries(ggtt) - first_entry;
 395         int i;
 396
 397         if (WARN(num_entries > max_entries,
 398                  "First entry = %d; Num entries = %d (max=%d)\n",
 399                  first_entry, num_entries, max_entries))
 400                 num_entries = max_entries;
 401
 402         scratch_pte = vm->scratch[0]->encode;
 403         for (i = 0; i < num_entries; i++)
 404                 iowrite32(scratch_pte, &gtt_base[i]);
 405 }
 406
 407 static void i915_ggtt_insert_page(struct i915_address_space *vm,
 408                                   dma_addr_t addr,
 409                                   u64 offset,
 410                                   enum i915_cache_level cache_level,
 411                                   u32 unused)
 412 {
 413         unsigned int flags = (cache_level == I915_CACHE_NONE) ?
 414                 AGP_USER_MEMORY : AGP_USER_CACHED_MEMORY;
 415
 416         intel_gtt_insert_page(addr, offset >> PAGE_SHIFT, flags);
 417 }
 418
 419 static void i915_ggtt_insert_entries(struct i915_address_space *vm,
 420                                      struct i915_vma *vma,
 421                                      enum i915_cache_level cache_level,
 422                                      u32 unused)
 423 {
 424         unsigned int flags = (cache_level == I915_CACHE_NONE) ?
 425                 AGP_USER_MEMORY : AGP_USER_CACHED_MEMORY;
 426
 427         intel_gtt_insert_sg_entries(vma->pages, vma->node.start >> PAGE_SHIFT,
 428                                     flags);
 429 }
 430
 431 static void i915_ggtt_clear_range(struct i915_address_space *vm,
 432                                   u64 start, u64 length)
 433 {
 434         intel_gtt_clear_range(start >> PAGE_SHIFT, length >> PAGE_SHIFT);
 435 }
 436
 437 static void ggtt_bind_vma(struct i915_address_space *vm,
 438                           struct i915_vm_pt_stash *stash,
 439                           struct i915_vma *vma,
 440                           enum i915_cache_level cache_level,
 441                           u32 flags)
 442 {
 443         struct drm_i915_gem_object *obj = vma->obj;
 444         u32 pte_flags;
 445
 446         if (i915_vma_is_bound(vma, ~flags & I915_VMA_BIND_MASK))
 447                 return;
 448
 449         /* Applicable to VLV (gen8+ do not support RO in the GGTT) */
 450         pte_flags = 0;
 451         if (i915_gem_object_is_readonly(obj))
 452                 pte_flags |= PTE_READ_ONLY;
 453
 454         vm->insert_entries(vm, vma, cache_level, pte_flags);
 455         vma->page_sizes.gtt = I915_GTT_PAGE_SIZE;
 456 }
 457
 458 static void ggtt_unbind_vma(struct i915_address_space *vm, struct i915_vma *vma)
 459 {
 460         vm->clear_range(vm, vma->node.start, vma->size);
 461 }
 462
 463 static int ggtt_reserve_guc_top(struct i915_ggtt *ggtt)
 464 {
 465         u64 size;
 466         int ret;
 467
 468         if (!intel_uc_uses_guc(&ggtt->vm.gt->uc))
 469                 return 0;
 470
 471         GEM_BUG_ON(ggtt->vm.total <= GUC_GGTT_TOP);
 472         size = ggtt->vm.total - GUC_GGTT_TOP;
 473
 474         ret = i915_gem_gtt_reserve(&ggtt->vm, &ggtt->uc_fw, size,
 475                                    GUC_GGTT_TOP, I915_COLOR_UNEVICTABLE,
 476                                    PIN_NOEVICT);
 477         if (ret)
 478                 drm_dbg(&ggtt->vm.i915->drm,
 479                         "Failed to reserve top of GGTT for GuC\n");
 480
 481         return ret;
 482 }
 483
 484 static void ggtt_release_guc_top(struct i915_ggtt *ggtt)
 485 {
 486         if (drm_mm_node_allocated(&ggtt->uc_fw))
 487                 drm_mm_remove_node(&ggtt->uc_fw);
 488 }
 489
 490 static void cleanup_init_ggtt(struct i915_ggtt *ggtt)
 491 {
 492         ggtt_release_guc_top(ggtt);
 493         if (drm_mm_node_allocated(&ggtt->error_capture))
 494                 drm_mm_remove_node(&ggtt->error_capture);
 495         mutex_destroy(&ggtt->error_mutex);
 496 }
 497
 498 static int init_ggtt(struct i915_ggtt *ggtt)
 499 {
 500         /*
 501          * Let GEM Manage all of the aperture.
 502          *
 503          * However, leave one page at the end still bound to the scratch page.
 504          * There are a number of places where the hardware apparently prefetches
 505          * past the end of the object, and we've seen multiple hangs with the
 506          * GPU head pointer stuck in a batchbuffer bound at the last page of the
 507          * aperture.  One page should be enough to keep any prefetching inside
 508          * of the aperture.
 509          */
 510         unsigned long hole_start, hole_end;
 511         struct drm_mm_node *entry;
 512         int ret;
 513
 514         /*
 515          * GuC requires all resources that we're sharing with it to be placed in
 516          * non-WOPCM memory. If GuC is not present or not in use we still need a
 517          * small bias as ring wraparound at offset 0 sometimes hangs. No idea
 518          * why.
 519          */
 520         ggtt->pin_bias = max_t(u32, I915_GTT_PAGE_SIZE,
 521                                intel_wopcm_guc_size(&ggtt->vm.i915->wopcm));
 522
 523         ret = intel_vgt_balloon(ggtt);
 524         if (ret)
 525                 return ret;
 526
 527         mutex_init(&ggtt->error_mutex);
 528         if (ggtt->mappable_end) {
 529                 /* Reserve a mappable slot for our lockless error capture */
 530                 ret = drm_mm_insert_node_in_range(&ggtt->vm.mm,
 531                                                   &ggtt->error_capture,
 532                                                   PAGE_SIZE, 0,
 533                                                   I915_COLOR_UNEVICTABLE,
 534                                                   0, ggtt->mappable_end,
 535                                                   DRM_MM_INSERT_LOW);
 536                 if (ret)
 537                         return ret;
 538         }
 539
 540         /*
 541          * The upper portion of the GuC address space has a sizeable hole
 542          * (several MB) that is inaccessible by GuC. Reserve this range within
 543          * GGTT as it can comfortably hold GuC/HuC firmware images.
 544          */
 545         ret = ggtt_reserve_guc_top(ggtt);
 546         if (ret)
 547                 goto err;
 548
 549         /* Clear any non-preallocated blocks */
 550         drm_mm_for_each_hole(entry, &ggtt->vm.mm, hole_start, hole_end) {
 551                 drm_dbg_kms(&ggtt->vm.i915->drm,
 552                             "clearing unused GTT space: [%lx, %lx]\n",
 553                             hole_start, hole_end);
 554                 ggtt->vm.clear_range(&ggtt->vm, hole_start,
 555                                      hole_end - hole_start);
 556         }
 557
 558         /* And finally clear the reserved guard page */
 559         ggtt->vm.clear_range(&ggtt->vm, ggtt->vm.total - PAGE_SIZE, PAGE_SIZE);
 560
 561         return 0;
 562
 563 err:
 564         cleanup_init_ggtt(ggtt);
 565         return ret;
 566 }
 567
 568 static void aliasing_gtt_bind_vma(struct i915_address_space *vm,
 569                                   struct i915_vm_pt_stash *stash,
 570                                   struct i915_vma *vma,
 571                                   enum i915_cache_level cache_level,
 572                                   u32 flags)
 573 {
 574         u32 pte_flags;
 575
 576         /* Currently applicable only to VLV */
 577         pte_flags = 0;
 578         if (i915_gem_object_is_readonly(vma->obj))
 579                 pte_flags |= PTE_READ_ONLY;
 580
 581         if (flags & I915_VMA_LOCAL_BIND)
 582                 ppgtt_bind_vma(&i915_vm_to_ggtt(vm)->alias->vm,
 583                                stash, vma, cache_level, flags);
 584
 585         if (flags & I915_VMA_GLOBAL_BIND)
 586                 vm->insert_entries(vm, vma, cache_level, pte_flags);
 587 }
 588
 589 static void aliasing_gtt_unbind_vma(struct i915_address_space *vm,
 590                                     struct i915_vma *vma)
 591 {
 592         if (i915_vma_is_bound(vma, I915_VMA_GLOBAL_BIND))
 593                 vm->clear_range(vm, vma->node.start, vma->size);
 594
 595         if (i915_vma_is_bound(vma, I915_VMA_LOCAL_BIND))
 596                 ppgtt_unbind_vma(&i915_vm_to_ggtt(vm)->alias->vm, vma);
 597 }
 598
 599 static int init_aliasing_ppgtt(struct i915_ggtt *ggtt)
 600 {
 601         struct i915_vm_pt_stash stash = {};
 602         struct i915_ppgtt *ppgtt;
 603         int err;
 604
 605         ppgtt = i915_ppgtt_create(ggtt->vm.gt);
 606         if (IS_ERR(ppgtt))
 607                 return PTR_ERR(ppgtt);
 608
 609         if (GEM_WARN_ON(ppgtt->vm.total < ggtt->vm.total)) {
 610                 err = -ENODEV;
 611                 goto err_ppgtt;
 612         }
 613
 614         err = i915_vm_alloc_pt_stash(&ppgtt->vm, &stash, ggtt->vm.total);
 615         if (err)
 616                 goto err_ppgtt;
 617
 618         err = i915_vm_pin_pt_stash(&ppgtt->vm, &stash);
 619         if (err)
 620                 goto err_stash;
 621
 622         /*
 623          * Note we only pre-allocate as far as the end of the global
 624          * GTT. On 48b / 4-level page-tables, the difference is very,
 625          * very significant! We have to preallocate as GVT/vgpu does
 626          * not like the page directory disappearing.
 627          */
 628         ppgtt->vm.allocate_va_range(&ppgtt->vm, &stash, 0, ggtt->vm.total);
 629
 630         ggtt->alias = ppgtt;
 631         ggtt->vm.bind_async_flags |= ppgtt->vm.bind_async_flags;
 632
 633         GEM_BUG_ON(ggtt->vm.vma_ops.bind_vma != ggtt_bind_vma);
 634         ggtt->vm.vma_ops.bind_vma = aliasing_gtt_bind_vma;
 635
 636         GEM_BUG_ON(ggtt->vm.vma_ops.unbind_vma != ggtt_unbind_vma);
 637         ggtt->vm.vma_ops.unbind_vma = aliasing_gtt_unbind_vma;
 638
 639         i915_vm_free_pt_stash(&ppgtt->vm, &stash);
 640         return 0;
 641
 642 err_stash:
 643         i915_vm_free_pt_stash(&ppgtt->vm, &stash);
 644 err_ppgtt:
 645         i915_vm_put(&ppgtt->vm);
 646         return err;
 647 }
 648
 649 static void fini_aliasing_ppgtt(struct i915_ggtt *ggtt)
 650 {
 651         struct i915_ppgtt *ppgtt;
 652
 653         ppgtt = fetch_and_zero(&ggtt->alias);
 654         if (!ppgtt)
 655                 return;
 656
 657         i915_vm_put(&ppgtt->vm);
 658
 659         ggtt->vm.vma_ops.bind_vma   = ggtt_bind_vma;
 660         ggtt->vm.vma_ops.unbind_vma = ggtt_unbind_vma;
 661 }
 662
 663 int i915_init_ggtt(struct drm_i915_private *i915)
 664 {
 665         int ret;
 666
 667         ret = init_ggtt(&i915->ggtt);
 668         if (ret)
 669                 return ret;
 670
 671         if (INTEL_PPGTT(i915) == INTEL_PPGTT_ALIASING) {
 672                 ret = init_aliasing_ppgtt(&i915->ggtt);
 673                 if (ret)
 674                         cleanup_init_ggtt(&i915->ggtt);
 675         }
 676
 677         return 0;
 678 }
 679
 680 static void ggtt_cleanup_hw(struct i915_ggtt *ggtt)
 681 {
 682         struct i915_vma *vma, *vn;
 683
 684         atomic_set(&ggtt->vm.open, 0);
 685
 686         rcu_barrier(); /* flush the RCU'ed__i915_vm_release */
 687         flush_workqueue(ggtt->vm.i915->wq);
 688
 689         mutex_lock(&ggtt->vm.mutex);
 690
 691         list_for_each_entry_safe(vma, vn, &ggtt->vm.bound_list, vm_link)
 692                 WARN_ON(__i915_vma_unbind(vma));
 693
 694         if (drm_mm_node_allocated(&ggtt->error_capture))
 695                 drm_mm_remove_node(&ggtt->error_capture);
 696         mutex_destroy(&ggtt->error_mutex);
 697
 698         ggtt_release_guc_top(ggtt);
 699         intel_vgt_deballoon(ggtt);
 700
 701         ggtt->vm.cleanup(&ggtt->vm);
 702
 703         mutex_unlock(&ggtt->vm.mutex);
 704         i915_address_space_fini(&ggtt->vm);
 705
 706         arch_phys_wc_del(ggtt->mtrr);
 707
 708         if (ggtt->iomap.size)
 709                 io_mapping_fini(&ggtt->iomap);
 710 }
 711
 712 /**
 713  * i915_ggtt_driver_release - Clean up GGTT hardware initialization
 714  * @i915: i915 device
 715  */
 716 void i915_ggtt_driver_release(struct drm_i915_private *i915)
 717 {
 718         struct i915_ggtt *ggtt = &i915->ggtt;
 719
 720         fini_aliasing_ppgtt(ggtt);
 721
 722         intel_ggtt_fini_fences(ggtt);
 723         ggtt_cleanup_hw(ggtt);
 724 }
 725
 726 static unsigned int gen6_get_total_gtt_size(u16 snb_gmch_ctl)
 727 {
 728         snb_gmch_ctl >>= SNB_GMCH_GGMS_SHIFT;
 729         snb_gmch_ctl &= SNB_GMCH_GGMS_MASK;
 730         return snb_gmch_ctl << 20;
 731 }
 732
 733 static unsigned int gen8_get_total_gtt_size(u16 bdw_gmch_ctl)
 734 {
 735         bdw_gmch_ctl >>= BDW_GMCH_GGMS_SHIFT;
 736         bdw_gmch_ctl &= BDW_GMCH_GGMS_MASK;
 737         if (bdw_gmch_ctl)
 738                 bdw_gmch_ctl = 1 << bdw_gmch_ctl;
 739
 740 #ifdef CONFIG_X86_32
 741         /* Limit 32b platforms to a 2GB GGTT: 4 << 20 / pte size * I915_GTT_PAGE_SIZE */
 742         if (bdw_gmch_ctl > 4)
 743                 bdw_gmch_ctl = 4;
 744 #endif
 745
 746         return bdw_gmch_ctl << 20;
 747 }
 748
 749 static unsigned int chv_get_total_gtt_size(u16 gmch_ctrl)
 750 {
 751         gmch_ctrl >>= SNB_GMCH_GGMS_SHIFT;
 752         gmch_ctrl &= SNB_GMCH_GGMS_MASK;
 753
 754         if (gmch_ctrl)
 755                 return 1 << (20 + gmch_ctrl);
 756
 757         return 0;
 758 }
 759
 760 static int ggtt_probe_common(struct i915_ggtt *ggtt, u64 size)
 761 {
 762         struct drm_i915_private *i915 = ggtt->vm.i915;
 763         struct pci_dev *pdev = i915->drm.pdev;
 764         phys_addr_t phys_addr;
 765         int ret;
 766
 767         /* For Modern GENs the PTEs and register space are split in the BAR */
 768         phys_addr = pci_resource_start(pdev, 0) + pci_resource_len(pdev, 0) / 2;
 769
 770         /*
 771          * On BXT+/CNL+ writes larger than 64 bit to the GTT pagetable range
 772          * will be dropped. For WC mappings in general we have 64 byte burst
 773          * writes when the WC buffer is flushed, so we can't use it, but have to
 774          * resort to an uncached mapping. The WC issue is easily caught by the
 775          * readback check when writing GTT PTE entries.
 776          */
 777         if (IS_GEN9_LP(i915) || INTEL_GEN(i915) >= 10)
 778                 ggtt->gsm = ioremap(phys_addr, size);
 779         else
 780                 ggtt->gsm = ioremap_wc(phys_addr, size);
 781         if (!ggtt->gsm) {
 782                 drm_err(&i915->drm, "Failed to map the ggtt page table\n");
 783                 return -ENOMEM;
 784         }
 785
 786         ret = setup_scratch_page(&ggtt->vm);
 787         if (ret) {
 788                 drm_err(&i915->drm, "Scratch setup failed\n");
 789                 /* iounmap will also get called at remove, but meh */
 790                 iounmap(ggtt->gsm);
 791                 return ret;
 792         }
 793
 794         ggtt->vm.scratch[0]->encode =
 795                 ggtt->vm.pte_encode(px_dma(ggtt->vm.scratch[0]),
 796                                     I915_CACHE_NONE, 0);
 797
 798         return 0;
 799 }
 800
 801 int ggtt_set_pages(struct i915_vma *vma)
 802 {
 803         int ret;
 804
 805         GEM_BUG_ON(vma->pages);
 806
 807         ret = i915_get_ggtt_vma_pages(vma);
 808         if (ret)
 809                 return ret;
 810
 811         vma->page_sizes = vma->obj->mm.page_sizes;
 812
 813         return 0;
 814 }
 815
 816 static void gen6_gmch_remove(struct i915_address_space *vm)
 817 {
 818         struct i915_ggtt *ggtt = i915_vm_to_ggtt(vm);
 819
 820         iounmap(ggtt->gsm);
 821         free_scratch(vm);
 822 }
 823
 824 static struct resource pci_resource(struct pci_dev *pdev, int bar)
 825 {
 826         return (struct resource)DEFINE_RES_MEM(pci_resource_start(pdev, bar),
 827                                                pci_resource_len(pdev, bar));
 828 }
 829
 830 static int gen8_gmch_probe(struct i915_ggtt *ggtt)
 831 {
 832         struct drm_i915_private *i915 = ggtt->vm.i915;
 833         struct pci_dev *pdev = i915->drm.pdev;
 834         unsigned int size;
 835         u16 snb_gmch_ctl;
 836
 837         /* TODO: We're not aware of mappable constraints on gen8 yet */
 838         if (!HAS_LMEM(i915)) {
 839                 ggtt->gmadr = pci_resource(pdev, 2);
 840                 ggtt->mappable_end = resource_size(&ggtt->gmadr);
 841         }
 842
 843         pci_read_config_word(pdev, SNB_GMCH_CTRL, &snb_gmch_ctl);
 844         if (IS_CHERRYVIEW(i915))
 845                 size = chv_get_total_gtt_size(snb_gmch_ctl);
 846         else
 847                 size = gen8_get_total_gtt_size(snb_gmch_ctl);
 848
 849         ggtt->vm.alloc_pt_dma = alloc_pt_dma;
 850
 851         ggtt->vm.total = (size / sizeof(gen8_pte_t)) * I915_GTT_PAGE_SIZE;
 852         ggtt->vm.cleanup = gen6_gmch_remove;
 853         ggtt->vm.insert_page = gen8_ggtt_insert_page;
 854         ggtt->vm.clear_range = nop_clear_range;
 855         if (intel_scanout_needs_vtd_wa(i915))
 856                 ggtt->vm.clear_range = gen8_ggtt_clear_range;
 857
 858         ggtt->vm.insert_entries = gen8_ggtt_insert_entries;
 859
 860         /* Serialize GTT updates with aperture access on BXT if VT-d is on. */
 861         if (intel_ggtt_update_needs_vtd_wa(i915) ||
 862             IS_CHERRYVIEW(i915) /* fails with concurrent use/update */) {
 863                 ggtt->vm.insert_entries = bxt_vtd_ggtt_insert_entries__BKL;
 864                 ggtt->vm.insert_page    = bxt_vtd_ggtt_insert_page__BKL;
 865                 ggtt->vm.bind_async_flags =
 866                         I915_VMA_GLOBAL_BIND | I915_VMA_LOCAL_BIND;
 867         }
 868
 869         ggtt->invalidate = gen8_ggtt_invalidate;
 870
 871         ggtt->vm.vma_ops.bind_vma    = ggtt_bind_vma;
 872         ggtt->vm.vma_ops.unbind_vma  = ggtt_unbind_vma;
 873         ggtt->vm.vma_ops.set_pages   = ggtt_set_pages;
 874         ggtt->vm.vma_ops.clear_pages = clear_pages;
 875
 876         ggtt->vm.pte_encode = gen8_ggtt_pte_encode;
 877
 878         setup_private_pat(ggtt->vm.gt->uncore);
 879
 880         return ggtt_probe_common(ggtt, size);
 881 }
 882
 883 static u64 snb_pte_encode(dma_addr_t addr,
 884                           enum i915_cache_level level,
 885                           u32 flags)
 886 {
 887         gen6_pte_t pte = GEN6_PTE_ADDR_ENCODE(addr) | GEN6_PTE_VALID;
 888
 889         switch (level) {
 890         case I915_CACHE_L3_LLC:
 891         case I915_CACHE_LLC:
 892                 pte |= GEN6_PTE_CACHE_LLC;
 893                 break;
 894         case I915_CACHE_NONE:
 895                 pte |= GEN6_PTE_UNCACHED;
 896                 break;
 897         default:
 898                 MISSING_CASE(level);
 899         }
 900
 901         return pte;
 902 }
 903
 904 static u64 ivb_pte_encode(dma_addr_t addr,
 905                           enum i915_cache_level level,
 906                           u32 flags)
 907 {
 908         gen6_pte_t pte = GEN6_PTE_ADDR_ENCODE(addr) | GEN6_PTE_VALID;
 909
 910         switch (level) {
 911         case I915_CACHE_L3_LLC:
 912                 pte |= GEN7_PTE_CACHE_L3_LLC;
 913                 break;
 914         case I915_CACHE_LLC:
 915                 pte |= GEN6_PTE_CACHE_LLC;
 916                 break;
 917         case I915_CACHE_NONE:
 918                 pte |= GEN6_PTE_UNCACHED;
 919                 break;
 920         default:
 921                 MISSING_CASE(level);
 922         }
 923
 924         return pte;
 925 }
 926
 927 static u64 byt_pte_encode(dma_addr_t addr,
 928                           enum i915_cache_level level,
 929                           u32 flags)
 930 {
 931         gen6_pte_t pte = GEN6_PTE_ADDR_ENCODE(addr) | GEN6_PTE_VALID;
 932
 933         if (!(flags & PTE_READ_ONLY))
 934                 pte |= BYT_PTE_WRITEABLE;
 935
 936         if (level != I915_CACHE_NONE)
 937                 pte |= BYT_PTE_SNOOPED_BY_CPU_CACHES;
 938
 939         return pte;
 940 }
 941
 942 static u64 hsw_pte_encode(dma_addr_t addr,
 943                           enum i915_cache_level level,
 944                           u32 flags)
 945 {
 946         gen6_pte_t pte = HSW_PTE_ADDR_ENCODE(addr) | GEN6_PTE_VALID;
 947
 948         if (level != I915_CACHE_NONE)
 949                 pte |= HSW_WB_LLC_AGE3;
 950
 951         return pte;
 952 }
 953
 954 static u64 iris_pte_encode(dma_addr_t addr,
 955                            enum i915_cache_level level,
 956                            u32 flags)
 957 {
 958         gen6_pte_t pte = HSW_PTE_ADDR_ENCODE(addr) | GEN6_PTE_VALID;
 959
 960         switch (level) {
 961         case I915_CACHE_NONE:
 962                 break;
 963         case I915_CACHE_WT:
 964                 pte |= HSW_WT_ELLC_LLC_AGE3;
 965                 break;
 966         default:
 967                 pte |= HSW_WB_ELLC_LLC_AGE3;
 968                 break;
 969         }
 970
 971         return pte;
 972 }
 973
 974 static int gen6_gmch_probe(struct i915_ggtt *ggtt)
 975 {
 976         struct drm_i915_private *i915 = ggtt->vm.i915;
 977         struct pci_dev *pdev = i915->drm.pdev;
 978         unsigned int size;
 979         u16 snb_gmch_ctl;
 980
 981         ggtt->gmadr = pci_resource(pdev, 2);
 982         ggtt->mappable_end = resource_size(&ggtt->gmadr);
 983
 984         /*
 985          * 64/512MB is the current min/max we actually know of, but this is
 986          * just a coarse sanity check.
 987          */
 988         if (ggtt->mappable_end < (64<<20) || ggtt->mappable_end > (512<<20)) {
 989                 drm_err(&i915->drm, "Unknown GMADR size (%pa)\n",
 990                         &ggtt->mappable_end);
 991                 return -ENXIO;
 992         }
 993
 994         pci_read_config_word(pdev, SNB_GMCH_CTRL, &snb_gmch_ctl);
 995
 996         size = gen6_get_total_gtt_size(snb_gmch_ctl);
 997         ggtt->vm.total = (size / sizeof(gen6_pte_t)) * I915_GTT_PAGE_SIZE;
 998
 999         ggtt->vm.alloc_pt_dma = alloc_pt_dma;
1000
1001         ggtt->vm.clear_range = nop_clear_range;
1002         if (!HAS_FULL_PPGTT(i915) || intel_scanout_needs_vtd_wa(i915))
1003                 ggtt->vm.clear_range = gen6_ggtt_clear_range;
1004         ggtt->vm.insert_page = gen6_ggtt_insert_page;
1005         ggtt->vm.insert_entries = gen6_ggtt_insert_entries;
1006         ggtt->vm.cleanup = gen6_gmch_remove;
1007
1008         ggtt->invalidate = gen6_ggtt_invalidate;
1009
1010         if (HAS_EDRAM(i915))
1011                 ggtt->vm.pte_encode = iris_pte_encode;
1012         else if (IS_HASWELL(i915))
1013                 ggtt->vm.pte_encode = hsw_pte_encode;
1014         else if (IS_VALLEYVIEW(i915))
1015                 ggtt->vm.pte_encode = byt_pte_encode;
1016         else if (INTEL_GEN(i915) >= 7)
1017                 ggtt->vm.pte_encode = ivb_pte_encode;
1018         else
1019                 ggtt->vm.pte_encode = snb_pte_encode;
1020
1021         ggtt->vm.vma_ops.bind_vma    = ggtt_bind_vma;
1022         ggtt->vm.vma_ops.unbind_vma  = ggtt_unbind_vma;
1023         ggtt->vm.vma_ops.set_pages   = ggtt_set_pages;
1024         ggtt->vm.vma_ops.clear_pages = clear_pages;
1025
1026         return ggtt_probe_common(ggtt, size);
1027 }
1028
1029 static void i915_gmch_remove(struct i915_address_space *vm)
1030 {
1031         intel_gmch_remove();
1032 }
1033
1034 static int i915_gmch_probe(struct i915_ggtt *ggtt)
1035 {
1036         struct drm_i915_private *i915 = ggtt->vm.i915;
1037         phys_addr_t gmadr_base;
1038         int ret;
1039
1040         ret = intel_gmch_probe(i915->bridge_dev, i915->drm.pdev, NULL);
1041         if (!ret) {
1042                 drm_err(&i915->drm, "failed to set up gmch\n");
1043                 return -EIO;
1044         }
1045
1046         intel_gtt_get(&ggtt->vm.total, &gmadr_base, &ggtt->mappable_end);
1047
1048         ggtt->gmadr =
1049                 (struct resource)DEFINE_RES_MEM(gmadr_base, ggtt->mappable_end);
1050
1051         ggtt->vm.alloc_pt_dma = alloc_pt_dma;
1052
1053         ggtt->do_idle_maps = needs_idle_maps(i915);
1054         ggtt->vm.insert_page = i915_ggtt_insert_page;
1055         ggtt->vm.insert_entries = i915_ggtt_insert_entries;
1056         ggtt->vm.clear_range = i915_ggtt_clear_range;
1057         ggtt->vm.cleanup = i915_gmch_remove;
1058
1059         ggtt->invalidate = gmch_ggtt_invalidate;
1060
1061         ggtt->vm.vma_ops.bind_vma    = ggtt_bind_vma;
1062         ggtt->vm.vma_ops.unbind_vma  = ggtt_unbind_vma;
1063         ggtt->vm.vma_ops.set_pages   = ggtt_set_pages;
1064         ggtt->vm.vma_ops.clear_pages = clear_pages;
1065
1066         if (unlikely(ggtt->do_idle_maps))
1067                 drm_notice(&i915->drm,
1068                            "Applying Ironlake quirks for intel_iommu\n");
1069
1070         return 0;
1071 }
1072
1073 static int ggtt_probe_hw(struct i915_ggtt *ggtt, struct intel_gt *gt)
1074 {
1075         struct drm_i915_private *i915 = gt->i915;
1076         int ret;
1077
1078         ggtt->vm.gt = gt;
1079         ggtt->vm.i915 = i915;
1080         ggtt->vm.dma = &i915->drm.pdev->dev;
1081
1082         if (INTEL_GEN(i915) <= 5)
1083                 ret = i915_gmch_probe(ggtt);
1084         else if (INTEL_GEN(i915) < 8)
1085                 ret = gen6_gmch_probe(ggtt);
1086         else
1087                 ret = gen8_gmch_probe(ggtt);
1088         if (ret)
1089                 return ret;
1090
1091         if ((ggtt->vm.total - 1) >> 32) {
1092                 drm_err(&i915->drm,
1093                         "We never expected a Global GTT with more than 32bits"
1094                         " of address space! Found %lldM!\n",
1095                         ggtt->vm.total >> 20);
1096                 ggtt->vm.total = 1ULL << 32;
1097                 ggtt->mappable_end =
1098                         min_t(u64, ggtt->mappable_end, ggtt->vm.total);
1099         }
1100
1101         if (ggtt->mappable_end > ggtt->vm.total) {
1102                 drm_err(&i915->drm,
1103                         "mappable aperture extends past end of GGTT,"
1104                         " aperture=%pa, total=%llx\n",
1105                         &ggtt->mappable_end, ggtt->vm.total);
1106                 ggtt->mappable_end = ggtt->vm.total;
1107         }
1108
1109         /* GMADR is the PCI mmio aperture into the global GTT. */
1110         drm_dbg(&i915->drm, "GGTT size = %lluM\n", ggtt->vm.total >> 20);
1111         drm_dbg(&i915->drm, "GMADR size = %lluM\n",
1112                 (u64)ggtt->mappable_end >> 20);
1113         drm_dbg(&i915->drm, "DSM size = %lluM\n",
1114                 (u64)resource_size(&intel_graphics_stolen_res) >> 20);
1115
1116         return 0;
1117 }
1118
1119 /**
1120  * i915_ggtt_probe_hw - Probe GGTT hardware location
1121  * @i915: i915 device
1122  */
1123 int i915_ggtt_probe_hw(struct drm_i915_private *i915)
1124 {
1125         int ret;
1126
1127         ret = ggtt_probe_hw(&i915->ggtt, &i915->gt);
1128         if (ret)
1129                 return ret;
1130
1131         if (intel_vtd_active())
1132                 drm_info(&i915->drm, "VT-d active for gfx access\n");
1133
1134         return 0;
1135 }
1136
1137 int i915_ggtt_enable_hw(struct drm_i915_private *i915)
1138 {
1139         if (INTEL_GEN(i915) < 6 && !intel_enable_gtt())
1140                 return -EIO;
1141
1142         return 0;
1143 }
1144
1145 void i915_ggtt_enable_guc(struct i915_ggtt *ggtt)
1146 {
1147         GEM_BUG_ON(ggtt->invalidate != gen8_ggtt_invalidate);
1148
1149         ggtt->invalidate = guc_ggtt_invalidate;
1150
1151         ggtt->invalidate(ggtt);
1152 }
1153
1154 void i915_ggtt_disable_guc(struct i915_ggtt *ggtt)
1155 {
1156         /* XXX Temporary pardon for error unload */
1157         if (ggtt->invalidate == gen8_ggtt_invalidate)
1158                 return;
1159
1160         /* We should only be called after i915_ggtt_enable_guc() */
1161         GEM_BUG_ON(ggtt->invalidate != guc_ggtt_invalidate);
1162
1163         ggtt->invalidate = gen8_ggtt_invalidate;
1164
1165         ggtt->invalidate(ggtt);
1166 }
1167
1168 void i915_ggtt_resume(struct i915_ggtt *ggtt)
1169 {
1170         struct i915_vma *vma;
1171         bool flush = false;
1172         int open;
1173
1174         intel_gt_check_and_clear_faults(ggtt->vm.gt);
1175
1176         /* First fill our portion of the GTT with scratch pages */
1177         ggtt->vm.clear_range(&ggtt->vm, 0, ggtt->vm.total);
1178
1179         /* Skip rewriting PTE on VMA unbind. */
1180         open = atomic_xchg(&ggtt->vm.open, 0);
1181
1182         /* clflush objects bound into the GGTT and rebind them. */
1183         list_for_each_entry(vma, &ggtt->vm.bound_list, vm_link) {
1184                 struct drm_i915_gem_object *obj = vma->obj;
1185                 unsigned int was_bound =
1186                         atomic_read(&vma->flags) & I915_VMA_BIND_MASK;
1187
1188                 GEM_BUG_ON(!was_bound);
1189                 vma->ops->bind_vma(&ggtt->vm, NULL, vma,
1190                                    obj ? obj->cache_level : 0,
1191                                    was_bound);
1192                 if (obj) { /* only used during resume => exclusive access */
1193                         flush |= fetch_and_zero(&obj->write_domain);
1194                         obj->read_domains |= I915_GEM_DOMAIN_GTT;
1195                 }
1196         }
1197
1198         atomic_set(&ggtt->vm.open, open);
1199         ggtt->invalidate(ggtt);
1200
1201         if (flush)
1202                 wbinvd_on_all_cpus();
1203
1204         if (INTEL_GEN(ggtt->vm.i915) >= 8)
1205                 setup_private_pat(ggtt->vm.gt->uncore);
1206
1207         intel_ggtt_restore_fences(ggtt);
1208 }
1209
1210 static struct scatterlist *
1211 rotate_pages(struct drm_i915_gem_object *obj, unsigned int offset,
1212              unsigned int width, unsigned int height,
1213              unsigned int stride,
1214              struct sg_table *st, struct scatterlist *sg)
1215 {
1216         unsigned int column, row;
1217         unsigned int src_idx;
1218
1219         for (column = 0; column < width; column++) {
1220                 src_idx = stride * (height - 1) + column + offset;
1221                 for (row = 0; row < height; row++) {
1222                         st->nents++;
1223                         /*
1224                          * We don't need the pages, but need to initialize
1225                          * the entries so the sg list can be happily traversed.
1226                          * The only thing we need are DMA addresses.
1227                          */
1228                         sg_set_page(sg, NULL, I915_GTT_PAGE_SIZE, 0);
1229                         sg_dma_address(sg) =
1230                                 i915_gem_object_get_dma_address(obj, src_idx);
1231                         sg_dma_len(sg) = I915_GTT_PAGE_SIZE;
1232                         sg = sg_next(sg);
1233                         src_idx -= stride;
1234                 }
1235         }
1236
1237         return sg;
1238 }
1239
1240 static noinline struct sg_table *
1241 intel_rotate_pages(struct intel_rotation_info *rot_info,
1242                    struct drm_i915_gem_object *obj)
1243 {
1244         unsigned int size = intel_rotation_info_size(rot_info);
1245         struct drm_i915_private *i915 = to_i915(obj->base.dev);
1246         struct sg_table *st;
1247         struct scatterlist *sg;
1248         int ret = -ENOMEM;
1249         int i;
1250
1251         /* Allocate target SG list. */
1252         st = kmalloc(sizeof(*st), GFP_KERNEL);
1253         if (!st)
1254                 goto err_st_alloc;
1255
1256         ret = sg_alloc_table(st, size, GFP_KERNEL);
1257         if (ret)
1258                 goto err_sg_alloc;
1259
1260         st->nents = 0;
1261         sg = st->sgl;
1262
1263         for (i = 0 ; i < ARRAY_SIZE(rot_info->plane); i++) {
1264                 sg = rotate_pages(obj, rot_info->plane[i].offset,
1265                                   rot_info->plane[i].width, rot_info->plane[i].height,
1266                                   rot_info->plane[i].stride, st, sg);
1267         }
1268
1269         return st;
1270
1271 err_sg_alloc:
1272         kfree(st);
1273 err_st_alloc:
1274
1275         drm_dbg(&i915->drm, "Failed to create rotated mapping for object size %zu! (%ux%u tiles, %u pages)\n",
1276                 obj->base.size, rot_info->plane[0].width,
1277                 rot_info->plane[0].height, size);
1278
1279         return ERR_PTR(ret);
1280 }
1281
1282 static struct scatterlist *
1283 remap_pages(struct drm_i915_gem_object *obj, unsigned int offset,
1284             unsigned int width, unsigned int height,
1285             unsigned int stride,
1286             struct sg_table *st, struct scatterlist *sg)
1287 {
1288         unsigned int row;
1289
1290         for (row = 0; row < height; row++) {
1291                 unsigned int left = width * I915_GTT_PAGE_SIZE;
1292
1293                 while (left) {
1294                         dma_addr_t addr;
1295                         unsigned int length;
1296
1297                         /*
1298                          * We don't need the pages, but need to initialize
1299                          * the entries so the sg list can be happily traversed.
1300                          * The only thing we need are DMA addresses.
1301                          */
1302
1303                         addr = i915_gem_object_get_dma_address_len(obj, offset, &length);
1304
1305                         length = min(left, length);
1306
1307                         st->nents++;
1308
1309                         sg_set_page(sg, NULL, length, 0);
1310                         sg_dma_address(sg) = addr;
1311                         sg_dma_len(sg) = length;
1312                         sg = sg_next(sg);
1313
1314                         offset += length / I915_GTT_PAGE_SIZE;
1315                         left -= length;
1316                 }
1317
1318                 offset += stride - width;
1319         }
1320
1321         return sg;
1322 }
1323
1324 static noinline struct sg_table *
1325 intel_remap_pages(struct intel_remapped_info *rem_info,
1326                   struct drm_i915_gem_object *obj)
1327 {
1328         unsigned int size = intel_remapped_info_size(rem_info);
1329         struct drm_i915_private *i915 = to_i915(obj->base.dev);
1330         struct sg_table *st;
1331         struct scatterlist *sg;
1332         int ret = -ENOMEM;
1333         int i;
1334
1335         /* Allocate target SG list. */
1336         st = kmalloc(sizeof(*st), GFP_KERNEL);
1337         if (!st)
1338                 goto err_st_alloc;
1339
1340         ret = sg_alloc_table(st, size, GFP_KERNEL);
1341         if (ret)
1342                 goto err_sg_alloc;
1343
1344         st->nents = 0;
1345         sg = st->sgl;
1346
1347         for (i = 0 ; i < ARRAY_SIZE(rem_info->plane); i++) {
1348                 sg = remap_pages(obj, rem_info->plane[i].offset,
1349                                  rem_info->plane[i].width, rem_info->plane[i].height,
1350                                  rem_info->plane[i].stride, st, sg);
1351         }
1352
1353         i915_sg_trim(st);
1354
1355         return st;
1356
1357 err_sg_alloc:
1358         kfree(st);
1359 err_st_alloc:
1360
1361         drm_dbg(&i915->drm, "Failed to create remapped mapping for object size %zu! (%ux%u tiles, %u pages)\n",
1362                 obj->base.size, rem_info->plane[0].width,
1363                 rem_info->plane[0].height, size);
1364
1365         return ERR_PTR(ret);
1366 }
1367
1368 static noinline struct sg_table *
1369 intel_partial_pages(const struct i915_ggtt_view *view,
1370                     struct drm_i915_gem_object *obj)
1371 {
1372         struct sg_table *st;
1373         struct scatterlist *sg, *iter;
1374         unsigned int count = view->partial.size;
1375         unsigned int offset;
1376         int ret = -ENOMEM;
1377
1378         st = kmalloc(sizeof(*st), GFP_KERNEL);
1379         if (!st)
1380                 goto err_st_alloc;
1381
1382         ret = sg_alloc_table(st, count, GFP_KERNEL);
1383         if (ret)
1384                 goto err_sg_alloc;
1385
1386         iter = i915_gem_object_get_sg_dma(obj, view->partial.offset, &offset);
1387         GEM_BUG_ON(!iter);
1388
1389         sg = st->sgl;
1390         st->nents = 0;
1391         do {
1392                 unsigned int len;
1393
1394                 len = min(sg_dma_len(iter) - (offset << PAGE_SHIFT),
1395                           count << PAGE_SHIFT);
1396                 sg_set_page(sg, NULL, len, 0);
1397                 sg_dma_address(sg) =
1398                         sg_dma_address(iter) + (offset << PAGE_SHIFT);
1399                 sg_dma_len(sg) = len;
1400
1401                 st->nents++;
1402                 count -= len >> PAGE_SHIFT;
1403                 if (count == 0) {
1404                         sg_mark_end(sg);
1405                         i915_sg_trim(st); /* Drop any unused tail entries. */
1406
1407                         return st;
1408                 }
1409
1410                 sg = __sg_next(sg);
1411                 iter = __sg_next(iter);
1412                 offset = 0;
1413         } while (1);
1414
1415 err_sg_alloc:
1416         kfree(st);
1417 err_st_alloc:
1418         return ERR_PTR(ret);
1419 }
1420
1421 static int
1422 i915_get_ggtt_vma_pages(struct i915_vma *vma)
1423 {
1424         int ret;
1425
1426         /*
1427          * The vma->pages are only valid within the lifespan of the borrowed
1428          * obj->mm.pages. When the obj->mm.pages sg_table is regenerated, so
1429          * must be the vma->pages. A simple rule is that vma->pages must only
1430          * be accessed when the obj->mm.pages are pinned.
1431          */
1432         GEM_BUG_ON(!i915_gem_object_has_pinned_pages(vma->obj));
1433
1434         switch (vma->ggtt_view.type) {
1435         default:
1436                 GEM_BUG_ON(vma->ggtt_view.type);
1437                 fallthrough;
1438         case I915_GGTT_VIEW_NORMAL:
1439                 vma->pages = vma->obj->mm.pages;
1440                 return 0;
1441
1442         case I915_GGTT_VIEW_ROTATED:
1443                 vma->pages =
1444                         intel_rotate_pages(&vma->ggtt_view.rotated, vma->obj);
1445                 break;
1446
1447         case I915_GGTT_VIEW_REMAPPED:
1448                 vma->pages =
1449                         intel_remap_pages(&vma->ggtt_view.remapped, vma->obj);
1450                 break;
1451
1452         case I915_GGTT_VIEW_PARTIAL:
1453                 vma->pages = intel_partial_pages(&vma->ggtt_view, vma->obj);
1454                 break;
1455         }
1456
1457         ret = 0;
1458         if (IS_ERR(vma->pages)) {
1459                 ret = PTR_ERR(vma->pages);
1460                 vma->pages = NULL;
1461                 drm_err(&vma->vm->i915->drm,
1462                         "Failed to get pages for VMA view type %u (%d)!\n",
1463                         vma->ggtt_view.type, ret);
1464         }
1465         return ret;
1466 }