drivers/gpu/drm/i915/gt/intel_migrate.c

   1 // SPDX-License-Identifier: MIT
   2 /*
   3  * Copyright © 2020 Intel Corporation
   4  */
   5
   6 #include "i915_drv.h"
   7 #include "intel_context.h"
   8 #include "intel_gpu_commands.h"
   9 #include "intel_gt.h"
  10 #include "intel_gtt.h"
  11 #include "intel_migrate.h"
  12 #include "intel_ring.h"
  13 #include "gem/i915_gem_lmem.h"
  14
  15 struct insert_pte_data {
  16         u64 offset;
  17 };
  18
  19 #define CHUNK_SZ SZ_8M /* ~1ms at 8GiB/s preemption delay */
  20
  21 #define GET_CCS_BYTES(i915, size)       (HAS_FLAT_CCS(i915) ? \
  22                                          DIV_ROUND_UP(size, NUM_BYTES_PER_CCS_BYTE) : 0)
  23 static bool engine_supports_migration(struct intel_engine_cs *engine)
  24 {
  25         if (!engine)
  26                 return false;
  27
  28         /*
  29          * We need the ability to prevent aribtration (MI_ARB_ON_OFF),
  30          * the ability to write PTE using inline data (MI_STORE_DATA)
  31          * and of course the ability to do the block transfer (blits).
  32          */
  33         GEM_BUG_ON(engine->class != COPY_ENGINE_CLASS);
  34
  35         return true;
  36 }
  37
  38 static void xehpsdv_toggle_pdes(struct i915_address_space *vm,
  39                                 struct i915_page_table *pt,
  40                                 void *data)
  41 {
  42         struct insert_pte_data *d = data;
  43
  44         /*
  45          * Insert a dummy PTE into every PT that will map to LMEM to ensure
  46          * we have a correctly setup PDE structure for later use.
  47          */
  48         vm->insert_page(vm, 0, d->offset, I915_CACHE_NONE, PTE_LM);
  49         GEM_BUG_ON(!pt->is_compact);
  50         d->offset += SZ_2M;
  51 }
  52
  53 static void xehpsdv_insert_pte(struct i915_address_space *vm,
  54                                struct i915_page_table *pt,
  55                                void *data)
  56 {
  57         struct insert_pte_data *d = data;
  58
  59         /*
  60          * We are playing tricks here, since the actual pt, from the hw
  61          * pov, is only 256bytes with 32 entries, or 4096bytes with 512
  62          * entries, but we are still guaranteed that the physical
  63          * alignment is 64K underneath for the pt, and we are careful
  64          * not to access the space in the void.
  65          */
  66         vm->insert_page(vm, px_dma(pt), d->offset, I915_CACHE_NONE, PTE_LM);
  67         d->offset += SZ_64K;
  68 }
  69
  70 static void insert_pte(struct i915_address_space *vm,
  71                        struct i915_page_table *pt,
  72                        void *data)
  73 {
  74         struct insert_pte_data *d = data;
  75
  76         vm->insert_page(vm, px_dma(pt), d->offset, I915_CACHE_NONE,
  77                         i915_gem_object_is_lmem(pt->base) ? PTE_LM : 0);
  78         d->offset += PAGE_SIZE;
  79 }
  80
  81 static struct i915_address_space *migrate_vm(struct intel_gt *gt)
  82 {
  83         struct i915_vm_pt_stash stash = {};
  84         struct i915_ppgtt *vm;
  85         int err;
  86         int i;
  87
  88         /*
  89          * We construct a very special VM for use by all migration contexts,
  90          * it is kept pinned so that it can be used at any time. As we need
  91          * to pre-allocate the page directories for the migration VM, this
  92          * limits us to only using a small number of prepared vma.
  93          *
  94          * To be able to pipeline and reschedule migration operations while
  95          * avoiding unnecessary contention on the vm itself, the PTE updates
  96          * are inline with the blits. All the blits use the same fixed
  97          * addresses, with the backing store redirection being updated on the
  98          * fly. Only 2 implicit vma are used for all migration operations.
  99          *
 100          * We lay the ppGTT out as:
 101          *
 102          *      [0, CHUNK_SZ) -> first object
 103          *      [CHUNK_SZ, 2 * CHUNK_SZ) -> second object
 104          *      [2 * CHUNK_SZ, 2 * CHUNK_SZ + 2 * CHUNK_SZ >> 9] -> PTE
 105          *
 106          * By exposing the dma addresses of the page directories themselves
 107          * within the ppGTT, we are then able to rewrite the PTE prior to use.
 108          * But the PTE update and subsequent migration operation must be atomic,
 109          * i.e. within the same non-preemptible window so that we do not switch
 110          * to another migration context that overwrites the PTE.
 111          *
 112          * This changes quite a bit on platforms with HAS_64K_PAGES support,
 113          * where we instead have three windows, each CHUNK_SIZE in size. The
 114          * first is reserved for mapping system-memory, and that just uses the
 115          * 512 entry layout using 4K GTT pages. The other two windows just map
 116          * lmem pages and must use the new compact 32 entry layout using 64K GTT
 117          * pages, which ensures we can address any lmem object that the user
 118          * throws at us. We then also use the xehpsdv_toggle_pdes as a way of
 119          * just toggling the PDE bit(GEN12_PDE_64K) for us, to enable the
 120          * compact layout for each of these page-tables, that fall within the
 121          * [CHUNK_SIZE, 3 * CHUNK_SIZE) range.
 122          *
 123          * We lay the ppGTT out as:
 124          *
 125          * [0, CHUNK_SZ) -> first window/object, maps smem
 126          * [CHUNK_SZ, 2 * CHUNK_SZ) -> second window/object, maps lmem src
 127          * [2 * CHUNK_SZ, 3 * CHUNK_SZ) -> third window/object, maps lmem dst
 128          *
 129          * For the PTE window it's also quite different, since each PTE must
 130          * point to some 64K page, one for each PT(since it's in lmem), and yet
 131          * each is only <= 4096bytes, but since the unused space within that PTE
 132          * range is never touched, this should be fine.
 133          *
 134          * So basically each PT now needs 64K of virtual memory, instead of 4K,
 135          * which looks like:
 136          *
 137          * [3 * CHUNK_SZ, 3 * CHUNK_SZ + ((3 * CHUNK_SZ / SZ_2M) * SZ_64K)] -> PTE
 138          */
 139
 140         vm = i915_ppgtt_create(gt, I915_BO_ALLOC_PM_EARLY);
 141         if (IS_ERR(vm))
 142                 return ERR_CAST(vm);
 143
 144         if (!vm->vm.allocate_va_range || !vm->vm.foreach) {
 145                 err = -ENODEV;
 146                 goto err_vm;
 147         }
 148
 149         if (HAS_64K_PAGES(gt->i915))
 150                 stash.pt_sz = I915_GTT_PAGE_SIZE_64K;
 151
 152         /*
 153          * Each engine instance is assigned its own chunk in the VM, so
 154          * that we can run multiple instances concurrently
 155          */
 156         for (i = 0; i < ARRAY_SIZE(gt->engine_class[COPY_ENGINE_CLASS]); i++) {
 157                 struct intel_engine_cs *engine;
 158                 u64 base = (u64)i << 32;
 159                 struct insert_pte_data d = {};
 160                 struct i915_gem_ww_ctx ww;
 161                 u64 sz;
 162
 163                 engine = gt->engine_class[COPY_ENGINE_CLASS][i];
 164                 if (!engine_supports_migration(engine))
 165                         continue;
 166
 167                 /*
 168                  * We copy in 8MiB chunks. Each PDE covers 2MiB, so we need
 169                  * 4x2 page directories for source/destination.
 170                  */
 171                 if (HAS_64K_PAGES(gt->i915))
 172                         sz = 3 * CHUNK_SZ;
 173                 else
 174                         sz = 2 * CHUNK_SZ;
 175                 d.offset = base + sz;
 176
 177                 /*
 178                  * We need another page directory setup so that we can write
 179                  * the 8x512 PTE in each chunk.
 180                  */
 181                 if (HAS_64K_PAGES(gt->i915))
 182                         sz += (sz / SZ_2M) * SZ_64K;
 183                 else
 184                         sz += (sz >> 12) * sizeof(u64);
 185
 186                 err = i915_vm_alloc_pt_stash(&vm->vm, &stash, sz);
 187                 if (err)
 188                         goto err_vm;
 189
 190                 for_i915_gem_ww(&ww, err, true) {
 191                         err = i915_vm_lock_objects(&vm->vm, &ww);
 192                         if (err)
 193                                 continue;
 194                         err = i915_vm_map_pt_stash(&vm->vm, &stash);
 195                         if (err)
 196                                 continue;
 197
 198                         vm->vm.allocate_va_range(&vm->vm, &stash, base, sz);
 199                 }
 200                 i915_vm_free_pt_stash(&vm->vm, &stash);
 201                 if (err)
 202                         goto err_vm;
 203
 204                 /* Now allow the GPU to rewrite the PTE via its own ppGTT */
 205                 if (HAS_64K_PAGES(gt->i915)) {
 206                         vm->vm.foreach(&vm->vm, base, d.offset - base,
 207                                        xehpsdv_insert_pte, &d);
 208                         d.offset = base + CHUNK_SZ;
 209                         vm->vm.foreach(&vm->vm,
 210                                        d.offset,
 211                                        2 * CHUNK_SZ,
 212                                        xehpsdv_toggle_pdes, &d);
 213                 } else {
 214                         vm->vm.foreach(&vm->vm, base, d.offset - base,
 215                                        insert_pte, &d);
 216                 }
 217         }
 218
 219         return &vm->vm;
 220
 221 err_vm:
 222         i915_vm_put(&vm->vm);
 223         return ERR_PTR(err);
 224 }
 225
 226 static struct intel_engine_cs *first_copy_engine(struct intel_gt *gt)
 227 {
 228         struct intel_engine_cs *engine;
 229         int i;
 230
 231         for (i = 0; i < ARRAY_SIZE(gt->engine_class[COPY_ENGINE_CLASS]); i++) {
 232                 engine = gt->engine_class[COPY_ENGINE_CLASS][i];
 233                 if (engine_supports_migration(engine))
 234                         return engine;
 235         }
 236
 237         return NULL;
 238 }
 239
 240 static struct intel_context *pinned_context(struct intel_gt *gt)
 241 {
 242         static struct lock_class_key key;
 243         struct intel_engine_cs *engine;
 244         struct i915_address_space *vm;
 245         struct intel_context *ce;
 246
 247         engine = first_copy_engine(gt);
 248         if (!engine)
 249                 return ERR_PTR(-ENODEV);
 250
 251         vm = migrate_vm(gt);
 252         if (IS_ERR(vm))
 253                 return ERR_CAST(vm);
 254
 255         ce = intel_engine_create_pinned_context(engine, vm, SZ_512K,
 256                                                 I915_GEM_HWS_MIGRATE,
 257                                                 &key, "migrate");
 258         i915_vm_put(vm);
 259         return ce;
 260 }
 261
 262 int intel_migrate_init(struct intel_migrate *m, struct intel_gt *gt)
 263 {
 264         struct intel_context *ce;
 265
 266         memset(m, 0, sizeof(*m));
 267
 268         ce = pinned_context(gt);
 269         if (IS_ERR(ce))
 270                 return PTR_ERR(ce);
 271
 272         m->context = ce;
 273         return 0;
 274 }
 275
 276 static int random_index(unsigned int max)
 277 {
 278         return upper_32_bits(mul_u32_u32(get_random_u32(), max));
 279 }
 280
 281 static struct intel_context *__migrate_engines(struct intel_gt *gt)
 282 {
 283         struct intel_engine_cs *engines[MAX_ENGINE_INSTANCE];
 284         struct intel_engine_cs *engine;
 285         unsigned int count, i;
 286
 287         count = 0;
 288         for (i = 0; i < ARRAY_SIZE(gt->engine_class[COPY_ENGINE_CLASS]); i++) {
 289                 engine = gt->engine_class[COPY_ENGINE_CLASS][i];
 290                 if (engine_supports_migration(engine))
 291                         engines[count++] = engine;
 292         }
 293
 294         return intel_context_create(engines[random_index(count)]);
 295 }
 296
 297 struct intel_context *intel_migrate_create_context(struct intel_migrate *m)
 298 {
 299         struct intel_context *ce;
 300
 301         /*
 302          * We randomly distribute contexts across the engines upon constrction,
 303          * as they all share the same pinned vm, and so in order to allow
 304          * multiple blits to run in parallel, we must construct each blit
 305          * to use a different range of the vm for its GTT. This has to be
 306          * known at construction, so we can not use the late greedy load
 307          * balancing of the virtual-engine.
 308          */
 309         ce = __migrate_engines(m->context->engine->gt);
 310         if (IS_ERR(ce))
 311                 return ce;
 312
 313         ce->ring = NULL;
 314         ce->ring_size = SZ_256K;
 315
 316         i915_vm_put(ce->vm);
 317         ce->vm = i915_vm_get(m->context->vm);
 318
 319         return ce;
 320 }
 321
 322 static inline struct sgt_dma sg_sgt(struct scatterlist *sg)
 323 {
 324         dma_addr_t addr = sg_dma_address(sg);
 325
 326         return (struct sgt_dma){ sg, addr, addr + sg_dma_len(sg) };
 327 }
 328
 329 static int emit_no_arbitration(struct i915_request *rq)
 330 {
 331         u32 *cs;
 332
 333         cs = intel_ring_begin(rq, 2);
 334         if (IS_ERR(cs))
 335                 return PTR_ERR(cs);
 336
 337         /* Explicitly disable preemption for this request. */
 338         *cs++ = MI_ARB_ON_OFF;
 339         *cs++ = MI_NOOP;
 340         intel_ring_advance(rq, cs);
 341
 342         return 0;
 343 }
 344
 345 static int max_pte_pkt_size(struct i915_request *rq, int pkt)
 346 {
 347         struct intel_ring *ring = rq->ring;
 348
 349         pkt = min_t(int, pkt, (ring->space - rq->reserved_space) / sizeof(u32) + 5);
 350         pkt = min_t(int, pkt, (ring->size - ring->emit) / sizeof(u32) + 5);
 351
 352         return pkt;
 353 }
 354
 355 #define I915_EMIT_PTE_NUM_DWORDS 6
 356
 357 static int emit_pte(struct i915_request *rq,
 358                     struct sgt_dma *it,
 359                     enum i915_cache_level cache_level,
 360                     bool is_lmem,
 361                     u64 offset,
 362                     int length)
 363 {
 364         bool has_64K_pages = HAS_64K_PAGES(rq->engine->i915);
 365         const u64 encode = rq->context->vm->pte_encode(0, cache_level,
 366                                                        is_lmem ? PTE_LM : 0);
 367         struct intel_ring *ring = rq->ring;
 368         int pkt, dword_length;
 369         u32 total = 0;
 370         u32 page_size;
 371         u32 *hdr, *cs;
 372
 373         GEM_BUG_ON(GRAPHICS_VER(rq->engine->i915) < 8);
 374
 375         page_size = I915_GTT_PAGE_SIZE;
 376         dword_length = 0x400;
 377
 378         /* Compute the page directory offset for the target address range */
 379         if (has_64K_pages) {
 380                 GEM_BUG_ON(!IS_ALIGNED(offset, SZ_2M));
 381
 382                 offset /= SZ_2M;
 383                 offset *= SZ_64K;
 384                 offset += 3 * CHUNK_SZ;
 385
 386                 if (is_lmem) {
 387                         page_size = I915_GTT_PAGE_SIZE_64K;
 388                         dword_length = 0x40;
 389                 }
 390         } else {
 391                 offset >>= 12;
 392                 offset *= sizeof(u64);
 393                 offset += 2 * CHUNK_SZ;
 394         }
 395
 396         offset += (u64)rq->engine->instance << 32;
 397
 398         cs = intel_ring_begin(rq, I915_EMIT_PTE_NUM_DWORDS);
 399         if (IS_ERR(cs))
 400                 return PTR_ERR(cs);
 401
 402         /* Pack as many PTE updates as possible into a single MI command */
 403         pkt = max_pte_pkt_size(rq, dword_length);
 404
 405         hdr = cs;
 406         *cs++ = MI_STORE_DATA_IMM | REG_BIT(21); /* as qword elements */
 407         *cs++ = lower_32_bits(offset);
 408         *cs++ = upper_32_bits(offset);
 409
 410         do {
 411                 if (cs - hdr >= pkt) {
 412                         int dword_rem;
 413
 414                         *hdr += cs - hdr - 2;
 415                         *cs++ = MI_NOOP;
 416
 417                         ring->emit = (void *)cs - ring->vaddr;
 418                         intel_ring_advance(rq, cs);
 419                         intel_ring_update_space(ring);
 420
 421                         cs = intel_ring_begin(rq, I915_EMIT_PTE_NUM_DWORDS);
 422                         if (IS_ERR(cs))
 423                                 return PTR_ERR(cs);
 424
 425                         dword_rem = dword_length;
 426                         if (has_64K_pages) {
 427                                 if (IS_ALIGNED(total, SZ_2M)) {
 428                                         offset = round_up(offset, SZ_64K);
 429                                 } else {
 430                                         dword_rem = SZ_2M - (total & (SZ_2M - 1));
 431                                         dword_rem /= page_size;
 432                                         dword_rem *= 2;
 433                                 }
 434                         }
 435
 436                         pkt = max_pte_pkt_size(rq, dword_rem);
 437
 438                         hdr = cs;
 439                         *cs++ = MI_STORE_DATA_IMM | REG_BIT(21);
 440                         *cs++ = lower_32_bits(offset);
 441                         *cs++ = upper_32_bits(offset);
 442                 }
 443
 444                 GEM_BUG_ON(!IS_ALIGNED(it->dma, page_size));
 445
 446                 *cs++ = lower_32_bits(encode | it->dma);
 447                 *cs++ = upper_32_bits(encode | it->dma);
 448
 449                 offset += 8;
 450                 total += page_size;
 451
 452                 it->dma += page_size;
 453                 if (it->dma >= it->max) {
 454                         it->sg = __sg_next(it->sg);
 455                         if (!it->sg || sg_dma_len(it->sg) == 0)
 456                                 break;
 457
 458                         it->dma = sg_dma_address(it->sg);
 459                         it->max = it->dma + sg_dma_len(it->sg);
 460                 }
 461         } while (total < length);
 462
 463         *hdr += cs - hdr - 2;
 464         *cs++ = MI_NOOP;
 465
 466         ring->emit = (void *)cs - ring->vaddr;
 467         intel_ring_advance(rq, cs);
 468         intel_ring_update_space(ring);
 469
 470         return total;
 471 }
 472
 473 static bool wa_1209644611_applies(int ver, u32 size)
 474 {
 475         u32 height = size >> PAGE_SHIFT;
 476
 477         if (ver != 11)
 478                 return false;
 479
 480         return height % 4 == 3 && height <= 8;
 481 }
 482
 483 /**
 484  * DOC: Flat-CCS - Memory compression for Local memory
 485  *
 486  * On Xe-HP and later devices, we use dedicated compression control state (CCS)
 487  * stored in local memory for each surface, to support the 3D and media
 488  * compression formats.
 489  *
 490  * The memory required for the CCS of the entire local memory is 1/256 of the
 491  * local memory size. So before the kernel boot, the required memory is reserved
 492  * for the CCS data and a secure register will be programmed with the CCS base
 493  * address.
 494  *
 495  * Flat CCS data needs to be cleared when a lmem object is allocated.
 496  * And CCS data can be copied in and out of CCS region through
 497  * XY_CTRL_SURF_COPY_BLT. CPU can't access the CCS data directly.
 498  *
 499  * I915 supports Flat-CCS on lmem only objects. When an objects has smem in
 500  * its preference list, on memory pressure, i915 needs to migrate the lmem
 501  * content into smem. If the lmem object is Flat-CCS compressed by userspace,
 502  * then i915 needs to decompress it. But I915 lack the required information
 503  * for such decompression. Hence I915 supports Flat-CCS only on lmem only objects.
 504  *
 505  * When we exhaust the lmem, Flat-CCS capable objects' lmem backing memory can
 506  * be temporarily evicted to smem, along with the auxiliary CCS state, where
 507  * it can be potentially swapped-out at a later point, if required.
 508  * If userspace later touches the evicted pages, then we always move
 509  * the backing memory back to lmem, which includes restoring the saved CCS state,
 510  * and potentially performing any required swap-in.
 511  *
 512  * For the migration of the lmem objects with smem in placement list, such as
 513  * {lmem, smem}, objects are treated as non Flat-CCS capable objects.
 514  */
 515
 516 static inline u32 *i915_flush_dw(u32 *cmd, u32 flags)
 517 {
 518         *cmd++ = MI_FLUSH_DW | flags;
 519         *cmd++ = 0;
 520         *cmd++ = 0;
 521
 522         return cmd;
 523 }
 524
 525 static int emit_copy_ccs(struct i915_request *rq,
 526                          u32 dst_offset, u8 dst_access,
 527                          u32 src_offset, u8 src_access, int size)
 528 {
 529         struct drm_i915_private *i915 = rq->engine->i915;
 530         int mocs = rq->engine->gt->mocs.uc_index << 1;
 531         u32 num_ccs_blks;
 532         u32 *cs;
 533
 534         cs = intel_ring_begin(rq, 12);
 535         if (IS_ERR(cs))
 536                 return PTR_ERR(cs);
 537
 538         num_ccs_blks = DIV_ROUND_UP(GET_CCS_BYTES(i915, size),
 539                                     NUM_CCS_BYTES_PER_BLOCK);
 540         GEM_BUG_ON(num_ccs_blks > NUM_CCS_BLKS_PER_XFER);
 541         cs = i915_flush_dw(cs, MI_FLUSH_DW_LLC | MI_FLUSH_DW_CCS);
 542
 543         /*
 544          * The XY_CTRL_SURF_COPY_BLT instruction is used to copy the CCS
 545          * data in and out of the CCS region.
 546          *
 547          * We can copy at most 1024 blocks of 256 bytes using one
 548          * XY_CTRL_SURF_COPY_BLT instruction.
 549          *
 550          * In case we need to copy more than 1024 blocks, we need to add
 551          * another instruction to the same batch buffer.
 552          *
 553          * 1024 blocks of 256 bytes of CCS represent a total 256KB of CCS.
 554          *
 555          * 256 KB of CCS represents 256 * 256 KB = 64 MB of LMEM.
 556          */
 557         *cs++ = XY_CTRL_SURF_COPY_BLT |
 558                 src_access << SRC_ACCESS_TYPE_SHIFT |
 559                 dst_access << DST_ACCESS_TYPE_SHIFT |
 560                 ((num_ccs_blks - 1) & CCS_SIZE_MASK) << CCS_SIZE_SHIFT;
 561         *cs++ = src_offset;
 562         *cs++ = rq->engine->instance |
 563                 FIELD_PREP(XY_CTRL_SURF_MOCS_MASK, mocs);
 564         *cs++ = dst_offset;
 565         *cs++ = rq->engine->instance |
 566                 FIELD_PREP(XY_CTRL_SURF_MOCS_MASK, mocs);
 567
 568         cs = i915_flush_dw(cs, MI_FLUSH_DW_LLC | MI_FLUSH_DW_CCS);
 569         *cs++ = MI_NOOP;
 570
 571         intel_ring_advance(rq, cs);
 572
 573         return 0;
 574 }
 575
 576 static int emit_copy(struct i915_request *rq,
 577                      u32 dst_offset, u32 src_offset, int size)
 578 {
 579         const int ver = GRAPHICS_VER(rq->engine->i915);
 580         u32 instance = rq->engine->instance;
 581         u32 *cs;
 582
 583         cs = intel_ring_begin(rq, ver >= 8 ? 10 : 6);
 584         if (IS_ERR(cs))
 585                 return PTR_ERR(cs);
 586
 587         if (ver >= 9 && !wa_1209644611_applies(ver, size)) {
 588                 *cs++ = GEN9_XY_FAST_COPY_BLT_CMD | (10 - 2);
 589                 *cs++ = BLT_DEPTH_32 | PAGE_SIZE;
 590                 *cs++ = 0;
 591                 *cs++ = size >> PAGE_SHIFT << 16 | PAGE_SIZE / 4;
 592                 *cs++ = dst_offset;
 593                 *cs++ = instance;
 594                 *cs++ = 0;
 595                 *cs++ = PAGE_SIZE;
 596                 *cs++ = src_offset;
 597                 *cs++ = instance;
 598         } else if (ver >= 8) {
 599                 *cs++ = XY_SRC_COPY_BLT_CMD | BLT_WRITE_RGBA | (10 - 2);
 600                 *cs++ = BLT_DEPTH_32 | BLT_ROP_SRC_COPY | PAGE_SIZE;
 601                 *cs++ = 0;
 602                 *cs++ = size >> PAGE_SHIFT << 16 | PAGE_SIZE / 4;
 603                 *cs++ = dst_offset;
 604                 *cs++ = instance;
 605                 *cs++ = 0;
 606                 *cs++ = PAGE_SIZE;
 607                 *cs++ = src_offset;
 608                 *cs++ = instance;
 609         } else {
 610                 GEM_BUG_ON(instance);
 611                 *cs++ = SRC_COPY_BLT_CMD | BLT_WRITE_RGBA | (6 - 2);
 612                 *cs++ = BLT_DEPTH_32 | BLT_ROP_SRC_COPY | PAGE_SIZE;
 613                 *cs++ = size >> PAGE_SHIFT << 16 | PAGE_SIZE;
 614                 *cs++ = dst_offset;
 615                 *cs++ = PAGE_SIZE;
 616                 *cs++ = src_offset;
 617         }
 618
 619         intel_ring_advance(rq, cs);
 620         return 0;
 621 }
 622
 623 static u64 scatter_list_length(struct scatterlist *sg)
 624 {
 625         u64 len = 0;
 626
 627         while (sg && sg_dma_len(sg)) {
 628                 len += sg_dma_len(sg);
 629                 sg = sg_next(sg);
 630         }
 631
 632         return len;
 633 }
 634
 635 static int
 636 calculate_chunk_sz(struct drm_i915_private *i915, bool src_is_lmem,
 637                    u64 bytes_to_cpy, u64 ccs_bytes_to_cpy)
 638 {
 639         if (ccs_bytes_to_cpy && !src_is_lmem)
 640                 /*
 641                  * When CHUNK_SZ is passed all the pages upto CHUNK_SZ
 642                  * will be taken for the blt. in Flat-ccs supported
 643                  * platform Smem obj will have more pages than required
 644                  * for main meory hence limit it to the required size
 645                  * for main memory
 646                  */
 647                 return min_t(u64, bytes_to_cpy, CHUNK_SZ);
 648         else
 649                 return CHUNK_SZ;
 650 }
 651
 652 static void get_ccs_sg_sgt(struct sgt_dma *it, u64 bytes_to_cpy)
 653 {
 654         u64 len;
 655
 656         do {
 657                 GEM_BUG_ON(!it->sg || !sg_dma_len(it->sg));
 658                 len = it->max - it->dma;
 659                 if (len > bytes_to_cpy) {
 660                         it->dma += bytes_to_cpy;
 661                         break;
 662                 }
 663
 664                 bytes_to_cpy -= len;
 665
 666                 it->sg = __sg_next(it->sg);
 667                 it->dma = sg_dma_address(it->sg);
 668                 it->max = it->dma + sg_dma_len(it->sg);
 669         } while (bytes_to_cpy);
 670 }
 671
 672 int
 673 intel_context_migrate_copy(struct intel_context *ce,
 674                            const struct i915_deps *deps,
 675                            struct scatterlist *src,
 676                            enum i915_cache_level src_cache_level,
 677                            bool src_is_lmem,
 678                            struct scatterlist *dst,
 679                            enum i915_cache_level dst_cache_level,
 680                            bool dst_is_lmem,
 681                            struct i915_request **out)
 682 {
 683         struct sgt_dma it_src = sg_sgt(src), it_dst = sg_sgt(dst), it_ccs;
 684         struct drm_i915_private *i915 = ce->engine->i915;
 685         u64 ccs_bytes_to_cpy = 0, bytes_to_cpy;
 686         enum i915_cache_level ccs_cache_level;
 687         u32 src_offset, dst_offset;
 688         u8 src_access, dst_access;
 689         struct i915_request *rq;
 690         u64 src_sz, dst_sz;
 691         bool ccs_is_src, overwrite_ccs;
 692         int err;
 693
 694         GEM_BUG_ON(ce->vm != ce->engine->gt->migrate.context->vm);
 695         GEM_BUG_ON(IS_DGFX(ce->engine->i915) && (!src_is_lmem && !dst_is_lmem));
 696         *out = NULL;
 697
 698         GEM_BUG_ON(ce->ring->size < SZ_64K);
 699
 700         src_sz = scatter_list_length(src);
 701         bytes_to_cpy = src_sz;
 702
 703         if (HAS_FLAT_CCS(i915) && src_is_lmem ^ dst_is_lmem) {
 704                 src_access = !src_is_lmem && dst_is_lmem;
 705                 dst_access = !src_access;
 706
 707                 dst_sz = scatter_list_length(dst);
 708                 if (src_is_lmem) {
 709                         it_ccs = it_dst;
 710                         ccs_cache_level = dst_cache_level;
 711                         ccs_is_src = false;
 712                 } else if (dst_is_lmem) {
 713                         bytes_to_cpy = dst_sz;
 714                         it_ccs = it_src;
 715                         ccs_cache_level = src_cache_level;
 716                         ccs_is_src = true;
 717                 }
 718
 719                 /*
 720                  * When there is a eviction of ccs needed smem will have the
 721                  * extra pages for the ccs data
 722                  *
 723                  * TO-DO: Want to move the size mismatch check to a WARN_ON,
 724                  * but still we have some requests of smem->lmem with same size.
 725                  * Need to fix it.
 726                  */
 727                 ccs_bytes_to_cpy = src_sz != dst_sz ? GET_CCS_BYTES(i915, bytes_to_cpy) : 0;
 728                 if (ccs_bytes_to_cpy)
 729                         get_ccs_sg_sgt(&it_ccs, bytes_to_cpy);
 730         }
 731
 732         overwrite_ccs = HAS_FLAT_CCS(i915) && !ccs_bytes_to_cpy && dst_is_lmem;
 733
 734         src_offset = 0;
 735         dst_offset = CHUNK_SZ;
 736         if (HAS_64K_PAGES(ce->engine->i915)) {
 737                 src_offset = 0;
 738                 dst_offset = 0;
 739                 if (src_is_lmem)
 740                         src_offset = CHUNK_SZ;
 741                 if (dst_is_lmem)
 742                         dst_offset = 2 * CHUNK_SZ;
 743         }
 744
 745         do {
 746                 int len;
 747
 748                 rq = i915_request_create(ce);
 749                 if (IS_ERR(rq)) {
 750                         err = PTR_ERR(rq);
 751                         goto out_ce;
 752                 }
 753
 754                 if (deps) {
 755                         err = i915_request_await_deps(rq, deps);
 756                         if (err)
 757                                 goto out_rq;
 758
 759                         if (rq->engine->emit_init_breadcrumb) {
 760                                 err = rq->engine->emit_init_breadcrumb(rq);
 761                                 if (err)
 762                                         goto out_rq;
 763                         }
 764
 765                         deps = NULL;
 766                 }
 767
 768                 /* The PTE updates + copy must not be interrupted. */
 769                 err = emit_no_arbitration(rq);
 770                 if (err)
 771                         goto out_rq;
 772
 773                 src_sz = calculate_chunk_sz(i915, src_is_lmem,
 774                                             bytes_to_cpy, ccs_bytes_to_cpy);
 775
 776                 len = emit_pte(rq, &it_src, src_cache_level, src_is_lmem,
 777                                src_offset, src_sz);
 778                 if (!len) {
 779                         err = -EINVAL;
 780                         goto out_rq;
 781                 }
 782                 if (len < 0) {
 783                         err = len;
 784                         goto out_rq;
 785                 }
 786
 787                 err = emit_pte(rq, &it_dst, dst_cache_level, dst_is_lmem,
 788                                dst_offset, len);
 789                 if (err < 0)
 790                         goto out_rq;
 791                 if (err < len) {
 792                         err = -EINVAL;
 793                         goto out_rq;
 794                 }
 795
 796                 err = rq->engine->emit_flush(rq, EMIT_INVALIDATE);
 797                 if (err)
 798                         goto out_rq;
 799
 800                 err = emit_copy(rq, dst_offset, src_offset, len);
 801                 if (err)
 802                         goto out_rq;
 803
 804                 bytes_to_cpy -= len;
 805
 806                 if (ccs_bytes_to_cpy) {
 807                         int ccs_sz;
 808
 809                         err = rq->engine->emit_flush(rq, EMIT_INVALIDATE);
 810                         if (err)
 811                                 goto out_rq;
 812
 813                         ccs_sz = GET_CCS_BYTES(i915, len);
 814                         err = emit_pte(rq, &it_ccs, ccs_cache_level, false,
 815                                        ccs_is_src ? src_offset : dst_offset,
 816                                        ccs_sz);
 817                         if (err < 0)
 818                                 goto out_rq;
 819                         if (err < ccs_sz) {
 820                                 err = -EINVAL;
 821                                 goto out_rq;
 822                         }
 823
 824                         err = rq->engine->emit_flush(rq, EMIT_INVALIDATE);
 825                         if (err)
 826                                 goto out_rq;
 827
 828                         err = emit_copy_ccs(rq, dst_offset, dst_access,
 829                                             src_offset, src_access, len);
 830                         if (err)
 831                                 goto out_rq;
 832
 833                         err = rq->engine->emit_flush(rq, EMIT_INVALIDATE);
 834                         if (err)
 835                                 goto out_rq;
 836                         ccs_bytes_to_cpy -= ccs_sz;
 837                 } else if (overwrite_ccs) {
 838                         err = rq->engine->emit_flush(rq, EMIT_INVALIDATE);
 839                         if (err)
 840                                 goto out_rq;
 841
 842                         if (src_is_lmem) {
 843                                 /*
 844                                  * If the src is already in lmem, then we must
 845                                  * be doing an lmem -> lmem transfer, and so
 846                                  * should be safe to directly copy the CCS
 847                                  * state. In this case we have either
 848                                  * initialised the CCS aux state when first
 849                                  * clearing the pages (since it is already
 850                                  * allocated in lmem), or the user has
 851                                  * potentially populated it, in which case we
 852                                  * need to copy the CCS state as-is.
 853                                  */
 854                                 err = emit_copy_ccs(rq,
 855                                                     dst_offset, INDIRECT_ACCESS,
 856                                                     src_offset, INDIRECT_ACCESS,
 857                                                     len);
 858                         } else {
 859                                 /*
 860                                  * While we can't always restore/manage the CCS
 861                                  * state, we still need to ensure we don't leak
 862                                  * the CCS state from the previous user, so make
 863                                  * sure we overwrite it with something.
 864                                  */
 865                                 err = emit_copy_ccs(rq,
 866                                                     dst_offset, INDIRECT_ACCESS,
 867                                                     dst_offset, DIRECT_ACCESS,
 868                                                     len);
 869                         }
 870
 871                         if (err)
 872                                 goto out_rq;
 873
 874                         err = rq->engine->emit_flush(rq, EMIT_INVALIDATE);
 875                         if (err)
 876                                 goto out_rq;
 877                 }
 878
 879                 /* Arbitration is re-enabled between requests. */
 880 out_rq:
 881                 if (*out)
 882                         i915_request_put(*out);
 883                 *out = i915_request_get(rq);
 884                 i915_request_add(rq);
 885
 886                 if (err)
 887                         break;
 888
 889                 if (!bytes_to_cpy && !ccs_bytes_to_cpy) {
 890                         if (src_is_lmem)
 891                                 WARN_ON(it_src.sg && sg_dma_len(it_src.sg));
 892                         else
 893                                 WARN_ON(it_dst.sg && sg_dma_len(it_dst.sg));
 894                         break;
 895                 }
 896
 897                 if (WARN_ON(!it_src.sg || !sg_dma_len(it_src.sg) ||
 898                             !it_dst.sg || !sg_dma_len(it_dst.sg) ||
 899                             (ccs_bytes_to_cpy && (!it_ccs.sg ||
 900                                                   !sg_dma_len(it_ccs.sg))))) {
 901                         err = -EINVAL;
 902                         break;
 903                 }
 904
 905                 cond_resched();
 906         } while (1);
 907
 908 out_ce:
 909         return err;
 910 }
 911
 912 static int emit_clear(struct i915_request *rq, u32 offset, int size,
 913                       u32 value, bool is_lmem)
 914 {
 915         struct drm_i915_private *i915 = rq->engine->i915;
 916         int mocs = rq->engine->gt->mocs.uc_index << 1;
 917         const int ver = GRAPHICS_VER(i915);
 918         int ring_sz;
 919         u32 *cs;
 920
 921         GEM_BUG_ON(size >> PAGE_SHIFT > S16_MAX);
 922
 923         if (HAS_FLAT_CCS(i915) && ver >= 12)
 924                 ring_sz = XY_FAST_COLOR_BLT_DW;
 925         else if (ver >= 8)
 926                 ring_sz = 8;
 927         else
 928                 ring_sz = 6;
 929
 930         cs = intel_ring_begin(rq, ring_sz);
 931         if (IS_ERR(cs))
 932                 return PTR_ERR(cs);
 933
 934         if (HAS_FLAT_CCS(i915) && ver >= 12) {
 935                 *cs++ = XY_FAST_COLOR_BLT_CMD | XY_FAST_COLOR_BLT_DEPTH_32 |
 936                         (XY_FAST_COLOR_BLT_DW - 2);
 937                 *cs++ = FIELD_PREP(XY_FAST_COLOR_BLT_MOCS_MASK, mocs) |
 938                         (PAGE_SIZE - 1);
 939                 *cs++ = 0;
 940                 *cs++ = size >> PAGE_SHIFT << 16 | PAGE_SIZE / 4;
 941                 *cs++ = offset;
 942                 *cs++ = rq->engine->instance;
 943                 *cs++ = !is_lmem << XY_FAST_COLOR_BLT_MEM_TYPE_SHIFT;
 944                 /* BG7 */
 945                 *cs++ = value;
 946                 *cs++ = 0;
 947                 *cs++ = 0;
 948                 *cs++ = 0;
 949                 /* BG11 */
 950                 *cs++ = 0;
 951                 *cs++ = 0;
 952                 /* BG13 */
 953                 *cs++ = 0;
 954                 *cs++ = 0;
 955                 *cs++ = 0;
 956         } else if (ver >= 8) {
 957                 *cs++ = XY_COLOR_BLT_CMD | BLT_WRITE_RGBA | (7 - 2);
 958                 *cs++ = BLT_DEPTH_32 | BLT_ROP_COLOR_COPY | PAGE_SIZE;
 959                 *cs++ = 0;
 960                 *cs++ = size >> PAGE_SHIFT << 16 | PAGE_SIZE / 4;
 961                 *cs++ = offset;
 962                 *cs++ = rq->engine->instance;
 963                 *cs++ = value;
 964                 *cs++ = MI_NOOP;
 965         } else {
 966                 *cs++ = XY_COLOR_BLT_CMD | BLT_WRITE_RGBA | (6 - 2);
 967                 *cs++ = BLT_DEPTH_32 | BLT_ROP_COLOR_COPY | PAGE_SIZE;
 968                 *cs++ = 0;
 969                 *cs++ = size >> PAGE_SHIFT << 16 | PAGE_SIZE / 4;
 970                 *cs++ = offset;
 971                 *cs++ = value;
 972         }
 973
 974         intel_ring_advance(rq, cs);
 975         return 0;
 976 }
 977
 978 int
 979 intel_context_migrate_clear(struct intel_context *ce,
 980                             const struct i915_deps *deps,
 981                             struct scatterlist *sg,
 982                             enum i915_cache_level cache_level,
 983                             bool is_lmem,
 984                             u32 value,
 985                             struct i915_request **out)
 986 {
 987         struct drm_i915_private *i915 = ce->engine->i915;
 988         struct sgt_dma it = sg_sgt(sg);
 989         struct i915_request *rq;
 990         u32 offset;
 991         int err;
 992
 993         GEM_BUG_ON(ce->vm != ce->engine->gt->migrate.context->vm);
 994         *out = NULL;
 995
 996         GEM_BUG_ON(ce->ring->size < SZ_64K);
 997
 998         offset = 0;
 999         if (HAS_64K_PAGES(i915) && is_lmem)
1000                 offset = CHUNK_SZ;
1001
1002         do {
1003                 int len;
1004
1005                 rq = i915_request_create(ce);
1006                 if (IS_ERR(rq)) {
1007                         err = PTR_ERR(rq);
1008                         goto out_ce;
1009                 }
1010
1011                 if (deps) {
1012                         err = i915_request_await_deps(rq, deps);
1013                         if (err)
1014                                 goto out_rq;
1015
1016                         if (rq->engine->emit_init_breadcrumb) {
1017                                 err = rq->engine->emit_init_breadcrumb(rq);
1018                                 if (err)
1019                                         goto out_rq;
1020                         }
1021
1022                         deps = NULL;
1023                 }
1024
1025                 /* The PTE updates + clear must not be interrupted. */
1026                 err = emit_no_arbitration(rq);
1027                 if (err)
1028                         goto out_rq;
1029
1030                 len = emit_pte(rq, &it, cache_level, is_lmem, offset, CHUNK_SZ);
1031                 if (len <= 0) {
1032                         err = len;
1033                         goto out_rq;
1034                 }
1035
1036                 err = rq->engine->emit_flush(rq, EMIT_INVALIDATE);
1037                 if (err)
1038                         goto out_rq;
1039
1040                 err = emit_clear(rq, offset, len, value, is_lmem);
1041                 if (err)
1042                         goto out_rq;
1043
1044                 if (HAS_FLAT_CCS(i915) && is_lmem && !value) {
1045                         /*
1046                          * copy the content of memory into corresponding
1047                          * ccs surface
1048                          */
1049                         err = emit_copy_ccs(rq, offset, INDIRECT_ACCESS, offset,
1050                                             DIRECT_ACCESS, len);
1051                         if (err)
1052                                 goto out_rq;
1053                 }
1054
1055                 err = rq->engine->emit_flush(rq, EMIT_INVALIDATE);
1056
1057                 /* Arbitration is re-enabled between requests. */
1058 out_rq:
1059                 if (*out)
1060                         i915_request_put(*out);
1061                 *out = i915_request_get(rq);
1062                 i915_request_add(rq);
1063                 if (err || !it.sg || !sg_dma_len(it.sg))
1064                         break;
1065
1066                 cond_resched();
1067         } while (1);
1068
1069 out_ce:
1070         return err;
1071 }
1072
1073 int intel_migrate_copy(struct intel_migrate *m,
1074                        struct i915_gem_ww_ctx *ww,
1075                        const struct i915_deps *deps,
1076                        struct scatterlist *src,
1077                        enum i915_cache_level src_cache_level,
1078                        bool src_is_lmem,
1079                        struct scatterlist *dst,
1080                        enum i915_cache_level dst_cache_level,
1081                        bool dst_is_lmem,
1082                        struct i915_request **out)
1083 {
1084         struct intel_context *ce;
1085         int err;
1086
1087         *out = NULL;
1088         if (!m->context)
1089                 return -ENODEV;
1090
1091         ce = intel_migrate_create_context(m);
1092         if (IS_ERR(ce))
1093                 ce = intel_context_get(m->context);
1094         GEM_BUG_ON(IS_ERR(ce));
1095
1096         err = intel_context_pin_ww(ce, ww);
1097         if (err)
1098                 goto out;
1099
1100         err = intel_context_migrate_copy(ce, deps,
1101                                          src, src_cache_level, src_is_lmem,
1102                                          dst, dst_cache_level, dst_is_lmem,
1103                                          out);
1104
1105         intel_context_unpin(ce);
1106 out:
1107         intel_context_put(ce);
1108         return err;
1109 }
1110
1111 int
1112 intel_migrate_clear(struct intel_migrate *m,
1113                     struct i915_gem_ww_ctx *ww,
1114                     const struct i915_deps *deps,
1115                     struct scatterlist *sg,
1116                     enum i915_cache_level cache_level,
1117                     bool is_lmem,
1118                     u32 value,
1119                     struct i915_request **out)
1120 {
1121         struct intel_context *ce;
1122         int err;
1123
1124         *out = NULL;
1125         if (!m->context)
1126                 return -ENODEV;
1127
1128         ce = intel_migrate_create_context(m);
1129         if (IS_ERR(ce))
1130                 ce = intel_context_get(m->context);
1131         GEM_BUG_ON(IS_ERR(ce));
1132
1133         err = intel_context_pin_ww(ce, ww);
1134         if (err)
1135                 goto out;
1136
1137         err = intel_context_migrate_clear(ce, deps, sg, cache_level,
1138                                           is_lmem, value, out);
1139
1140         intel_context_unpin(ce);
1141 out:
1142         intel_context_put(ce);
1143         return err;
1144 }
1145
1146 void intel_migrate_fini(struct intel_migrate *m)
1147 {
1148         struct intel_context *ce;
1149
1150         ce = fetch_and_zero(&m->context);
1151         if (!ce)
1152                 return;
1153
1154         intel_engine_destroy_pinned_context(ce);
1155 }
1156
1157 #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
1158 #include "selftest_migrate.c"
1159 #endif