drivers/gpu/drm/amd/amdgpu/amdgpu_umsch_mm.c

   1 // SPDX-License-Identifier: MIT
   2 /*
   3  * Copyright 2023 Advanced Micro Devices, Inc.
   4  *
   5  * Permission is hereby granted, free of charge, to any person obtaining a
   6  * copy of this software and associated documentation files (the "Software"),
   7  * to deal in the Software without restriction, including without limitation
   8  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   9  * and/or sell copies of the Software, and to permit persons to whom the
  10  * Software is furnished to do so, subject to the following conditions:
  11  *
  12  * The above copyright notice and this permission notice shall be included in
  13  * all copies or substantial portions of the Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  19  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  20  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  21  * OTHER DEALINGS IN THE SOFTWARE.
  22  *
  23  */
  24
  25 #include <linux/firmware.h>
  26 #include <drm/drm_exec.h>
  27
  28 #include "amdgpu.h"
  29 #include "amdgpu_umsch_mm.h"
  30 #include "umsch_mm_v4_0.h"
  31
  32 struct umsch_mm_test_ctx_data {
  33         uint8_t process_csa[PAGE_SIZE];
  34         uint8_t vpe_ctx_csa[PAGE_SIZE];
  35         uint8_t vcn_ctx_csa[PAGE_SIZE];
  36 };
  37
  38 struct umsch_mm_test_mqd_data {
  39         uint8_t vpe_mqd[PAGE_SIZE];
  40         uint8_t vcn_mqd[PAGE_SIZE];
  41 };
  42
  43 struct umsch_mm_test_ring_data {
  44         uint8_t vpe_ring[PAGE_SIZE];
  45         uint8_t vpe_ib[PAGE_SIZE];
  46         uint8_t vcn_ring[PAGE_SIZE];
  47         uint8_t vcn_ib[PAGE_SIZE];
  48 };
  49
  50 struct umsch_mm_test_queue_info {
  51         uint64_t mqd_addr;
  52         uint64_t csa_addr;
  53         uint32_t doorbell_offset_0;
  54         uint32_t doorbell_offset_1;
  55         enum UMSCH_SWIP_ENGINE_TYPE engine;
  56 };
  57
  58 struct umsch_mm_test {
  59         struct amdgpu_bo        *ctx_data_obj;
  60         uint64_t                ctx_data_gpu_addr;
  61         uint32_t                *ctx_data_cpu_addr;
  62
  63         struct amdgpu_bo        *mqd_data_obj;
  64         uint64_t                mqd_data_gpu_addr;
  65         uint32_t                *mqd_data_cpu_addr;
  66
  67         struct amdgpu_bo        *ring_data_obj;
  68         uint64_t                ring_data_gpu_addr;
  69         uint32_t                *ring_data_cpu_addr;
  70
  71
  72         struct amdgpu_vm        *vm;
  73         struct amdgpu_bo_va     *bo_va;
  74         uint32_t                pasid;
  75         uint32_t                vm_cntx_cntl;
  76         uint32_t                num_queues;
  77 };
  78
  79 int umsch_mm_psp_update_sram(struct amdgpu_device *adev, u32 ucode_size)
  80 {
  81         struct amdgpu_firmware_info ucode = {
  82                 .ucode_id = AMDGPU_UCODE_ID_UMSCH_MM_CMD_BUFFER,
  83                 .mc_addr = adev->umsch_mm.cmd_buf_gpu_addr,
  84                 .ucode_size = ucode_size,
  85         };
  86
  87         return psp_execute_ip_fw_load(&adev->psp, &ucode);
  88 }
  89
  90 static int map_ring_data(struct amdgpu_device *adev, struct amdgpu_vm *vm,
  91                           struct amdgpu_bo *bo, struct amdgpu_bo_va **bo_va,
  92                           uint64_t addr, uint32_t size)
  93 {
  94         struct amdgpu_sync sync;
  95         struct drm_exec exec;
  96         int r;
  97
  98         amdgpu_sync_create(&sync);
  99
 100         drm_exec_init(&exec, 0);
 101         drm_exec_until_all_locked(&exec) {
 102                 r = drm_exec_lock_obj(&exec, &bo->tbo.base);
 103                 drm_exec_retry_on_contention(&exec);
 104                 if (unlikely(r))
 105                         goto error_fini_exec;
 106
 107                 r = amdgpu_vm_lock_pd(vm, &exec, 0);
 108                 drm_exec_retry_on_contention(&exec);
 109                 if (unlikely(r))
 110                         goto error_fini_exec;
 111         }
 112
 113         *bo_va = amdgpu_vm_bo_add(adev, vm, bo);
 114         if (!*bo_va) {
 115                 r = -ENOMEM;
 116                 goto error_fini_exec;
 117         }
 118
 119         r = amdgpu_vm_bo_map(adev, *bo_va, addr, 0, size,
 120                              AMDGPU_PTE_READABLE | AMDGPU_PTE_WRITEABLE |
 121                              AMDGPU_PTE_EXECUTABLE);
 122
 123         if (r)
 124                 goto error_del_bo_va;
 125
 126
 127         r = amdgpu_vm_bo_update(adev, *bo_va, false);
 128         if (r)
 129                 goto error_del_bo_va;
 130
 131         amdgpu_sync_fence(&sync, (*bo_va)->last_pt_update);
 132
 133         r = amdgpu_vm_update_pdes(adev, vm, false);
 134         if (r)
 135                 goto error_del_bo_va;
 136
 137         amdgpu_sync_fence(&sync, vm->last_update);
 138
 139         amdgpu_sync_wait(&sync, false);
 140         drm_exec_fini(&exec);
 141
 142         amdgpu_sync_free(&sync);
 143
 144         return 0;
 145
 146 error_del_bo_va:
 147         amdgpu_vm_bo_del(adev, *bo_va);
 148         amdgpu_sync_free(&sync);
 149
 150 error_fini_exec:
 151         drm_exec_fini(&exec);
 152         amdgpu_sync_free(&sync);
 153         return r;
 154 }
 155
 156 static int unmap_ring_data(struct amdgpu_device *adev, struct amdgpu_vm *vm,
 157                             struct amdgpu_bo *bo, struct amdgpu_bo_va *bo_va,
 158                             uint64_t addr)
 159 {
 160         struct drm_exec exec;
 161         long r;
 162
 163         drm_exec_init(&exec, 0);
 164         drm_exec_until_all_locked(&exec) {
 165                 r = drm_exec_lock_obj(&exec, &bo->tbo.base);
 166                 drm_exec_retry_on_contention(&exec);
 167                 if (unlikely(r))
 168                         goto out_unlock;
 169
 170                 r = amdgpu_vm_lock_pd(vm, &exec, 0);
 171                 drm_exec_retry_on_contention(&exec);
 172                 if (unlikely(r))
 173                         goto out_unlock;
 174         }
 175
 176
 177         r = amdgpu_vm_bo_unmap(adev, bo_va, addr);
 178         if (r)
 179                 goto out_unlock;
 180
 181         amdgpu_vm_bo_del(adev, bo_va);
 182
 183 out_unlock:
 184         drm_exec_fini(&exec);
 185
 186         return r;
 187 }
 188
 189 static void setup_vpe_queue(struct amdgpu_device *adev,
 190                             struct umsch_mm_test *test,
 191                             struct umsch_mm_test_queue_info *qinfo)
 192 {
 193         struct MQD_INFO *mqd = (struct MQD_INFO *)test->mqd_data_cpu_addr;
 194         uint64_t ring_gpu_addr = test->ring_data_gpu_addr;
 195
 196         mqd->rb_base_lo = (ring_gpu_addr >> 8);
 197         mqd->rb_base_hi = (ring_gpu_addr >> 40);
 198         mqd->rb_size = PAGE_SIZE / 4;
 199         mqd->wptr_val = 0;
 200         mqd->rptr_val = 0;
 201         mqd->unmapped = 1;
 202
 203         qinfo->mqd_addr = test->mqd_data_gpu_addr;
 204         qinfo->csa_addr = test->ctx_data_gpu_addr +
 205                 offsetof(struct umsch_mm_test_ctx_data, vpe_ctx_csa);
 206         qinfo->doorbell_offset_0 = (adev->doorbell_index.vpe_ring + 1) << 1;
 207         qinfo->doorbell_offset_1 = 0;
 208 }
 209
 210 static void setup_vcn_queue(struct amdgpu_device *adev,
 211                             struct umsch_mm_test *test,
 212                             struct umsch_mm_test_queue_info *qinfo)
 213 {
 214 }
 215
 216 static int add_test_queue(struct amdgpu_device *adev,
 217                           struct umsch_mm_test *test,
 218                           struct umsch_mm_test_queue_info *qinfo)
 219 {
 220         struct umsch_mm_add_queue_input queue_input = {};
 221         int r;
 222
 223         queue_input.process_id = test->pasid;
 224         queue_input.page_table_base_addr = amdgpu_gmc_pd_addr(test->vm->root.bo);
 225
 226         queue_input.process_va_start = 0;
 227         queue_input.process_va_end = (adev->vm_manager.max_pfn - 1) << AMDGPU_GPU_PAGE_SHIFT;
 228
 229         queue_input.process_quantum = 100000; /* 10ms */
 230         queue_input.process_csa_addr = test->ctx_data_gpu_addr +
 231                                        offsetof(struct umsch_mm_test_ctx_data, process_csa);
 232
 233         queue_input.context_quantum = 10000; /* 1ms */
 234         queue_input.context_csa_addr = qinfo->csa_addr;
 235
 236         queue_input.inprocess_context_priority = CONTEXT_PRIORITY_LEVEL_NORMAL;
 237         queue_input.context_global_priority_level = CONTEXT_PRIORITY_LEVEL_NORMAL;
 238         queue_input.doorbell_offset_0 = qinfo->doorbell_offset_0;
 239         queue_input.doorbell_offset_1 = qinfo->doorbell_offset_1;
 240
 241         queue_input.engine_type = qinfo->engine;
 242         queue_input.mqd_addr = qinfo->mqd_addr;
 243         queue_input.vm_context_cntl = test->vm_cntx_cntl;
 244
 245         amdgpu_umsch_mm_lock(&adev->umsch_mm);
 246         r = adev->umsch_mm.funcs->add_queue(&adev->umsch_mm, &queue_input);
 247         amdgpu_umsch_mm_unlock(&adev->umsch_mm);
 248         if (r)
 249                 return r;
 250
 251         return 0;
 252 }
 253
 254 static int remove_test_queue(struct amdgpu_device *adev,
 255                              struct umsch_mm_test *test,
 256                              struct umsch_mm_test_queue_info *qinfo)
 257 {
 258         struct umsch_mm_remove_queue_input queue_input = {};
 259         int r;
 260
 261         queue_input.doorbell_offset_0 = qinfo->doorbell_offset_0;
 262         queue_input.doorbell_offset_1 = qinfo->doorbell_offset_1;
 263         queue_input.context_csa_addr = qinfo->csa_addr;
 264
 265         amdgpu_umsch_mm_lock(&adev->umsch_mm);
 266         r = adev->umsch_mm.funcs->remove_queue(&adev->umsch_mm, &queue_input);
 267         amdgpu_umsch_mm_unlock(&adev->umsch_mm);
 268         if (r)
 269                 return r;
 270
 271         return 0;
 272 }
 273
 274 static int submit_vpe_queue(struct amdgpu_device *adev, struct umsch_mm_test *test)
 275 {
 276         struct MQD_INFO *mqd = (struct MQD_INFO *)test->mqd_data_cpu_addr;
 277         uint32_t *ring = test->ring_data_cpu_addr +
 278                 offsetof(struct umsch_mm_test_ring_data, vpe_ring) / 4;
 279         uint32_t *ib = test->ring_data_cpu_addr +
 280                 offsetof(struct umsch_mm_test_ring_data, vpe_ib) / 4;
 281         uint64_t ib_gpu_addr = test->ring_data_gpu_addr +
 282                 offsetof(struct umsch_mm_test_ring_data, vpe_ib);
 283         uint32_t *fence = ib + 2048 / 4;
 284         uint64_t fence_gpu_addr = ib_gpu_addr + 2048;
 285         const uint32_t test_pattern = 0xdeadbeef;
 286         int i;
 287
 288         ib[0] = VPE_CMD_HEADER(VPE_CMD_OPCODE_FENCE, 0);
 289         ib[1] = lower_32_bits(fence_gpu_addr);
 290         ib[2] = upper_32_bits(fence_gpu_addr);
 291         ib[3] = test_pattern;
 292
 293         ring[0] = VPE_CMD_HEADER(VPE_CMD_OPCODE_INDIRECT, 0);
 294         ring[1] = (ib_gpu_addr & 0xffffffe0);
 295         ring[2] = upper_32_bits(ib_gpu_addr);
 296         ring[3] = 4;
 297         ring[4] = 0;
 298         ring[5] = 0;
 299
 300         mqd->wptr_val = (6 << 2);
 301         // WDOORBELL32(adev->umsch_mm.agdb_index[CONTEXT_PRIORITY_LEVEL_NORMAL], mqd->wptr_val);
 302
 303         for (i = 0; i < adev->usec_timeout; i++) {
 304                 if (*fence == test_pattern)
 305                         return 0;
 306                 udelay(1);
 307         }
 308
 309         dev_err(adev->dev, "vpe queue submission timeout\n");
 310
 311         return -ETIMEDOUT;
 312 }
 313
 314 static int submit_vcn_queue(struct amdgpu_device *adev, struct umsch_mm_test *test)
 315 {
 316         return 0;
 317 }
 318
 319 static int setup_umsch_mm_test(struct amdgpu_device *adev,
 320                           struct umsch_mm_test *test)
 321 {
 322         struct amdgpu_vmhub *hub = &adev->vmhub[AMDGPU_MMHUB0(0)];
 323         int r;
 324
 325         test->vm_cntx_cntl = hub->vm_cntx_cntl;
 326
 327         test->vm = kzalloc(sizeof(*test->vm), GFP_KERNEL);
 328         if (!test->vm) {
 329                 r = -ENOMEM;
 330                 return r;
 331         }
 332
 333         r = amdgpu_vm_init(adev, test->vm, -1);
 334         if (r)
 335                 goto error_free_vm;
 336
 337         r = amdgpu_pasid_alloc(16);
 338         if (r < 0)
 339                 goto error_fini_vm;
 340         test->pasid = r;
 341
 342         r = amdgpu_bo_create_kernel(adev, sizeof(struct umsch_mm_test_ctx_data),
 343                                     PAGE_SIZE, AMDGPU_GEM_DOMAIN_GTT,
 344                                     &test->ctx_data_obj,
 345                                     &test->ctx_data_gpu_addr,
 346                                     (void **)&test->ctx_data_cpu_addr);
 347         if (r)
 348                 goto error_free_pasid;
 349
 350         memset(test->ctx_data_cpu_addr, 0, sizeof(struct umsch_mm_test_ctx_data));
 351
 352         r = amdgpu_bo_create_kernel(adev, PAGE_SIZE,
 353                                     PAGE_SIZE, AMDGPU_GEM_DOMAIN_GTT,
 354                                     &test->mqd_data_obj,
 355                                     &test->mqd_data_gpu_addr,
 356                                     (void **)&test->mqd_data_cpu_addr);
 357         if (r)
 358                 goto error_free_ctx_data_obj;
 359
 360         memset(test->mqd_data_cpu_addr, 0, PAGE_SIZE);
 361
 362         r = amdgpu_bo_create_kernel(adev, sizeof(struct umsch_mm_test_ring_data),
 363                                     PAGE_SIZE, AMDGPU_GEM_DOMAIN_GTT,
 364                                     &test->ring_data_obj,
 365                                     NULL,
 366                                     (void **)&test->ring_data_cpu_addr);
 367         if (r)
 368                 goto error_free_mqd_data_obj;
 369
 370         memset(test->ring_data_cpu_addr, 0, sizeof(struct umsch_mm_test_ring_data));
 371
 372         test->ring_data_gpu_addr = AMDGPU_VA_RESERVED_SIZE;
 373         r = map_ring_data(adev, test->vm, test->ring_data_obj, &test->bo_va,
 374                           test->ring_data_gpu_addr, sizeof(struct umsch_mm_test_ring_data));
 375         if (r)
 376                 goto error_free_ring_data_obj;
 377
 378         return 0;
 379
 380 error_free_ring_data_obj:
 381         amdgpu_bo_free_kernel(&test->ring_data_obj, NULL,
 382                               (void **)&test->ring_data_cpu_addr);
 383 error_free_mqd_data_obj:
 384         amdgpu_bo_free_kernel(&test->mqd_data_obj, &test->mqd_data_gpu_addr,
 385                               (void **)&test->mqd_data_cpu_addr);
 386 error_free_ctx_data_obj:
 387         amdgpu_bo_free_kernel(&test->ctx_data_obj, &test->ctx_data_gpu_addr,
 388                               (void **)&test->ctx_data_cpu_addr);
 389 error_free_pasid:
 390         amdgpu_pasid_free(test->pasid);
 391 error_fini_vm:
 392         amdgpu_vm_fini(adev, test->vm);
 393 error_free_vm:
 394         kfree(test->vm);
 395
 396         return r;
 397 }
 398
 399 static void cleanup_umsch_mm_test(struct amdgpu_device *adev,
 400                                   struct umsch_mm_test *test)
 401 {
 402         unmap_ring_data(adev, test->vm, test->ring_data_obj,
 403                         test->bo_va, test->ring_data_gpu_addr);
 404         amdgpu_bo_free_kernel(&test->mqd_data_obj, &test->mqd_data_gpu_addr,
 405                               (void **)&test->mqd_data_cpu_addr);
 406         amdgpu_bo_free_kernel(&test->ring_data_obj, NULL,
 407                               (void **)&test->ring_data_cpu_addr);
 408         amdgpu_bo_free_kernel(&test->ctx_data_obj, &test->ctx_data_gpu_addr,
 409                                (void **)&test->ctx_data_cpu_addr);
 410         amdgpu_pasid_free(test->pasid);
 411         amdgpu_vm_fini(adev, test->vm);
 412         kfree(test->vm);
 413 }
 414
 415 static int setup_test_queues(struct amdgpu_device *adev,
 416                              struct umsch_mm_test *test,
 417                              struct umsch_mm_test_queue_info *qinfo)
 418 {
 419         int i, r;
 420
 421         for (i = 0; i < test->num_queues; i++) {
 422                 if (qinfo[i].engine == UMSCH_SWIP_ENGINE_TYPE_VPE)
 423                         setup_vpe_queue(adev, test, &qinfo[i]);
 424                 else
 425                         setup_vcn_queue(adev, test, &qinfo[i]);
 426
 427                 r = add_test_queue(adev, test, &qinfo[i]);
 428                 if (r)
 429                         return r;
 430         }
 431
 432         return 0;
 433 }
 434
 435 static int submit_test_queues(struct amdgpu_device *adev,
 436                               struct umsch_mm_test *test,
 437                               struct umsch_mm_test_queue_info *qinfo)
 438 {
 439         int i, r;
 440
 441         for (i = 0; i < test->num_queues; i++) {
 442                 if (qinfo[i].engine == UMSCH_SWIP_ENGINE_TYPE_VPE)
 443                         r = submit_vpe_queue(adev, test);
 444                 else
 445                         r = submit_vcn_queue(adev, test);
 446                 if (r)
 447                         return r;
 448         }
 449
 450         return 0;
 451 }
 452
 453 static void cleanup_test_queues(struct amdgpu_device *adev,
 454                               struct umsch_mm_test *test,
 455                               struct umsch_mm_test_queue_info *qinfo)
 456 {
 457         int i;
 458
 459         for (i = 0; i < test->num_queues; i++)
 460                 remove_test_queue(adev, test, &qinfo[i]);
 461 }
 462
 463 static int umsch_mm_test(struct amdgpu_device *adev)
 464 {
 465         struct umsch_mm_test_queue_info qinfo[] = {
 466                 { .engine = UMSCH_SWIP_ENGINE_TYPE_VPE },
 467         };
 468         struct umsch_mm_test test = { .num_queues = ARRAY_SIZE(qinfo) };
 469         int r;
 470
 471         r = setup_umsch_mm_test(adev, &test);
 472         if (r)
 473                 return r;
 474
 475         r = setup_test_queues(adev, &test, qinfo);
 476         if (r)
 477                 goto cleanup;
 478
 479         r = submit_test_queues(adev, &test, qinfo);
 480         if (r)
 481                 goto cleanup;
 482
 483         cleanup_test_queues(adev, &test, qinfo);
 484         cleanup_umsch_mm_test(adev, &test);
 485
 486         return 0;
 487
 488 cleanup:
 489         cleanup_test_queues(adev, &test, qinfo);
 490         cleanup_umsch_mm_test(adev, &test);
 491         return r;
 492 }
 493
 494 int amdgpu_umsch_mm_submit_pkt(struct amdgpu_umsch_mm *umsch, void *pkt, int ndws)
 495 {
 496         struct amdgpu_ring *ring = &umsch->ring;
 497
 498         if (amdgpu_ring_alloc(ring, ndws))
 499                 return -ENOMEM;
 500
 501         amdgpu_ring_write_multiple(ring, pkt, ndws);
 502         amdgpu_ring_commit(ring);
 503
 504         return 0;
 505 }
 506
 507 int amdgpu_umsch_mm_query_fence(struct amdgpu_umsch_mm *umsch)
 508 {
 509         struct amdgpu_ring *ring = &umsch->ring;
 510         struct amdgpu_device *adev = ring->adev;
 511         int r;
 512
 513         r = amdgpu_fence_wait_polling(ring, ring->fence_drv.sync_seq, adev->usec_timeout);
 514         if (r < 1) {
 515                 dev_err(adev->dev, "ring umsch timeout, emitted fence %u\n",
 516                         ring->fence_drv.sync_seq);
 517                 return -ETIMEDOUT;
 518         }
 519
 520         return 0;
 521 }
 522
 523 static void umsch_mm_ring_set_wptr(struct amdgpu_ring *ring)
 524 {
 525         struct amdgpu_umsch_mm *umsch = (struct amdgpu_umsch_mm *)ring;
 526         struct amdgpu_device *adev = ring->adev;
 527
 528         if (ring->use_doorbell)
 529                 WDOORBELL32(ring->doorbell_index, ring->wptr << 2);
 530         else
 531                 WREG32(umsch->rb_wptr, ring->wptr << 2);
 532 }
 533
 534 static u64 umsch_mm_ring_get_rptr(struct amdgpu_ring *ring)
 535 {
 536         struct amdgpu_umsch_mm *umsch = (struct amdgpu_umsch_mm *)ring;
 537         struct amdgpu_device *adev = ring->adev;
 538
 539         return RREG32(umsch->rb_rptr);
 540 }
 541
 542 static u64 umsch_mm_ring_get_wptr(struct amdgpu_ring *ring)
 543 {
 544         struct amdgpu_umsch_mm *umsch = (struct amdgpu_umsch_mm *)ring;
 545         struct amdgpu_device *adev = ring->adev;
 546
 547         return RREG32(umsch->rb_wptr);
 548 }
 549
 550 static const struct amdgpu_ring_funcs umsch_v4_0_ring_funcs = {
 551         .type = AMDGPU_RING_TYPE_UMSCH_MM,
 552         .align_mask = 0,
 553         .nop = 0,
 554         .support_64bit_ptrs = false,
 555         .get_rptr = umsch_mm_ring_get_rptr,
 556         .get_wptr = umsch_mm_ring_get_wptr,
 557         .set_wptr = umsch_mm_ring_set_wptr,
 558         .insert_nop = amdgpu_ring_insert_nop,
 559 };
 560
 561 int amdgpu_umsch_mm_ring_init(struct amdgpu_umsch_mm *umsch)
 562 {
 563         struct amdgpu_device *adev = container_of(umsch, struct amdgpu_device, umsch_mm);
 564         struct amdgpu_ring *ring = &umsch->ring;
 565
 566         ring->vm_hub = AMDGPU_MMHUB0(0);
 567         ring->use_doorbell = 0;
 568         ring->no_scheduler = true;
 569         ring->doorbell_index = (AMDGPU_NAVI10_DOORBELL64_VCN0_1 << 1) + 6;
 570
 571         snprintf(ring->name, sizeof(ring->name), "umsch");
 572
 573         return amdgpu_ring_init(adev, ring, 1024, NULL, 0, AMDGPU_RING_PRIO_DEFAULT, NULL);
 574 }
 575
 576 int amdgpu_umsch_mm_init_microcode(struct amdgpu_umsch_mm *umsch)
 577 {
 578         const struct umsch_mm_firmware_header_v1_0 *umsch_mm_hdr;
 579         struct amdgpu_device *adev = umsch->ring.adev;
 580         const char *fw_name = NULL;
 581         int r;
 582
 583         switch (amdgpu_ip_version(adev, VCN_HWIP, 0)) {
 584         case IP_VERSION(4, 0, 5):
 585                 fw_name = "amdgpu/umsch_mm_4_0_0.bin";
 586                 break;
 587         default:
 588                 break;
 589         }
 590
 591         r = amdgpu_ucode_request(adev, &adev->umsch_mm.fw, fw_name);
 592         if (r) {
 593                 release_firmware(adev->umsch_mm.fw);
 594                 adev->umsch_mm.fw = NULL;
 595                 return r;
 596         }
 597
 598         umsch_mm_hdr = (const struct umsch_mm_firmware_header_v1_0 *)adev->umsch_mm.fw->data;
 599
 600         adev->umsch_mm.ucode_size = le32_to_cpu(umsch_mm_hdr->umsch_mm_ucode_size_bytes);
 601         adev->umsch_mm.data_size = le32_to_cpu(umsch_mm_hdr->umsch_mm_ucode_data_size_bytes);
 602
 603         adev->umsch_mm.irq_start_addr =
 604                 le32_to_cpu(umsch_mm_hdr->umsch_mm_irq_start_addr_lo) |
 605                 ((uint64_t)(le32_to_cpu(umsch_mm_hdr->umsch_mm_irq_start_addr_hi)) << 32);
 606         adev->umsch_mm.uc_start_addr =
 607                 le32_to_cpu(umsch_mm_hdr->umsch_mm_uc_start_addr_lo) |
 608                 ((uint64_t)(le32_to_cpu(umsch_mm_hdr->umsch_mm_uc_start_addr_hi)) << 32);
 609         adev->umsch_mm.data_start_addr =
 610                 le32_to_cpu(umsch_mm_hdr->umsch_mm_data_start_addr_lo) |
 611                 ((uint64_t)(le32_to_cpu(umsch_mm_hdr->umsch_mm_data_start_addr_hi)) << 32);
 612
 613         if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) {
 614                 struct amdgpu_firmware_info *info;
 615
 616                 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_UMSCH_MM_UCODE];
 617                 info->ucode_id = AMDGPU_UCODE_ID_UMSCH_MM_UCODE;
 618                 info->fw = adev->umsch_mm.fw;
 619                 adev->firmware.fw_size +=
 620                         ALIGN(le32_to_cpu(umsch_mm_hdr->umsch_mm_ucode_size_bytes), PAGE_SIZE);
 621
 622                 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_UMSCH_MM_DATA];
 623                 info->ucode_id = AMDGPU_UCODE_ID_UMSCH_MM_DATA;
 624                 info->fw = adev->umsch_mm.fw;
 625                 adev->firmware.fw_size +=
 626                         ALIGN(le32_to_cpu(umsch_mm_hdr->umsch_mm_ucode_data_size_bytes), PAGE_SIZE);
 627         }
 628
 629         return 0;
 630 }
 631
 632 int amdgpu_umsch_mm_allocate_ucode_buffer(struct amdgpu_umsch_mm *umsch)
 633 {
 634         const struct umsch_mm_firmware_header_v1_0 *umsch_mm_hdr;
 635         struct amdgpu_device *adev = umsch->ring.adev;
 636         const __le32 *fw_data;
 637         uint32_t fw_size;
 638         int r;
 639
 640         umsch_mm_hdr = (const struct umsch_mm_firmware_header_v1_0 *)
 641                        adev->umsch_mm.fw->data;
 642
 643         fw_data = (const __le32 *)(adev->umsch_mm.fw->data +
 644                   le32_to_cpu(umsch_mm_hdr->umsch_mm_ucode_offset_bytes));
 645         fw_size = le32_to_cpu(umsch_mm_hdr->umsch_mm_ucode_size_bytes);
 646
 647         r = amdgpu_bo_create_reserved(adev, fw_size,
 648                                       4 * 1024, AMDGPU_GEM_DOMAIN_VRAM,
 649                                       &adev->umsch_mm.ucode_fw_obj,
 650                                       &adev->umsch_mm.ucode_fw_gpu_addr,
 651                                       (void **)&adev->umsch_mm.ucode_fw_ptr);
 652         if (r) {
 653                 dev_err(adev->dev, "(%d) failed to create umsch_mm fw ucode bo\n", r);
 654                 return r;
 655         }
 656
 657         memcpy(adev->umsch_mm.ucode_fw_ptr, fw_data, fw_size);
 658
 659         amdgpu_bo_kunmap(adev->umsch_mm.ucode_fw_obj);
 660         amdgpu_bo_unreserve(adev->umsch_mm.ucode_fw_obj);
 661         return 0;
 662 }
 663
 664 int amdgpu_umsch_mm_allocate_ucode_data_buffer(struct amdgpu_umsch_mm *umsch)
 665 {
 666         const struct umsch_mm_firmware_header_v1_0 *umsch_mm_hdr;
 667         struct amdgpu_device *adev = umsch->ring.adev;
 668         const __le32 *fw_data;
 669         uint32_t fw_size;
 670         int r;
 671
 672         umsch_mm_hdr = (const struct umsch_mm_firmware_header_v1_0 *)
 673                        adev->umsch_mm.fw->data;
 674
 675         fw_data = (const __le32 *)(adev->umsch_mm.fw->data +
 676                   le32_to_cpu(umsch_mm_hdr->umsch_mm_ucode_data_offset_bytes));
 677         fw_size = le32_to_cpu(umsch_mm_hdr->umsch_mm_ucode_data_size_bytes);
 678
 679         r = amdgpu_bo_create_reserved(adev, fw_size,
 680                                       64 * 1024, AMDGPU_GEM_DOMAIN_VRAM,
 681                                       &adev->umsch_mm.data_fw_obj,
 682                                       &adev->umsch_mm.data_fw_gpu_addr,
 683                                       (void **)&adev->umsch_mm.data_fw_ptr);
 684         if (r) {
 685                 dev_err(adev->dev, "(%d) failed to create umsch_mm fw data bo\n", r);
 686                 return r;
 687         }
 688
 689         memcpy(adev->umsch_mm.data_fw_ptr, fw_data, fw_size);
 690
 691         amdgpu_bo_kunmap(adev->umsch_mm.data_fw_obj);
 692         amdgpu_bo_unreserve(adev->umsch_mm.data_fw_obj);
 693         return 0;
 694 }
 695
 696 void* amdgpu_umsch_mm_add_cmd(struct amdgpu_umsch_mm *umsch,
 697                               void* cmd_ptr, uint32_t reg_offset, uint32_t reg_data)
 698 {
 699         uint32_t* ptr = (uint32_t *)cmd_ptr;
 700
 701         *ptr++ = (reg_offset << 2);
 702         *ptr++ = reg_data;
 703
 704         return ptr;
 705 }
 706
 707 static void umsch_mm_agdb_index_init(struct amdgpu_device *adev)
 708 {
 709         uint32_t umsch_mm_agdb_start;
 710         int i;
 711
 712         umsch_mm_agdb_start = adev->doorbell_index.max_assignment + 1;
 713         umsch_mm_agdb_start = roundup(umsch_mm_agdb_start, 1024);
 714         umsch_mm_agdb_start += (AMDGPU_NAVI10_DOORBELL64_VCN0_1 << 1);
 715
 716         for (i = 0; i < CONTEXT_PRIORITY_NUM_LEVELS; i++)
 717                 adev->umsch_mm.agdb_index[i] = umsch_mm_agdb_start + i;
 718 }
 719
 720 static int umsch_mm_init(struct amdgpu_device *adev)
 721 {
 722         int r;
 723
 724         adev->umsch_mm.vmid_mask_mm_vpe = 0xf00;
 725         adev->umsch_mm.engine_mask = (1 << UMSCH_SWIP_ENGINE_TYPE_VPE);
 726         adev->umsch_mm.vpe_hqd_mask = 0xfe;
 727
 728         r = amdgpu_device_wb_get(adev, &adev->umsch_mm.wb_index);
 729         if (r) {
 730                 dev_err(adev->dev, "failed to alloc wb for umsch: %d\n", r);
 731                 return r;
 732         }
 733
 734         adev->umsch_mm.sch_ctx_gpu_addr = adev->wb.gpu_addr +
 735                                           (adev->umsch_mm.wb_index * 4);
 736
 737         r = amdgpu_bo_create_kernel(adev, PAGE_SIZE, PAGE_SIZE,
 738                                     AMDGPU_GEM_DOMAIN_GTT,
 739                                     &adev->umsch_mm.cmd_buf_obj,
 740                                     &adev->umsch_mm.cmd_buf_gpu_addr,
 741                                     (void **)&adev->umsch_mm.cmd_buf_ptr);
 742         if (r) {
 743                 dev_err(adev->dev, "failed to allocate cmdbuf bo %d\n", r);
 744                 amdgpu_device_wb_free(adev, adev->umsch_mm.wb_index);
 745                 return r;
 746         }
 747
 748         mutex_init(&adev->umsch_mm.mutex_hidden);
 749
 750         umsch_mm_agdb_index_init(adev);
 751
 752         return 0;
 753 }
 754
 755
 756 static int umsch_mm_early_init(void *handle)
 757 {
 758         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 759
 760         switch (amdgpu_ip_version(adev, VCN_HWIP, 0)) {
 761         case IP_VERSION(4, 0, 5):
 762                 umsch_mm_v4_0_set_funcs(&adev->umsch_mm);
 763                 break;
 764         default:
 765                 return -EINVAL;
 766         }
 767
 768         adev->umsch_mm.ring.funcs = &umsch_v4_0_ring_funcs;
 769         umsch_mm_set_regs(&adev->umsch_mm);
 770
 771         return 0;
 772 }
 773
 774 static int umsch_mm_late_init(void *handle)
 775 {
 776         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 777
 778         return umsch_mm_test(adev);
 779 }
 780
 781 static int umsch_mm_sw_init(void *handle)
 782 {
 783         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 784         int r;
 785
 786         r = umsch_mm_init(adev);
 787         if (r)
 788                 return r;
 789
 790         r = umsch_mm_ring_init(&adev->umsch_mm);
 791         if (r)
 792                 return r;
 793
 794         r = umsch_mm_init_microcode(&adev->umsch_mm);
 795         if (r)
 796                 return r;
 797
 798         return 0;
 799 }
 800
 801 static int umsch_mm_sw_fini(void *handle)
 802 {
 803         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 804
 805         release_firmware(adev->umsch_mm.fw);
 806         adev->umsch_mm.fw = NULL;
 807
 808         amdgpu_ring_fini(&adev->umsch_mm.ring);
 809
 810         mutex_destroy(&adev->umsch_mm.mutex_hidden);
 811
 812         amdgpu_bo_free_kernel(&adev->umsch_mm.cmd_buf_obj,
 813                               &adev->umsch_mm.cmd_buf_gpu_addr,
 814                               (void **)&adev->umsch_mm.cmd_buf_ptr);
 815
 816         amdgpu_device_wb_free(adev, adev->umsch_mm.wb_index);
 817
 818         return 0;
 819 }
 820
 821 static int umsch_mm_hw_init(void *handle)
 822 {
 823         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 824         int r;
 825
 826         if (adev->firmware.load_type == AMDGPU_FW_LOAD_DIRECT) {
 827                 r = umsch_mm_load_microcode(&adev->umsch_mm);
 828                 if (r)
 829                         return r;
 830         }
 831
 832         umsch_mm_ring_start(&adev->umsch_mm);
 833
 834         r = umsch_mm_set_hw_resources(&adev->umsch_mm);
 835         if (r)
 836                 return r;
 837
 838         return 0;
 839 }
 840
 841 static int umsch_mm_hw_fini(void *handle)
 842 {
 843         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 844
 845         umsch_mm_ring_stop(&adev->umsch_mm);
 846
 847         amdgpu_bo_free_kernel(&adev->umsch_mm.data_fw_obj,
 848                               &adev->umsch_mm.data_fw_gpu_addr,
 849                               (void **)&adev->umsch_mm.data_fw_ptr);
 850
 851         amdgpu_bo_free_kernel(&adev->umsch_mm.ucode_fw_obj,
 852                               &adev->umsch_mm.ucode_fw_gpu_addr,
 853                               (void **)&adev->umsch_mm.ucode_fw_ptr);
 854         return 0;
 855 }
 856
 857 static const struct amd_ip_funcs umsch_mm_v4_0_ip_funcs = {
 858         .name = "umsch_mm_v4_0",
 859         .early_init = umsch_mm_early_init,
 860         .late_init = umsch_mm_late_init,
 861         .sw_init = umsch_mm_sw_init,
 862         .sw_fini = umsch_mm_sw_fini,
 863         .hw_init = umsch_mm_hw_init,
 864         .hw_fini = umsch_mm_hw_fini,
 865 };
 866
 867 const struct amdgpu_ip_block_version umsch_mm_v4_0_ip_block = {
 868         .type = AMD_IP_BLOCK_TYPE_UMSCH_MM,
 869         .major = 4,
 870         .minor = 0,
 871         .rev = 0,
 872         .funcs = &umsch_mm_v4_0_ip_funcs,
 873 };