drivers/gpu/drm/amd/amdgpu/amdgpu_umsch_mm.c

   1 // SPDX-License-Identifier: MIT
   2 /*
   3  * Copyright 2023 Advanced Micro Devices, Inc.
   4  *
   5  * Permission is hereby granted, free of charge, to any person obtaining a
   6  * copy of this software and associated documentation files (the "Software"),
   7  * to deal in the Software without restriction, including without limitation
   8  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   9  * and/or sell copies of the Software, and to permit persons to whom the
  10  * Software is furnished to do so, subject to the following conditions:
  11  *
  12  * The above copyright notice and this permission notice shall be included in
  13  * all copies or substantial portions of the Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  19  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  20  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  21  * OTHER DEALINGS IN THE SOFTWARE.
  22  *
  23  */
  24
  25 #include <linux/firmware.h>
  26 #include <drm/drm_exec.h>
  27
  28 #include "amdgpu.h"
  29 #include "amdgpu_umsch_mm.h"
  30 #include "umsch_mm_v4_0.h"
  31
  32 struct umsch_mm_test_ctx_data {
  33         uint8_t process_csa[PAGE_SIZE];
  34         uint8_t vpe_ctx_csa[PAGE_SIZE];
  35         uint8_t vcn_ctx_csa[PAGE_SIZE];
  36 };
  37
  38 struct umsch_mm_test_mqd_data {
  39         uint8_t vpe_mqd[PAGE_SIZE];
  40         uint8_t vcn_mqd[PAGE_SIZE];
  41 };
  42
  43 struct umsch_mm_test_ring_data {
  44         uint8_t vpe_ring[PAGE_SIZE];
  45         uint8_t vpe_ib[PAGE_SIZE];
  46         uint8_t vcn_ring[PAGE_SIZE];
  47         uint8_t vcn_ib[PAGE_SIZE];
  48 };
  49
  50 struct umsch_mm_test_queue_info {
  51         uint64_t mqd_addr;
  52         uint64_t csa_addr;
  53         uint32_t doorbell_offset_0;
  54         uint32_t doorbell_offset_1;
  55         enum UMSCH_SWIP_ENGINE_TYPE engine;
  56 };
  57
  58 struct umsch_mm_test {
  59         struct amdgpu_bo        *ctx_data_obj;
  60         uint64_t                ctx_data_gpu_addr;
  61         uint32_t                *ctx_data_cpu_addr;
  62
  63         struct amdgpu_bo        *mqd_data_obj;
  64         uint64_t                mqd_data_gpu_addr;
  65         uint32_t                *mqd_data_cpu_addr;
  66
  67         struct amdgpu_bo        *ring_data_obj;
  68         uint64_t                ring_data_gpu_addr;
  69         uint32_t                *ring_data_cpu_addr;
  70
  71
  72         struct amdgpu_vm        *vm;
  73         struct amdgpu_bo_va     *bo_va;
  74         uint32_t                pasid;
  75         uint32_t                vm_cntx_cntl;
  76         uint32_t                num_queues;
  77 };
  78
  79 static int map_ring_data(struct amdgpu_device *adev, struct amdgpu_vm *vm,
  80                           struct amdgpu_bo *bo, struct amdgpu_bo_va **bo_va,
  81                           uint64_t addr, uint32_t size)
  82 {
  83         struct amdgpu_sync sync;
  84         struct drm_exec exec;
  85         int r;
  86
  87         amdgpu_sync_create(&sync);
  88
  89         drm_exec_init(&exec, 0);
  90         drm_exec_until_all_locked(&exec) {
  91                 r = drm_exec_lock_obj(&exec, &bo->tbo.base);
  92                 drm_exec_retry_on_contention(&exec);
  93                 if (unlikely(r))
  94                         goto error_fini_exec;
  95
  96                 r = amdgpu_vm_lock_pd(vm, &exec, 0);
  97                 drm_exec_retry_on_contention(&exec);
  98                 if (unlikely(r))
  99                         goto error_fini_exec;
 100         }
 101
 102         *bo_va = amdgpu_vm_bo_add(adev, vm, bo);
 103         if (!*bo_va) {
 104                 r = -ENOMEM;
 105                 goto error_fini_exec;
 106         }
 107
 108         r = amdgpu_vm_bo_map(adev, *bo_va, addr, 0, size,
 109                              AMDGPU_PTE_READABLE | AMDGPU_PTE_WRITEABLE |
 110                              AMDGPU_PTE_EXECUTABLE);
 111
 112         if (r)
 113                 goto error_del_bo_va;
 114
 115
 116         r = amdgpu_vm_bo_update(adev, *bo_va, false);
 117         if (r)
 118                 goto error_del_bo_va;
 119
 120         amdgpu_sync_fence(&sync, (*bo_va)->last_pt_update);
 121
 122         r = amdgpu_vm_update_pdes(adev, vm, false);
 123         if (r)
 124                 goto error_del_bo_va;
 125
 126         amdgpu_sync_fence(&sync, vm->last_update);
 127
 128         amdgpu_sync_wait(&sync, false);
 129         drm_exec_fini(&exec);
 130
 131         amdgpu_sync_free(&sync);
 132
 133         return 0;
 134
 135 error_del_bo_va:
 136         amdgpu_vm_bo_del(adev, *bo_va);
 137         amdgpu_sync_free(&sync);
 138
 139 error_fini_exec:
 140         drm_exec_fini(&exec);
 141         amdgpu_sync_free(&sync);
 142         return r;
 143 }
 144
 145 static int unmap_ring_data(struct amdgpu_device *adev, struct amdgpu_vm *vm,
 146                             struct amdgpu_bo *bo, struct amdgpu_bo_va *bo_va,
 147                             uint64_t addr)
 148 {
 149         struct drm_exec exec;
 150         long r;
 151
 152         drm_exec_init(&exec, 0);
 153         drm_exec_until_all_locked(&exec) {
 154                 r = drm_exec_lock_obj(&exec, &bo->tbo.base);
 155                 drm_exec_retry_on_contention(&exec);
 156                 if (unlikely(r))
 157                         goto out_unlock;
 158
 159                 r = amdgpu_vm_lock_pd(vm, &exec, 0);
 160                 drm_exec_retry_on_contention(&exec);
 161                 if (unlikely(r))
 162                         goto out_unlock;
 163         }
 164
 165
 166         r = amdgpu_vm_bo_unmap(adev, bo_va, addr);
 167         if (r)
 168                 goto out_unlock;
 169
 170         amdgpu_vm_bo_del(adev, bo_va);
 171
 172 out_unlock:
 173         drm_exec_fini(&exec);
 174
 175         return r;
 176 }
 177
 178 static void setup_vpe_queue(struct amdgpu_device *adev,
 179                             struct umsch_mm_test *test,
 180                             struct umsch_mm_test_queue_info *qinfo)
 181 {
 182         struct MQD_INFO *mqd = (struct MQD_INFO *)test->mqd_data_cpu_addr;
 183         uint64_t ring_gpu_addr = test->ring_data_gpu_addr;
 184
 185         mqd->rb_base_lo = (ring_gpu_addr >> 8);
 186         mqd->rb_base_hi = (ring_gpu_addr >> 40);
 187         mqd->rb_size = PAGE_SIZE / 4;
 188         mqd->wptr_val = 0;
 189         mqd->rptr_val = 0;
 190         mqd->unmapped = 1;
 191
 192         qinfo->mqd_addr = test->mqd_data_gpu_addr;
 193         qinfo->csa_addr = test->ctx_data_gpu_addr +
 194                 offsetof(struct umsch_mm_test_ctx_data, vpe_ctx_csa);
 195         qinfo->doorbell_offset_0 = (adev->doorbell_index.vpe_ring + 1) << 1;
 196         qinfo->doorbell_offset_1 = 0;
 197 }
 198
 199 static void setup_vcn_queue(struct amdgpu_device *adev,
 200                             struct umsch_mm_test *test,
 201                             struct umsch_mm_test_queue_info *qinfo)
 202 {
 203 }
 204
 205 static int add_test_queue(struct amdgpu_device *adev,
 206                           struct umsch_mm_test *test,
 207                           struct umsch_mm_test_queue_info *qinfo)
 208 {
 209         struct umsch_mm_add_queue_input queue_input = {};
 210         int r;
 211
 212         queue_input.process_id = test->pasid;
 213         queue_input.page_table_base_addr = amdgpu_gmc_pd_addr(test->vm->root.bo);
 214
 215         queue_input.process_va_start = 0;
 216         queue_input.process_va_end = (adev->vm_manager.max_pfn - 1) << AMDGPU_GPU_PAGE_SHIFT;
 217
 218         queue_input.process_quantum = 100000; /* 10ms */
 219         queue_input.process_csa_addr = test->ctx_data_gpu_addr +
 220                                        offsetof(struct umsch_mm_test_ctx_data, process_csa);
 221
 222         queue_input.context_quantum = 10000; /* 1ms */
 223         queue_input.context_csa_addr = qinfo->csa_addr;
 224
 225         queue_input.inprocess_context_priority = CONTEXT_PRIORITY_LEVEL_NORMAL;
 226         queue_input.context_global_priority_level = CONTEXT_PRIORITY_LEVEL_NORMAL;
 227         queue_input.doorbell_offset_0 = qinfo->doorbell_offset_0;
 228         queue_input.doorbell_offset_1 = qinfo->doorbell_offset_1;
 229
 230         queue_input.engine_type = qinfo->engine;
 231         queue_input.mqd_addr = qinfo->mqd_addr;
 232         queue_input.vm_context_cntl = test->vm_cntx_cntl;
 233
 234         amdgpu_umsch_mm_lock(&adev->umsch_mm);
 235         r = adev->umsch_mm.funcs->add_queue(&adev->umsch_mm, &queue_input);
 236         amdgpu_umsch_mm_unlock(&adev->umsch_mm);
 237         if (r)
 238                 return r;
 239
 240         return 0;
 241 }
 242
 243 static int remove_test_queue(struct amdgpu_device *adev,
 244                              struct umsch_mm_test *test,
 245                              struct umsch_mm_test_queue_info *qinfo)
 246 {
 247         struct umsch_mm_remove_queue_input queue_input = {};
 248         int r;
 249
 250         queue_input.doorbell_offset_0 = qinfo->doorbell_offset_0;
 251         queue_input.doorbell_offset_1 = qinfo->doorbell_offset_1;
 252         queue_input.context_csa_addr = qinfo->csa_addr;
 253
 254         amdgpu_umsch_mm_lock(&adev->umsch_mm);
 255         r = adev->umsch_mm.funcs->remove_queue(&adev->umsch_mm, &queue_input);
 256         amdgpu_umsch_mm_unlock(&adev->umsch_mm);
 257         if (r)
 258                 return r;
 259
 260         return 0;
 261 }
 262
 263 static int submit_vpe_queue(struct amdgpu_device *adev, struct umsch_mm_test *test)
 264 {
 265         struct MQD_INFO *mqd = (struct MQD_INFO *)test->mqd_data_cpu_addr;
 266         uint32_t *ring = test->ring_data_cpu_addr +
 267                 offsetof(struct umsch_mm_test_ring_data, vpe_ring) / 4;
 268         uint32_t *ib = test->ring_data_cpu_addr +
 269                 offsetof(struct umsch_mm_test_ring_data, vpe_ib) / 4;
 270         uint64_t ib_gpu_addr = test->ring_data_gpu_addr +
 271                 offsetof(struct umsch_mm_test_ring_data, vpe_ib);
 272         uint32_t *fence = ib + 2048 / 4;
 273         uint64_t fence_gpu_addr = ib_gpu_addr + 2048;
 274         const uint32_t test_pattern = 0xdeadbeef;
 275         int i;
 276
 277         ib[0] = VPE_CMD_HEADER(VPE_CMD_OPCODE_FENCE, 0);
 278         ib[1] = lower_32_bits(fence_gpu_addr);
 279         ib[2] = upper_32_bits(fence_gpu_addr);
 280         ib[3] = test_pattern;
 281
 282         ring[0] = VPE_CMD_HEADER(VPE_CMD_OPCODE_INDIRECT, 0);
 283         ring[1] = (ib_gpu_addr & 0xffffffe0);
 284         ring[2] = upper_32_bits(ib_gpu_addr);
 285         ring[3] = 4;
 286         ring[4] = 0;
 287         ring[5] = 0;
 288
 289         mqd->wptr_val = (6 << 2);
 290         // WDOORBELL32(adev->umsch_mm.agdb_index[CONTEXT_PRIORITY_LEVEL_NORMAL], mqd->wptr_val);
 291
 292         for (i = 0; i < adev->usec_timeout; i++) {
 293                 if (*fence == test_pattern)
 294                         return 0;
 295                 udelay(1);
 296         }
 297
 298         dev_err(adev->dev, "vpe queue submission timeout\n");
 299
 300         return -ETIMEDOUT;
 301 }
 302
 303 static int submit_vcn_queue(struct amdgpu_device *adev, struct umsch_mm_test *test)
 304 {
 305         return 0;
 306 }
 307
 308 static int setup_umsch_mm_test(struct amdgpu_device *adev,
 309                           struct umsch_mm_test *test)
 310 {
 311         struct amdgpu_vmhub *hub = &adev->vmhub[AMDGPU_MMHUB0(0)];
 312         int r;
 313
 314         test->vm_cntx_cntl = hub->vm_cntx_cntl;
 315
 316         test->vm = kzalloc(sizeof(*test->vm), GFP_KERNEL);
 317         if (!test->vm) {
 318                 r = -ENOMEM;
 319                 return r;
 320         }
 321
 322         r = amdgpu_vm_init(adev, test->vm, -1);
 323         if (r)
 324                 goto error_free_vm;
 325
 326         r = amdgpu_pasid_alloc(16);
 327         if (r < 0)
 328                 goto error_fini_vm;
 329         test->pasid = r;
 330
 331         r = amdgpu_bo_create_kernel(adev, sizeof(struct umsch_mm_test_ctx_data),
 332                                     PAGE_SIZE, AMDGPU_GEM_DOMAIN_GTT,
 333                                     &test->ctx_data_obj,
 334                                     &test->ctx_data_gpu_addr,
 335                                     (void **)&test->ctx_data_cpu_addr);
 336         if (r)
 337                 goto error_free_pasid;
 338
 339         memset(test->ctx_data_cpu_addr, 0, sizeof(struct umsch_mm_test_ctx_data));
 340
 341         r = amdgpu_bo_create_kernel(adev, PAGE_SIZE,
 342                                     PAGE_SIZE, AMDGPU_GEM_DOMAIN_GTT,
 343                                     &test->mqd_data_obj,
 344                                     &test->mqd_data_gpu_addr,
 345                                     (void **)&test->mqd_data_cpu_addr);
 346         if (r)
 347                 goto error_free_ctx_data_obj;
 348
 349         memset(test->mqd_data_cpu_addr, 0, PAGE_SIZE);
 350
 351         r = amdgpu_bo_create_kernel(adev, sizeof(struct umsch_mm_test_ring_data),
 352                                     PAGE_SIZE, AMDGPU_GEM_DOMAIN_GTT,
 353                                     &test->ring_data_obj,
 354                                     NULL,
 355                                     (void **)&test->ring_data_cpu_addr);
 356         if (r)
 357                 goto error_free_mqd_data_obj;
 358
 359         memset(test->ring_data_cpu_addr, 0, sizeof(struct umsch_mm_test_ring_data));
 360
 361         test->ring_data_gpu_addr = AMDGPU_VA_RESERVED_SIZE;
 362         r = map_ring_data(adev, test->vm, test->ring_data_obj, &test->bo_va,
 363                           test->ring_data_gpu_addr, sizeof(struct umsch_mm_test_ring_data));
 364         if (r)
 365                 goto error_free_ring_data_obj;
 366
 367         return 0;
 368
 369 error_free_ring_data_obj:
 370         amdgpu_bo_free_kernel(&test->ring_data_obj, NULL,
 371                               (void **)&test->ring_data_cpu_addr);
 372 error_free_mqd_data_obj:
 373         amdgpu_bo_free_kernel(&test->mqd_data_obj, &test->mqd_data_gpu_addr,
 374                               (void **)&test->mqd_data_cpu_addr);
 375 error_free_ctx_data_obj:
 376         amdgpu_bo_free_kernel(&test->ctx_data_obj, &test->ctx_data_gpu_addr,
 377                               (void **)&test->ctx_data_cpu_addr);
 378 error_free_pasid:
 379         amdgpu_pasid_free(test->pasid);
 380 error_fini_vm:
 381         amdgpu_vm_fini(adev, test->vm);
 382 error_free_vm:
 383         kfree(test->vm);
 384
 385         return r;
 386 }
 387
 388 static void cleanup_umsch_mm_test(struct amdgpu_device *adev,
 389                                   struct umsch_mm_test *test)
 390 {
 391         unmap_ring_data(adev, test->vm, test->ring_data_obj,
 392                         test->bo_va, test->ring_data_gpu_addr);
 393         amdgpu_bo_free_kernel(&test->mqd_data_obj, &test->mqd_data_gpu_addr,
 394                               (void **)&test->mqd_data_cpu_addr);
 395         amdgpu_bo_free_kernel(&test->ring_data_obj, NULL,
 396                               (void **)&test->ring_data_cpu_addr);
 397         amdgpu_bo_free_kernel(&test->ctx_data_obj, &test->ctx_data_gpu_addr,
 398                                (void **)&test->ctx_data_cpu_addr);
 399         amdgpu_pasid_free(test->pasid);
 400         amdgpu_vm_fini(adev, test->vm);
 401         kfree(test->vm);
 402 }
 403
 404 static int setup_test_queues(struct amdgpu_device *adev,
 405                              struct umsch_mm_test *test,
 406                              struct umsch_mm_test_queue_info *qinfo)
 407 {
 408         int i, r;
 409
 410         for (i = 0; i < test->num_queues; i++) {
 411                 if (qinfo[i].engine == UMSCH_SWIP_ENGINE_TYPE_VPE)
 412                         setup_vpe_queue(adev, test, &qinfo[i]);
 413                 else
 414                         setup_vcn_queue(adev, test, &qinfo[i]);
 415
 416                 r = add_test_queue(adev, test, &qinfo[i]);
 417                 if (r)
 418                         return r;
 419         }
 420
 421         return 0;
 422 }
 423
 424 static int submit_test_queues(struct amdgpu_device *adev,
 425                               struct umsch_mm_test *test,
 426                               struct umsch_mm_test_queue_info *qinfo)
 427 {
 428         int i, r;
 429
 430         for (i = 0; i < test->num_queues; i++) {
 431                 if (qinfo[i].engine == UMSCH_SWIP_ENGINE_TYPE_VPE)
 432                         r = submit_vpe_queue(adev, test);
 433                 else
 434                         r = submit_vcn_queue(adev, test);
 435                 if (r)
 436                         return r;
 437         }
 438
 439         return 0;
 440 }
 441
 442 static void cleanup_test_queues(struct amdgpu_device *adev,
 443                               struct umsch_mm_test *test,
 444                               struct umsch_mm_test_queue_info *qinfo)
 445 {
 446         int i;
 447
 448         for (i = 0; i < test->num_queues; i++)
 449                 remove_test_queue(adev, test, &qinfo[i]);
 450 }
 451
 452 static int umsch_mm_test(struct amdgpu_device *adev)
 453 {
 454         struct umsch_mm_test_queue_info qinfo[] = {
 455                 { .engine = UMSCH_SWIP_ENGINE_TYPE_VPE },
 456         };
 457         struct umsch_mm_test test = { .num_queues = ARRAY_SIZE(qinfo) };
 458         int r;
 459
 460         r = setup_umsch_mm_test(adev, &test);
 461         if (r)
 462                 return r;
 463
 464         r = setup_test_queues(adev, &test, qinfo);
 465         if (r)
 466                 goto cleanup;
 467
 468         r = submit_test_queues(adev, &test, qinfo);
 469         if (r)
 470                 goto cleanup;
 471
 472         cleanup_test_queues(adev, &test, qinfo);
 473         cleanup_umsch_mm_test(adev, &test);
 474
 475         return 0;
 476
 477 cleanup:
 478         cleanup_test_queues(adev, &test, qinfo);
 479         cleanup_umsch_mm_test(adev, &test);
 480         return r;
 481 }
 482
 483 int amdgpu_umsch_mm_submit_pkt(struct amdgpu_umsch_mm *umsch, void *pkt, int ndws)
 484 {
 485         struct amdgpu_ring *ring = &umsch->ring;
 486
 487         if (amdgpu_ring_alloc(ring, ndws))
 488                 return -ENOMEM;
 489
 490         amdgpu_ring_write_multiple(ring, pkt, ndws);
 491         amdgpu_ring_commit(ring);
 492
 493         return 0;
 494 }
 495
 496 int amdgpu_umsch_mm_query_fence(struct amdgpu_umsch_mm *umsch)
 497 {
 498         struct amdgpu_ring *ring = &umsch->ring;
 499         struct amdgpu_device *adev = ring->adev;
 500         int r;
 501
 502         r = amdgpu_fence_wait_polling(ring, ring->fence_drv.sync_seq, adev->usec_timeout);
 503         if (r < 1) {
 504                 dev_err(adev->dev, "ring umsch timeout, emitted fence %u\n",
 505                         ring->fence_drv.sync_seq);
 506                 return -ETIMEDOUT;
 507         }
 508
 509         return 0;
 510 }
 511
 512 static void umsch_mm_ring_set_wptr(struct amdgpu_ring *ring)
 513 {
 514         struct amdgpu_umsch_mm *umsch = (struct amdgpu_umsch_mm *)ring;
 515         struct amdgpu_device *adev = ring->adev;
 516
 517         if (ring->use_doorbell)
 518                 WDOORBELL32(ring->doorbell_index, ring->wptr << 2);
 519         else
 520                 WREG32(umsch->rb_wptr, ring->wptr << 2);
 521 }
 522
 523 static u64 umsch_mm_ring_get_rptr(struct amdgpu_ring *ring)
 524 {
 525         struct amdgpu_umsch_mm *umsch = (struct amdgpu_umsch_mm *)ring;
 526         struct amdgpu_device *adev = ring->adev;
 527
 528         return RREG32(umsch->rb_rptr);
 529 }
 530
 531 static u64 umsch_mm_ring_get_wptr(struct amdgpu_ring *ring)
 532 {
 533         struct amdgpu_umsch_mm *umsch = (struct amdgpu_umsch_mm *)ring;
 534         struct amdgpu_device *adev = ring->adev;
 535
 536         return RREG32(umsch->rb_wptr);
 537 }
 538
 539 static const struct amdgpu_ring_funcs umsch_v4_0_ring_funcs = {
 540         .type = AMDGPU_RING_TYPE_UMSCH_MM,
 541         .align_mask = 0,
 542         .nop = 0,
 543         .support_64bit_ptrs = false,
 544         .get_rptr = umsch_mm_ring_get_rptr,
 545         .get_wptr = umsch_mm_ring_get_wptr,
 546         .set_wptr = umsch_mm_ring_set_wptr,
 547         .insert_nop = amdgpu_ring_insert_nop,
 548 };
 549
 550 int amdgpu_umsch_mm_ring_init(struct amdgpu_umsch_mm *umsch)
 551 {
 552         struct amdgpu_device *adev = container_of(umsch, struct amdgpu_device, umsch_mm);
 553         struct amdgpu_ring *ring = &umsch->ring;
 554
 555         ring->vm_hub = AMDGPU_MMHUB0(0);
 556         ring->use_doorbell = true;
 557         ring->no_scheduler = true;
 558         ring->doorbell_index = (AMDGPU_NAVI10_DOORBELL64_VCN0_1 << 1) + 6;
 559
 560         snprintf(ring->name, sizeof(ring->name), "umsch");
 561
 562         return amdgpu_ring_init(adev, ring, 1024, NULL, 0, AMDGPU_RING_PRIO_DEFAULT, NULL);
 563 }
 564
 565 int amdgpu_umsch_mm_init_microcode(struct amdgpu_umsch_mm *umsch)
 566 {
 567         const struct umsch_mm_firmware_header_v1_0 *umsch_mm_hdr;
 568         struct amdgpu_device *adev = umsch->ring.adev;
 569         const char *fw_name = NULL;
 570         int r;
 571
 572         switch (amdgpu_ip_version(adev, VCN_HWIP, 0)) {
 573         case IP_VERSION(4, 0, 5):
 574                 fw_name = "amdgpu/umsch_mm_4_0_0.bin";
 575                 break;
 576         default:
 577                 break;
 578         }
 579
 580         r = amdgpu_ucode_request(adev, &adev->umsch_mm.fw, fw_name);
 581         if (r) {
 582                 release_firmware(adev->umsch_mm.fw);
 583                 adev->umsch_mm.fw = NULL;
 584                 return r;
 585         }
 586
 587         umsch_mm_hdr = (const struct umsch_mm_firmware_header_v1_0 *)adev->umsch_mm.fw->data;
 588
 589         adev->umsch_mm.ucode_size = le32_to_cpu(umsch_mm_hdr->umsch_mm_ucode_size_bytes);
 590         adev->umsch_mm.data_size = le32_to_cpu(umsch_mm_hdr->umsch_mm_ucode_data_size_bytes);
 591
 592         adev->umsch_mm.irq_start_addr =
 593                 le32_to_cpu(umsch_mm_hdr->umsch_mm_irq_start_addr_lo) |
 594                 ((uint64_t)(le32_to_cpu(umsch_mm_hdr->umsch_mm_irq_start_addr_hi)) << 32);
 595         adev->umsch_mm.uc_start_addr =
 596                 le32_to_cpu(umsch_mm_hdr->umsch_mm_uc_start_addr_lo) |
 597                 ((uint64_t)(le32_to_cpu(umsch_mm_hdr->umsch_mm_uc_start_addr_hi)) << 32);
 598         adev->umsch_mm.data_start_addr =
 599                 le32_to_cpu(umsch_mm_hdr->umsch_mm_data_start_addr_lo) |
 600                 ((uint64_t)(le32_to_cpu(umsch_mm_hdr->umsch_mm_data_start_addr_hi)) << 32);
 601
 602         if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) {
 603                 struct amdgpu_firmware_info *info;
 604
 605                 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_UMSCH_MM_UCODE];
 606                 info->ucode_id = AMDGPU_UCODE_ID_UMSCH_MM_UCODE;
 607                 info->fw = adev->umsch_mm.fw;
 608                 adev->firmware.fw_size +=
 609                         ALIGN(le32_to_cpu(umsch_mm_hdr->umsch_mm_ucode_size_bytes), PAGE_SIZE);
 610
 611                 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_UMSCH_MM_DATA];
 612                 info->ucode_id = AMDGPU_UCODE_ID_UMSCH_MM_DATA;
 613                 info->fw = adev->umsch_mm.fw;
 614                 adev->firmware.fw_size +=
 615                         ALIGN(le32_to_cpu(umsch_mm_hdr->umsch_mm_ucode_data_size_bytes), PAGE_SIZE);
 616         }
 617
 618         return 0;
 619 }
 620
 621 int amdgpu_umsch_mm_allocate_ucode_buffer(struct amdgpu_umsch_mm *umsch)
 622 {
 623         const struct umsch_mm_firmware_header_v1_0 *umsch_mm_hdr;
 624         struct amdgpu_device *adev = umsch->ring.adev;
 625         const __le32 *fw_data;
 626         uint32_t fw_size;
 627         int r;
 628
 629         umsch_mm_hdr = (const struct umsch_mm_firmware_header_v1_0 *)
 630                        adev->umsch_mm.fw->data;
 631
 632         fw_data = (const __le32 *)(adev->umsch_mm.fw->data +
 633                   le32_to_cpu(umsch_mm_hdr->umsch_mm_ucode_offset_bytes));
 634         fw_size = le32_to_cpu(umsch_mm_hdr->umsch_mm_ucode_size_bytes);
 635
 636         r = amdgpu_bo_create_reserved(adev, fw_size,
 637                                       4 * 1024, AMDGPU_GEM_DOMAIN_VRAM,
 638                                       &adev->umsch_mm.ucode_fw_obj,
 639                                       &adev->umsch_mm.ucode_fw_gpu_addr,
 640                                       (void **)&adev->umsch_mm.ucode_fw_ptr);
 641         if (r) {
 642                 dev_err(adev->dev, "(%d) failed to create umsch_mm fw ucode bo\n", r);
 643                 return r;
 644         }
 645
 646         memcpy(adev->umsch_mm.ucode_fw_ptr, fw_data, fw_size);
 647
 648         amdgpu_bo_kunmap(adev->umsch_mm.ucode_fw_obj);
 649         amdgpu_bo_unreserve(adev->umsch_mm.ucode_fw_obj);
 650         return 0;
 651 }
 652
 653 int amdgpu_umsch_mm_allocate_ucode_data_buffer(struct amdgpu_umsch_mm *umsch)
 654 {
 655         const struct umsch_mm_firmware_header_v1_0 *umsch_mm_hdr;
 656         struct amdgpu_device *adev = umsch->ring.adev;
 657         const __le32 *fw_data;
 658         uint32_t fw_size;
 659         int r;
 660
 661         umsch_mm_hdr = (const struct umsch_mm_firmware_header_v1_0 *)
 662                        adev->umsch_mm.fw->data;
 663
 664         fw_data = (const __le32 *)(adev->umsch_mm.fw->data +
 665                   le32_to_cpu(umsch_mm_hdr->umsch_mm_ucode_data_offset_bytes));
 666         fw_size = le32_to_cpu(umsch_mm_hdr->umsch_mm_ucode_data_size_bytes);
 667
 668         r = amdgpu_bo_create_reserved(adev, fw_size,
 669                                       64 * 1024, AMDGPU_GEM_DOMAIN_VRAM,
 670                                       &adev->umsch_mm.data_fw_obj,
 671                                       &adev->umsch_mm.data_fw_gpu_addr,
 672                                       (void **)&adev->umsch_mm.data_fw_ptr);
 673         if (r) {
 674                 dev_err(adev->dev, "(%d) failed to create umsch_mm fw data bo\n", r);
 675                 return r;
 676         }
 677
 678         memcpy(adev->umsch_mm.data_fw_ptr, fw_data, fw_size);
 679
 680         amdgpu_bo_kunmap(adev->umsch_mm.data_fw_obj);
 681         amdgpu_bo_unreserve(adev->umsch_mm.data_fw_obj);
 682         return 0;
 683 }
 684
 685 int amdgpu_umsch_mm_psp_execute_cmd_buf(struct amdgpu_umsch_mm *umsch)
 686 {
 687         struct amdgpu_device *adev = umsch->ring.adev;
 688         struct amdgpu_firmware_info ucode = {
 689                 .ucode_id = AMDGPU_UCODE_ID_UMSCH_MM_CMD_BUFFER,
 690                 .mc_addr = adev->umsch_mm.cmd_buf_gpu_addr,
 691                 .ucode_size = ((uintptr_t)adev->umsch_mm.cmd_buf_curr_ptr -
 692                               (uintptr_t)adev->umsch_mm.cmd_buf_ptr),
 693         };
 694
 695         return psp_execute_ip_fw_load(&adev->psp, &ucode);
 696 }
 697
 698 static void umsch_mm_agdb_index_init(struct amdgpu_device *adev)
 699 {
 700         uint32_t umsch_mm_agdb_start;
 701         int i;
 702
 703         umsch_mm_agdb_start = adev->doorbell_index.max_assignment + 1;
 704         umsch_mm_agdb_start = roundup(umsch_mm_agdb_start, 1024);
 705         umsch_mm_agdb_start += (AMDGPU_NAVI10_DOORBELL64_VCN0_1 << 1);
 706
 707         for (i = 0; i < CONTEXT_PRIORITY_NUM_LEVELS; i++)
 708                 adev->umsch_mm.agdb_index[i] = umsch_mm_agdb_start + i;
 709 }
 710
 711 static int umsch_mm_init(struct amdgpu_device *adev)
 712 {
 713         int r;
 714
 715         adev->umsch_mm.vmid_mask_mm_vpe = 0xf00;
 716         adev->umsch_mm.engine_mask = (1 << UMSCH_SWIP_ENGINE_TYPE_VPE);
 717         adev->umsch_mm.vpe_hqd_mask = 0xfe;
 718
 719         r = amdgpu_device_wb_get(adev, &adev->umsch_mm.wb_index);
 720         if (r) {
 721                 dev_err(adev->dev, "failed to alloc wb for umsch: %d\n", r);
 722                 return r;
 723         }
 724
 725         adev->umsch_mm.sch_ctx_gpu_addr = adev->wb.gpu_addr +
 726                                           (adev->umsch_mm.wb_index * 4);
 727
 728         r = amdgpu_bo_create_kernel(adev, PAGE_SIZE, PAGE_SIZE,
 729                                     AMDGPU_GEM_DOMAIN_GTT,
 730                                     &adev->umsch_mm.cmd_buf_obj,
 731                                     &adev->umsch_mm.cmd_buf_gpu_addr,
 732                                     (void **)&adev->umsch_mm.cmd_buf_ptr);
 733         if (r) {
 734                 dev_err(adev->dev, "failed to allocate cmdbuf bo %d\n", r);
 735                 amdgpu_device_wb_free(adev, adev->umsch_mm.wb_index);
 736                 return r;
 737         }
 738
 739         mutex_init(&adev->umsch_mm.mutex_hidden);
 740
 741         umsch_mm_agdb_index_init(adev);
 742
 743         return 0;
 744 }
 745
 746
 747 static int umsch_mm_early_init(void *handle)
 748 {
 749         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 750
 751         switch (amdgpu_ip_version(adev, VCN_HWIP, 0)) {
 752         case IP_VERSION(4, 0, 5):
 753                 umsch_mm_v4_0_set_funcs(&adev->umsch_mm);
 754                 break;
 755         default:
 756                 return -EINVAL;
 757         }
 758
 759         adev->umsch_mm.ring.funcs = &umsch_v4_0_ring_funcs;
 760         umsch_mm_set_regs(&adev->umsch_mm);
 761
 762         return 0;
 763 }
 764
 765 static int umsch_mm_late_init(void *handle)
 766 {
 767         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 768
 769         return umsch_mm_test(adev);
 770 }
 771
 772 static int umsch_mm_sw_init(void *handle)
 773 {
 774         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 775         int r;
 776
 777         r = umsch_mm_init(adev);
 778         if (r)
 779                 return r;
 780
 781         r = umsch_mm_ring_init(&adev->umsch_mm);
 782         if (r)
 783                 return r;
 784
 785         r = umsch_mm_init_microcode(&adev->umsch_mm);
 786         if (r)
 787                 return r;
 788
 789         return 0;
 790 }
 791
 792 static int umsch_mm_sw_fini(void *handle)
 793 {
 794         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 795
 796         release_firmware(adev->umsch_mm.fw);
 797         adev->umsch_mm.fw = NULL;
 798
 799         amdgpu_ring_fini(&adev->umsch_mm.ring);
 800
 801         mutex_destroy(&adev->umsch_mm.mutex_hidden);
 802
 803         amdgpu_bo_free_kernel(&adev->umsch_mm.cmd_buf_obj,
 804                               &adev->umsch_mm.cmd_buf_gpu_addr,
 805                               (void **)&adev->umsch_mm.cmd_buf_ptr);
 806
 807         amdgpu_device_wb_free(adev, adev->umsch_mm.wb_index);
 808
 809         return 0;
 810 }
 811
 812 static int umsch_mm_hw_init(void *handle)
 813 {
 814         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 815         int r;
 816
 817         r = umsch_mm_load_microcode(&adev->umsch_mm);
 818         if (r)
 819                 return r;
 820
 821         umsch_mm_ring_start(&adev->umsch_mm);
 822
 823         r = umsch_mm_set_hw_resources(&adev->umsch_mm);
 824         if (r)
 825                 return r;
 826
 827         return 0;
 828 }
 829
 830 static int umsch_mm_hw_fini(void *handle)
 831 {
 832         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 833
 834         umsch_mm_ring_stop(&adev->umsch_mm);
 835
 836         amdgpu_bo_free_kernel(&adev->umsch_mm.data_fw_obj,
 837                               &adev->umsch_mm.data_fw_gpu_addr,
 838                               (void **)&adev->umsch_mm.data_fw_ptr);
 839
 840         amdgpu_bo_free_kernel(&adev->umsch_mm.ucode_fw_obj,
 841                               &adev->umsch_mm.ucode_fw_gpu_addr,
 842                               (void **)&adev->umsch_mm.ucode_fw_ptr);
 843         return 0;
 844 }
 845
 846 static int umsch_mm_suspend(void *handle)
 847 {
 848         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 849
 850         return umsch_mm_hw_fini(adev);
 851 }
 852
 853 static int umsch_mm_resume(void *handle)
 854 {
 855         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 856
 857         return umsch_mm_hw_init(adev);
 858 }
 859
 860 static const struct amd_ip_funcs umsch_mm_v4_0_ip_funcs = {
 861         .name = "umsch_mm_v4_0",
 862         .early_init = umsch_mm_early_init,
 863         .late_init = umsch_mm_late_init,
 864         .sw_init = umsch_mm_sw_init,
 865         .sw_fini = umsch_mm_sw_fini,
 866         .hw_init = umsch_mm_hw_init,
 867         .hw_fini = umsch_mm_hw_fini,
 868         .suspend = umsch_mm_suspend,
 869         .resume = umsch_mm_resume,
 870 };
 871
 872 const struct amdgpu_ip_block_version umsch_mm_v4_0_ip_block = {
 873         .type = AMD_IP_BLOCK_TYPE_UMSCH_MM,
 874         .major = 4,
 875         .minor = 0,
 876         .rev = 0,
 877         .funcs = &umsch_mm_v4_0_ip_funcs,
 878 };