drivers/gpu/drm/amd/amdgpu/amdgpu_umsch_mm.c

   1 // SPDX-License-Identifier: MIT
   2 /*
   3  * Copyright 2023 Advanced Micro Devices, Inc.
   4  *
   5  * Permission is hereby granted, free of charge, to any person obtaining a
   6  * copy of this software and associated documentation files (the "Software"),
   7  * to deal in the Software without restriction, including without limitation
   8  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   9  * and/or sell copies of the Software, and to permit persons to whom the
  10  * Software is furnished to do so, subject to the following conditions:
  11  *
  12  * The above copyright notice and this permission notice shall be included in
  13  * all copies or substantial portions of the Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  19  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  20  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  21  * OTHER DEALINGS IN THE SOFTWARE.
  22  *
  23  */
  24
  25 #include <linux/firmware.h>
  26 #include <linux/module.h>
  27 #include <linux/debugfs.h>
  28 #include <drm/drm_exec.h>
  29 #include <drm/drm_drv.h>
  30
  31 #include "amdgpu.h"
  32 #include "amdgpu_umsch_mm.h"
  33 #include "umsch_mm_v4_0.h"
  34
  35 struct umsch_mm_test_ctx_data {
  36         uint8_t process_csa[PAGE_SIZE];
  37         uint8_t vpe_ctx_csa[PAGE_SIZE];
  38         uint8_t vcn_ctx_csa[PAGE_SIZE];
  39 };
  40
  41 struct umsch_mm_test_mqd_data {
  42         uint8_t vpe_mqd[PAGE_SIZE];
  43         uint8_t vcn_mqd[PAGE_SIZE];
  44 };
  45
  46 struct umsch_mm_test_ring_data {
  47         uint8_t vpe_ring[PAGE_SIZE];
  48         uint8_t vpe_ib[PAGE_SIZE];
  49         uint8_t vcn_ring[PAGE_SIZE];
  50         uint8_t vcn_ib[PAGE_SIZE];
  51 };
  52
  53 struct umsch_mm_test_queue_info {
  54         uint64_t mqd_addr;
  55         uint64_t csa_addr;
  56         uint32_t doorbell_offset_0;
  57         uint32_t doorbell_offset_1;
  58         enum UMSCH_SWIP_ENGINE_TYPE engine;
  59 };
  60
  61 struct umsch_mm_test {
  62         struct amdgpu_bo        *ctx_data_obj;
  63         uint64_t                ctx_data_gpu_addr;
  64         uint32_t                *ctx_data_cpu_addr;
  65
  66         struct amdgpu_bo        *mqd_data_obj;
  67         uint64_t                mqd_data_gpu_addr;
  68         uint32_t                *mqd_data_cpu_addr;
  69
  70         struct amdgpu_bo        *ring_data_obj;
  71         uint64_t                ring_data_gpu_addr;
  72         uint32_t                *ring_data_cpu_addr;
  73
  74
  75         struct amdgpu_vm        *vm;
  76         struct amdgpu_bo_va     *bo_va;
  77         uint32_t                pasid;
  78         uint32_t                vm_cntx_cntl;
  79         uint32_t                num_queues;
  80 };
  81
  82 static int map_ring_data(struct amdgpu_device *adev, struct amdgpu_vm *vm,
  83                           struct amdgpu_bo *bo, struct amdgpu_bo_va **bo_va,
  84                           uint64_t addr, uint32_t size)
  85 {
  86         struct amdgpu_sync sync;
  87         struct drm_exec exec;
  88         int r;
  89
  90         amdgpu_sync_create(&sync);
  91
  92         drm_exec_init(&exec, 0, 0);
  93         drm_exec_until_all_locked(&exec) {
  94                 r = drm_exec_lock_obj(&exec, &bo->tbo.base);
  95                 drm_exec_retry_on_contention(&exec);
  96                 if (unlikely(r))
  97                         goto error_fini_exec;
  98
  99                 r = amdgpu_vm_lock_pd(vm, &exec, 0);
 100                 drm_exec_retry_on_contention(&exec);
 101                 if (unlikely(r))
 102                         goto error_fini_exec;
 103         }
 104
 105         *bo_va = amdgpu_vm_bo_add(adev, vm, bo);
 106         if (!*bo_va) {
 107                 r = -ENOMEM;
 108                 goto error_fini_exec;
 109         }
 110
 111         r = amdgpu_vm_bo_map(adev, *bo_va, addr, 0, size,
 112                              AMDGPU_PTE_READABLE | AMDGPU_PTE_WRITEABLE |
 113                              AMDGPU_PTE_EXECUTABLE);
 114
 115         if (r)
 116                 goto error_del_bo_va;
 117
 118
 119         r = amdgpu_vm_bo_update(adev, *bo_va, false);
 120         if (r)
 121                 goto error_del_bo_va;
 122
 123         amdgpu_sync_fence(&sync, (*bo_va)->last_pt_update);
 124
 125         r = amdgpu_vm_update_pdes(adev, vm, false);
 126         if (r)
 127                 goto error_del_bo_va;
 128
 129         amdgpu_sync_fence(&sync, vm->last_update);
 130
 131         amdgpu_sync_wait(&sync, false);
 132         drm_exec_fini(&exec);
 133
 134         amdgpu_sync_free(&sync);
 135
 136         return 0;
 137
 138 error_del_bo_va:
 139         amdgpu_vm_bo_del(adev, *bo_va);
 140         amdgpu_sync_free(&sync);
 141
 142 error_fini_exec:
 143         drm_exec_fini(&exec);
 144         amdgpu_sync_free(&sync);
 145         return r;
 146 }
 147
 148 static int unmap_ring_data(struct amdgpu_device *adev, struct amdgpu_vm *vm,
 149                             struct amdgpu_bo *bo, struct amdgpu_bo_va *bo_va,
 150                             uint64_t addr)
 151 {
 152         struct drm_exec exec;
 153         long r;
 154
 155         drm_exec_init(&exec, 0, 0);
 156         drm_exec_until_all_locked(&exec) {
 157                 r = drm_exec_lock_obj(&exec, &bo->tbo.base);
 158                 drm_exec_retry_on_contention(&exec);
 159                 if (unlikely(r))
 160                         goto out_unlock;
 161
 162                 r = amdgpu_vm_lock_pd(vm, &exec, 0);
 163                 drm_exec_retry_on_contention(&exec);
 164                 if (unlikely(r))
 165                         goto out_unlock;
 166         }
 167
 168
 169         r = amdgpu_vm_bo_unmap(adev, bo_va, addr);
 170         if (r)
 171                 goto out_unlock;
 172
 173         amdgpu_vm_bo_del(adev, bo_va);
 174
 175 out_unlock:
 176         drm_exec_fini(&exec);
 177
 178         return r;
 179 }
 180
 181 static void setup_vpe_queue(struct amdgpu_device *adev,
 182                             struct umsch_mm_test *test,
 183                             struct umsch_mm_test_queue_info *qinfo)
 184 {
 185         struct MQD_INFO *mqd = (struct MQD_INFO *)test->mqd_data_cpu_addr;
 186         uint64_t ring_gpu_addr = test->ring_data_gpu_addr;
 187
 188         mqd->rb_base_lo = (ring_gpu_addr >> 8);
 189         mqd->rb_base_hi = (ring_gpu_addr >> 40);
 190         mqd->rb_size = PAGE_SIZE / 4;
 191         mqd->wptr_val = 0;
 192         mqd->rptr_val = 0;
 193         mqd->unmapped = 1;
 194
 195         if (adev->vpe.collaborate_mode)
 196                 memcpy(++mqd, test->mqd_data_cpu_addr, sizeof(struct MQD_INFO));
 197
 198         qinfo->mqd_addr = test->mqd_data_gpu_addr;
 199         qinfo->csa_addr = test->ctx_data_gpu_addr +
 200                 offsetof(struct umsch_mm_test_ctx_data, vpe_ctx_csa);
 201         qinfo->doorbell_offset_0 = 0;
 202         qinfo->doorbell_offset_1 = 0;
 203 }
 204
 205 static void setup_vcn_queue(struct amdgpu_device *adev,
 206                             struct umsch_mm_test *test,
 207                             struct umsch_mm_test_queue_info *qinfo)
 208 {
 209 }
 210
 211 static int add_test_queue(struct amdgpu_device *adev,
 212                           struct umsch_mm_test *test,
 213                           struct umsch_mm_test_queue_info *qinfo)
 214 {
 215         struct umsch_mm_add_queue_input queue_input = {};
 216         int r;
 217
 218         queue_input.process_id = test->pasid;
 219         queue_input.page_table_base_addr = amdgpu_gmc_pd_addr(test->vm->root.bo);
 220
 221         queue_input.process_va_start = 0;
 222         queue_input.process_va_end = (adev->vm_manager.max_pfn - 1) << AMDGPU_GPU_PAGE_SHIFT;
 223
 224         queue_input.process_quantum = 100000; /* 10ms */
 225         queue_input.process_csa_addr = test->ctx_data_gpu_addr +
 226                                        offsetof(struct umsch_mm_test_ctx_data, process_csa);
 227
 228         queue_input.context_quantum = 10000; /* 1ms */
 229         queue_input.context_csa_addr = qinfo->csa_addr;
 230
 231         queue_input.inprocess_context_priority = CONTEXT_PRIORITY_LEVEL_NORMAL;
 232         queue_input.context_global_priority_level = CONTEXT_PRIORITY_LEVEL_NORMAL;
 233         queue_input.doorbell_offset_0 = qinfo->doorbell_offset_0;
 234         queue_input.doorbell_offset_1 = qinfo->doorbell_offset_1;
 235
 236         queue_input.engine_type = qinfo->engine;
 237         queue_input.mqd_addr = qinfo->mqd_addr;
 238         queue_input.vm_context_cntl = test->vm_cntx_cntl;
 239
 240         amdgpu_umsch_mm_lock(&adev->umsch_mm);
 241         r = adev->umsch_mm.funcs->add_queue(&adev->umsch_mm, &queue_input);
 242         amdgpu_umsch_mm_unlock(&adev->umsch_mm);
 243         if (r)
 244                 return r;
 245
 246         return 0;
 247 }
 248
 249 static int remove_test_queue(struct amdgpu_device *adev,
 250                              struct umsch_mm_test *test,
 251                              struct umsch_mm_test_queue_info *qinfo)
 252 {
 253         struct umsch_mm_remove_queue_input queue_input = {};
 254         int r;
 255
 256         queue_input.doorbell_offset_0 = qinfo->doorbell_offset_0;
 257         queue_input.doorbell_offset_1 = qinfo->doorbell_offset_1;
 258         queue_input.context_csa_addr = qinfo->csa_addr;
 259
 260         amdgpu_umsch_mm_lock(&adev->umsch_mm);
 261         r = adev->umsch_mm.funcs->remove_queue(&adev->umsch_mm, &queue_input);
 262         amdgpu_umsch_mm_unlock(&adev->umsch_mm);
 263         if (r)
 264                 return r;
 265
 266         return 0;
 267 }
 268
 269 static int submit_vpe_queue(struct amdgpu_device *adev, struct umsch_mm_test *test)
 270 {
 271         struct MQD_INFO *mqd = (struct MQD_INFO *)test->mqd_data_cpu_addr;
 272         uint32_t *ring = test->ring_data_cpu_addr +
 273                 offsetof(struct umsch_mm_test_ring_data, vpe_ring) / 4;
 274         uint32_t *ib = test->ring_data_cpu_addr +
 275                 offsetof(struct umsch_mm_test_ring_data, vpe_ib) / 4;
 276         uint64_t ib_gpu_addr = test->ring_data_gpu_addr +
 277                 offsetof(struct umsch_mm_test_ring_data, vpe_ib);
 278         uint32_t *fence = ib + 2048 / 4;
 279         uint64_t fence_gpu_addr = ib_gpu_addr + 2048;
 280         const uint32_t test_pattern = 0xdeadbeef;
 281         int i;
 282
 283         ib[0] = VPE_CMD_HEADER(VPE_CMD_OPCODE_FENCE, 0);
 284         ib[1] = lower_32_bits(fence_gpu_addr);
 285         ib[2] = upper_32_bits(fence_gpu_addr);
 286         ib[3] = test_pattern;
 287
 288         ring[0] = VPE_CMD_HEADER(VPE_CMD_OPCODE_INDIRECT, 0);
 289         ring[1] = (ib_gpu_addr & 0xffffffe0);
 290         ring[2] = upper_32_bits(ib_gpu_addr);
 291         ring[3] = 4;
 292         ring[4] = 0;
 293         ring[5] = 0;
 294
 295         mqd->wptr_val = (6 << 2);
 296         if (adev->vpe.collaborate_mode)
 297                 (++mqd)->wptr_val = (6 << 2);
 298
 299         WDOORBELL32(adev->umsch_mm.agdb_index[CONTEXT_PRIORITY_LEVEL_NORMAL], mqd->wptr_val);
 300
 301         for (i = 0; i < adev->usec_timeout; i++) {
 302                 if (*fence == test_pattern)
 303                         return 0;
 304                 udelay(1);
 305         }
 306
 307         dev_err(adev->dev, "vpe queue submission timeout\n");
 308
 309         return -ETIMEDOUT;
 310 }
 311
 312 static int submit_vcn_queue(struct amdgpu_device *adev, struct umsch_mm_test *test)
 313 {
 314         return 0;
 315 }
 316
 317 static int setup_umsch_mm_test(struct amdgpu_device *adev,
 318                           struct umsch_mm_test *test)
 319 {
 320         struct amdgpu_vmhub *hub = &adev->vmhub[AMDGPU_MMHUB0(0)];
 321         int r;
 322
 323         test->vm_cntx_cntl = hub->vm_cntx_cntl;
 324
 325         test->vm = kzalloc(sizeof(*test->vm), GFP_KERNEL);
 326         if (!test->vm) {
 327                 r = -ENOMEM;
 328                 return r;
 329         }
 330
 331         r = amdgpu_vm_init(adev, test->vm, -1);
 332         if (r)
 333                 goto error_free_vm;
 334
 335         r = amdgpu_pasid_alloc(16);
 336         if (r < 0)
 337                 goto error_fini_vm;
 338         test->pasid = r;
 339
 340         r = amdgpu_bo_create_kernel(adev, sizeof(struct umsch_mm_test_ctx_data),
 341                                     PAGE_SIZE, AMDGPU_GEM_DOMAIN_GTT,
 342                                     &test->ctx_data_obj,
 343                                     &test->ctx_data_gpu_addr,
 344                                     (void **)&test->ctx_data_cpu_addr);
 345         if (r)
 346                 goto error_free_pasid;
 347
 348         memset(test->ctx_data_cpu_addr, 0, sizeof(struct umsch_mm_test_ctx_data));
 349
 350         r = amdgpu_bo_create_kernel(adev, PAGE_SIZE,
 351                                     PAGE_SIZE, AMDGPU_GEM_DOMAIN_GTT,
 352                                     &test->mqd_data_obj,
 353                                     &test->mqd_data_gpu_addr,
 354                                     (void **)&test->mqd_data_cpu_addr);
 355         if (r)
 356                 goto error_free_ctx_data_obj;
 357
 358         memset(test->mqd_data_cpu_addr, 0, PAGE_SIZE);
 359
 360         r = amdgpu_bo_create_kernel(adev, sizeof(struct umsch_mm_test_ring_data),
 361                                     PAGE_SIZE, AMDGPU_GEM_DOMAIN_GTT,
 362                                     &test->ring_data_obj,
 363                                     NULL,
 364                                     (void **)&test->ring_data_cpu_addr);
 365         if (r)
 366                 goto error_free_mqd_data_obj;
 367
 368         memset(test->ring_data_cpu_addr, 0, sizeof(struct umsch_mm_test_ring_data));
 369
 370         test->ring_data_gpu_addr = AMDGPU_VA_RESERVED_BOTTOM;
 371         r = map_ring_data(adev, test->vm, test->ring_data_obj, &test->bo_va,
 372                           test->ring_data_gpu_addr, sizeof(struct umsch_mm_test_ring_data));
 373         if (r)
 374                 goto error_free_ring_data_obj;
 375
 376         return 0;
 377
 378 error_free_ring_data_obj:
 379         amdgpu_bo_free_kernel(&test->ring_data_obj, NULL,
 380                               (void **)&test->ring_data_cpu_addr);
 381 error_free_mqd_data_obj:
 382         amdgpu_bo_free_kernel(&test->mqd_data_obj, &test->mqd_data_gpu_addr,
 383                               (void **)&test->mqd_data_cpu_addr);
 384 error_free_ctx_data_obj:
 385         amdgpu_bo_free_kernel(&test->ctx_data_obj, &test->ctx_data_gpu_addr,
 386                               (void **)&test->ctx_data_cpu_addr);
 387 error_free_pasid:
 388         amdgpu_pasid_free(test->pasid);
 389 error_fini_vm:
 390         amdgpu_vm_fini(adev, test->vm);
 391 error_free_vm:
 392         kfree(test->vm);
 393
 394         return r;
 395 }
 396
 397 static void cleanup_umsch_mm_test(struct amdgpu_device *adev,
 398                                   struct umsch_mm_test *test)
 399 {
 400         unmap_ring_data(adev, test->vm, test->ring_data_obj,
 401                         test->bo_va, test->ring_data_gpu_addr);
 402         amdgpu_bo_free_kernel(&test->mqd_data_obj, &test->mqd_data_gpu_addr,
 403                               (void **)&test->mqd_data_cpu_addr);
 404         amdgpu_bo_free_kernel(&test->ring_data_obj, NULL,
 405                               (void **)&test->ring_data_cpu_addr);
 406         amdgpu_bo_free_kernel(&test->ctx_data_obj, &test->ctx_data_gpu_addr,
 407                                (void **)&test->ctx_data_cpu_addr);
 408         amdgpu_pasid_free(test->pasid);
 409         amdgpu_vm_fini(adev, test->vm);
 410         kfree(test->vm);
 411 }
 412
 413 static int setup_test_queues(struct amdgpu_device *adev,
 414                              struct umsch_mm_test *test,
 415                              struct umsch_mm_test_queue_info *qinfo)
 416 {
 417         int i, r;
 418
 419         for (i = 0; i < test->num_queues; i++) {
 420                 if (qinfo[i].engine == UMSCH_SWIP_ENGINE_TYPE_VPE)
 421                         setup_vpe_queue(adev, test, &qinfo[i]);
 422                 else
 423                         setup_vcn_queue(adev, test, &qinfo[i]);
 424
 425                 r = add_test_queue(adev, test, &qinfo[i]);
 426                 if (r)
 427                         return r;
 428         }
 429
 430         return 0;
 431 }
 432
 433 static int submit_test_queues(struct amdgpu_device *adev,
 434                               struct umsch_mm_test *test,
 435                               struct umsch_mm_test_queue_info *qinfo)
 436 {
 437         int i, r;
 438
 439         for (i = 0; i < test->num_queues; i++) {
 440                 if (qinfo[i].engine == UMSCH_SWIP_ENGINE_TYPE_VPE)
 441                         r = submit_vpe_queue(adev, test);
 442                 else
 443                         r = submit_vcn_queue(adev, test);
 444                 if (r)
 445                         return r;
 446         }
 447
 448         return 0;
 449 }
 450
 451 static void cleanup_test_queues(struct amdgpu_device *adev,
 452                               struct umsch_mm_test *test,
 453                               struct umsch_mm_test_queue_info *qinfo)
 454 {
 455         int i;
 456
 457         for (i = 0; i < test->num_queues; i++)
 458                 remove_test_queue(adev, test, &qinfo[i]);
 459 }
 460
 461 static int umsch_mm_test(struct amdgpu_device *adev)
 462 {
 463         struct umsch_mm_test_queue_info qinfo[] = {
 464                 { .engine = UMSCH_SWIP_ENGINE_TYPE_VPE },
 465         };
 466         struct umsch_mm_test test = { .num_queues = ARRAY_SIZE(qinfo) };
 467         int r;
 468
 469         r = setup_umsch_mm_test(adev, &test);
 470         if (r)
 471                 return r;
 472
 473         r = setup_test_queues(adev, &test, qinfo);
 474         if (r)
 475                 goto cleanup;
 476
 477         r = submit_test_queues(adev, &test, qinfo);
 478         if (r)
 479                 goto cleanup;
 480
 481         cleanup_test_queues(adev, &test, qinfo);
 482         cleanup_umsch_mm_test(adev, &test);
 483
 484         return 0;
 485
 486 cleanup:
 487         cleanup_test_queues(adev, &test, qinfo);
 488         cleanup_umsch_mm_test(adev, &test);
 489         return r;
 490 }
 491
 492 int amdgpu_umsch_mm_submit_pkt(struct amdgpu_umsch_mm *umsch, void *pkt, int ndws)
 493 {
 494         struct amdgpu_ring *ring = &umsch->ring;
 495
 496         if (amdgpu_ring_alloc(ring, ndws))
 497                 return -ENOMEM;
 498
 499         amdgpu_ring_write_multiple(ring, pkt, ndws);
 500         amdgpu_ring_commit(ring);
 501
 502         return 0;
 503 }
 504
 505 int amdgpu_umsch_mm_query_fence(struct amdgpu_umsch_mm *umsch)
 506 {
 507         struct amdgpu_ring *ring = &umsch->ring;
 508         struct amdgpu_device *adev = ring->adev;
 509         int r;
 510
 511         r = amdgpu_fence_wait_polling(ring, ring->fence_drv.sync_seq, adev->usec_timeout);
 512         if (r < 1) {
 513                 dev_err(adev->dev, "ring umsch timeout, emitted fence %u\n",
 514                         ring->fence_drv.sync_seq);
 515                 return -ETIMEDOUT;
 516         }
 517
 518         return 0;
 519 }
 520
 521 static void umsch_mm_ring_set_wptr(struct amdgpu_ring *ring)
 522 {
 523         struct amdgpu_umsch_mm *umsch = (struct amdgpu_umsch_mm *)ring;
 524         struct amdgpu_device *adev = ring->adev;
 525
 526         if (ring->use_doorbell)
 527                 WDOORBELL32(ring->doorbell_index, ring->wptr << 2);
 528         else
 529                 WREG32(umsch->rb_wptr, ring->wptr << 2);
 530 }
 531
 532 static u64 umsch_mm_ring_get_rptr(struct amdgpu_ring *ring)
 533 {
 534         struct amdgpu_umsch_mm *umsch = (struct amdgpu_umsch_mm *)ring;
 535         struct amdgpu_device *adev = ring->adev;
 536
 537         return RREG32(umsch->rb_rptr);
 538 }
 539
 540 static u64 umsch_mm_ring_get_wptr(struct amdgpu_ring *ring)
 541 {
 542         struct amdgpu_umsch_mm *umsch = (struct amdgpu_umsch_mm *)ring;
 543         struct amdgpu_device *adev = ring->adev;
 544
 545         return RREG32(umsch->rb_wptr);
 546 }
 547
 548 static const struct amdgpu_ring_funcs umsch_v4_0_ring_funcs = {
 549         .type = AMDGPU_RING_TYPE_UMSCH_MM,
 550         .align_mask = 0,
 551         .nop = 0,
 552         .support_64bit_ptrs = false,
 553         .get_rptr = umsch_mm_ring_get_rptr,
 554         .get_wptr = umsch_mm_ring_get_wptr,
 555         .set_wptr = umsch_mm_ring_set_wptr,
 556         .insert_nop = amdgpu_ring_insert_nop,
 557 };
 558
 559 int amdgpu_umsch_mm_ring_init(struct amdgpu_umsch_mm *umsch)
 560 {
 561         struct amdgpu_device *adev = container_of(umsch, struct amdgpu_device, umsch_mm);
 562         struct amdgpu_ring *ring = &umsch->ring;
 563
 564         ring->vm_hub = AMDGPU_MMHUB0(0);
 565         ring->use_doorbell = true;
 566         ring->no_scheduler = true;
 567         ring->doorbell_index = (AMDGPU_NAVI10_DOORBELL64_VCN0_1 << 1) + 6;
 568
 569         snprintf(ring->name, sizeof(ring->name), "umsch");
 570
 571         return amdgpu_ring_init(adev, ring, 1024, NULL, 0, AMDGPU_RING_PRIO_DEFAULT, NULL);
 572 }
 573
 574 int amdgpu_umsch_mm_init_microcode(struct amdgpu_umsch_mm *umsch)
 575 {
 576         const struct umsch_mm_firmware_header_v1_0 *umsch_mm_hdr;
 577         struct amdgpu_device *adev = umsch->ring.adev;
 578         const char *fw_name = NULL;
 579         int r;
 580
 581         switch (amdgpu_ip_version(adev, VCN_HWIP, 0)) {
 582         case IP_VERSION(4, 0, 5):
 583         case IP_VERSION(4, 0, 6):
 584                 fw_name = "amdgpu/umsch_mm_4_0_0.bin";
 585                 break;
 586         default:
 587                 break;
 588         }
 589
 590         r = amdgpu_ucode_request(adev, &adev->umsch_mm.fw, AMDGPU_UCODE_REQUIRED,
 591                                  "%s", fw_name);
 592         if (r) {
 593                 release_firmware(adev->umsch_mm.fw);
 594                 adev->umsch_mm.fw = NULL;
 595                 return r;
 596         }
 597
 598         umsch_mm_hdr = (const struct umsch_mm_firmware_header_v1_0 *)adev->umsch_mm.fw->data;
 599
 600         adev->umsch_mm.ucode_size = le32_to_cpu(umsch_mm_hdr->umsch_mm_ucode_size_bytes);
 601         adev->umsch_mm.data_size = le32_to_cpu(umsch_mm_hdr->umsch_mm_ucode_data_size_bytes);
 602
 603         adev->umsch_mm.irq_start_addr =
 604                 le32_to_cpu(umsch_mm_hdr->umsch_mm_irq_start_addr_lo) |
 605                 ((uint64_t)(le32_to_cpu(umsch_mm_hdr->umsch_mm_irq_start_addr_hi)) << 32);
 606         adev->umsch_mm.uc_start_addr =
 607                 le32_to_cpu(umsch_mm_hdr->umsch_mm_uc_start_addr_lo) |
 608                 ((uint64_t)(le32_to_cpu(umsch_mm_hdr->umsch_mm_uc_start_addr_hi)) << 32);
 609         adev->umsch_mm.data_start_addr =
 610                 le32_to_cpu(umsch_mm_hdr->umsch_mm_data_start_addr_lo) |
 611                 ((uint64_t)(le32_to_cpu(umsch_mm_hdr->umsch_mm_data_start_addr_hi)) << 32);
 612
 613         if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) {
 614                 struct amdgpu_firmware_info *info;
 615
 616                 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_UMSCH_MM_UCODE];
 617                 info->ucode_id = AMDGPU_UCODE_ID_UMSCH_MM_UCODE;
 618                 info->fw = adev->umsch_mm.fw;
 619                 adev->firmware.fw_size +=
 620                         ALIGN(le32_to_cpu(umsch_mm_hdr->umsch_mm_ucode_size_bytes), PAGE_SIZE);
 621
 622                 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_UMSCH_MM_DATA];
 623                 info->ucode_id = AMDGPU_UCODE_ID_UMSCH_MM_DATA;
 624                 info->fw = adev->umsch_mm.fw;
 625                 adev->firmware.fw_size +=
 626                         ALIGN(le32_to_cpu(umsch_mm_hdr->umsch_mm_ucode_data_size_bytes), PAGE_SIZE);
 627         }
 628
 629         return 0;
 630 }
 631
 632 int amdgpu_umsch_mm_allocate_ucode_buffer(struct amdgpu_umsch_mm *umsch)
 633 {
 634         const struct umsch_mm_firmware_header_v1_0 *umsch_mm_hdr;
 635         struct amdgpu_device *adev = umsch->ring.adev;
 636         const __le32 *fw_data;
 637         uint32_t fw_size;
 638         int r;
 639
 640         umsch_mm_hdr = (const struct umsch_mm_firmware_header_v1_0 *)
 641                        adev->umsch_mm.fw->data;
 642
 643         fw_data = (const __le32 *)(adev->umsch_mm.fw->data +
 644                   le32_to_cpu(umsch_mm_hdr->umsch_mm_ucode_offset_bytes));
 645         fw_size = le32_to_cpu(umsch_mm_hdr->umsch_mm_ucode_size_bytes);
 646
 647         r = amdgpu_bo_create_reserved(adev, fw_size,
 648                                       4 * 1024, AMDGPU_GEM_DOMAIN_VRAM,
 649                                       &adev->umsch_mm.ucode_fw_obj,
 650                                       &adev->umsch_mm.ucode_fw_gpu_addr,
 651                                       (void **)&adev->umsch_mm.ucode_fw_ptr);
 652         if (r) {
 653                 dev_err(adev->dev, "(%d) failed to create umsch_mm fw ucode bo\n", r);
 654                 return r;
 655         }
 656
 657         memcpy(adev->umsch_mm.ucode_fw_ptr, fw_data, fw_size);
 658
 659         amdgpu_bo_kunmap(adev->umsch_mm.ucode_fw_obj);
 660         amdgpu_bo_unreserve(adev->umsch_mm.ucode_fw_obj);
 661         return 0;
 662 }
 663
 664 int amdgpu_umsch_mm_allocate_ucode_data_buffer(struct amdgpu_umsch_mm *umsch)
 665 {
 666         const struct umsch_mm_firmware_header_v1_0 *umsch_mm_hdr;
 667         struct amdgpu_device *adev = umsch->ring.adev;
 668         const __le32 *fw_data;
 669         uint32_t fw_size;
 670         int r;
 671
 672         umsch_mm_hdr = (const struct umsch_mm_firmware_header_v1_0 *)
 673                        adev->umsch_mm.fw->data;
 674
 675         fw_data = (const __le32 *)(adev->umsch_mm.fw->data +
 676                   le32_to_cpu(umsch_mm_hdr->umsch_mm_ucode_data_offset_bytes));
 677         fw_size = le32_to_cpu(umsch_mm_hdr->umsch_mm_ucode_data_size_bytes);
 678
 679         r = amdgpu_bo_create_reserved(adev, fw_size,
 680                                       64 * 1024, AMDGPU_GEM_DOMAIN_VRAM,
 681                                       &adev->umsch_mm.data_fw_obj,
 682                                       &adev->umsch_mm.data_fw_gpu_addr,
 683                                       (void **)&adev->umsch_mm.data_fw_ptr);
 684         if (r) {
 685                 dev_err(adev->dev, "(%d) failed to create umsch_mm fw data bo\n", r);
 686                 return r;
 687         }
 688
 689         memcpy(adev->umsch_mm.data_fw_ptr, fw_data, fw_size);
 690
 691         amdgpu_bo_kunmap(adev->umsch_mm.data_fw_obj);
 692         amdgpu_bo_unreserve(adev->umsch_mm.data_fw_obj);
 693         return 0;
 694 }
 695
 696 int amdgpu_umsch_mm_psp_execute_cmd_buf(struct amdgpu_umsch_mm *umsch)
 697 {
 698         struct amdgpu_device *adev = umsch->ring.adev;
 699         struct amdgpu_firmware_info ucode = {
 700                 .ucode_id = AMDGPU_UCODE_ID_UMSCH_MM_CMD_BUFFER,
 701                 .mc_addr = adev->umsch_mm.cmd_buf_gpu_addr,
 702                 .ucode_size = ((uintptr_t)adev->umsch_mm.cmd_buf_curr_ptr -
 703                               (uintptr_t)adev->umsch_mm.cmd_buf_ptr),
 704         };
 705
 706         return psp_execute_ip_fw_load(&adev->psp, &ucode);
 707 }
 708
 709 static void umsch_mm_agdb_index_init(struct amdgpu_device *adev)
 710 {
 711         uint32_t umsch_mm_agdb_start;
 712         int i;
 713
 714         umsch_mm_agdb_start = adev->doorbell_index.max_assignment + 1;
 715         umsch_mm_agdb_start = roundup(umsch_mm_agdb_start, 1024);
 716         umsch_mm_agdb_start += (AMDGPU_NAVI10_DOORBELL64_VCN0_1 << 1);
 717
 718         for (i = 0; i < CONTEXT_PRIORITY_NUM_LEVELS; i++)
 719                 adev->umsch_mm.agdb_index[i] = umsch_mm_agdb_start + i;
 720 }
 721
 722 static int umsch_mm_init(struct amdgpu_device *adev)
 723 {
 724         int r;
 725
 726         adev->umsch_mm.vmid_mask_mm_vpe = 0xf00;
 727         adev->umsch_mm.engine_mask = (1 << UMSCH_SWIP_ENGINE_TYPE_VPE);
 728         adev->umsch_mm.vpe_hqd_mask = 0xfe;
 729
 730         r = amdgpu_device_wb_get(adev, &adev->umsch_mm.wb_index);
 731         if (r) {
 732                 dev_err(adev->dev, "failed to alloc wb for umsch: %d\n", r);
 733                 return r;
 734         }
 735
 736         adev->umsch_mm.sch_ctx_gpu_addr = adev->wb.gpu_addr +
 737                                           (adev->umsch_mm.wb_index * 4);
 738
 739         r = amdgpu_bo_create_kernel(adev, PAGE_SIZE, PAGE_SIZE,
 740                                     AMDGPU_GEM_DOMAIN_GTT,
 741                                     &adev->umsch_mm.cmd_buf_obj,
 742                                     &adev->umsch_mm.cmd_buf_gpu_addr,
 743                                     (void **)&adev->umsch_mm.cmd_buf_ptr);
 744         if (r) {
 745                 dev_err(adev->dev, "failed to allocate cmdbuf bo %d\n", r);
 746                 amdgpu_device_wb_free(adev, adev->umsch_mm.wb_index);
 747                 return r;
 748         }
 749
 750         r = amdgpu_bo_create_kernel(adev, AMDGPU_UMSCHFW_LOG_SIZE, PAGE_SIZE,
 751                                     AMDGPU_GEM_DOMAIN_VRAM |
 752                                     AMDGPU_GEM_DOMAIN_GTT,
 753                                     &adev->umsch_mm.dbglog_bo,
 754                                     &adev->umsch_mm.log_gpu_addr,
 755                                     &adev->umsch_mm.log_cpu_addr);
 756         if (r) {
 757                 dev_err(adev->dev, "(%d) failed to allocate umsch debug bo\n", r);
 758                 return r;
 759         }
 760
 761         mutex_init(&adev->umsch_mm.mutex_hidden);
 762
 763         umsch_mm_agdb_index_init(adev);
 764
 765         return 0;
 766 }
 767
 768
 769 static int umsch_mm_early_init(struct amdgpu_ip_block *ip_block)
 770 {
 771         struct amdgpu_device *adev = ip_block->adev;
 772
 773         switch (amdgpu_ip_version(adev, VCN_HWIP, 0)) {
 774         case IP_VERSION(4, 0, 5):
 775         case IP_VERSION(4, 0, 6):
 776                 umsch_mm_v4_0_set_funcs(&adev->umsch_mm);
 777                 break;
 778         default:
 779                 return -EINVAL;
 780         }
 781
 782         adev->umsch_mm.ring.funcs = &umsch_v4_0_ring_funcs;
 783         umsch_mm_set_regs(&adev->umsch_mm);
 784
 785         return 0;
 786 }
 787
 788 static int umsch_mm_late_init(struct amdgpu_ip_block *ip_block)
 789 {
 790         struct amdgpu_device *adev = ip_block->adev;
 791
 792         if (amdgpu_in_reset(adev) || adev->in_s0ix || adev->in_suspend)
 793                 return 0;
 794
 795         return umsch_mm_test(adev);
 796 }
 797
 798 static int umsch_mm_sw_init(struct amdgpu_ip_block *ip_block)
 799 {
 800         struct amdgpu_device *adev = ip_block->adev;
 801         int r;
 802
 803         r = umsch_mm_init(adev);
 804         if (r)
 805                 return r;
 806
 807         amdgpu_umsch_fwlog_init(&adev->umsch_mm);
 808         r = umsch_mm_ring_init(&adev->umsch_mm);
 809         if (r)
 810                 return r;
 811
 812         r = umsch_mm_init_microcode(&adev->umsch_mm);
 813         if (r)
 814                 return r;
 815
 816         return 0;
 817 }
 818
 819 static int umsch_mm_sw_fini(struct amdgpu_ip_block *ip_block)
 820 {
 821         struct amdgpu_device *adev = ip_block->adev;
 822
 823         release_firmware(adev->umsch_mm.fw);
 824         adev->umsch_mm.fw = NULL;
 825
 826         amdgpu_ring_fini(&adev->umsch_mm.ring);
 827
 828         mutex_destroy(&adev->umsch_mm.mutex_hidden);
 829
 830         amdgpu_bo_free_kernel(&adev->umsch_mm.cmd_buf_obj,
 831                               &adev->umsch_mm.cmd_buf_gpu_addr,
 832                               (void **)&adev->umsch_mm.cmd_buf_ptr);
 833
 834         amdgpu_bo_free_kernel(&adev->umsch_mm.dbglog_bo,
 835                                     &adev->umsch_mm.log_gpu_addr,
 836                                     (void **)&adev->umsch_mm.log_cpu_addr);
 837
 838         amdgpu_device_wb_free(adev, adev->umsch_mm.wb_index);
 839
 840         return 0;
 841 }
 842
 843 static int umsch_mm_hw_init(struct amdgpu_ip_block *ip_block)
 844 {
 845         struct amdgpu_device *adev = ip_block->adev;
 846         int r;
 847
 848         r = umsch_mm_load_microcode(&adev->umsch_mm);
 849         if (r)
 850                 return r;
 851
 852         umsch_mm_ring_start(&adev->umsch_mm);
 853
 854         r = umsch_mm_set_hw_resources(&adev->umsch_mm);
 855         if (r)
 856                 return r;
 857
 858         return 0;
 859 }
 860
 861 static int umsch_mm_hw_fini(struct amdgpu_ip_block *ip_block)
 862 {
 863         struct amdgpu_device *adev = ip_block->adev;
 864
 865         umsch_mm_ring_stop(&adev->umsch_mm);
 866
 867         amdgpu_bo_free_kernel(&adev->umsch_mm.data_fw_obj,
 868                               &adev->umsch_mm.data_fw_gpu_addr,
 869                               (void **)&adev->umsch_mm.data_fw_ptr);
 870
 871         amdgpu_bo_free_kernel(&adev->umsch_mm.ucode_fw_obj,
 872                               &adev->umsch_mm.ucode_fw_gpu_addr,
 873                               (void **)&adev->umsch_mm.ucode_fw_ptr);
 874         return 0;
 875 }
 876
 877 static int umsch_mm_suspend(struct amdgpu_ip_block *ip_block)
 878 {
 879         return umsch_mm_hw_fini(ip_block);
 880 }
 881
 882 static int umsch_mm_resume(struct amdgpu_ip_block *ip_block)
 883 {
 884         return umsch_mm_hw_init(ip_block);
 885 }
 886
 887 void amdgpu_umsch_fwlog_init(struct amdgpu_umsch_mm *umsch_mm)
 888 {
 889 #if defined(CONFIG_DEBUG_FS)
 890         void *fw_log_cpu_addr = umsch_mm->log_cpu_addr;
 891         volatile struct amdgpu_umsch_fwlog *log_buf = fw_log_cpu_addr;
 892
 893         log_buf->header_size = sizeof(struct amdgpu_umsch_fwlog);
 894         log_buf->buffer_size = AMDGPU_UMSCHFW_LOG_SIZE;
 895         log_buf->rptr = log_buf->header_size;
 896         log_buf->wptr = log_buf->header_size;
 897         log_buf->wrapped = 0;
 898 #endif
 899 }
 900
 901 /*
 902  * debugfs for mapping umsch firmware log buffer.
 903  */
 904 #if defined(CONFIG_DEBUG_FS)
 905 static ssize_t amdgpu_debugfs_umsch_fwlog_read(struct file *f, char __user *buf,
 906                                              size_t size, loff_t *pos)
 907 {
 908         struct amdgpu_umsch_mm *umsch_mm;
 909         void *log_buf;
 910         volatile struct amdgpu_umsch_fwlog *plog;
 911         unsigned int read_pos, write_pos, available, i, read_bytes = 0;
 912         unsigned int read_num[2] = {0};
 913
 914         umsch_mm = file_inode(f)->i_private;
 915         if (!umsch_mm)
 916                 return -ENODEV;
 917
 918         if (!umsch_mm->log_cpu_addr)
 919                 return -EFAULT;
 920
 921         log_buf = umsch_mm->log_cpu_addr;
 922
 923         plog = (volatile struct amdgpu_umsch_fwlog *)log_buf;
 924         read_pos = plog->rptr;
 925         write_pos = plog->wptr;
 926
 927         if (read_pos > AMDGPU_UMSCHFW_LOG_SIZE || write_pos > AMDGPU_UMSCHFW_LOG_SIZE)
 928                 return -EFAULT;
 929
 930         if (!size || (read_pos == write_pos))
 931                 return 0;
 932
 933         if (write_pos > read_pos) {
 934                 available = write_pos - read_pos;
 935                 read_num[0] = min_t(size_t, size, available);
 936         } else {
 937                 read_num[0] = AMDGPU_UMSCHFW_LOG_SIZE - read_pos;
 938                 available = read_num[0] + write_pos - plog->header_size;
 939                 if (size > available)
 940                         read_num[1] = write_pos - plog->header_size;
 941                 else if (size > read_num[0])
 942                         read_num[1] = size - read_num[0];
 943                 else
 944                         read_num[0] = size;
 945         }
 946
 947         for (i = 0; i < 2; i++) {
 948                 if (read_num[i]) {
 949                         if (read_pos == AMDGPU_UMSCHFW_LOG_SIZE)
 950                                 read_pos = plog->header_size;
 951                         if (read_num[i] == copy_to_user((buf + read_bytes),
 952                                                         (log_buf + read_pos), read_num[i]))
 953                                 return -EFAULT;
 954
 955                         read_bytes += read_num[i];
 956                         read_pos += read_num[i];
 957                 }
 958         }
 959
 960         plog->rptr = read_pos;
 961         *pos += read_bytes;
 962         return read_bytes;
 963 }
 964
 965 static const struct file_operations amdgpu_debugfs_umschfwlog_fops = {
 966         .owner = THIS_MODULE,
 967         .read = amdgpu_debugfs_umsch_fwlog_read,
 968         .llseek = default_llseek
 969 };
 970 #endif
 971
 972 void amdgpu_debugfs_umsch_fwlog_init(struct amdgpu_device *adev,
 973                         struct amdgpu_umsch_mm *umsch_mm)
 974 {
 975 #if defined(CONFIG_DEBUG_FS)
 976         struct drm_minor *minor = adev_to_drm(adev)->primary;
 977         struct dentry *root = minor->debugfs_root;
 978         char name[32];
 979
 980         sprintf(name, "amdgpu_umsch_fwlog");
 981         debugfs_create_file_size(name, S_IFREG | 0444, root, umsch_mm,
 982                                  &amdgpu_debugfs_umschfwlog_fops,
 983                                  AMDGPU_UMSCHFW_LOG_SIZE);
 984 #endif
 985 }
 986
 987 static const struct amd_ip_funcs umsch_mm_v4_0_ip_funcs = {
 988         .name = "umsch_mm_v4_0",
 989         .early_init = umsch_mm_early_init,
 990         .late_init = umsch_mm_late_init,
 991         .sw_init = umsch_mm_sw_init,
 992         .sw_fini = umsch_mm_sw_fini,
 993         .hw_init = umsch_mm_hw_init,
 994         .hw_fini = umsch_mm_hw_fini,
 995         .suspend = umsch_mm_suspend,
 996         .resume = umsch_mm_resume,
 997 };
 998
 999 const struct amdgpu_ip_block_version umsch_mm_v4_0_ip_block = {
1000         .type = AMD_IP_BLOCK_TYPE_UMSCH_MM,
1001         .major = 4,
1002         .minor = 0,
1003         .rev = 0,
1004         .funcs = &umsch_mm_v4_0_ip_funcs,
1005 };