drivers/gpu/drm/amd/amdgpu/amdgpu_mes.c

   1 /*
   2  * Copyright 2019 Advanced Micro Devices, Inc.
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice shall be included in
  12  * all copies or substantial portions of the Software.
  13  *
  14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  20  * OTHER DEALINGS IN THE SOFTWARE.
  21  *
  22  */
  23
  24 #include <linux/firmware.h>
  25 #include <drm/drm_exec.h>
  26
  27 #include "amdgpu_mes.h"
  28 #include "amdgpu.h"
  29 #include "soc15_common.h"
  30 #include "amdgpu_mes_ctx.h"
  31
  32 #define AMDGPU_MES_MAX_NUM_OF_QUEUES_PER_PROCESS 1024
  33 #define AMDGPU_ONE_DOORBELL_SIZE 8
  34
  35 int amdgpu_mes_doorbell_process_slice(struct amdgpu_device *adev)
  36 {
  37         return roundup(AMDGPU_ONE_DOORBELL_SIZE *
  38                        AMDGPU_MES_MAX_NUM_OF_QUEUES_PER_PROCESS,
  39                        PAGE_SIZE);
  40 }
  41
  42 static int amdgpu_mes_kernel_doorbell_get(struct amdgpu_device *adev,
  43                                          int ip_type, uint64_t *doorbell_index)
  44 {
  45         unsigned int offset, found;
  46         struct amdgpu_mes *mes = &adev->mes;
  47
  48         if (ip_type == AMDGPU_RING_TYPE_SDMA)
  49                 offset = adev->doorbell_index.sdma_engine[0];
  50         else
  51                 offset = 0;
  52
  53         found = find_next_zero_bit(mes->doorbell_bitmap, mes->num_mes_dbs, offset);
  54         if (found >= mes->num_mes_dbs) {
  55                 DRM_WARN("No doorbell available\n");
  56                 return -ENOSPC;
  57         }
  58
  59         set_bit(found, mes->doorbell_bitmap);
  60
  61         /* Get the absolute doorbell index on BAR */
  62         *doorbell_index = mes->db_start_dw_offset + found * 2;
  63         return 0;
  64 }
  65
  66 static void amdgpu_mes_kernel_doorbell_free(struct amdgpu_device *adev,
  67                                            uint32_t doorbell_index)
  68 {
  69         unsigned int old, rel_index;
  70         struct amdgpu_mes *mes = &adev->mes;
  71
  72         /* Find the relative index of the doorbell in this object */
  73         rel_index = (doorbell_index - mes->db_start_dw_offset) / 2;
  74         old = test_and_clear_bit(rel_index, mes->doorbell_bitmap);
  75         WARN_ON(!old);
  76 }
  77
  78 static int amdgpu_mes_doorbell_init(struct amdgpu_device *adev)
  79 {
  80         int i;
  81         struct amdgpu_mes *mes = &adev->mes;
  82
  83         /* Bitmap for dynamic allocation of kernel doorbells */
  84         mes->doorbell_bitmap = bitmap_zalloc(PAGE_SIZE / sizeof(u32), GFP_KERNEL);
  85         if (!mes->doorbell_bitmap) {
  86                 DRM_ERROR("Failed to allocate MES doorbell bitmap\n");
  87                 return -ENOMEM;
  88         }
  89
  90         mes->num_mes_dbs = PAGE_SIZE / AMDGPU_ONE_DOORBELL_SIZE;
  91         for (i = 0; i < AMDGPU_MES_PRIORITY_NUM_LEVELS; i++) {
  92                 adev->mes.aggregated_doorbells[i] = mes->db_start_dw_offset + i * 2;
  93                 set_bit(i, mes->doorbell_bitmap);
  94         }
  95
  96         return 0;
  97 }
  98
  99 static int amdgpu_mes_event_log_init(struct amdgpu_device *adev)
 100 {
 101         int r;
 102
 103         if (!amdgpu_mes_log_enable)
 104                 return 0;
 105
 106         r = amdgpu_bo_create_kernel(adev, adev->mes.event_log_size, PAGE_SIZE,
 107                                     AMDGPU_GEM_DOMAIN_GTT,
 108                                     &adev->mes.event_log_gpu_obj,
 109                                     &adev->mes.event_log_gpu_addr,
 110                                     &adev->mes.event_log_cpu_addr);
 111         if (r) {
 112                 dev_warn(adev->dev, "failed to create MES event log buffer (%d)", r);
 113                 return r;
 114         }
 115
 116         memset(adev->mes.event_log_cpu_addr, 0, adev->mes.event_log_size);
 117
 118         return  0;
 119
 120 }
 121
 122 static void amdgpu_mes_doorbell_free(struct amdgpu_device *adev)
 123 {
 124         bitmap_free(adev->mes.doorbell_bitmap);
 125 }
 126
 127 int amdgpu_mes_init(struct amdgpu_device *adev)
 128 {
 129         int i, r;
 130
 131         adev->mes.adev = adev;
 132
 133         idr_init(&adev->mes.pasid_idr);
 134         idr_init(&adev->mes.gang_id_idr);
 135         idr_init(&adev->mes.queue_id_idr);
 136         ida_init(&adev->mes.doorbell_ida);
 137         spin_lock_init(&adev->mes.queue_id_lock);
 138         mutex_init(&adev->mes.mutex_hidden);
 139
 140         for (i = 0; i < AMDGPU_MAX_MES_PIPES; i++)
 141                 spin_lock_init(&adev->mes.ring_lock[i]);
 142
 143         adev->mes.total_max_queue = AMDGPU_FENCE_MES_QUEUE_ID_MASK;
 144         adev->mes.vmid_mask_mmhub = 0xffffff00;
 145         adev->mes.vmid_mask_gfxhub = 0xffffff00;
 146
 147         for (i = 0; i < AMDGPU_MES_MAX_COMPUTE_PIPES; i++) {
 148                 /* use only 1st MEC pipes */
 149                 if (i >= adev->gfx.mec.num_pipe_per_mec)
 150                         continue;
 151                 adev->mes.compute_hqd_mask[i] = 0xc;
 152         }
 153
 154         for (i = 0; i < AMDGPU_MES_MAX_GFX_PIPES; i++)
 155                 adev->mes.gfx_hqd_mask[i] = i ? 0 : 0xfffffffe;
 156
 157         for (i = 0; i < AMDGPU_MES_MAX_SDMA_PIPES; i++) {
 158                 if (amdgpu_ip_version(adev, SDMA0_HWIP, 0) <
 159                     IP_VERSION(6, 0, 0))
 160                         adev->mes.sdma_hqd_mask[i] = i ? 0 : 0x3fc;
 161                 /* zero sdma_hqd_mask for non-existent engine */
 162                 else if (adev->sdma.num_instances == 1)
 163                         adev->mes.sdma_hqd_mask[i] = i ? 0 : 0xfc;
 164                 else
 165                         adev->mes.sdma_hqd_mask[i] = 0xfc;
 166         }
 167
 168         for (i = 0; i < AMDGPU_MAX_MES_PIPES; i++) {
 169                 r = amdgpu_device_wb_get(adev, &adev->mes.sch_ctx_offs[i]);
 170                 if (r) {
 171                         dev_err(adev->dev,
 172                                 "(%d) ring trail_fence_offs wb alloc failed\n",
 173                                 r);
 174                         goto error;
 175                 }
 176                 adev->mes.sch_ctx_gpu_addr[i] =
 177                         adev->wb.gpu_addr + (adev->mes.sch_ctx_offs[i] * 4);
 178                 adev->mes.sch_ctx_ptr[i] =
 179                         (uint64_t *)&adev->wb.wb[adev->mes.sch_ctx_offs[i]];
 180
 181                 r = amdgpu_device_wb_get(adev,
 182                                  &adev->mes.query_status_fence_offs[i]);
 183                 if (r) {
 184                         dev_err(adev->dev,
 185                               "(%d) query_status_fence_offs wb alloc failed\n",
 186                               r);
 187                         goto error;
 188                 }
 189                 adev->mes.query_status_fence_gpu_addr[i] = adev->wb.gpu_addr +
 190                         (adev->mes.query_status_fence_offs[i] * 4);
 191                 adev->mes.query_status_fence_ptr[i] =
 192                         (uint64_t *)&adev->wb.wb[adev->mes.query_status_fence_offs[i]];
 193         }
 194
 195         r = amdgpu_device_wb_get(adev, &adev->mes.read_val_offs);
 196         if (r) {
 197                 dev_err(adev->dev,
 198                         "(%d) read_val_offs alloc failed\n", r);
 199                 goto error;
 200         }
 201         adev->mes.read_val_gpu_addr =
 202                 adev->wb.gpu_addr + (adev->mes.read_val_offs * 4);
 203         adev->mes.read_val_ptr =
 204                 (uint32_t *)&adev->wb.wb[adev->mes.read_val_offs];
 205
 206         r = amdgpu_mes_doorbell_init(adev);
 207         if (r)
 208                 goto error;
 209
 210         r = amdgpu_mes_event_log_init(adev);
 211         if (r)
 212                 goto error_doorbell;
 213
 214         return 0;
 215
 216 error_doorbell:
 217         amdgpu_mes_doorbell_free(adev);
 218 error:
 219         for (i = 0; i < AMDGPU_MAX_MES_PIPES; i++) {
 220                 if (adev->mes.sch_ctx_ptr[i])
 221                         amdgpu_device_wb_free(adev, adev->mes.sch_ctx_offs[i]);
 222                 if (adev->mes.query_status_fence_ptr[i])
 223                         amdgpu_device_wb_free(adev,
 224                                       adev->mes.query_status_fence_offs[i]);
 225         }
 226         if (adev->mes.read_val_ptr)
 227                 amdgpu_device_wb_free(adev, adev->mes.read_val_offs);
 228
 229         idr_destroy(&adev->mes.pasid_idr);
 230         idr_destroy(&adev->mes.gang_id_idr);
 231         idr_destroy(&adev->mes.queue_id_idr);
 232         ida_destroy(&adev->mes.doorbell_ida);
 233         mutex_destroy(&adev->mes.mutex_hidden);
 234         return r;
 235 }
 236
 237 void amdgpu_mes_fini(struct amdgpu_device *adev)
 238 {
 239         int i;
 240
 241         amdgpu_bo_free_kernel(&adev->mes.event_log_gpu_obj,
 242                               &adev->mes.event_log_gpu_addr,
 243                               &adev->mes.event_log_cpu_addr);
 244
 245         for (i = 0; i < AMDGPU_MAX_MES_PIPES; i++) {
 246                 if (adev->mes.sch_ctx_ptr[i])
 247                         amdgpu_device_wb_free(adev, adev->mes.sch_ctx_offs[i]);
 248                 if (adev->mes.query_status_fence_ptr[i])
 249                         amdgpu_device_wb_free(adev,
 250                                       adev->mes.query_status_fence_offs[i]);
 251         }
 252         if (adev->mes.read_val_ptr)
 253                 amdgpu_device_wb_free(adev, adev->mes.read_val_offs);
 254
 255         amdgpu_mes_doorbell_free(adev);
 256
 257         idr_destroy(&adev->mes.pasid_idr);
 258         idr_destroy(&adev->mes.gang_id_idr);
 259         idr_destroy(&adev->mes.queue_id_idr);
 260         ida_destroy(&adev->mes.doorbell_ida);
 261         mutex_destroy(&adev->mes.mutex_hidden);
 262 }
 263
 264 static void amdgpu_mes_queue_free_mqd(struct amdgpu_mes_queue *q)
 265 {
 266         amdgpu_bo_free_kernel(&q->mqd_obj,
 267                               &q->mqd_gpu_addr,
 268                               &q->mqd_cpu_ptr);
 269 }
 270
 271 int amdgpu_mes_create_process(struct amdgpu_device *adev, int pasid,
 272                               struct amdgpu_vm *vm)
 273 {
 274         struct amdgpu_mes_process *process;
 275         int r;
 276
 277         /* allocate the mes process buffer */
 278         process = kzalloc(sizeof(struct amdgpu_mes_process), GFP_KERNEL);
 279         if (!process) {
 280                 DRM_ERROR("no more memory to create mes process\n");
 281                 return -ENOMEM;
 282         }
 283
 284         /* allocate the process context bo and map it */
 285         r = amdgpu_bo_create_kernel(adev, AMDGPU_MES_PROC_CTX_SIZE, PAGE_SIZE,
 286                                     AMDGPU_GEM_DOMAIN_GTT,
 287                                     &process->proc_ctx_bo,
 288                                     &process->proc_ctx_gpu_addr,
 289                                     &process->proc_ctx_cpu_ptr);
 290         if (r) {
 291                 DRM_ERROR("failed to allocate process context bo\n");
 292                 goto clean_up_memory;
 293         }
 294         memset(process->proc_ctx_cpu_ptr, 0, AMDGPU_MES_PROC_CTX_SIZE);
 295
 296         /*
 297          * Avoid taking any other locks under MES lock to avoid circular
 298          * lock dependencies.
 299          */
 300         amdgpu_mes_lock(&adev->mes);
 301
 302         /* add the mes process to idr list */
 303         r = idr_alloc(&adev->mes.pasid_idr, process, pasid, pasid + 1,
 304                       GFP_KERNEL);
 305         if (r < 0) {
 306                 DRM_ERROR("failed to lock pasid=%d\n", pasid);
 307                 goto clean_up_ctx;
 308         }
 309
 310         INIT_LIST_HEAD(&process->gang_list);
 311         process->vm = vm;
 312         process->pasid = pasid;
 313         process->process_quantum = adev->mes.default_process_quantum;
 314         process->pd_gpu_addr = amdgpu_bo_gpu_offset(vm->root.bo);
 315
 316         amdgpu_mes_unlock(&adev->mes);
 317         return 0;
 318
 319 clean_up_ctx:
 320         amdgpu_mes_unlock(&adev->mes);
 321         amdgpu_bo_free_kernel(&process->proc_ctx_bo,
 322                               &process->proc_ctx_gpu_addr,
 323                               &process->proc_ctx_cpu_ptr);
 324 clean_up_memory:
 325         kfree(process);
 326         return r;
 327 }
 328
 329 void amdgpu_mes_destroy_process(struct amdgpu_device *adev, int pasid)
 330 {
 331         struct amdgpu_mes_process *process;
 332         struct amdgpu_mes_gang *gang, *tmp1;
 333         struct amdgpu_mes_queue *queue, *tmp2;
 334         struct mes_remove_queue_input queue_input;
 335         unsigned long flags;
 336         int r;
 337
 338         /*
 339          * Avoid taking any other locks under MES lock to avoid circular
 340          * lock dependencies.
 341          */
 342         amdgpu_mes_lock(&adev->mes);
 343
 344         process = idr_find(&adev->mes.pasid_idr, pasid);
 345         if (!process) {
 346                 DRM_WARN("pasid %d doesn't exist\n", pasid);
 347                 amdgpu_mes_unlock(&adev->mes);
 348                 return;
 349         }
 350
 351         /* Remove all queues from hardware */
 352         list_for_each_entry_safe(gang, tmp1, &process->gang_list, list) {
 353                 list_for_each_entry_safe(queue, tmp2, &gang->queue_list, list) {
 354                         spin_lock_irqsave(&adev->mes.queue_id_lock, flags);
 355                         idr_remove(&adev->mes.queue_id_idr, queue->queue_id);
 356                         spin_unlock_irqrestore(&adev->mes.queue_id_lock, flags);
 357
 358                         queue_input.doorbell_offset = queue->doorbell_off;
 359                         queue_input.gang_context_addr = gang->gang_ctx_gpu_addr;
 360
 361                         r = adev->mes.funcs->remove_hw_queue(&adev->mes,
 362                                                              &queue_input);
 363                         if (r)
 364                                 DRM_WARN("failed to remove hardware queue\n");
 365                 }
 366
 367                 idr_remove(&adev->mes.gang_id_idr, gang->gang_id);
 368         }
 369
 370         idr_remove(&adev->mes.pasid_idr, pasid);
 371         amdgpu_mes_unlock(&adev->mes);
 372
 373         /* free all memory allocated by the process */
 374         list_for_each_entry_safe(gang, tmp1, &process->gang_list, list) {
 375                 /* free all queues in the gang */
 376                 list_for_each_entry_safe(queue, tmp2, &gang->queue_list, list) {
 377                         amdgpu_mes_queue_free_mqd(queue);
 378                         list_del(&queue->list);
 379                         kfree(queue);
 380                 }
 381                 amdgpu_bo_free_kernel(&gang->gang_ctx_bo,
 382                                       &gang->gang_ctx_gpu_addr,
 383                                       &gang->gang_ctx_cpu_ptr);
 384                 list_del(&gang->list);
 385                 kfree(gang);
 386
 387         }
 388         amdgpu_bo_free_kernel(&process->proc_ctx_bo,
 389                               &process->proc_ctx_gpu_addr,
 390                               &process->proc_ctx_cpu_ptr);
 391         kfree(process);
 392 }
 393
 394 int amdgpu_mes_add_gang(struct amdgpu_device *adev, int pasid,
 395                         struct amdgpu_mes_gang_properties *gprops,
 396                         int *gang_id)
 397 {
 398         struct amdgpu_mes_process *process;
 399         struct amdgpu_mes_gang *gang;
 400         int r;
 401
 402         /* allocate the mes gang buffer */
 403         gang = kzalloc(sizeof(struct amdgpu_mes_gang), GFP_KERNEL);
 404         if (!gang) {
 405                 return -ENOMEM;
 406         }
 407
 408         /* allocate the gang context bo and map it to cpu space */
 409         r = amdgpu_bo_create_kernel(adev, AMDGPU_MES_GANG_CTX_SIZE, PAGE_SIZE,
 410                                     AMDGPU_GEM_DOMAIN_GTT,
 411                                     &gang->gang_ctx_bo,
 412                                     &gang->gang_ctx_gpu_addr,
 413                                     &gang->gang_ctx_cpu_ptr);
 414         if (r) {
 415                 DRM_ERROR("failed to allocate process context bo\n");
 416                 goto clean_up_mem;
 417         }
 418         memset(gang->gang_ctx_cpu_ptr, 0, AMDGPU_MES_GANG_CTX_SIZE);
 419
 420         /*
 421          * Avoid taking any other locks under MES lock to avoid circular
 422          * lock dependencies.
 423          */
 424         amdgpu_mes_lock(&adev->mes);
 425
 426         process = idr_find(&adev->mes.pasid_idr, pasid);
 427         if (!process) {
 428                 DRM_ERROR("pasid %d doesn't exist\n", pasid);
 429                 r = -EINVAL;
 430                 goto clean_up_ctx;
 431         }
 432
 433         /* add the mes gang to idr list */
 434         r = idr_alloc(&adev->mes.gang_id_idr, gang, 1, 0,
 435                       GFP_KERNEL);
 436         if (r < 0) {
 437                 DRM_ERROR("failed to allocate idr for gang\n");
 438                 goto clean_up_ctx;
 439         }
 440
 441         gang->gang_id = r;
 442         *gang_id = r;
 443
 444         INIT_LIST_HEAD(&gang->queue_list);
 445         gang->process = process;
 446         gang->priority = gprops->priority;
 447         gang->gang_quantum = gprops->gang_quantum ?
 448                 gprops->gang_quantum : adev->mes.default_gang_quantum;
 449         gang->global_priority_level = gprops->global_priority_level;
 450         gang->inprocess_gang_priority = gprops->inprocess_gang_priority;
 451         list_add_tail(&gang->list, &process->gang_list);
 452
 453         amdgpu_mes_unlock(&adev->mes);
 454         return 0;
 455
 456 clean_up_ctx:
 457         amdgpu_mes_unlock(&adev->mes);
 458         amdgpu_bo_free_kernel(&gang->gang_ctx_bo,
 459                               &gang->gang_ctx_gpu_addr,
 460                               &gang->gang_ctx_cpu_ptr);
 461 clean_up_mem:
 462         kfree(gang);
 463         return r;
 464 }
 465
 466 int amdgpu_mes_remove_gang(struct amdgpu_device *adev, int gang_id)
 467 {
 468         struct amdgpu_mes_gang *gang;
 469
 470         /*
 471          * Avoid taking any other locks under MES lock to avoid circular
 472          * lock dependencies.
 473          */
 474         amdgpu_mes_lock(&adev->mes);
 475
 476         gang = idr_find(&adev->mes.gang_id_idr, gang_id);
 477         if (!gang) {
 478                 DRM_ERROR("gang id %d doesn't exist\n", gang_id);
 479                 amdgpu_mes_unlock(&adev->mes);
 480                 return -EINVAL;
 481         }
 482
 483         if (!list_empty(&gang->queue_list)) {
 484                 DRM_ERROR("queue list is not empty\n");
 485                 amdgpu_mes_unlock(&adev->mes);
 486                 return -EBUSY;
 487         }
 488
 489         idr_remove(&adev->mes.gang_id_idr, gang->gang_id);
 490         list_del(&gang->list);
 491         amdgpu_mes_unlock(&adev->mes);
 492
 493         amdgpu_bo_free_kernel(&gang->gang_ctx_bo,
 494                               &gang->gang_ctx_gpu_addr,
 495                               &gang->gang_ctx_cpu_ptr);
 496
 497         kfree(gang);
 498
 499         return 0;
 500 }
 501
 502 int amdgpu_mes_suspend(struct amdgpu_device *adev)
 503 {
 504         struct idr *idp;
 505         struct amdgpu_mes_process *process;
 506         struct amdgpu_mes_gang *gang;
 507         struct mes_suspend_gang_input input;
 508         int r, pasid;
 509
 510         /*
 511          * Avoid taking any other locks under MES lock to avoid circular
 512          * lock dependencies.
 513          */
 514         amdgpu_mes_lock(&adev->mes);
 515
 516         idp = &adev->mes.pasid_idr;
 517
 518         idr_for_each_entry(idp, process, pasid) {
 519                 list_for_each_entry(gang, &process->gang_list, list) {
 520                         r = adev->mes.funcs->suspend_gang(&adev->mes, &input);
 521                         if (r)
 522                                 DRM_ERROR("failed to suspend pasid %d gangid %d",
 523                                          pasid, gang->gang_id);
 524                 }
 525         }
 526
 527         amdgpu_mes_unlock(&adev->mes);
 528         return 0;
 529 }
 530
 531 int amdgpu_mes_resume(struct amdgpu_device *adev)
 532 {
 533         struct idr *idp;
 534         struct amdgpu_mes_process *process;
 535         struct amdgpu_mes_gang *gang;
 536         struct mes_resume_gang_input input;
 537         int r, pasid;
 538
 539         /*
 540          * Avoid taking any other locks under MES lock to avoid circular
 541          * lock dependencies.
 542          */
 543         amdgpu_mes_lock(&adev->mes);
 544
 545         idp = &adev->mes.pasid_idr;
 546
 547         idr_for_each_entry(idp, process, pasid) {
 548                 list_for_each_entry(gang, &process->gang_list, list) {
 549                         r = adev->mes.funcs->resume_gang(&adev->mes, &input);
 550                         if (r)
 551                                 DRM_ERROR("failed to resume pasid %d gangid %d",
 552                                          pasid, gang->gang_id);
 553                 }
 554         }
 555
 556         amdgpu_mes_unlock(&adev->mes);
 557         return 0;
 558 }
 559
 560 static int amdgpu_mes_queue_alloc_mqd(struct amdgpu_device *adev,
 561                                      struct amdgpu_mes_queue *q,
 562                                      struct amdgpu_mes_queue_properties *p)
 563 {
 564         struct amdgpu_mqd *mqd_mgr = &adev->mqds[p->queue_type];
 565         u32 mqd_size = mqd_mgr->mqd_size;
 566         int r;
 567
 568         r = amdgpu_bo_create_kernel(adev, mqd_size, PAGE_SIZE,
 569                                     AMDGPU_GEM_DOMAIN_GTT,
 570                                     &q->mqd_obj,
 571                                     &q->mqd_gpu_addr, &q->mqd_cpu_ptr);
 572         if (r) {
 573                 dev_warn(adev->dev, "failed to create queue mqd bo (%d)", r);
 574                 return r;
 575         }
 576         memset(q->mqd_cpu_ptr, 0, mqd_size);
 577
 578         r = amdgpu_bo_reserve(q->mqd_obj, false);
 579         if (unlikely(r != 0))
 580                 goto clean_up;
 581
 582         return 0;
 583
 584 clean_up:
 585         amdgpu_bo_free_kernel(&q->mqd_obj,
 586                               &q->mqd_gpu_addr,
 587                               &q->mqd_cpu_ptr);
 588         return r;
 589 }
 590
 591 static void amdgpu_mes_queue_init_mqd(struct amdgpu_device *adev,
 592                                      struct amdgpu_mes_queue *q,
 593                                      struct amdgpu_mes_queue_properties *p)
 594 {
 595         struct amdgpu_mqd *mqd_mgr = &adev->mqds[p->queue_type];
 596         struct amdgpu_mqd_prop mqd_prop = {0};
 597
 598         mqd_prop.mqd_gpu_addr = q->mqd_gpu_addr;
 599         mqd_prop.hqd_base_gpu_addr = p->hqd_base_gpu_addr;
 600         mqd_prop.rptr_gpu_addr = p->rptr_gpu_addr;
 601         mqd_prop.wptr_gpu_addr = p->wptr_gpu_addr;
 602         mqd_prop.queue_size = p->queue_size;
 603         mqd_prop.use_doorbell = true;
 604         mqd_prop.doorbell_index = p->doorbell_off;
 605         mqd_prop.eop_gpu_addr = p->eop_gpu_addr;
 606         mqd_prop.hqd_pipe_priority = p->hqd_pipe_priority;
 607         mqd_prop.hqd_queue_priority = p->hqd_queue_priority;
 608         mqd_prop.hqd_active = false;
 609
 610         if (p->queue_type == AMDGPU_RING_TYPE_GFX ||
 611             p->queue_type == AMDGPU_RING_TYPE_COMPUTE) {
 612                 mutex_lock(&adev->srbm_mutex);
 613                 amdgpu_gfx_select_me_pipe_q(adev, p->ring->me, p->ring->pipe, 0, 0, 0);
 614         }
 615
 616         mqd_mgr->init_mqd(adev, q->mqd_cpu_ptr, &mqd_prop);
 617
 618         if (p->queue_type == AMDGPU_RING_TYPE_GFX ||
 619             p->queue_type == AMDGPU_RING_TYPE_COMPUTE) {
 620                 amdgpu_gfx_select_me_pipe_q(adev, 0, 0, 0, 0, 0);
 621                 mutex_unlock(&adev->srbm_mutex);
 622         }
 623
 624         amdgpu_bo_unreserve(q->mqd_obj);
 625 }
 626
 627 int amdgpu_mes_add_hw_queue(struct amdgpu_device *adev, int gang_id,
 628                             struct amdgpu_mes_queue_properties *qprops,
 629                             int *queue_id)
 630 {
 631         struct amdgpu_mes_queue *queue;
 632         struct amdgpu_mes_gang *gang;
 633         struct mes_add_queue_input queue_input;
 634         unsigned long flags;
 635         int r;
 636
 637         memset(&queue_input, 0, sizeof(struct mes_add_queue_input));
 638
 639         /* allocate the mes queue buffer */
 640         queue = kzalloc(sizeof(struct amdgpu_mes_queue), GFP_KERNEL);
 641         if (!queue) {
 642                 DRM_ERROR("Failed to allocate memory for queue\n");
 643                 return -ENOMEM;
 644         }
 645
 646         /* Allocate the queue mqd */
 647         r = amdgpu_mes_queue_alloc_mqd(adev, queue, qprops);
 648         if (r)
 649                 goto clean_up_memory;
 650
 651         /*
 652          * Avoid taking any other locks under MES lock to avoid circular
 653          * lock dependencies.
 654          */
 655         amdgpu_mes_lock(&adev->mes);
 656
 657         gang = idr_find(&adev->mes.gang_id_idr, gang_id);
 658         if (!gang) {
 659                 DRM_ERROR("gang id %d doesn't exist\n", gang_id);
 660                 r = -EINVAL;
 661                 goto clean_up_mqd;
 662         }
 663
 664         /* add the mes gang to idr list */
 665         spin_lock_irqsave(&adev->mes.queue_id_lock, flags);
 666         r = idr_alloc(&adev->mes.queue_id_idr, queue, 1, 0,
 667                       GFP_ATOMIC);
 668         if (r < 0) {
 669                 spin_unlock_irqrestore(&adev->mes.queue_id_lock, flags);
 670                 goto clean_up_mqd;
 671         }
 672         spin_unlock_irqrestore(&adev->mes.queue_id_lock, flags);
 673         *queue_id = queue->queue_id = r;
 674
 675         /* allocate a doorbell index for the queue */
 676         r = amdgpu_mes_kernel_doorbell_get(adev,
 677                                           qprops->queue_type,
 678                                           &qprops->doorbell_off);
 679         if (r)
 680                 goto clean_up_queue_id;
 681
 682         /* initialize the queue mqd */
 683         amdgpu_mes_queue_init_mqd(adev, queue, qprops);
 684
 685         /* add hw queue to mes */
 686         queue_input.process_id = gang->process->pasid;
 687
 688         queue_input.page_table_base_addr =
 689                 adev->vm_manager.vram_base_offset + gang->process->pd_gpu_addr -
 690                 adev->gmc.vram_start;
 691
 692         queue_input.process_va_start = 0;
 693         queue_input.process_va_end =
 694                 (adev->vm_manager.max_pfn - 1) << AMDGPU_GPU_PAGE_SHIFT;
 695         queue_input.process_quantum = gang->process->process_quantum;
 696         queue_input.process_context_addr = gang->process->proc_ctx_gpu_addr;
 697         queue_input.gang_quantum = gang->gang_quantum;
 698         queue_input.gang_context_addr = gang->gang_ctx_gpu_addr;
 699         queue_input.inprocess_gang_priority = gang->inprocess_gang_priority;
 700         queue_input.gang_global_priority_level = gang->global_priority_level;
 701         queue_input.doorbell_offset = qprops->doorbell_off;
 702         queue_input.mqd_addr = queue->mqd_gpu_addr;
 703         queue_input.wptr_addr = qprops->wptr_gpu_addr;
 704         queue_input.wptr_mc_addr = qprops->wptr_mc_addr;
 705         queue_input.queue_type = qprops->queue_type;
 706         queue_input.paging = qprops->paging;
 707         queue_input.is_kfd_process = 0;
 708
 709         r = adev->mes.funcs->add_hw_queue(&adev->mes, &queue_input);
 710         if (r) {
 711                 DRM_ERROR("failed to add hardware queue to MES, doorbell=0x%llx\n",
 712                           qprops->doorbell_off);
 713                 goto clean_up_doorbell;
 714         }
 715
 716         DRM_DEBUG("MES hw queue was added, pasid=%d, gang id=%d, "
 717                   "queue type=%d, doorbell=0x%llx\n",
 718                   gang->process->pasid, gang_id, qprops->queue_type,
 719                   qprops->doorbell_off);
 720
 721         queue->ring = qprops->ring;
 722         queue->doorbell_off = qprops->doorbell_off;
 723         queue->wptr_gpu_addr = qprops->wptr_gpu_addr;
 724         queue->queue_type = qprops->queue_type;
 725         queue->paging = qprops->paging;
 726         queue->gang = gang;
 727         queue->ring->mqd_ptr = queue->mqd_cpu_ptr;
 728         list_add_tail(&queue->list, &gang->queue_list);
 729
 730         amdgpu_mes_unlock(&adev->mes);
 731         return 0;
 732
 733 clean_up_doorbell:
 734         amdgpu_mes_kernel_doorbell_free(adev, qprops->doorbell_off);
 735 clean_up_queue_id:
 736         spin_lock_irqsave(&adev->mes.queue_id_lock, flags);
 737         idr_remove(&adev->mes.queue_id_idr, queue->queue_id);
 738         spin_unlock_irqrestore(&adev->mes.queue_id_lock, flags);
 739 clean_up_mqd:
 740         amdgpu_mes_unlock(&adev->mes);
 741         amdgpu_mes_queue_free_mqd(queue);
 742 clean_up_memory:
 743         kfree(queue);
 744         return r;
 745 }
 746
 747 int amdgpu_mes_remove_hw_queue(struct amdgpu_device *adev, int queue_id)
 748 {
 749         unsigned long flags;
 750         struct amdgpu_mes_queue *queue;
 751         struct amdgpu_mes_gang *gang;
 752         struct mes_remove_queue_input queue_input;
 753         int r;
 754
 755         /*
 756          * Avoid taking any other locks under MES lock to avoid circular
 757          * lock dependencies.
 758          */
 759         amdgpu_mes_lock(&adev->mes);
 760
 761         /* remove the mes gang from idr list */
 762         spin_lock_irqsave(&adev->mes.queue_id_lock, flags);
 763
 764         queue = idr_find(&adev->mes.queue_id_idr, queue_id);
 765         if (!queue) {
 766                 spin_unlock_irqrestore(&adev->mes.queue_id_lock, flags);
 767                 amdgpu_mes_unlock(&adev->mes);
 768                 DRM_ERROR("queue id %d doesn't exist\n", queue_id);
 769                 return -EINVAL;
 770         }
 771
 772         idr_remove(&adev->mes.queue_id_idr, queue_id);
 773         spin_unlock_irqrestore(&adev->mes.queue_id_lock, flags);
 774
 775         DRM_DEBUG("try to remove queue, doorbell off = 0x%llx\n",
 776                   queue->doorbell_off);
 777
 778         gang = queue->gang;
 779         queue_input.doorbell_offset = queue->doorbell_off;
 780         queue_input.gang_context_addr = gang->gang_ctx_gpu_addr;
 781
 782         r = adev->mes.funcs->remove_hw_queue(&adev->mes, &queue_input);
 783         if (r)
 784                 DRM_ERROR("failed to remove hardware queue, queue id = %d\n",
 785                           queue_id);
 786
 787         list_del(&queue->list);
 788         amdgpu_mes_kernel_doorbell_free(adev, queue->doorbell_off);
 789         amdgpu_mes_unlock(&adev->mes);
 790
 791         amdgpu_mes_queue_free_mqd(queue);
 792         kfree(queue);
 793         return 0;
 794 }
 795
 796 int amdgpu_mes_map_legacy_queue(struct amdgpu_device *adev,
 797                                 struct amdgpu_ring *ring)
 798 {
 799         struct mes_map_legacy_queue_input queue_input;
 800         int r;
 801
 802         memset(&queue_input, 0, sizeof(queue_input));
 803
 804         queue_input.queue_type = ring->funcs->type;
 805         queue_input.doorbell_offset = ring->doorbell_index;
 806         queue_input.pipe_id = ring->pipe;
 807         queue_input.queue_id = ring->queue;
 808         queue_input.mqd_addr = amdgpu_bo_gpu_offset(ring->mqd_obj);
 809         queue_input.wptr_addr = ring->wptr_gpu_addr;
 810
 811         r = adev->mes.funcs->map_legacy_queue(&adev->mes, &queue_input);
 812         if (r)
 813                 DRM_ERROR("failed to map legacy queue\n");
 814
 815         return r;
 816 }
 817
 818 int amdgpu_mes_unmap_legacy_queue(struct amdgpu_device *adev,
 819                                   struct amdgpu_ring *ring,
 820                                   enum amdgpu_unmap_queues_action action,
 821                                   u64 gpu_addr, u64 seq)
 822 {
 823         struct mes_unmap_legacy_queue_input queue_input;
 824         int r;
 825
 826         queue_input.action = action;
 827         queue_input.queue_type = ring->funcs->type;
 828         queue_input.doorbell_offset = ring->doorbell_index;
 829         queue_input.pipe_id = ring->pipe;
 830         queue_input.queue_id = ring->queue;
 831         queue_input.trail_fence_addr = gpu_addr;
 832         queue_input.trail_fence_data = seq;
 833
 834         r = adev->mes.funcs->unmap_legacy_queue(&adev->mes, &queue_input);
 835         if (r)
 836                 DRM_ERROR("failed to unmap legacy queue\n");
 837
 838         return r;
 839 }
 840
 841 uint32_t amdgpu_mes_rreg(struct amdgpu_device *adev, uint32_t reg)
 842 {
 843         struct mes_misc_op_input op_input;
 844         int r, val = 0;
 845
 846         op_input.op = MES_MISC_OP_READ_REG;
 847         op_input.read_reg.reg_offset = reg;
 848         op_input.read_reg.buffer_addr = adev->mes.read_val_gpu_addr;
 849
 850         if (!adev->mes.funcs->misc_op) {
 851                 DRM_ERROR("mes rreg is not supported!\n");
 852                 goto error;
 853         }
 854
 855         r = adev->mes.funcs->misc_op(&adev->mes, &op_input);
 856         if (r)
 857                 DRM_ERROR("failed to read reg (0x%x)\n", reg);
 858         else
 859                 val = *(adev->mes.read_val_ptr);
 860
 861 error:
 862         return val;
 863 }
 864
 865 int amdgpu_mes_wreg(struct amdgpu_device *adev,
 866                     uint32_t reg, uint32_t val)
 867 {
 868         struct mes_misc_op_input op_input;
 869         int r;
 870
 871         op_input.op = MES_MISC_OP_WRITE_REG;
 872         op_input.write_reg.reg_offset = reg;
 873         op_input.write_reg.reg_value = val;
 874
 875         if (!adev->mes.funcs->misc_op) {
 876                 DRM_ERROR("mes wreg is not supported!\n");
 877                 r = -EINVAL;
 878                 goto error;
 879         }
 880
 881         r = adev->mes.funcs->misc_op(&adev->mes, &op_input);
 882         if (r)
 883                 DRM_ERROR("failed to write reg (0x%x)\n", reg);
 884
 885 error:
 886         return r;
 887 }
 888
 889 int amdgpu_mes_reg_write_reg_wait(struct amdgpu_device *adev,
 890                                   uint32_t reg0, uint32_t reg1,
 891                                   uint32_t ref, uint32_t mask)
 892 {
 893         struct mes_misc_op_input op_input;
 894         int r;
 895
 896         op_input.op = MES_MISC_OP_WRM_REG_WR_WAIT;
 897         op_input.wrm_reg.reg0 = reg0;
 898         op_input.wrm_reg.reg1 = reg1;
 899         op_input.wrm_reg.ref = ref;
 900         op_input.wrm_reg.mask = mask;
 901
 902         if (!adev->mes.funcs->misc_op) {
 903                 DRM_ERROR("mes reg_write_reg_wait is not supported!\n");
 904                 r = -EINVAL;
 905                 goto error;
 906         }
 907
 908         r = adev->mes.funcs->misc_op(&adev->mes, &op_input);
 909         if (r)
 910                 DRM_ERROR("failed to reg_write_reg_wait\n");
 911
 912 error:
 913         return r;
 914 }
 915
 916 int amdgpu_mes_reg_wait(struct amdgpu_device *adev, uint32_t reg,
 917                         uint32_t val, uint32_t mask)
 918 {
 919         struct mes_misc_op_input op_input;
 920         int r;
 921
 922         op_input.op = MES_MISC_OP_WRM_REG_WAIT;
 923         op_input.wrm_reg.reg0 = reg;
 924         op_input.wrm_reg.ref = val;
 925         op_input.wrm_reg.mask = mask;
 926
 927         if (!adev->mes.funcs->misc_op) {
 928                 DRM_ERROR("mes reg wait is not supported!\n");
 929                 r = -EINVAL;
 930                 goto error;
 931         }
 932
 933         r = adev->mes.funcs->misc_op(&adev->mes, &op_input);
 934         if (r)
 935                 DRM_ERROR("failed to reg_write_reg_wait\n");
 936
 937 error:
 938         return r;
 939 }
 940
 941 int amdgpu_mes_set_shader_debugger(struct amdgpu_device *adev,
 942                                 uint64_t process_context_addr,
 943                                 uint32_t spi_gdbg_per_vmid_cntl,
 944                                 const uint32_t *tcp_watch_cntl,
 945                                 uint32_t flags,
 946                                 bool trap_en)
 947 {
 948         struct mes_misc_op_input op_input = {0};
 949         int r;
 950
 951         if (!adev->mes.funcs->misc_op) {
 952                 DRM_ERROR("mes set shader debugger is not supported!\n");
 953                 return -EINVAL;
 954         }
 955
 956         op_input.op = MES_MISC_OP_SET_SHADER_DEBUGGER;
 957         op_input.set_shader_debugger.process_context_addr = process_context_addr;
 958         op_input.set_shader_debugger.flags.u32all = flags;
 959
 960         /* use amdgpu mes_flush_shader_debugger instead */
 961         if (op_input.set_shader_debugger.flags.process_ctx_flush)
 962                 return -EINVAL;
 963
 964         op_input.set_shader_debugger.spi_gdbg_per_vmid_cntl = spi_gdbg_per_vmid_cntl;
 965         memcpy(op_input.set_shader_debugger.tcp_watch_cntl, tcp_watch_cntl,
 966                         sizeof(op_input.set_shader_debugger.tcp_watch_cntl));
 967
 968         if (((adev->mes.sched_version & AMDGPU_MES_API_VERSION_MASK) >>
 969                         AMDGPU_MES_API_VERSION_SHIFT) >= 14)
 970                 op_input.set_shader_debugger.trap_en = trap_en;
 971
 972         amdgpu_mes_lock(&adev->mes);
 973
 974         r = adev->mes.funcs->misc_op(&adev->mes, &op_input);
 975         if (r)
 976                 DRM_ERROR("failed to set_shader_debugger\n");
 977
 978         amdgpu_mes_unlock(&adev->mes);
 979
 980         return r;
 981 }
 982
 983 int amdgpu_mes_flush_shader_debugger(struct amdgpu_device *adev,
 984                                      uint64_t process_context_addr)
 985 {
 986         struct mes_misc_op_input op_input = {0};
 987         int r;
 988
 989         if (!adev->mes.funcs->misc_op) {
 990                 DRM_ERROR("mes flush shader debugger is not supported!\n");
 991                 return -EINVAL;
 992         }
 993
 994         op_input.op = MES_MISC_OP_SET_SHADER_DEBUGGER;
 995         op_input.set_shader_debugger.process_context_addr = process_context_addr;
 996         op_input.set_shader_debugger.flags.process_ctx_flush = true;
 997
 998         amdgpu_mes_lock(&adev->mes);
 999
1000         r = adev->mes.funcs->misc_op(&adev->mes, &op_input);
1001         if (r)
1002                 DRM_ERROR("failed to set_shader_debugger\n");
1003
1004         amdgpu_mes_unlock(&adev->mes);
1005
1006         return r;
1007 }
1008
1009 static void
1010 amdgpu_mes_ring_to_queue_props(struct amdgpu_device *adev,
1011                                struct amdgpu_ring *ring,
1012                                struct amdgpu_mes_queue_properties *props)
1013 {
1014         props->queue_type = ring->funcs->type;
1015         props->hqd_base_gpu_addr = ring->gpu_addr;
1016         props->rptr_gpu_addr = ring->rptr_gpu_addr;
1017         props->wptr_gpu_addr = ring->wptr_gpu_addr;
1018         props->wptr_mc_addr =
1019                 ring->mes_ctx->meta_data_mc_addr + ring->wptr_offs;
1020         props->queue_size = ring->ring_size;
1021         props->eop_gpu_addr = ring->eop_gpu_addr;
1022         props->hqd_pipe_priority = AMDGPU_GFX_PIPE_PRIO_NORMAL;
1023         props->hqd_queue_priority = AMDGPU_GFX_QUEUE_PRIORITY_MINIMUM;
1024         props->paging = false;
1025         props->ring = ring;
1026 }
1027
1028 #define DEFINE_AMDGPU_MES_CTX_GET_OFFS_ENG(_eng)                        \
1029 do {                                                                    \
1030        if (id_offs < AMDGPU_MES_CTX_MAX_OFFS)                           \
1031                 return offsetof(struct amdgpu_mes_ctx_meta_data,        \
1032                                 _eng[ring->idx].slots[id_offs]);        \
1033        else if (id_offs == AMDGPU_MES_CTX_RING_OFFS)                    \
1034                 return offsetof(struct amdgpu_mes_ctx_meta_data,        \
1035                                 _eng[ring->idx].ring);                  \
1036        else if (id_offs == AMDGPU_MES_CTX_IB_OFFS)                      \
1037                 return offsetof(struct amdgpu_mes_ctx_meta_data,        \
1038                                 _eng[ring->idx].ib);                    \
1039        else if (id_offs == AMDGPU_MES_CTX_PADDING_OFFS)                 \
1040                 return offsetof(struct amdgpu_mes_ctx_meta_data,        \
1041                                 _eng[ring->idx].padding);               \
1042 } while(0)
1043
1044 int amdgpu_mes_ctx_get_offs(struct amdgpu_ring *ring, unsigned int id_offs)
1045 {
1046         switch (ring->funcs->type) {
1047         case AMDGPU_RING_TYPE_GFX:
1048                 DEFINE_AMDGPU_MES_CTX_GET_OFFS_ENG(gfx);
1049                 break;
1050         case AMDGPU_RING_TYPE_COMPUTE:
1051                 DEFINE_AMDGPU_MES_CTX_GET_OFFS_ENG(compute);
1052                 break;
1053         case AMDGPU_RING_TYPE_SDMA:
1054                 DEFINE_AMDGPU_MES_CTX_GET_OFFS_ENG(sdma);
1055                 break;
1056         default:
1057                 break;
1058         }
1059
1060         WARN_ON(1);
1061         return -EINVAL;
1062 }
1063
1064 int amdgpu_mes_add_ring(struct amdgpu_device *adev, int gang_id,
1065                         int queue_type, int idx,
1066                         struct amdgpu_mes_ctx_data *ctx_data,
1067                         struct amdgpu_ring **out)
1068 {
1069         struct amdgpu_ring *ring;
1070         struct amdgpu_mes_gang *gang;
1071         struct amdgpu_mes_queue_properties qprops = {0};
1072         int r, queue_id, pasid;
1073
1074         /*
1075          * Avoid taking any other locks under MES lock to avoid circular
1076          * lock dependencies.
1077          */
1078         amdgpu_mes_lock(&adev->mes);
1079         gang = idr_find(&adev->mes.gang_id_idr, gang_id);
1080         if (!gang) {
1081                 DRM_ERROR("gang id %d doesn't exist\n", gang_id);
1082                 amdgpu_mes_unlock(&adev->mes);
1083                 return -EINVAL;
1084         }
1085         pasid = gang->process->pasid;
1086
1087         ring = kzalloc(sizeof(struct amdgpu_ring), GFP_KERNEL);
1088         if (!ring) {
1089                 amdgpu_mes_unlock(&adev->mes);
1090                 return -ENOMEM;
1091         }
1092
1093         ring->ring_obj = NULL;
1094         ring->use_doorbell = true;
1095         ring->is_mes_queue = true;
1096         ring->mes_ctx = ctx_data;
1097         ring->idx = idx;
1098         ring->no_scheduler = true;
1099
1100         if (queue_type == AMDGPU_RING_TYPE_COMPUTE) {
1101                 int offset = offsetof(struct amdgpu_mes_ctx_meta_data,
1102                                       compute[ring->idx].mec_hpd);
1103                 ring->eop_gpu_addr =
1104                         amdgpu_mes_ctx_get_offs_gpu_addr(ring, offset);
1105         }
1106
1107         switch (queue_type) {
1108         case AMDGPU_RING_TYPE_GFX:
1109                 ring->funcs = adev->gfx.gfx_ring[0].funcs;
1110                 ring->me = adev->gfx.gfx_ring[0].me;
1111                 ring->pipe = adev->gfx.gfx_ring[0].pipe;
1112                 break;
1113         case AMDGPU_RING_TYPE_COMPUTE:
1114                 ring->funcs = adev->gfx.compute_ring[0].funcs;
1115                 ring->me = adev->gfx.compute_ring[0].me;
1116                 ring->pipe = adev->gfx.compute_ring[0].pipe;
1117                 break;
1118         case AMDGPU_RING_TYPE_SDMA:
1119                 ring->funcs = adev->sdma.instance[0].ring.funcs;
1120                 break;
1121         default:
1122                 BUG();
1123         }
1124
1125         r = amdgpu_ring_init(adev, ring, 1024, NULL, 0,
1126                              AMDGPU_RING_PRIO_DEFAULT, NULL);
1127         if (r)
1128                 goto clean_up_memory;
1129
1130         amdgpu_mes_ring_to_queue_props(adev, ring, &qprops);
1131
1132         dma_fence_wait(gang->process->vm->last_update, false);
1133         dma_fence_wait(ctx_data->meta_data_va->last_pt_update, false);
1134         amdgpu_mes_unlock(&adev->mes);
1135
1136         r = amdgpu_mes_add_hw_queue(adev, gang_id, &qprops, &queue_id);
1137         if (r)
1138                 goto clean_up_ring;
1139
1140         ring->hw_queue_id = queue_id;
1141         ring->doorbell_index = qprops.doorbell_off;
1142
1143         if (queue_type == AMDGPU_RING_TYPE_GFX)
1144                 sprintf(ring->name, "gfx_%d.%d.%d", pasid, gang_id, queue_id);
1145         else if (queue_type == AMDGPU_RING_TYPE_COMPUTE)
1146                 sprintf(ring->name, "compute_%d.%d.%d", pasid, gang_id,
1147                         queue_id);
1148         else if (queue_type == AMDGPU_RING_TYPE_SDMA)
1149                 sprintf(ring->name, "sdma_%d.%d.%d", pasid, gang_id,
1150                         queue_id);
1151         else
1152                 BUG();
1153
1154         *out = ring;
1155         return 0;
1156
1157 clean_up_ring:
1158         amdgpu_ring_fini(ring);
1159 clean_up_memory:
1160         kfree(ring);
1161         amdgpu_mes_unlock(&adev->mes);
1162         return r;
1163 }
1164
1165 void amdgpu_mes_remove_ring(struct amdgpu_device *adev,
1166                             struct amdgpu_ring *ring)
1167 {
1168         if (!ring)
1169                 return;
1170
1171         amdgpu_mes_remove_hw_queue(adev, ring->hw_queue_id);
1172         del_timer_sync(&ring->fence_drv.fallback_timer);
1173         amdgpu_ring_fini(ring);
1174         kfree(ring);
1175 }
1176
1177 uint32_t amdgpu_mes_get_aggregated_doorbell_index(struct amdgpu_device *adev,
1178                                                    enum amdgpu_mes_priority_level prio)
1179 {
1180         return adev->mes.aggregated_doorbells[prio];
1181 }
1182
1183 int amdgpu_mes_ctx_alloc_meta_data(struct amdgpu_device *adev,
1184                                    struct amdgpu_mes_ctx_data *ctx_data)
1185 {
1186         int r;
1187
1188         r = amdgpu_bo_create_kernel(adev,
1189                             sizeof(struct amdgpu_mes_ctx_meta_data),
1190                             PAGE_SIZE, AMDGPU_GEM_DOMAIN_GTT,
1191                             &ctx_data->meta_data_obj,
1192                             &ctx_data->meta_data_mc_addr,
1193                             &ctx_data->meta_data_ptr);
1194         if (r) {
1195                 dev_warn(adev->dev, "(%d) create CTX bo failed\n", r);
1196                 return r;
1197         }
1198
1199         if (!ctx_data->meta_data_obj)
1200                 return -ENOMEM;
1201
1202         memset(ctx_data->meta_data_ptr, 0,
1203                sizeof(struct amdgpu_mes_ctx_meta_data));
1204
1205         return 0;
1206 }
1207
1208 void amdgpu_mes_ctx_free_meta_data(struct amdgpu_mes_ctx_data *ctx_data)
1209 {
1210         if (ctx_data->meta_data_obj)
1211                 amdgpu_bo_free_kernel(&ctx_data->meta_data_obj,
1212                                       &ctx_data->meta_data_mc_addr,
1213                                       &ctx_data->meta_data_ptr);
1214 }
1215
1216 int amdgpu_mes_ctx_map_meta_data(struct amdgpu_device *adev,
1217                                  struct amdgpu_vm *vm,
1218                                  struct amdgpu_mes_ctx_data *ctx_data)
1219 {
1220         struct amdgpu_bo_va *bo_va;
1221         struct amdgpu_sync sync;
1222         struct drm_exec exec;
1223         int r;
1224
1225         amdgpu_sync_create(&sync);
1226
1227         drm_exec_init(&exec, 0, 0);
1228         drm_exec_until_all_locked(&exec) {
1229                 r = drm_exec_lock_obj(&exec,
1230                                       &ctx_data->meta_data_obj->tbo.base);
1231                 drm_exec_retry_on_contention(&exec);
1232                 if (unlikely(r))
1233                         goto error_fini_exec;
1234
1235                 r = amdgpu_vm_lock_pd(vm, &exec, 0);
1236                 drm_exec_retry_on_contention(&exec);
1237                 if (unlikely(r))
1238                         goto error_fini_exec;
1239         }
1240
1241         bo_va = amdgpu_vm_bo_add(adev, vm, ctx_data->meta_data_obj);
1242         if (!bo_va) {
1243                 DRM_ERROR("failed to create bo_va for meta data BO\n");
1244                 r = -ENOMEM;
1245                 goto error_fini_exec;
1246         }
1247
1248         r = amdgpu_vm_bo_map(adev, bo_va, ctx_data->meta_data_gpu_addr, 0,
1249                              sizeof(struct amdgpu_mes_ctx_meta_data),
1250                              AMDGPU_PTE_READABLE | AMDGPU_PTE_WRITEABLE |
1251                              AMDGPU_PTE_EXECUTABLE);
1252
1253         if (r) {
1254                 DRM_ERROR("failed to do bo_map on meta data, err=%d\n", r);
1255                 goto error_del_bo_va;
1256         }
1257
1258         r = amdgpu_vm_bo_update(adev, bo_va, false);
1259         if (r) {
1260                 DRM_ERROR("failed to do vm_bo_update on meta data\n");
1261                 goto error_del_bo_va;
1262         }
1263         amdgpu_sync_fence(&sync, bo_va->last_pt_update);
1264
1265         r = amdgpu_vm_update_pdes(adev, vm, false);
1266         if (r) {
1267                 DRM_ERROR("failed to update pdes on meta data\n");
1268                 goto error_del_bo_va;
1269         }
1270         amdgpu_sync_fence(&sync, vm->last_update);
1271
1272         amdgpu_sync_wait(&sync, false);
1273         drm_exec_fini(&exec);
1274
1275         amdgpu_sync_free(&sync);
1276         ctx_data->meta_data_va = bo_va;
1277         return 0;
1278
1279 error_del_bo_va:
1280         amdgpu_vm_bo_del(adev, bo_va);
1281
1282 error_fini_exec:
1283         drm_exec_fini(&exec);
1284         amdgpu_sync_free(&sync);
1285         return r;
1286 }
1287
1288 int amdgpu_mes_ctx_unmap_meta_data(struct amdgpu_device *adev,
1289                                    struct amdgpu_mes_ctx_data *ctx_data)
1290 {
1291         struct amdgpu_bo_va *bo_va = ctx_data->meta_data_va;
1292         struct amdgpu_bo *bo = ctx_data->meta_data_obj;
1293         struct amdgpu_vm *vm = bo_va->base.vm;
1294         struct dma_fence *fence;
1295         struct drm_exec exec;
1296         long r;
1297
1298         drm_exec_init(&exec, 0, 0);
1299         drm_exec_until_all_locked(&exec) {
1300                 r = drm_exec_lock_obj(&exec,
1301                                       &ctx_data->meta_data_obj->tbo.base);
1302                 drm_exec_retry_on_contention(&exec);
1303                 if (unlikely(r))
1304                         goto out_unlock;
1305
1306                 r = amdgpu_vm_lock_pd(vm, &exec, 0);
1307                 drm_exec_retry_on_contention(&exec);
1308                 if (unlikely(r))
1309                         goto out_unlock;
1310         }
1311
1312         amdgpu_vm_bo_del(adev, bo_va);
1313         if (!amdgpu_vm_ready(vm))
1314                 goto out_unlock;
1315
1316         r = dma_resv_get_singleton(bo->tbo.base.resv, DMA_RESV_USAGE_BOOKKEEP,
1317                                    &fence);
1318         if (r)
1319                 goto out_unlock;
1320         if (fence) {
1321                 amdgpu_bo_fence(bo, fence, true);
1322                 fence = NULL;
1323         }
1324
1325         r = amdgpu_vm_clear_freed(adev, vm, &fence);
1326         if (r || !fence)
1327                 goto out_unlock;
1328
1329         dma_fence_wait(fence, false);
1330         amdgpu_bo_fence(bo, fence, true);
1331         dma_fence_put(fence);
1332
1333 out_unlock:
1334         if (unlikely(r < 0))
1335                 dev_err(adev->dev, "failed to clear page tables (%ld)\n", r);
1336         drm_exec_fini(&exec);
1337
1338         return r;
1339 }
1340
1341 static int amdgpu_mes_test_create_gang_and_queues(struct amdgpu_device *adev,
1342                                           int pasid, int *gang_id,
1343                                           int queue_type, int num_queue,
1344                                           struct amdgpu_ring **added_rings,
1345                                           struct amdgpu_mes_ctx_data *ctx_data)
1346 {
1347         struct amdgpu_ring *ring;
1348         struct amdgpu_mes_gang_properties gprops = {0};
1349         int r, j;
1350
1351         /* create a gang for the process */
1352         gprops.priority = AMDGPU_MES_PRIORITY_LEVEL_NORMAL;
1353         gprops.gang_quantum = adev->mes.default_gang_quantum;
1354         gprops.inprocess_gang_priority = AMDGPU_MES_PRIORITY_LEVEL_NORMAL;
1355         gprops.priority_level = AMDGPU_MES_PRIORITY_LEVEL_NORMAL;
1356         gprops.global_priority_level = AMDGPU_MES_PRIORITY_LEVEL_NORMAL;
1357
1358         r = amdgpu_mes_add_gang(adev, pasid, &gprops, gang_id);
1359         if (r) {
1360                 DRM_ERROR("failed to add gang\n");
1361                 return r;
1362         }
1363
1364         /* create queues for the gang */
1365         for (j = 0; j < num_queue; j++) {
1366                 r = amdgpu_mes_add_ring(adev, *gang_id, queue_type, j,
1367                                         ctx_data, &ring);
1368                 if (r) {
1369                         DRM_ERROR("failed to add ring\n");
1370                         break;
1371                 }
1372
1373                 DRM_INFO("ring %s was added\n", ring->name);
1374                 added_rings[j] = ring;
1375         }
1376
1377         return 0;
1378 }
1379
1380 static int amdgpu_mes_test_queues(struct amdgpu_ring **added_rings)
1381 {
1382         struct amdgpu_ring *ring;
1383         int i, r;
1384
1385         for (i = 0; i < AMDGPU_MES_CTX_MAX_RINGS; i++) {
1386                 ring = added_rings[i];
1387                 if (!ring)
1388                         continue;
1389
1390                 r = amdgpu_ring_test_helper(ring);
1391                 if (r)
1392                         return r;
1393
1394                 r = amdgpu_ring_test_ib(ring, 1000 * 10);
1395                 if (r) {
1396                         DRM_DEV_ERROR(ring->adev->dev,
1397                                       "ring %s ib test failed (%d)\n",
1398                                       ring->name, r);
1399                         return r;
1400                 } else
1401                         DRM_INFO("ring %s ib test pass\n", ring->name);
1402         }
1403
1404         return 0;
1405 }
1406
1407 int amdgpu_mes_self_test(struct amdgpu_device *adev)
1408 {
1409         struct amdgpu_vm *vm = NULL;
1410         struct amdgpu_mes_ctx_data ctx_data = {0};
1411         struct amdgpu_ring *added_rings[AMDGPU_MES_CTX_MAX_RINGS] = { NULL };
1412         int gang_ids[3] = {0};
1413         int queue_types[][2] = { { AMDGPU_RING_TYPE_GFX, 1 },
1414                                  { AMDGPU_RING_TYPE_COMPUTE, 1 },
1415                                  { AMDGPU_RING_TYPE_SDMA, 1} };
1416         int i, r, pasid, k = 0;
1417
1418         pasid = amdgpu_pasid_alloc(16);
1419         if (pasid < 0) {
1420                 dev_warn(adev->dev, "No more PASIDs available!");
1421                 pasid = 0;
1422         }
1423
1424         vm = kzalloc(sizeof(*vm), GFP_KERNEL);
1425         if (!vm) {
1426                 r = -ENOMEM;
1427                 goto error_pasid;
1428         }
1429
1430         r = amdgpu_vm_init(adev, vm, -1);
1431         if (r) {
1432                 DRM_ERROR("failed to initialize vm\n");
1433                 goto error_pasid;
1434         }
1435
1436         r = amdgpu_mes_ctx_alloc_meta_data(adev, &ctx_data);
1437         if (r) {
1438                 DRM_ERROR("failed to alloc ctx meta data\n");
1439                 goto error_fini;
1440         }
1441
1442         ctx_data.meta_data_gpu_addr = AMDGPU_VA_RESERVED_BOTTOM;
1443         r = amdgpu_mes_ctx_map_meta_data(adev, vm, &ctx_data);
1444         if (r) {
1445                 DRM_ERROR("failed to map ctx meta data\n");
1446                 goto error_vm;
1447         }
1448
1449         r = amdgpu_mes_create_process(adev, pasid, vm);
1450         if (r) {
1451                 DRM_ERROR("failed to create MES process\n");
1452                 goto error_vm;
1453         }
1454
1455         for (i = 0; i < ARRAY_SIZE(queue_types); i++) {
1456                 /* On GFX v10.3, fw hasn't supported to map sdma queue. */
1457                 if (amdgpu_ip_version(adev, GC_HWIP, 0) >=
1458                             IP_VERSION(10, 3, 0) &&
1459                     amdgpu_ip_version(adev, GC_HWIP, 0) <
1460                             IP_VERSION(11, 0, 0) &&
1461                     queue_types[i][0] == AMDGPU_RING_TYPE_SDMA)
1462                         continue;
1463
1464                 r = amdgpu_mes_test_create_gang_and_queues(adev, pasid,
1465                                                            &gang_ids[i],
1466                                                            queue_types[i][0],
1467                                                            queue_types[i][1],
1468                                                            &added_rings[k],
1469                                                            &ctx_data);
1470                 if (r)
1471                         goto error_queues;
1472
1473                 k += queue_types[i][1];
1474         }
1475
1476         /* start ring test and ib test for MES queues */
1477         amdgpu_mes_test_queues(added_rings);
1478
1479 error_queues:
1480         /* remove all queues */
1481         for (i = 0; i < ARRAY_SIZE(added_rings); i++) {
1482                 if (!added_rings[i])
1483                         continue;
1484                 amdgpu_mes_remove_ring(adev, added_rings[i]);
1485         }
1486
1487         for (i = 0; i < ARRAY_SIZE(gang_ids); i++) {
1488                 if (!gang_ids[i])
1489                         continue;
1490                 amdgpu_mes_remove_gang(adev, gang_ids[i]);
1491         }
1492
1493         amdgpu_mes_destroy_process(adev, pasid);
1494
1495 error_vm:
1496         amdgpu_mes_ctx_unmap_meta_data(adev, &ctx_data);
1497
1498 error_fini:
1499         amdgpu_vm_fini(adev, vm);
1500
1501 error_pasid:
1502         if (pasid)
1503                 amdgpu_pasid_free(pasid);
1504
1505         amdgpu_mes_ctx_free_meta_data(&ctx_data);
1506         kfree(vm);
1507         return 0;
1508 }
1509
1510 int amdgpu_mes_init_microcode(struct amdgpu_device *adev, int pipe)
1511 {
1512         const struct mes_firmware_header_v1_0 *mes_hdr;
1513         struct amdgpu_firmware_info *info;
1514         char ucode_prefix[30];
1515         char fw_name[50];
1516         bool need_retry = false;
1517         int r;
1518
1519         amdgpu_ucode_ip_version_decode(adev, GC_HWIP, ucode_prefix,
1520                                        sizeof(ucode_prefix));
1521         if (adev->enable_uni_mes) {
1522                 snprintf(fw_name, sizeof(fw_name),
1523                          "amdgpu/%s_uni_mes.bin", ucode_prefix);
1524         } else if (amdgpu_ip_version(adev, GC_HWIP, 0) >= IP_VERSION(11, 0, 0) &&
1525             amdgpu_ip_version(adev, GC_HWIP, 0) < IP_VERSION(12, 0, 0)) {
1526                 snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_mes%s.bin",
1527                          ucode_prefix,
1528                          pipe == AMDGPU_MES_SCHED_PIPE ? "_2" : "1");
1529                 need_retry = true;
1530         } else {
1531                 snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_mes%s.bin",
1532                          ucode_prefix,
1533                          pipe == AMDGPU_MES_SCHED_PIPE ? "" : "1");
1534         }
1535
1536         r = amdgpu_ucode_request(adev, &adev->mes.fw[pipe], fw_name);
1537         if (r && need_retry && pipe == AMDGPU_MES_SCHED_PIPE) {
1538                 dev_info(adev->dev, "try to fall back to %s_mes.bin\n", ucode_prefix);
1539                 r = amdgpu_ucode_request(adev, &adev->mes.fw[pipe],
1540                                          "amdgpu/%s_mes.bin", ucode_prefix);
1541         }
1542
1543         if (r)
1544                 goto out;
1545
1546         mes_hdr = (const struct mes_firmware_header_v1_0 *)
1547                 adev->mes.fw[pipe]->data;
1548         adev->mes.uc_start_addr[pipe] =
1549                 le32_to_cpu(mes_hdr->mes_uc_start_addr_lo) |
1550                 ((uint64_t)(le32_to_cpu(mes_hdr->mes_uc_start_addr_hi)) << 32);
1551         adev->mes.data_start_addr[pipe] =
1552                 le32_to_cpu(mes_hdr->mes_data_start_addr_lo) |
1553                 ((uint64_t)(le32_to_cpu(mes_hdr->mes_data_start_addr_hi)) << 32);
1554
1555         if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) {
1556                 int ucode, ucode_data;
1557
1558                 if (pipe == AMDGPU_MES_SCHED_PIPE) {
1559                         ucode = AMDGPU_UCODE_ID_CP_MES;
1560                         ucode_data = AMDGPU_UCODE_ID_CP_MES_DATA;
1561                 } else {
1562                         ucode = AMDGPU_UCODE_ID_CP_MES1;
1563                         ucode_data = AMDGPU_UCODE_ID_CP_MES1_DATA;
1564                 }
1565
1566                 info = &adev->firmware.ucode[ucode];
1567                 info->ucode_id = ucode;
1568                 info->fw = adev->mes.fw[pipe];
1569                 adev->firmware.fw_size +=
1570                         ALIGN(le32_to_cpu(mes_hdr->mes_ucode_size_bytes),
1571                               PAGE_SIZE);
1572
1573                 info = &adev->firmware.ucode[ucode_data];
1574                 info->ucode_id = ucode_data;
1575                 info->fw = adev->mes.fw[pipe];
1576                 adev->firmware.fw_size +=
1577                         ALIGN(le32_to_cpu(mes_hdr->mes_ucode_data_size_bytes),
1578                               PAGE_SIZE);
1579         }
1580
1581         return 0;
1582 out:
1583         amdgpu_ucode_release(&adev->mes.fw[pipe]);
1584         return r;
1585 }
1586
1587 #if defined(CONFIG_DEBUG_FS)
1588
1589 static int amdgpu_debugfs_mes_event_log_show(struct seq_file *m, void *unused)
1590 {
1591         struct amdgpu_device *adev = m->private;
1592         uint32_t *mem = (uint32_t *)(adev->mes.event_log_cpu_addr);
1593
1594         seq_hex_dump(m, "", DUMP_PREFIX_OFFSET, 32, 4,
1595                      mem, adev->mes.event_log_size, false);
1596
1597         return 0;
1598 }
1599
1600 DEFINE_SHOW_ATTRIBUTE(amdgpu_debugfs_mes_event_log);
1601
1602 #endif
1603
1604 void amdgpu_debugfs_mes_event_log_init(struct amdgpu_device *adev)
1605 {
1606
1607 #if defined(CONFIG_DEBUG_FS)
1608         struct drm_minor *minor = adev_to_drm(adev)->primary;
1609         struct dentry *root = minor->debugfs_root;
1610         if (adev->enable_mes && amdgpu_mes_log_enable)
1611                 debugfs_create_file("amdgpu_mes_event_log", 0444, root,
1612                                     adev, &amdgpu_debugfs_mes_event_log_fops);
1613
1614 #endif
1615 }