drivers/gpu/drm/amd/amdgpu/mes_v12_0.c

   1 /*
   2  * Copyright 2023 Advanced Micro Devices, Inc.
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice shall be included in
  12  * all copies or substantial portions of the Software.
  13  *
  14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  20  * OTHER DEALINGS IN THE SOFTWARE.
  21  *
  22  */
  23
  24 #include <linux/firmware.h>
  25 #include <linux/module.h>
  26 #include "amdgpu.h"
  27 #include "gfx_v12_0.h"
  28 #include "soc15_common.h"
  29 #include "soc21.h"
  30 #include "gc/gc_12_0_0_offset.h"
  31 #include "gc/gc_12_0_0_sh_mask.h"
  32 #include "gc/gc_11_0_0_default.h"
  33 #include "v12_structs.h"
  34 #include "mes_v12_api_def.h"
  35
  36 MODULE_FIRMWARE("amdgpu/gc_12_0_0_mes.bin");
  37 MODULE_FIRMWARE("amdgpu/gc_12_0_0_mes1.bin");
  38 MODULE_FIRMWARE("amdgpu/gc_12_0_0_uni_mes.bin");
  39 MODULE_FIRMWARE("amdgpu/gc_12_0_1_mes.bin");
  40 MODULE_FIRMWARE("amdgpu/gc_12_0_1_mes1.bin");
  41 MODULE_FIRMWARE("amdgpu/gc_12_0_1_uni_mes.bin");
  42
  43 static int mes_v12_0_hw_init(struct amdgpu_ip_block *ip_block);
  44 static int mes_v12_0_hw_fini(struct amdgpu_ip_block *ip_block);
  45 static int mes_v12_0_kiq_hw_init(struct amdgpu_device *adev);
  46 static int mes_v12_0_kiq_hw_fini(struct amdgpu_device *adev);
  47
  48 #define MES_EOP_SIZE   2048
  49
  50 static void mes_v12_0_ring_set_wptr(struct amdgpu_ring *ring)
  51 {
  52         struct amdgpu_device *adev = ring->adev;
  53
  54         if (ring->use_doorbell) {
  55                 atomic64_set((atomic64_t *)ring->wptr_cpu_addr,
  56                              ring->wptr);
  57                 WDOORBELL64(ring->doorbell_index, ring->wptr);
  58         } else {
  59                 BUG();
  60         }
  61 }
  62
  63 static u64 mes_v12_0_ring_get_rptr(struct amdgpu_ring *ring)
  64 {
  65         return *ring->rptr_cpu_addr;
  66 }
  67
  68 static u64 mes_v12_0_ring_get_wptr(struct amdgpu_ring *ring)
  69 {
  70         u64 wptr;
  71
  72         if (ring->use_doorbell)
  73                 wptr = atomic64_read((atomic64_t *)ring->wptr_cpu_addr);
  74         else
  75                 BUG();
  76         return wptr;
  77 }
  78
  79 static const struct amdgpu_ring_funcs mes_v12_0_ring_funcs = {
  80         .type = AMDGPU_RING_TYPE_MES,
  81         .align_mask = 1,
  82         .nop = 0,
  83         .support_64bit_ptrs = true,
  84         .get_rptr = mes_v12_0_ring_get_rptr,
  85         .get_wptr = mes_v12_0_ring_get_wptr,
  86         .set_wptr = mes_v12_0_ring_set_wptr,
  87         .insert_nop = amdgpu_ring_insert_nop,
  88 };
  89
  90 static const char *mes_v12_0_opcodes[] = {
  91         "SET_HW_RSRC",
  92         "SET_SCHEDULING_CONFIG",
  93         "ADD_QUEUE",
  94         "REMOVE_QUEUE",
  95         "PERFORM_YIELD",
  96         "SET_GANG_PRIORITY_LEVEL",
  97         "SUSPEND",
  98         "RESUME",
  99         "RESET",
 100         "SET_LOG_BUFFER",
 101         "CHANGE_GANG_PRORITY",
 102         "QUERY_SCHEDULER_STATUS",
 103         "unused",
 104         "SET_DEBUG_VMID",
 105         "MISC",
 106         "UPDATE_ROOT_PAGE_TABLE",
 107         "AMD_LOG",
 108         "SET_SE_MODE",
 109         "SET_GANG_SUBMIT",
 110         "SET_HW_RSRC_1",
 111 };
 112
 113 static const char *mes_v12_0_misc_opcodes[] = {
 114         "WRITE_REG",
 115         "INV_GART",
 116         "QUERY_STATUS",
 117         "READ_REG",
 118         "WAIT_REG_MEM",
 119         "SET_SHADER_DEBUGGER",
 120         "NOTIFY_WORK_ON_UNMAPPED_QUEUE",
 121         "NOTIFY_TO_UNMAP_PROCESSES",
 122 };
 123
 124 static const char *mes_v12_0_get_op_string(union MESAPI__MISC *x_pkt)
 125 {
 126         const char *op_str = NULL;
 127
 128         if (x_pkt->header.opcode < ARRAY_SIZE(mes_v12_0_opcodes))
 129                 op_str = mes_v12_0_opcodes[x_pkt->header.opcode];
 130
 131         return op_str;
 132 }
 133
 134 static const char *mes_v12_0_get_misc_op_string(union MESAPI__MISC *x_pkt)
 135 {
 136         const char *op_str = NULL;
 137
 138         if ((x_pkt->header.opcode == MES_SCH_API_MISC) &&
 139             (x_pkt->opcode < ARRAY_SIZE(mes_v12_0_misc_opcodes)))
 140                 op_str = mes_v12_0_misc_opcodes[x_pkt->opcode];
 141
 142         return op_str;
 143 }
 144
 145 static int mes_v12_0_submit_pkt_and_poll_completion(struct amdgpu_mes *mes,
 146                                             int pipe, void *pkt, int size,
 147                                             int api_status_off)
 148 {
 149         union MESAPI__QUERY_MES_STATUS mes_status_pkt;
 150         signed long timeout = 2100000; /* 2100 ms */
 151         struct amdgpu_device *adev = mes->adev;
 152         struct amdgpu_ring *ring = &mes->ring[pipe];
 153         spinlock_t *ring_lock = &mes->ring_lock[pipe];
 154         struct MES_API_STATUS *api_status;
 155         union MESAPI__MISC *x_pkt = pkt;
 156         const char *op_str, *misc_op_str;
 157         unsigned long flags;
 158         u64 status_gpu_addr;
 159         u32 seq, status_offset;
 160         u64 *status_ptr;
 161         signed long r;
 162         int ret;
 163
 164         if (x_pkt->header.opcode >= MES_SCH_API_MAX)
 165                 return -EINVAL;
 166
 167         if (amdgpu_emu_mode) {
 168                 timeout *= 100;
 169         } else if (amdgpu_sriov_vf(adev)) {
 170                 /* Worst case in sriov where all other 15 VF timeout, each VF needs about 600ms */
 171                 timeout = 15 * 600 * 1000;
 172         }
 173
 174         ret = amdgpu_device_wb_get(adev, &status_offset);
 175         if (ret)
 176                 return ret;
 177
 178         status_gpu_addr = adev->wb.gpu_addr + (status_offset * 4);
 179         status_ptr = (u64 *)&adev->wb.wb[status_offset];
 180         *status_ptr = 0;
 181
 182         spin_lock_irqsave(ring_lock, flags);
 183         r = amdgpu_ring_alloc(ring, (size + sizeof(mes_status_pkt)) / 4);
 184         if (r)
 185                 goto error_unlock_free;
 186
 187         seq = ++ring->fence_drv.sync_seq;
 188         r = amdgpu_fence_wait_polling(ring,
 189                                       seq - ring->fence_drv.num_fences_mask,
 190                                       timeout);
 191         if (r < 1)
 192                 goto error_undo;
 193
 194         api_status = (struct MES_API_STATUS *)((char *)pkt + api_status_off);
 195         api_status->api_completion_fence_addr = status_gpu_addr;
 196         api_status->api_completion_fence_value = 1;
 197
 198         amdgpu_ring_write_multiple(ring, pkt, size / 4);
 199
 200         memset(&mes_status_pkt, 0, sizeof(mes_status_pkt));
 201         mes_status_pkt.header.type = MES_API_TYPE_SCHEDULER;
 202         mes_status_pkt.header.opcode = MES_SCH_API_QUERY_SCHEDULER_STATUS;
 203         mes_status_pkt.header.dwsize = API_FRAME_SIZE_IN_DWORDS;
 204         mes_status_pkt.api_status.api_completion_fence_addr =
 205                 ring->fence_drv.gpu_addr;
 206         mes_status_pkt.api_status.api_completion_fence_value = seq;
 207
 208         amdgpu_ring_write_multiple(ring, &mes_status_pkt,
 209                                    sizeof(mes_status_pkt) / 4);
 210
 211         amdgpu_ring_commit(ring);
 212         spin_unlock_irqrestore(ring_lock, flags);
 213
 214         op_str = mes_v12_0_get_op_string(x_pkt);
 215         misc_op_str = mes_v12_0_get_misc_op_string(x_pkt);
 216
 217         if (misc_op_str)
 218                 dev_dbg(adev->dev, "MES(%d) msg=%s (%s) was emitted\n",
 219                         pipe, op_str, misc_op_str);
 220         else if (op_str)
 221                 dev_dbg(adev->dev, "MES(%d) msg=%s was emitted\n",
 222                         pipe, op_str);
 223         else
 224                 dev_dbg(adev->dev, "MES(%d) msg=%d was emitted\n",
 225                         pipe, x_pkt->header.opcode);
 226
 227         r = amdgpu_fence_wait_polling(ring, seq, timeout);
 228         if (r < 1 || !*status_ptr) {
 229
 230                 if (misc_op_str)
 231                         dev_err(adev->dev, "MES(%d) failed to respond to msg=%s (%s)\n",
 232                                 pipe, op_str, misc_op_str);
 233                 else if (op_str)
 234                         dev_err(adev->dev, "MES(%d) failed to respond to msg=%s\n",
 235                                 pipe, op_str);
 236                 else
 237                         dev_err(adev->dev, "MES(%d) failed to respond to msg=%d\n",
 238                                 pipe, x_pkt->header.opcode);
 239
 240                 while (halt_if_hws_hang)
 241                         schedule();
 242
 243                 r = -ETIMEDOUT;
 244                 goto error_wb_free;
 245         }
 246
 247         amdgpu_device_wb_free(adev, status_offset);
 248         return 0;
 249
 250 error_undo:
 251         dev_err(adev->dev, "MES ring buffer is full.\n");
 252         amdgpu_ring_undo(ring);
 253
 254 error_unlock_free:
 255         spin_unlock_irqrestore(ring_lock, flags);
 256
 257 error_wb_free:
 258         amdgpu_device_wb_free(adev, status_offset);
 259         return r;
 260 }
 261
 262 static int convert_to_mes_queue_type(int queue_type)
 263 {
 264         if (queue_type == AMDGPU_RING_TYPE_GFX)
 265                 return MES_QUEUE_TYPE_GFX;
 266         else if (queue_type == AMDGPU_RING_TYPE_COMPUTE)
 267                 return MES_QUEUE_TYPE_COMPUTE;
 268         else if (queue_type == AMDGPU_RING_TYPE_SDMA)
 269                 return MES_QUEUE_TYPE_SDMA;
 270         else if (queue_type == AMDGPU_RING_TYPE_MES)
 271                 return MES_QUEUE_TYPE_SCHQ;
 272         else
 273                 BUG();
 274         return -1;
 275 }
 276
 277 static int mes_v12_0_add_hw_queue(struct amdgpu_mes *mes,
 278                                   struct mes_add_queue_input *input)
 279 {
 280         struct amdgpu_device *adev = mes->adev;
 281         union MESAPI__ADD_QUEUE mes_add_queue_pkt;
 282         struct amdgpu_vmhub *hub = &adev->vmhub[AMDGPU_GFXHUB(0)];
 283         uint32_t vm_cntx_cntl = hub->vm_cntx_cntl;
 284
 285         memset(&mes_add_queue_pkt, 0, sizeof(mes_add_queue_pkt));
 286
 287         mes_add_queue_pkt.header.type = MES_API_TYPE_SCHEDULER;
 288         mes_add_queue_pkt.header.opcode = MES_SCH_API_ADD_QUEUE;
 289         mes_add_queue_pkt.header.dwsize = API_FRAME_SIZE_IN_DWORDS;
 290
 291         mes_add_queue_pkt.process_id = input->process_id;
 292         mes_add_queue_pkt.page_table_base_addr = input->page_table_base_addr;
 293         mes_add_queue_pkt.process_va_start = input->process_va_start;
 294         mes_add_queue_pkt.process_va_end = input->process_va_end;
 295         mes_add_queue_pkt.process_quantum = input->process_quantum;
 296         mes_add_queue_pkt.process_context_addr = input->process_context_addr;
 297         mes_add_queue_pkt.gang_quantum = input->gang_quantum;
 298         mes_add_queue_pkt.gang_context_addr = input->gang_context_addr;
 299         mes_add_queue_pkt.inprocess_gang_priority =
 300                 input->inprocess_gang_priority;
 301         mes_add_queue_pkt.gang_global_priority_level =
 302                 input->gang_global_priority_level;
 303         mes_add_queue_pkt.doorbell_offset = input->doorbell_offset;
 304         mes_add_queue_pkt.mqd_addr = input->mqd_addr;
 305
 306         mes_add_queue_pkt.wptr_addr = input->wptr_mc_addr;
 307
 308         mes_add_queue_pkt.queue_type =
 309                 convert_to_mes_queue_type(input->queue_type);
 310         mes_add_queue_pkt.paging = input->paging;
 311         mes_add_queue_pkt.vm_context_cntl = vm_cntx_cntl;
 312         mes_add_queue_pkt.gws_base = input->gws_base;
 313         mes_add_queue_pkt.gws_size = input->gws_size;
 314         mes_add_queue_pkt.trap_handler_addr = input->tba_addr;
 315         mes_add_queue_pkt.tma_addr = input->tma_addr;
 316         mes_add_queue_pkt.trap_en = input->trap_en;
 317         mes_add_queue_pkt.skip_process_ctx_clear = input->skip_process_ctx_clear;
 318         mes_add_queue_pkt.is_kfd_process = input->is_kfd_process;
 319
 320         /* For KFD, gds_size is re-used for queue size (needed in MES for AQL queues) */
 321         mes_add_queue_pkt.is_aql_queue = input->is_aql_queue;
 322         mes_add_queue_pkt.gds_size = input->queue_size;
 323
 324         /* For KFD, gds_size is re-used for queue size (needed in MES for AQL queues) */
 325         mes_add_queue_pkt.is_aql_queue = input->is_aql_queue;
 326         mes_add_queue_pkt.gds_size = input->queue_size;
 327
 328         return mes_v12_0_submit_pkt_and_poll_completion(mes,
 329                         AMDGPU_MES_SCHED_PIPE,
 330                         &mes_add_queue_pkt, sizeof(mes_add_queue_pkt),
 331                         offsetof(union MESAPI__ADD_QUEUE, api_status));
 332 }
 333
 334 static int mes_v12_0_remove_hw_queue(struct amdgpu_mes *mes,
 335                                      struct mes_remove_queue_input *input)
 336 {
 337         union MESAPI__REMOVE_QUEUE mes_remove_queue_pkt;
 338
 339         memset(&mes_remove_queue_pkt, 0, sizeof(mes_remove_queue_pkt));
 340
 341         mes_remove_queue_pkt.header.type = MES_API_TYPE_SCHEDULER;
 342         mes_remove_queue_pkt.header.opcode = MES_SCH_API_REMOVE_QUEUE;
 343         mes_remove_queue_pkt.header.dwsize = API_FRAME_SIZE_IN_DWORDS;
 344
 345         mes_remove_queue_pkt.doorbell_offset = input->doorbell_offset;
 346         mes_remove_queue_pkt.gang_context_addr = input->gang_context_addr;
 347
 348         return mes_v12_0_submit_pkt_and_poll_completion(mes,
 349                         AMDGPU_MES_SCHED_PIPE,
 350                         &mes_remove_queue_pkt, sizeof(mes_remove_queue_pkt),
 351                         offsetof(union MESAPI__REMOVE_QUEUE, api_status));
 352 }
 353
 354 int gfx_v12_0_request_gfx_index_mutex(struct amdgpu_device *adev,
 355                                       bool req)
 356 {
 357         u32 i, tmp, val;
 358
 359         for (i = 0; i < adev->usec_timeout; i++) {
 360                 /* Request with MeId=2, PipeId=0 */
 361                 tmp = REG_SET_FIELD(0, CP_GFX_INDEX_MUTEX, REQUEST, req);
 362                 tmp = REG_SET_FIELD(tmp, CP_GFX_INDEX_MUTEX, CLIENTID, 4);
 363                 WREG32_SOC15(GC, 0, regCP_GFX_INDEX_MUTEX, tmp);
 364
 365                 val = RREG32_SOC15(GC, 0, regCP_GFX_INDEX_MUTEX);
 366                 if (req) {
 367                         if (val == tmp)
 368                                 break;
 369                 } else {
 370                         tmp = REG_SET_FIELD(tmp, CP_GFX_INDEX_MUTEX,
 371                                             REQUEST, 1);
 372
 373                         /* unlocked or locked by firmware */
 374                         if (val != tmp)
 375                                 break;
 376                 }
 377                 udelay(1);
 378         }
 379
 380         if (i >= adev->usec_timeout)
 381                 return -EINVAL;
 382
 383         return 0;
 384 }
 385
 386 static int mes_v12_0_reset_queue_mmio(struct amdgpu_mes *mes, uint32_t queue_type,
 387                                       uint32_t me_id, uint32_t pipe_id,
 388                                       uint32_t queue_id, uint32_t vmid)
 389 {
 390         struct amdgpu_device *adev = mes->adev;
 391         uint32_t value, reg;
 392         int i, r = 0;
 393
 394         amdgpu_gfx_rlc_enter_safe_mode(adev, 0);
 395
 396         if (queue_type == AMDGPU_RING_TYPE_GFX) {
 397                 dev_info(adev->dev, "reset gfx queue (%d:%d:%d: vmid:%d)\n",
 398                          me_id, pipe_id, queue_id, vmid);
 399
 400                 mutex_lock(&adev->gfx.reset_sem_mutex);
 401                 gfx_v12_0_request_gfx_index_mutex(adev, true);
 402                 /* all se allow writes */
 403                 WREG32_SOC15(GC, 0, regGRBM_GFX_INDEX,
 404                              (uint32_t)(0x1 << GRBM_GFX_INDEX__SE_BROADCAST_WRITES__SHIFT));
 405                 value = REG_SET_FIELD(0, CP_VMID_RESET, RESET_REQUEST, 1 << vmid);
 406                 if (pipe_id == 0)
 407                         value = REG_SET_FIELD(value, CP_VMID_RESET, PIPE0_QUEUES, 1 << queue_id);
 408                 else
 409                         value = REG_SET_FIELD(value, CP_VMID_RESET, PIPE1_QUEUES, 1 << queue_id);
 410                 WREG32_SOC15(GC, 0, regCP_VMID_RESET, value);
 411                 gfx_v12_0_request_gfx_index_mutex(adev, false);
 412                 mutex_unlock(&adev->gfx.reset_sem_mutex);
 413
 414                 mutex_lock(&adev->srbm_mutex);
 415                 soc21_grbm_select(adev, me_id, pipe_id, queue_id, 0);
 416                 /* wait till dequeue take effects */
 417                 for (i = 0; i < adev->usec_timeout; i++) {
 418                         if (!(RREG32_SOC15(GC, 0, regCP_GFX_HQD_ACTIVE) & 1))
 419                                 break;
 420                         udelay(1);
 421                 }
 422                 if (i >= adev->usec_timeout) {
 423                         dev_err(adev->dev, "failed to wait on gfx hqd deactivate\n");
 424                         r = -ETIMEDOUT;
 425                 }
 426
 427                 soc21_grbm_select(adev, 0, 0, 0, 0);
 428                 mutex_unlock(&adev->srbm_mutex);
 429         } else if (queue_type == AMDGPU_RING_TYPE_COMPUTE) {
 430                 dev_info(adev->dev, "reset compute queue (%d:%d:%d)\n",
 431                          me_id, pipe_id, queue_id);
 432                 mutex_lock(&adev->srbm_mutex);
 433                 soc21_grbm_select(adev, me_id, pipe_id, queue_id, 0);
 434                 WREG32_SOC15(GC, 0, regCP_HQD_DEQUEUE_REQUEST, 0x2);
 435                 WREG32_SOC15(GC, 0, regSPI_COMPUTE_QUEUE_RESET, 0x1);
 436
 437                 /* wait till dequeue take effects */
 438                 for (i = 0; i < adev->usec_timeout; i++) {
 439                         if (!(RREG32_SOC15(GC, 0, regCP_HQD_ACTIVE) & 1))
 440                                 break;
 441                         udelay(1);
 442                 }
 443                 if (i >= adev->usec_timeout) {
 444                         dev_err(adev->dev, "failed to wait on hqd deactivate\n");
 445                         r = -ETIMEDOUT;
 446                 }
 447                 soc21_grbm_select(adev, 0, 0, 0, 0);
 448                 mutex_unlock(&adev->srbm_mutex);
 449         } else if (queue_type == AMDGPU_RING_TYPE_SDMA) {
 450                 dev_info(adev->dev, "reset sdma queue (%d:%d:%d)\n",
 451                          me_id, pipe_id, queue_id);
 452                 switch (me_id) {
 453                 case 1:
 454                         reg = SOC15_REG_OFFSET(GC, 0, regSDMA1_QUEUE_RESET_REQ);
 455                         break;
 456                 case 0:
 457                 default:
 458                         reg = SOC15_REG_OFFSET(GC, 0, regSDMA0_QUEUE_RESET_REQ);
 459                         break;
 460                 }
 461
 462                 value = 1 << queue_id;
 463                 WREG32(reg, value);
 464                 /* wait for queue reset done */
 465                 for (i = 0; i < adev->usec_timeout; i++) {
 466                         if (!(RREG32(reg) & value))
 467                                 break;
 468                         udelay(1);
 469                 }
 470                 if (i >= adev->usec_timeout) {
 471                         dev_err(adev->dev, "failed to wait on sdma queue reset done\n");
 472                         r = -ETIMEDOUT;
 473                 }
 474         }
 475
 476         amdgpu_gfx_rlc_exit_safe_mode(adev, 0);
 477         return r;
 478 }
 479
 480 static int mes_v12_0_reset_hw_queue(struct amdgpu_mes *mes,
 481                                     struct mes_reset_queue_input *input)
 482 {
 483         union MESAPI__RESET mes_reset_queue_pkt;
 484         int pipe;
 485
 486         memset(&mes_reset_queue_pkt, 0, sizeof(mes_reset_queue_pkt));
 487
 488         mes_reset_queue_pkt.header.type = MES_API_TYPE_SCHEDULER;
 489         mes_reset_queue_pkt.header.opcode = MES_SCH_API_RESET;
 490         mes_reset_queue_pkt.header.dwsize = API_FRAME_SIZE_IN_DWORDS;
 491
 492         mes_reset_queue_pkt.doorbell_offset = input->doorbell_offset;
 493         mes_reset_queue_pkt.gang_context_addr = input->gang_context_addr;
 494         /*mes_reset_queue_pkt.reset_queue_only = 1;*/
 495
 496         if (mes->adev->enable_uni_mes)
 497                 pipe = AMDGPU_MES_KIQ_PIPE;
 498         else
 499                 pipe = AMDGPU_MES_SCHED_PIPE;
 500
 501         return mes_v12_0_submit_pkt_and_poll_completion(mes, pipe,
 502                         &mes_reset_queue_pkt, sizeof(mes_reset_queue_pkt),
 503                         offsetof(union MESAPI__REMOVE_QUEUE, api_status));
 504 }
 505
 506 static int mes_v12_0_map_legacy_queue(struct amdgpu_mes *mes,
 507                                       struct mes_map_legacy_queue_input *input)
 508 {
 509         union MESAPI__ADD_QUEUE mes_add_queue_pkt;
 510         int pipe;
 511
 512         memset(&mes_add_queue_pkt, 0, sizeof(mes_add_queue_pkt));
 513
 514         mes_add_queue_pkt.header.type = MES_API_TYPE_SCHEDULER;
 515         mes_add_queue_pkt.header.opcode = MES_SCH_API_ADD_QUEUE;
 516         mes_add_queue_pkt.header.dwsize = API_FRAME_SIZE_IN_DWORDS;
 517
 518         mes_add_queue_pkt.pipe_id = input->pipe_id;
 519         mes_add_queue_pkt.queue_id = input->queue_id;
 520         mes_add_queue_pkt.doorbell_offset = input->doorbell_offset;
 521         mes_add_queue_pkt.mqd_addr = input->mqd_addr;
 522         mes_add_queue_pkt.wptr_addr = input->wptr_addr;
 523         mes_add_queue_pkt.queue_type =
 524                 convert_to_mes_queue_type(input->queue_type);
 525         mes_add_queue_pkt.map_legacy_kq = 1;
 526
 527         if (mes->adev->enable_uni_mes)
 528                 pipe = AMDGPU_MES_KIQ_PIPE;
 529         else
 530                 pipe = AMDGPU_MES_SCHED_PIPE;
 531
 532         return mes_v12_0_submit_pkt_and_poll_completion(mes, pipe,
 533                         &mes_add_queue_pkt, sizeof(mes_add_queue_pkt),
 534                         offsetof(union MESAPI__ADD_QUEUE, api_status));
 535 }
 536
 537 static int mes_v12_0_unmap_legacy_queue(struct amdgpu_mes *mes,
 538                         struct mes_unmap_legacy_queue_input *input)
 539 {
 540         union MESAPI__REMOVE_QUEUE mes_remove_queue_pkt;
 541         int pipe;
 542
 543         memset(&mes_remove_queue_pkt, 0, sizeof(mes_remove_queue_pkt));
 544
 545         mes_remove_queue_pkt.header.type = MES_API_TYPE_SCHEDULER;
 546         mes_remove_queue_pkt.header.opcode = MES_SCH_API_REMOVE_QUEUE;
 547         mes_remove_queue_pkt.header.dwsize = API_FRAME_SIZE_IN_DWORDS;
 548
 549         mes_remove_queue_pkt.doorbell_offset = input->doorbell_offset;
 550         mes_remove_queue_pkt.gang_context_addr = 0;
 551
 552         mes_remove_queue_pkt.pipe_id = input->pipe_id;
 553         mes_remove_queue_pkt.queue_id = input->queue_id;
 554
 555         if (input->action == PREEMPT_QUEUES_NO_UNMAP) {
 556                 mes_remove_queue_pkt.preempt_legacy_gfx_queue = 1;
 557                 mes_remove_queue_pkt.tf_addr = input->trail_fence_addr;
 558                 mes_remove_queue_pkt.tf_data =
 559                         lower_32_bits(input->trail_fence_data);
 560         } else {
 561                 mes_remove_queue_pkt.unmap_legacy_queue = 1;
 562                 mes_remove_queue_pkt.queue_type =
 563                         convert_to_mes_queue_type(input->queue_type);
 564         }
 565
 566         if (mes->adev->enable_uni_mes)
 567                 pipe = AMDGPU_MES_KIQ_PIPE;
 568         else
 569                 pipe = AMDGPU_MES_SCHED_PIPE;
 570
 571         return mes_v12_0_submit_pkt_and_poll_completion(mes, pipe,
 572                         &mes_remove_queue_pkt, sizeof(mes_remove_queue_pkt),
 573                         offsetof(union MESAPI__REMOVE_QUEUE, api_status));
 574 }
 575
 576 static int mes_v12_0_suspend_gang(struct amdgpu_mes *mes,
 577                                   struct mes_suspend_gang_input *input)
 578 {
 579         return 0;
 580 }
 581
 582 static int mes_v12_0_resume_gang(struct amdgpu_mes *mes,
 583                                  struct mes_resume_gang_input *input)
 584 {
 585         return 0;
 586 }
 587
 588 static int mes_v12_0_query_sched_status(struct amdgpu_mes *mes, int pipe)
 589 {
 590         union MESAPI__QUERY_MES_STATUS mes_status_pkt;
 591
 592         memset(&mes_status_pkt, 0, sizeof(mes_status_pkt));
 593
 594         mes_status_pkt.header.type = MES_API_TYPE_SCHEDULER;
 595         mes_status_pkt.header.opcode = MES_SCH_API_QUERY_SCHEDULER_STATUS;
 596         mes_status_pkt.header.dwsize = API_FRAME_SIZE_IN_DWORDS;
 597
 598         return mes_v12_0_submit_pkt_and_poll_completion(mes, pipe,
 599                         &mes_status_pkt, sizeof(mes_status_pkt),
 600                         offsetof(union MESAPI__QUERY_MES_STATUS, api_status));
 601 }
 602
 603 static int mes_v12_0_misc_op(struct amdgpu_mes *mes,
 604                              struct mes_misc_op_input *input)
 605 {
 606         union MESAPI__MISC misc_pkt;
 607         int pipe;
 608
 609         if (mes->adev->enable_uni_mes)
 610                 pipe = AMDGPU_MES_KIQ_PIPE;
 611         else
 612                 pipe = AMDGPU_MES_SCHED_PIPE;
 613
 614         memset(&misc_pkt, 0, sizeof(misc_pkt));
 615
 616         misc_pkt.header.type = MES_API_TYPE_SCHEDULER;
 617         misc_pkt.header.opcode = MES_SCH_API_MISC;
 618         misc_pkt.header.dwsize = API_FRAME_SIZE_IN_DWORDS;
 619
 620         switch (input->op) {
 621         case MES_MISC_OP_READ_REG:
 622                 misc_pkt.opcode = MESAPI_MISC__READ_REG;
 623                 misc_pkt.read_reg.reg_offset = input->read_reg.reg_offset;
 624                 misc_pkt.read_reg.buffer_addr = input->read_reg.buffer_addr;
 625                 break;
 626         case MES_MISC_OP_WRITE_REG:
 627                 misc_pkt.opcode = MESAPI_MISC__WRITE_REG;
 628                 misc_pkt.write_reg.reg_offset = input->write_reg.reg_offset;
 629                 misc_pkt.write_reg.reg_value = input->write_reg.reg_value;
 630                 break;
 631         case MES_MISC_OP_WRM_REG_WAIT:
 632                 misc_pkt.opcode = MESAPI_MISC__WAIT_REG_MEM;
 633                 misc_pkt.wait_reg_mem.op = WRM_OPERATION__WAIT_REG_MEM;
 634                 misc_pkt.wait_reg_mem.reference = input->wrm_reg.ref;
 635                 misc_pkt.wait_reg_mem.mask = input->wrm_reg.mask;
 636                 misc_pkt.wait_reg_mem.reg_offset1 = input->wrm_reg.reg0;
 637                 misc_pkt.wait_reg_mem.reg_offset2 = 0;
 638                 break;
 639         case MES_MISC_OP_WRM_REG_WR_WAIT:
 640                 misc_pkt.opcode = MESAPI_MISC__WAIT_REG_MEM;
 641                 misc_pkt.wait_reg_mem.op = WRM_OPERATION__WR_WAIT_WR_REG;
 642                 misc_pkt.wait_reg_mem.reference = input->wrm_reg.ref;
 643                 misc_pkt.wait_reg_mem.mask = input->wrm_reg.mask;
 644                 misc_pkt.wait_reg_mem.reg_offset1 = input->wrm_reg.reg0;
 645                 misc_pkt.wait_reg_mem.reg_offset2 = input->wrm_reg.reg1;
 646                 break;
 647         case MES_MISC_OP_SET_SHADER_DEBUGGER:
 648                 pipe = AMDGPU_MES_SCHED_PIPE;
 649                 misc_pkt.opcode = MESAPI_MISC__SET_SHADER_DEBUGGER;
 650                 misc_pkt.set_shader_debugger.process_context_addr =
 651                                 input->set_shader_debugger.process_context_addr;
 652                 misc_pkt.set_shader_debugger.flags.u32all =
 653                                 input->set_shader_debugger.flags.u32all;
 654                 misc_pkt.set_shader_debugger.spi_gdbg_per_vmid_cntl =
 655                                 input->set_shader_debugger.spi_gdbg_per_vmid_cntl;
 656                 memcpy(misc_pkt.set_shader_debugger.tcp_watch_cntl,
 657                                 input->set_shader_debugger.tcp_watch_cntl,
 658                                 sizeof(misc_pkt.set_shader_debugger.tcp_watch_cntl));
 659                 misc_pkt.set_shader_debugger.trap_en = input->set_shader_debugger.trap_en;
 660                 break;
 661         case MES_MISC_OP_CHANGE_CONFIG:
 662                 misc_pkt.opcode = MESAPI_MISC__CHANGE_CONFIG;
 663                 misc_pkt.change_config.opcode =
 664                                 MESAPI_MISC__CHANGE_CONFIG_OPTION_LIMIT_SINGLE_PROCESS;
 665                 misc_pkt.change_config.option.bits.limit_single_process =
 666                                 input->change_config.option.limit_single_process;
 667                 break;
 668
 669         default:
 670                 DRM_ERROR("unsupported misc op (%d) \n", input->op);
 671                 return -EINVAL;
 672         }
 673
 674         return mes_v12_0_submit_pkt_and_poll_completion(mes, pipe,
 675                         &misc_pkt, sizeof(misc_pkt),
 676                         offsetof(union MESAPI__MISC, api_status));
 677 }
 678
 679 static int mes_v12_0_set_hw_resources_1(struct amdgpu_mes *mes, int pipe)
 680 {
 681         union MESAPI_SET_HW_RESOURCES_1 mes_set_hw_res_1_pkt;
 682
 683         memset(&mes_set_hw_res_1_pkt, 0, sizeof(mes_set_hw_res_1_pkt));
 684
 685         mes_set_hw_res_1_pkt.header.type = MES_API_TYPE_SCHEDULER;
 686         mes_set_hw_res_1_pkt.header.opcode = MES_SCH_API_SET_HW_RSRC_1;
 687         mes_set_hw_res_1_pkt.header.dwsize = API_FRAME_SIZE_IN_DWORDS;
 688         mes_set_hw_res_1_pkt.mes_kiq_unmap_timeout = 0xa;
 689
 690         return mes_v12_0_submit_pkt_and_poll_completion(mes, pipe,
 691                         &mes_set_hw_res_1_pkt, sizeof(mes_set_hw_res_1_pkt),
 692                         offsetof(union MESAPI_SET_HW_RESOURCES_1, api_status));
 693 }
 694
 695 static int mes_v12_0_set_hw_resources(struct amdgpu_mes *mes, int pipe)
 696 {
 697         int i;
 698         struct amdgpu_device *adev = mes->adev;
 699         union MESAPI_SET_HW_RESOURCES mes_set_hw_res_pkt;
 700
 701         memset(&mes_set_hw_res_pkt, 0, sizeof(mes_set_hw_res_pkt));
 702
 703         mes_set_hw_res_pkt.header.type = MES_API_TYPE_SCHEDULER;
 704         mes_set_hw_res_pkt.header.opcode = MES_SCH_API_SET_HW_RSRC;
 705         mes_set_hw_res_pkt.header.dwsize = API_FRAME_SIZE_IN_DWORDS;
 706
 707         if (pipe == AMDGPU_MES_SCHED_PIPE) {
 708                 mes_set_hw_res_pkt.vmid_mask_mmhub = mes->vmid_mask_mmhub;
 709                 mes_set_hw_res_pkt.vmid_mask_gfxhub = mes->vmid_mask_gfxhub;
 710                 mes_set_hw_res_pkt.gds_size = adev->gds.gds_size;
 711                 mes_set_hw_res_pkt.paging_vmid = 0;
 712
 713                 for (i = 0; i < MAX_COMPUTE_PIPES; i++)
 714                         mes_set_hw_res_pkt.compute_hqd_mask[i] =
 715                                 mes->compute_hqd_mask[i];
 716
 717                 for (i = 0; i < MAX_GFX_PIPES; i++)
 718                         mes_set_hw_res_pkt.gfx_hqd_mask[i] =
 719                                 mes->gfx_hqd_mask[i];
 720
 721                 for (i = 0; i < MAX_SDMA_PIPES; i++)
 722                         mes_set_hw_res_pkt.sdma_hqd_mask[i] =
 723                                 mes->sdma_hqd_mask[i];
 724
 725                 for (i = 0; i < AMD_PRIORITY_NUM_LEVELS; i++)
 726                         mes_set_hw_res_pkt.aggregated_doorbells[i] =
 727                                 mes->aggregated_doorbells[i];
 728         }
 729
 730         mes_set_hw_res_pkt.g_sch_ctx_gpu_mc_ptr =
 731                 mes->sch_ctx_gpu_addr[pipe];
 732         mes_set_hw_res_pkt.query_status_fence_gpu_mc_ptr =
 733                 mes->query_status_fence_gpu_addr[pipe];
 734
 735         for (i = 0; i < 5; i++) {
 736                 mes_set_hw_res_pkt.gc_base[i] = adev->reg_offset[GC_HWIP][0][i];
 737                 mes_set_hw_res_pkt.mmhub_base[i] =
 738                                 adev->reg_offset[MMHUB_HWIP][0][i];
 739                 mes_set_hw_res_pkt.osssys_base[i] =
 740                 adev->reg_offset[OSSSYS_HWIP][0][i];
 741         }
 742
 743         mes_set_hw_res_pkt.disable_reset = 1;
 744         mes_set_hw_res_pkt.disable_mes_log = 1;
 745         mes_set_hw_res_pkt.use_different_vmid_compute = 1;
 746         mes_set_hw_res_pkt.enable_reg_active_poll = 1;
 747         mes_set_hw_res_pkt.enable_level_process_quantum_check = 1;
 748
 749         /*
 750          * Keep oversubscribe timer for sdma . When we have unmapped doorbell
 751          * handling support, other queue will not use the oversubscribe timer.
 752          * handling  mode - 0: disabled; 1: basic version; 2: basic+ version
 753          */
 754         mes_set_hw_res_pkt.oversubscription_timer = 50;
 755         mes_set_hw_res_pkt.unmapped_doorbell_handling = 1;
 756
 757         if (amdgpu_mes_log_enable) {
 758                 mes_set_hw_res_pkt.enable_mes_event_int_logging = 1;
 759                 mes_set_hw_res_pkt.event_intr_history_gpu_mc_ptr = mes->event_log_gpu_addr + pipe * AMDGPU_MES_LOG_BUFFER_SIZE;
 760         }
 761
 762         if (enforce_isolation)
 763                 mes_set_hw_res_pkt.limit_single_process = 1;
 764
 765         return mes_v12_0_submit_pkt_and_poll_completion(mes, pipe,
 766                         &mes_set_hw_res_pkt, sizeof(mes_set_hw_res_pkt),
 767                         offsetof(union MESAPI_SET_HW_RESOURCES, api_status));
 768 }
 769
 770 static void mes_v12_0_init_aggregated_doorbell(struct amdgpu_mes *mes)
 771 {
 772         struct amdgpu_device *adev = mes->adev;
 773         uint32_t data;
 774
 775         data = RREG32_SOC15(GC, 0, regCP_MES_DOORBELL_CONTROL1);
 776         data &= ~(CP_MES_DOORBELL_CONTROL1__DOORBELL_OFFSET_MASK |
 777                   CP_MES_DOORBELL_CONTROL1__DOORBELL_EN_MASK |
 778                   CP_MES_DOORBELL_CONTROL1__DOORBELL_HIT_MASK);
 779         data |= mes->aggregated_doorbells[AMDGPU_MES_PRIORITY_LEVEL_LOW] <<
 780                 CP_MES_DOORBELL_CONTROL1__DOORBELL_OFFSET__SHIFT;
 781         data |= 1 << CP_MES_DOORBELL_CONTROL1__DOORBELL_EN__SHIFT;
 782         WREG32_SOC15(GC, 0, regCP_MES_DOORBELL_CONTROL1, data);
 783
 784         data = RREG32_SOC15(GC, 0, regCP_MES_DOORBELL_CONTROL2);
 785         data &= ~(CP_MES_DOORBELL_CONTROL2__DOORBELL_OFFSET_MASK |
 786                   CP_MES_DOORBELL_CONTROL2__DOORBELL_EN_MASK |
 787                   CP_MES_DOORBELL_CONTROL2__DOORBELL_HIT_MASK);
 788         data |= mes->aggregated_doorbells[AMDGPU_MES_PRIORITY_LEVEL_NORMAL] <<
 789                 CP_MES_DOORBELL_CONTROL2__DOORBELL_OFFSET__SHIFT;
 790         data |= 1 << CP_MES_DOORBELL_CONTROL2__DOORBELL_EN__SHIFT;
 791         WREG32_SOC15(GC, 0, regCP_MES_DOORBELL_CONTROL2, data);
 792
 793         data = RREG32_SOC15(GC, 0, regCP_MES_DOORBELL_CONTROL3);
 794         data &= ~(CP_MES_DOORBELL_CONTROL3__DOORBELL_OFFSET_MASK |
 795                   CP_MES_DOORBELL_CONTROL3__DOORBELL_EN_MASK |
 796                   CP_MES_DOORBELL_CONTROL3__DOORBELL_HIT_MASK);
 797         data |= mes->aggregated_doorbells[AMDGPU_MES_PRIORITY_LEVEL_MEDIUM] <<
 798                 CP_MES_DOORBELL_CONTROL3__DOORBELL_OFFSET__SHIFT;
 799         data |= 1 << CP_MES_DOORBELL_CONTROL3__DOORBELL_EN__SHIFT;
 800         WREG32_SOC15(GC, 0, regCP_MES_DOORBELL_CONTROL3, data);
 801
 802         data = RREG32_SOC15(GC, 0, regCP_MES_DOORBELL_CONTROL4);
 803         data &= ~(CP_MES_DOORBELL_CONTROL4__DOORBELL_OFFSET_MASK |
 804                   CP_MES_DOORBELL_CONTROL4__DOORBELL_EN_MASK |
 805                   CP_MES_DOORBELL_CONTROL4__DOORBELL_HIT_MASK);
 806         data |= mes->aggregated_doorbells[AMDGPU_MES_PRIORITY_LEVEL_HIGH] <<
 807                 CP_MES_DOORBELL_CONTROL4__DOORBELL_OFFSET__SHIFT;
 808         data |= 1 << CP_MES_DOORBELL_CONTROL4__DOORBELL_EN__SHIFT;
 809         WREG32_SOC15(GC, 0, regCP_MES_DOORBELL_CONTROL4, data);
 810
 811         data = RREG32_SOC15(GC, 0, regCP_MES_DOORBELL_CONTROL5);
 812         data &= ~(CP_MES_DOORBELL_CONTROL5__DOORBELL_OFFSET_MASK |
 813                   CP_MES_DOORBELL_CONTROL5__DOORBELL_EN_MASK |
 814                   CP_MES_DOORBELL_CONTROL5__DOORBELL_HIT_MASK);
 815         data |= mes->aggregated_doorbells[AMDGPU_MES_PRIORITY_LEVEL_REALTIME] <<
 816                 CP_MES_DOORBELL_CONTROL5__DOORBELL_OFFSET__SHIFT;
 817         data |= 1 << CP_MES_DOORBELL_CONTROL5__DOORBELL_EN__SHIFT;
 818         WREG32_SOC15(GC, 0, regCP_MES_DOORBELL_CONTROL5, data);
 819
 820         data = 1 << CP_HQD_GFX_CONTROL__DB_UPDATED_MSG_EN__SHIFT;
 821         WREG32_SOC15(GC, 0, regCP_HQD_GFX_CONTROL, data);
 822 }
 823
 824
 825 static void mes_v12_0_enable_unmapped_doorbell_handling(
 826                 struct amdgpu_mes *mes, bool enable)
 827 {
 828         struct amdgpu_device *adev = mes->adev;
 829         uint32_t data = RREG32_SOC15(GC, 0, regCP_UNMAPPED_DOORBELL);
 830
 831         /*
 832          * The default PROC_LSB settng is 0xc which means doorbell
 833          * addr[16:12] gives the doorbell page number. For kfd, each
 834          * process will use 2 pages of doorbell, we need to change the
 835          * setting to 0xd
 836          */
 837         data &= ~CP_UNMAPPED_DOORBELL__PROC_LSB_MASK;
 838         data |= 0xd <<  CP_UNMAPPED_DOORBELL__PROC_LSB__SHIFT;
 839
 840         data |= (enable ? 1 : 0) << CP_UNMAPPED_DOORBELL__ENABLE__SHIFT;
 841
 842         WREG32_SOC15(GC, 0, regCP_UNMAPPED_DOORBELL, data);
 843 }
 844
 845 static int mes_v12_0_reset_legacy_queue(struct amdgpu_mes *mes,
 846                                         struct mes_reset_legacy_queue_input *input)
 847 {
 848         union MESAPI__RESET mes_reset_queue_pkt;
 849         int pipe;
 850
 851         if (input->use_mmio)
 852                 return mes_v12_0_reset_queue_mmio(mes, input->queue_type,
 853                                                   input->me_id, input->pipe_id,
 854                                                   input->queue_id, input->vmid);
 855
 856         memset(&mes_reset_queue_pkt, 0, sizeof(mes_reset_queue_pkt));
 857
 858         mes_reset_queue_pkt.header.type = MES_API_TYPE_SCHEDULER;
 859         mes_reset_queue_pkt.header.opcode = MES_SCH_API_RESET;
 860         mes_reset_queue_pkt.header.dwsize = API_FRAME_SIZE_IN_DWORDS;
 861
 862         mes_reset_queue_pkt.queue_type =
 863                 convert_to_mes_queue_type(input->queue_type);
 864
 865         if (mes_reset_queue_pkt.queue_type == MES_QUEUE_TYPE_GFX) {
 866                 mes_reset_queue_pkt.reset_legacy_gfx = 1;
 867                 mes_reset_queue_pkt.pipe_id_lp = input->pipe_id;
 868                 mes_reset_queue_pkt.queue_id_lp = input->queue_id;
 869                 mes_reset_queue_pkt.mqd_mc_addr_lp = input->mqd_addr;
 870                 mes_reset_queue_pkt.doorbell_offset_lp = input->doorbell_offset;
 871                 mes_reset_queue_pkt.wptr_addr_lp = input->wptr_addr;
 872                 mes_reset_queue_pkt.vmid_id_lp = input->vmid;
 873         } else {
 874                 mes_reset_queue_pkt.reset_queue_only = 1;
 875                 mes_reset_queue_pkt.doorbell_offset = input->doorbell_offset;
 876         }
 877
 878         if (mes->adev->enable_uni_mes)
 879                 pipe = AMDGPU_MES_KIQ_PIPE;
 880         else
 881                 pipe = AMDGPU_MES_SCHED_PIPE;
 882
 883         return mes_v12_0_submit_pkt_and_poll_completion(mes, pipe,
 884                         &mes_reset_queue_pkt, sizeof(mes_reset_queue_pkt),
 885                         offsetof(union MESAPI__RESET, api_status));
 886 }
 887
 888 static const struct amdgpu_mes_funcs mes_v12_0_funcs = {
 889         .add_hw_queue = mes_v12_0_add_hw_queue,
 890         .remove_hw_queue = mes_v12_0_remove_hw_queue,
 891         .map_legacy_queue = mes_v12_0_map_legacy_queue,
 892         .unmap_legacy_queue = mes_v12_0_unmap_legacy_queue,
 893         .suspend_gang = mes_v12_0_suspend_gang,
 894         .resume_gang = mes_v12_0_resume_gang,
 895         .misc_op = mes_v12_0_misc_op,
 896         .reset_legacy_queue = mes_v12_0_reset_legacy_queue,
 897         .reset_hw_queue = mes_v12_0_reset_hw_queue,
 898 };
 899
 900 static int mes_v12_0_allocate_ucode_buffer(struct amdgpu_device *adev,
 901                                            enum admgpu_mes_pipe pipe)
 902 {
 903         int r;
 904         const struct mes_firmware_header_v1_0 *mes_hdr;
 905         const __le32 *fw_data;
 906         unsigned fw_size;
 907
 908         mes_hdr = (const struct mes_firmware_header_v1_0 *)
 909                 adev->mes.fw[pipe]->data;
 910
 911         fw_data = (const __le32 *)(adev->mes.fw[pipe]->data +
 912                    le32_to_cpu(mes_hdr->mes_ucode_offset_bytes));
 913         fw_size = le32_to_cpu(mes_hdr->mes_ucode_size_bytes);
 914
 915         r = amdgpu_bo_create_reserved(adev, fw_size,
 916                                       PAGE_SIZE,
 917                                       AMDGPU_GEM_DOMAIN_VRAM,
 918                                       &adev->mes.ucode_fw_obj[pipe],
 919                                       &adev->mes.ucode_fw_gpu_addr[pipe],
 920                                       (void **)&adev->mes.ucode_fw_ptr[pipe]);
 921         if (r) {
 922                 dev_err(adev->dev, "(%d) failed to create mes fw bo\n", r);
 923                 return r;
 924         }
 925
 926         memcpy(adev->mes.ucode_fw_ptr[pipe], fw_data, fw_size);
 927
 928         amdgpu_bo_kunmap(adev->mes.ucode_fw_obj[pipe]);
 929         amdgpu_bo_unreserve(adev->mes.ucode_fw_obj[pipe]);
 930
 931         return 0;
 932 }
 933
 934 static int mes_v12_0_allocate_ucode_data_buffer(struct amdgpu_device *adev,
 935                                                 enum admgpu_mes_pipe pipe)
 936 {
 937         int r;
 938         const struct mes_firmware_header_v1_0 *mes_hdr;
 939         const __le32 *fw_data;
 940         unsigned fw_size;
 941
 942         mes_hdr = (const struct mes_firmware_header_v1_0 *)
 943                 adev->mes.fw[pipe]->data;
 944
 945         fw_data = (const __le32 *)(adev->mes.fw[pipe]->data +
 946                    le32_to_cpu(mes_hdr->mes_ucode_data_offset_bytes));
 947         fw_size = le32_to_cpu(mes_hdr->mes_ucode_data_size_bytes);
 948
 949         r = amdgpu_bo_create_reserved(adev, fw_size,
 950                                       64 * 1024,
 951                                       AMDGPU_GEM_DOMAIN_VRAM,
 952                                       &adev->mes.data_fw_obj[pipe],
 953                                       &adev->mes.data_fw_gpu_addr[pipe],
 954                                       (void **)&adev->mes.data_fw_ptr[pipe]);
 955         if (r) {
 956                 dev_err(adev->dev, "(%d) failed to create mes data fw bo\n", r);
 957                 return r;
 958         }
 959
 960         memcpy(adev->mes.data_fw_ptr[pipe], fw_data, fw_size);
 961
 962         amdgpu_bo_kunmap(adev->mes.data_fw_obj[pipe]);
 963         amdgpu_bo_unreserve(adev->mes.data_fw_obj[pipe]);
 964
 965         return 0;
 966 }
 967
 968 static void mes_v12_0_free_ucode_buffers(struct amdgpu_device *adev,
 969                                          enum admgpu_mes_pipe pipe)
 970 {
 971         amdgpu_bo_free_kernel(&adev->mes.data_fw_obj[pipe],
 972                               &adev->mes.data_fw_gpu_addr[pipe],
 973                               (void **)&adev->mes.data_fw_ptr[pipe]);
 974
 975         amdgpu_bo_free_kernel(&adev->mes.ucode_fw_obj[pipe],
 976                               &adev->mes.ucode_fw_gpu_addr[pipe],
 977                               (void **)&adev->mes.ucode_fw_ptr[pipe]);
 978 }
 979
 980 static void mes_v12_0_enable(struct amdgpu_device *adev, bool enable)
 981 {
 982         uint64_t ucode_addr;
 983         uint32_t pipe, data = 0;
 984
 985         if (enable) {
 986                 data = RREG32_SOC15(GC, 0, regCP_MES_CNTL);
 987                 data = REG_SET_FIELD(data, CP_MES_CNTL, MES_PIPE0_RESET, 1);
 988                 data = REG_SET_FIELD(data, CP_MES_CNTL, MES_PIPE1_RESET, 1);
 989                 WREG32_SOC15(GC, 0, regCP_MES_CNTL, data);
 990
 991                 mutex_lock(&adev->srbm_mutex);
 992                 for (pipe = 0; pipe < AMDGPU_MAX_MES_PIPES; pipe++) {
 993                         soc21_grbm_select(adev, 3, pipe, 0, 0);
 994
 995                         ucode_addr = adev->mes.uc_start_addr[pipe] >> 2;
 996                         WREG32_SOC15(GC, 0, regCP_MES_PRGRM_CNTR_START,
 997                                      lower_32_bits(ucode_addr));
 998                         WREG32_SOC15(GC, 0, regCP_MES_PRGRM_CNTR_START_HI,
 999                                      upper_32_bits(ucode_addr));
1000                 }
1001                 soc21_grbm_select(adev, 0, 0, 0, 0);
1002                 mutex_unlock(&adev->srbm_mutex);
1003
1004                 /* unhalt MES and activate pipe0 */
1005                 data = REG_SET_FIELD(0, CP_MES_CNTL, MES_PIPE0_ACTIVE, 1);
1006                 data = REG_SET_FIELD(data, CP_MES_CNTL, MES_PIPE1_ACTIVE, 1);
1007                 WREG32_SOC15(GC, 0, regCP_MES_CNTL, data);
1008
1009                 if (amdgpu_emu_mode)
1010                         msleep(100);
1011                 else if (adev->enable_uni_mes)
1012                         udelay(500);
1013                 else
1014                         udelay(50);
1015         } else {
1016                 data = RREG32_SOC15(GC, 0, regCP_MES_CNTL);
1017                 data = REG_SET_FIELD(data, CP_MES_CNTL, MES_PIPE0_ACTIVE, 0);
1018                 data = REG_SET_FIELD(data, CP_MES_CNTL, MES_PIPE1_ACTIVE, 0);
1019                 data = REG_SET_FIELD(data, CP_MES_CNTL,
1020                                      MES_INVALIDATE_ICACHE, 1);
1021                 data = REG_SET_FIELD(data, CP_MES_CNTL, MES_PIPE0_RESET, 1);
1022                 data = REG_SET_FIELD(data, CP_MES_CNTL, MES_PIPE1_RESET, 1);
1023                 data = REG_SET_FIELD(data, CP_MES_CNTL, MES_HALT, 1);
1024                 WREG32_SOC15(GC, 0, regCP_MES_CNTL, data);
1025         }
1026 }
1027
1028 static void mes_v12_0_set_ucode_start_addr(struct amdgpu_device *adev)
1029 {
1030         uint64_t ucode_addr;
1031         int pipe;
1032
1033         mes_v12_0_enable(adev, false);
1034
1035         mutex_lock(&adev->srbm_mutex);
1036         for (pipe = 0; pipe < AMDGPU_MAX_MES_PIPES; pipe++) {
1037                 /* me=3, queue=0 */
1038                 soc21_grbm_select(adev, 3, pipe, 0, 0);
1039
1040                 /* set ucode start address */
1041                 ucode_addr = adev->mes.uc_start_addr[pipe] >> 2;
1042                 WREG32_SOC15(GC, 0, regCP_MES_PRGRM_CNTR_START,
1043                                 lower_32_bits(ucode_addr));
1044                 WREG32_SOC15(GC, 0, regCP_MES_PRGRM_CNTR_START_HI,
1045                                 upper_32_bits(ucode_addr));
1046
1047                 soc21_grbm_select(adev, 0, 0, 0, 0);
1048         }
1049         mutex_unlock(&adev->srbm_mutex);
1050 }
1051
1052 /* This function is for backdoor MES firmware */
1053 static int mes_v12_0_load_microcode(struct amdgpu_device *adev,
1054                                     enum admgpu_mes_pipe pipe, bool prime_icache)
1055 {
1056         int r;
1057         uint32_t data;
1058
1059         mes_v12_0_enable(adev, false);
1060
1061         if (!adev->mes.fw[pipe])
1062                 return -EINVAL;
1063
1064         r = mes_v12_0_allocate_ucode_buffer(adev, pipe);
1065         if (r)
1066                 return r;
1067
1068         r = mes_v12_0_allocate_ucode_data_buffer(adev, pipe);
1069         if (r) {
1070                 mes_v12_0_free_ucode_buffers(adev, pipe);
1071                 return r;
1072         }
1073
1074         mutex_lock(&adev->srbm_mutex);
1075         /* me=3, pipe=0, queue=0 */
1076         soc21_grbm_select(adev, 3, pipe, 0, 0);
1077
1078         WREG32_SOC15(GC, 0, regCP_MES_IC_BASE_CNTL, 0);
1079
1080         /* set ucode fimrware address */
1081         WREG32_SOC15(GC, 0, regCP_MES_IC_BASE_LO,
1082                      lower_32_bits(adev->mes.ucode_fw_gpu_addr[pipe]));
1083         WREG32_SOC15(GC, 0, regCP_MES_IC_BASE_HI,
1084                      upper_32_bits(adev->mes.ucode_fw_gpu_addr[pipe]));
1085
1086         /* set ucode instruction cache boundary to 2M-1 */
1087         WREG32_SOC15(GC, 0, regCP_MES_MIBOUND_LO, 0x1FFFFF);
1088
1089         /* set ucode data firmware address */
1090         WREG32_SOC15(GC, 0, regCP_MES_MDBASE_LO,
1091                      lower_32_bits(adev->mes.data_fw_gpu_addr[pipe]));
1092         WREG32_SOC15(GC, 0, regCP_MES_MDBASE_HI,
1093                      upper_32_bits(adev->mes.data_fw_gpu_addr[pipe]));
1094
1095         /* Set data cache boundary CP_MES_MDBOUND_LO */
1096         WREG32_SOC15(GC, 0, regCP_MES_MDBOUND_LO, 0x7FFFF);
1097
1098         if (prime_icache) {
1099                 /* invalidate ICACHE */
1100                 data = RREG32_SOC15(GC, 0, regCP_MES_IC_OP_CNTL);
1101                 data = REG_SET_FIELD(data, CP_MES_IC_OP_CNTL, PRIME_ICACHE, 0);
1102                 data = REG_SET_FIELD(data, CP_MES_IC_OP_CNTL, INVALIDATE_CACHE, 1);
1103                 WREG32_SOC15(GC, 0, regCP_MES_IC_OP_CNTL, data);
1104
1105                 /* prime the ICACHE. */
1106                 data = RREG32_SOC15(GC, 0, regCP_MES_IC_OP_CNTL);
1107                 data = REG_SET_FIELD(data, CP_MES_IC_OP_CNTL, PRIME_ICACHE, 1);
1108                 WREG32_SOC15(GC, 0, regCP_MES_IC_OP_CNTL, data);
1109         }
1110
1111         soc21_grbm_select(adev, 0, 0, 0, 0);
1112         mutex_unlock(&adev->srbm_mutex);
1113
1114         return 0;
1115 }
1116
1117 static int mes_v12_0_allocate_eop_buf(struct amdgpu_device *adev,
1118                                       enum admgpu_mes_pipe pipe)
1119 {
1120         int r;
1121         u32 *eop;
1122
1123         r = amdgpu_bo_create_reserved(adev, MES_EOP_SIZE, PAGE_SIZE,
1124                               AMDGPU_GEM_DOMAIN_GTT,
1125                               &adev->mes.eop_gpu_obj[pipe],
1126                               &adev->mes.eop_gpu_addr[pipe],
1127                               (void **)&eop);
1128         if (r) {
1129                 dev_warn(adev->dev, "(%d) create EOP bo failed\n", r);
1130                 return r;
1131         }
1132
1133         memset(eop, 0,
1134                adev->mes.eop_gpu_obj[pipe]->tbo.base.size);
1135
1136         amdgpu_bo_kunmap(adev->mes.eop_gpu_obj[pipe]);
1137         amdgpu_bo_unreserve(adev->mes.eop_gpu_obj[pipe]);
1138
1139         return 0;
1140 }
1141
1142 static int mes_v12_0_mqd_init(struct amdgpu_ring *ring)
1143 {
1144         struct v12_compute_mqd *mqd = ring->mqd_ptr;
1145         uint64_t hqd_gpu_addr, wb_gpu_addr, eop_base_addr;
1146         uint32_t tmp;
1147
1148         mqd->header = 0xC0310800;
1149         mqd->compute_pipelinestat_enable = 0x00000001;
1150         mqd->compute_static_thread_mgmt_se0 = 0xffffffff;
1151         mqd->compute_static_thread_mgmt_se1 = 0xffffffff;
1152         mqd->compute_static_thread_mgmt_se2 = 0xffffffff;
1153         mqd->compute_static_thread_mgmt_se3 = 0xffffffff;
1154         mqd->compute_misc_reserved = 0x00000007;
1155
1156         eop_base_addr = ring->eop_gpu_addr >> 8;
1157
1158         /* set the EOP size, register value is 2^(EOP_SIZE+1) dwords */
1159         tmp = regCP_HQD_EOP_CONTROL_DEFAULT;
1160         tmp = REG_SET_FIELD(tmp, CP_HQD_EOP_CONTROL, EOP_SIZE,
1161                         (order_base_2(MES_EOP_SIZE / 4) - 1));
1162
1163         mqd->cp_hqd_eop_base_addr_lo = lower_32_bits(eop_base_addr);
1164         mqd->cp_hqd_eop_base_addr_hi = upper_32_bits(eop_base_addr);
1165         mqd->cp_hqd_eop_control = tmp;
1166
1167         /* disable the queue if it's active */
1168         ring->wptr = 0;
1169         mqd->cp_hqd_pq_rptr = 0;
1170         mqd->cp_hqd_pq_wptr_lo = 0;
1171         mqd->cp_hqd_pq_wptr_hi = 0;
1172
1173         /* set the pointer to the MQD */
1174         mqd->cp_mqd_base_addr_lo = ring->mqd_gpu_addr & 0xfffffffc;
1175         mqd->cp_mqd_base_addr_hi = upper_32_bits(ring->mqd_gpu_addr);
1176
1177         /* set MQD vmid to 0 */
1178         tmp = regCP_MQD_CONTROL_DEFAULT;
1179         tmp = REG_SET_FIELD(tmp, CP_MQD_CONTROL, VMID, 0);
1180         mqd->cp_mqd_control = tmp;
1181
1182         /* set the pointer to the HQD, this is similar CP_RB0_BASE/_HI */
1183         hqd_gpu_addr = ring->gpu_addr >> 8;
1184         mqd->cp_hqd_pq_base_lo = lower_32_bits(hqd_gpu_addr);
1185         mqd->cp_hqd_pq_base_hi = upper_32_bits(hqd_gpu_addr);
1186
1187         /* set the wb address whether it's enabled or not */
1188         wb_gpu_addr = ring->rptr_gpu_addr;
1189         mqd->cp_hqd_pq_rptr_report_addr_lo = wb_gpu_addr & 0xfffffffc;
1190         mqd->cp_hqd_pq_rptr_report_addr_hi =
1191                 upper_32_bits(wb_gpu_addr) & 0xffff;
1192
1193         /* only used if CP_PQ_WPTR_POLL_CNTL.CP_PQ_WPTR_POLL_CNTL__EN_MASK=1 */
1194         wb_gpu_addr = ring->wptr_gpu_addr;
1195         mqd->cp_hqd_pq_wptr_poll_addr_lo = wb_gpu_addr & 0xfffffff8;
1196         mqd->cp_hqd_pq_wptr_poll_addr_hi = upper_32_bits(wb_gpu_addr) & 0xffff;
1197
1198         /* set up the HQD, this is similar to CP_RB0_CNTL */
1199         tmp = regCP_HQD_PQ_CONTROL_DEFAULT;
1200         tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, QUEUE_SIZE,
1201                             (order_base_2(ring->ring_size / 4) - 1));
1202         tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, RPTR_BLOCK_SIZE,
1203                             ((order_base_2(AMDGPU_GPU_PAGE_SIZE / 4) - 1) << 8));
1204         tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, UNORD_DISPATCH, 1);
1205         tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, TUNNEL_DISPATCH, 0);
1206         tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, PRIV_STATE, 1);
1207         tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, KMD_QUEUE, 1);
1208         tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, NO_UPDATE_RPTR, 1);
1209         mqd->cp_hqd_pq_control = tmp;
1210
1211         /* enable doorbell */
1212         tmp = 0;
1213         if (ring->use_doorbell) {
1214                 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
1215                                     DOORBELL_OFFSET, ring->doorbell_index);
1216                 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
1217                                     DOORBELL_EN, 1);
1218                 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
1219                                     DOORBELL_SOURCE, 0);
1220                 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
1221                                     DOORBELL_HIT, 0);
1222         } else {
1223                 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
1224                                     DOORBELL_EN, 0);
1225         }
1226         mqd->cp_hqd_pq_doorbell_control = tmp;
1227
1228         mqd->cp_hqd_vmid = 0;
1229         /* activate the queue */
1230         mqd->cp_hqd_active = 1;
1231
1232         tmp = regCP_HQD_PERSISTENT_STATE_DEFAULT;
1233         tmp = REG_SET_FIELD(tmp, CP_HQD_PERSISTENT_STATE,
1234                             PRELOAD_SIZE, 0x55);
1235         mqd->cp_hqd_persistent_state = tmp;
1236
1237         mqd->cp_hqd_ib_control = regCP_HQD_IB_CONTROL_DEFAULT;
1238         mqd->cp_hqd_iq_timer = regCP_HQD_IQ_TIMER_DEFAULT;
1239         mqd->cp_hqd_quantum = regCP_HQD_QUANTUM_DEFAULT;
1240
1241         /*
1242          * Set CP_HQD_GFX_CONTROL.DB_UPDATED_MSG_EN[15] to enable unmapped
1243          * doorbell handling. This is a reserved CP internal register can
1244          * not be accesss by others
1245          */
1246         mqd->reserved_184 = BIT(15);
1247
1248         return 0;
1249 }
1250
1251 static void mes_v12_0_queue_init_register(struct amdgpu_ring *ring)
1252 {
1253         struct v12_compute_mqd *mqd = ring->mqd_ptr;
1254         struct amdgpu_device *adev = ring->adev;
1255         uint32_t data = 0;
1256
1257         mutex_lock(&adev->srbm_mutex);
1258         soc21_grbm_select(adev, 3, ring->pipe, 0, 0);
1259
1260         /* set CP_HQD_VMID.VMID = 0. */
1261         data = RREG32_SOC15(GC, 0, regCP_HQD_VMID);
1262         data = REG_SET_FIELD(data, CP_HQD_VMID, VMID, 0);
1263         WREG32_SOC15(GC, 0, regCP_HQD_VMID, data);
1264
1265         /* set CP_HQD_PQ_DOORBELL_CONTROL.DOORBELL_EN=0 */
1266         data = RREG32_SOC15(GC, 0, regCP_HQD_PQ_DOORBELL_CONTROL);
1267         data = REG_SET_FIELD(data, CP_HQD_PQ_DOORBELL_CONTROL,
1268                              DOORBELL_EN, 0);
1269         WREG32_SOC15(GC, 0, regCP_HQD_PQ_DOORBELL_CONTROL, data);
1270
1271         /* set CP_MQD_BASE_ADDR/HI with the MQD base address */
1272         WREG32_SOC15(GC, 0, regCP_MQD_BASE_ADDR, mqd->cp_mqd_base_addr_lo);
1273         WREG32_SOC15(GC, 0, regCP_MQD_BASE_ADDR_HI, mqd->cp_mqd_base_addr_hi);
1274
1275         /* set CP_MQD_CONTROL.VMID=0 */
1276         data = RREG32_SOC15(GC, 0, regCP_MQD_CONTROL);
1277         data = REG_SET_FIELD(data, CP_MQD_CONTROL, VMID, 0);
1278         WREG32_SOC15(GC, 0, regCP_MQD_CONTROL, 0);
1279
1280         /* set CP_HQD_PQ_BASE/HI with the ring buffer base address */
1281         WREG32_SOC15(GC, 0, regCP_HQD_PQ_BASE, mqd->cp_hqd_pq_base_lo);
1282         WREG32_SOC15(GC, 0, regCP_HQD_PQ_BASE_HI, mqd->cp_hqd_pq_base_hi);
1283
1284         /* set CP_HQD_PQ_RPTR_REPORT_ADDR/HI */
1285         WREG32_SOC15(GC, 0, regCP_HQD_PQ_RPTR_REPORT_ADDR,
1286                      mqd->cp_hqd_pq_rptr_report_addr_lo);
1287         WREG32_SOC15(GC, 0, regCP_HQD_PQ_RPTR_REPORT_ADDR_HI,
1288                      mqd->cp_hqd_pq_rptr_report_addr_hi);
1289
1290         /* set CP_HQD_PQ_CONTROL */
1291         WREG32_SOC15(GC, 0, regCP_HQD_PQ_CONTROL, mqd->cp_hqd_pq_control);
1292
1293         /* set CP_HQD_PQ_WPTR_POLL_ADDR/HI */
1294         WREG32_SOC15(GC, 0, regCP_HQD_PQ_WPTR_POLL_ADDR,
1295                      mqd->cp_hqd_pq_wptr_poll_addr_lo);
1296         WREG32_SOC15(GC, 0, regCP_HQD_PQ_WPTR_POLL_ADDR_HI,
1297                      mqd->cp_hqd_pq_wptr_poll_addr_hi);
1298
1299         /* set CP_HQD_PQ_DOORBELL_CONTROL */
1300         WREG32_SOC15(GC, 0, regCP_HQD_PQ_DOORBELL_CONTROL,
1301                      mqd->cp_hqd_pq_doorbell_control);
1302
1303         /* set CP_HQD_PERSISTENT_STATE.PRELOAD_SIZE=0x53 */
1304         WREG32_SOC15(GC, 0, regCP_HQD_PERSISTENT_STATE, mqd->cp_hqd_persistent_state);
1305
1306         /* set CP_HQD_ACTIVE.ACTIVE=1 */
1307         WREG32_SOC15(GC, 0, regCP_HQD_ACTIVE, mqd->cp_hqd_active);
1308
1309         soc21_grbm_select(adev, 0, 0, 0, 0);
1310         mutex_unlock(&adev->srbm_mutex);
1311 }
1312
1313 static int mes_v12_0_kiq_enable_queue(struct amdgpu_device *adev)
1314 {
1315         struct amdgpu_kiq *kiq = &adev->gfx.kiq[0];
1316         struct amdgpu_ring *kiq_ring = &adev->gfx.kiq[0].ring;
1317         int r;
1318
1319         if (!kiq->pmf || !kiq->pmf->kiq_map_queues)
1320                 return -EINVAL;
1321
1322         r = amdgpu_ring_alloc(kiq_ring, kiq->pmf->map_queues_size);
1323         if (r) {
1324                 DRM_ERROR("Failed to lock KIQ (%d).\n", r);
1325                 return r;
1326         }
1327
1328         kiq->pmf->kiq_map_queues(kiq_ring, &adev->mes.ring[0]);
1329
1330         r = amdgpu_ring_test_ring(kiq_ring);
1331         if (r) {
1332                 DRM_ERROR("kfq enable failed\n");
1333                 kiq_ring->sched.ready = false;
1334         }
1335         return r;
1336 }
1337
1338 static int mes_v12_0_queue_init(struct amdgpu_device *adev,
1339                                 enum admgpu_mes_pipe pipe)
1340 {
1341         struct amdgpu_ring *ring;
1342         int r;
1343
1344         if (!adev->enable_uni_mes && pipe == AMDGPU_MES_KIQ_PIPE)
1345                 ring = &adev->gfx.kiq[0].ring;
1346         else
1347                 ring = &adev->mes.ring[pipe];
1348
1349         if ((adev->enable_uni_mes || pipe == AMDGPU_MES_SCHED_PIPE) &&
1350             (amdgpu_in_reset(adev) || adev->in_suspend)) {
1351                 *(ring->wptr_cpu_addr) = 0;
1352                 *(ring->rptr_cpu_addr) = 0;
1353                 amdgpu_ring_clear_ring(ring);
1354         }
1355
1356         r = mes_v12_0_mqd_init(ring);
1357         if (r)
1358                 return r;
1359
1360         if (pipe == AMDGPU_MES_SCHED_PIPE) {
1361                 if (adev->enable_uni_mes)
1362                         r = amdgpu_mes_map_legacy_queue(adev, ring);
1363                 else
1364                         r = mes_v12_0_kiq_enable_queue(adev);
1365                 if (r)
1366                         return r;
1367         } else {
1368                 mes_v12_0_queue_init_register(ring);
1369         }
1370
1371         /* get MES scheduler/KIQ versions */
1372         mutex_lock(&adev->srbm_mutex);
1373         soc21_grbm_select(adev, 3, pipe, 0, 0);
1374
1375         if (pipe == AMDGPU_MES_SCHED_PIPE)
1376                 adev->mes.sched_version = RREG32_SOC15(GC, 0, regCP_MES_GP3_LO);
1377         else if (pipe == AMDGPU_MES_KIQ_PIPE && adev->enable_mes_kiq)
1378                 adev->mes.kiq_version = RREG32_SOC15(GC, 0, regCP_MES_GP3_LO);
1379
1380         soc21_grbm_select(adev, 0, 0, 0, 0);
1381         mutex_unlock(&adev->srbm_mutex);
1382
1383         return 0;
1384 }
1385
1386 static int mes_v12_0_ring_init(struct amdgpu_device *adev, int pipe)
1387 {
1388         struct amdgpu_ring *ring;
1389
1390         ring = &adev->mes.ring[pipe];
1391
1392         ring->funcs = &mes_v12_0_ring_funcs;
1393
1394         ring->me = 3;
1395         ring->pipe = pipe;
1396         ring->queue = 0;
1397
1398         ring->ring_obj = NULL;
1399         ring->use_doorbell = true;
1400         ring->eop_gpu_addr = adev->mes.eop_gpu_addr[pipe];
1401         ring->no_scheduler = true;
1402         sprintf(ring->name, "mes_%d.%d.%d", ring->me, ring->pipe, ring->queue);
1403
1404         if (pipe == AMDGPU_MES_SCHED_PIPE)
1405                 ring->doorbell_index = adev->doorbell_index.mes_ring0 << 1;
1406         else
1407                 ring->doorbell_index = adev->doorbell_index.mes_ring1 << 1;
1408
1409         return amdgpu_ring_init(adev, ring, 1024, NULL, 0,
1410                                 AMDGPU_RING_PRIO_DEFAULT, NULL);
1411 }
1412
1413 static int mes_v12_0_kiq_ring_init(struct amdgpu_device *adev)
1414 {
1415         struct amdgpu_ring *ring;
1416
1417         spin_lock_init(&adev->gfx.kiq[0].ring_lock);
1418
1419         ring = &adev->gfx.kiq[0].ring;
1420
1421         ring->me = 3;
1422         ring->pipe = 1;
1423         ring->queue = 0;
1424
1425         ring->adev = NULL;
1426         ring->ring_obj = NULL;
1427         ring->use_doorbell = true;
1428         ring->doorbell_index = adev->doorbell_index.mes_ring1 << 1;
1429         ring->eop_gpu_addr = adev->mes.eop_gpu_addr[AMDGPU_MES_KIQ_PIPE];
1430         ring->no_scheduler = true;
1431         sprintf(ring->name, "mes_kiq_%d.%d.%d",
1432                 ring->me, ring->pipe, ring->queue);
1433
1434         return amdgpu_ring_init(adev, ring, 1024, NULL, 0,
1435                                 AMDGPU_RING_PRIO_DEFAULT, NULL);
1436 }
1437
1438 static int mes_v12_0_mqd_sw_init(struct amdgpu_device *adev,
1439                                  enum admgpu_mes_pipe pipe)
1440 {
1441         int r, mqd_size = sizeof(struct v12_compute_mqd);
1442         struct amdgpu_ring *ring;
1443
1444         if (!adev->enable_uni_mes && pipe == AMDGPU_MES_KIQ_PIPE)
1445                 ring = &adev->gfx.kiq[0].ring;
1446         else
1447                 ring = &adev->mes.ring[pipe];
1448
1449         if (ring->mqd_obj)
1450                 return 0;
1451
1452         r = amdgpu_bo_create_kernel(adev, mqd_size, PAGE_SIZE,
1453                                     AMDGPU_GEM_DOMAIN_GTT, &ring->mqd_obj,
1454                                     &ring->mqd_gpu_addr, &ring->mqd_ptr);
1455         if (r) {
1456                 dev_warn(adev->dev, "failed to create ring mqd bo (%d)", r);
1457                 return r;
1458         }
1459
1460         memset(ring->mqd_ptr, 0, mqd_size);
1461
1462         /* prepare MQD backup */
1463         adev->mes.mqd_backup[pipe] = kmalloc(mqd_size, GFP_KERNEL);
1464         if (!adev->mes.mqd_backup[pipe])
1465                 dev_warn(adev->dev,
1466                          "no memory to create MQD backup for ring %s\n",
1467                          ring->name);
1468
1469         return 0;
1470 }
1471
1472 static int mes_v12_0_sw_init(struct amdgpu_ip_block *ip_block)
1473 {
1474         struct amdgpu_device *adev = ip_block->adev;
1475         int pipe, r;
1476
1477         adev->mes.funcs = &mes_v12_0_funcs;
1478         adev->mes.kiq_hw_init = &mes_v12_0_kiq_hw_init;
1479         adev->mes.kiq_hw_fini = &mes_v12_0_kiq_hw_fini;
1480         adev->mes.enable_legacy_queue_map = true;
1481
1482         adev->mes.event_log_size = adev->enable_uni_mes ? (AMDGPU_MAX_MES_PIPES * AMDGPU_MES_LOG_BUFFER_SIZE) : AMDGPU_MES_LOG_BUFFER_SIZE;
1483
1484         r = amdgpu_mes_init(adev);
1485         if (r)
1486                 return r;
1487
1488         for (pipe = 0; pipe < AMDGPU_MAX_MES_PIPES; pipe++) {
1489                 r = mes_v12_0_allocate_eop_buf(adev, pipe);
1490                 if (r)
1491                         return r;
1492
1493                 r = mes_v12_0_mqd_sw_init(adev, pipe);
1494                 if (r)
1495                         return r;
1496
1497                 if (!adev->enable_uni_mes && pipe == AMDGPU_MES_KIQ_PIPE)
1498                         r = mes_v12_0_kiq_ring_init(adev);
1499                 else
1500                         r = mes_v12_0_ring_init(adev, pipe);
1501                 if (r)
1502                         return r;
1503         }
1504
1505         return 0;
1506 }
1507
1508 static int mes_v12_0_sw_fini(struct amdgpu_ip_block *ip_block)
1509 {
1510         struct amdgpu_device *adev = ip_block->adev;
1511         int pipe;
1512
1513         for (pipe = 0; pipe < AMDGPU_MAX_MES_PIPES; pipe++) {
1514                 kfree(adev->mes.mqd_backup[pipe]);
1515
1516                 amdgpu_bo_free_kernel(&adev->mes.eop_gpu_obj[pipe],
1517                                       &adev->mes.eop_gpu_addr[pipe],
1518                                       NULL);
1519                 amdgpu_ucode_release(&adev->mes.fw[pipe]);
1520
1521                 if (adev->enable_uni_mes || pipe == AMDGPU_MES_SCHED_PIPE) {
1522                         amdgpu_bo_free_kernel(&adev->mes.ring[pipe].mqd_obj,
1523                                               &adev->mes.ring[pipe].mqd_gpu_addr,
1524                                               &adev->mes.ring[pipe].mqd_ptr);
1525                         amdgpu_ring_fini(&adev->mes.ring[pipe]);
1526                 }
1527         }
1528
1529         if (!adev->enable_uni_mes) {
1530                 amdgpu_bo_free_kernel(&adev->gfx.kiq[0].ring.mqd_obj,
1531                                       &adev->gfx.kiq[0].ring.mqd_gpu_addr,
1532                                       &adev->gfx.kiq[0].ring.mqd_ptr);
1533                 amdgpu_ring_fini(&adev->gfx.kiq[0].ring);
1534         }
1535
1536         if (adev->firmware.load_type == AMDGPU_FW_LOAD_DIRECT) {
1537                 mes_v12_0_free_ucode_buffers(adev, AMDGPU_MES_KIQ_PIPE);
1538                 mes_v12_0_free_ucode_buffers(adev, AMDGPU_MES_SCHED_PIPE);
1539         }
1540
1541         amdgpu_mes_fini(adev);
1542         return 0;
1543 }
1544
1545 static void mes_v12_0_kiq_dequeue_sched(struct amdgpu_device *adev)
1546 {
1547         uint32_t data;
1548         int i;
1549
1550         mutex_lock(&adev->srbm_mutex);
1551         soc21_grbm_select(adev, 3, AMDGPU_MES_SCHED_PIPE, 0, 0);
1552
1553         /* disable the queue if it's active */
1554         if (RREG32_SOC15(GC, 0, regCP_HQD_ACTIVE) & 1) {
1555                 WREG32_SOC15(GC, 0, regCP_HQD_DEQUEUE_REQUEST, 1);
1556                 for (i = 0; i < adev->usec_timeout; i++) {
1557                         if (!(RREG32_SOC15(GC, 0, regCP_HQD_ACTIVE) & 1))
1558                                 break;
1559                         udelay(1);
1560                 }
1561         }
1562         data = RREG32_SOC15(GC, 0, regCP_HQD_PQ_DOORBELL_CONTROL);
1563         data = REG_SET_FIELD(data, CP_HQD_PQ_DOORBELL_CONTROL,
1564                                 DOORBELL_EN, 0);
1565         data = REG_SET_FIELD(data, CP_HQD_PQ_DOORBELL_CONTROL,
1566                                 DOORBELL_HIT, 1);
1567         WREG32_SOC15(GC, 0, regCP_HQD_PQ_DOORBELL_CONTROL, data);
1568
1569         WREG32_SOC15(GC, 0, regCP_HQD_PQ_DOORBELL_CONTROL, 0);
1570
1571         WREG32_SOC15(GC, 0, regCP_HQD_PQ_WPTR_LO, 0);
1572         WREG32_SOC15(GC, 0, regCP_HQD_PQ_WPTR_HI, 0);
1573         WREG32_SOC15(GC, 0, regCP_HQD_PQ_RPTR, 0);
1574
1575         soc21_grbm_select(adev, 0, 0, 0, 0);
1576         mutex_unlock(&adev->srbm_mutex);
1577
1578         adev->mes.ring[0].sched.ready = false;
1579 }
1580
1581 static void mes_v12_0_kiq_setting(struct amdgpu_ring *ring)
1582 {
1583         uint32_t tmp;
1584         struct amdgpu_device *adev = ring->adev;
1585
1586         /* tell RLC which is KIQ queue */
1587         tmp = RREG32_SOC15(GC, 0, regRLC_CP_SCHEDULERS);
1588         tmp &= 0xffffff00;
1589         tmp |= (ring->me << 5) | (ring->pipe << 3) | (ring->queue);
1590         WREG32_SOC15(GC, 0, regRLC_CP_SCHEDULERS, tmp | 0x80);
1591 }
1592
1593 static int mes_v12_0_kiq_hw_init(struct amdgpu_device *adev)
1594 {
1595         int r = 0;
1596         struct amdgpu_ip_block *ip_block;
1597
1598         if (adev->enable_uni_mes)
1599                 mes_v12_0_kiq_setting(&adev->mes.ring[AMDGPU_MES_KIQ_PIPE]);
1600         else
1601                 mes_v12_0_kiq_setting(&adev->gfx.kiq[0].ring);
1602
1603         if (adev->firmware.load_type == AMDGPU_FW_LOAD_DIRECT) {
1604
1605                 r = mes_v12_0_load_microcode(adev, AMDGPU_MES_SCHED_PIPE, false);
1606                 if (r) {
1607                         DRM_ERROR("failed to load MES fw, r=%d\n", r);
1608                         return r;
1609                 }
1610
1611                 r = mes_v12_0_load_microcode(adev, AMDGPU_MES_KIQ_PIPE, true);
1612                 if (r) {
1613                         DRM_ERROR("failed to load MES kiq fw, r=%d\n", r);
1614                         return r;
1615                 }
1616
1617                 mes_v12_0_set_ucode_start_addr(adev);
1618
1619         } else if (adev->firmware.load_type == AMDGPU_FW_LOAD_RLC_BACKDOOR_AUTO)
1620                 mes_v12_0_set_ucode_start_addr(adev);
1621
1622         mes_v12_0_enable(adev, true);
1623
1624         ip_block = amdgpu_device_ip_get_ip_block(adev, AMD_IP_BLOCK_TYPE_MES);
1625         if (unlikely(!ip_block)) {
1626                 dev_err(adev->dev, "Failed to get MES handle\n");
1627                 return -EINVAL;
1628         }
1629
1630         r = mes_v12_0_queue_init(adev, AMDGPU_MES_KIQ_PIPE);
1631         if (r)
1632                 goto failure;
1633
1634         if (adev->enable_uni_mes) {
1635                 r = mes_v12_0_set_hw_resources(&adev->mes, AMDGPU_MES_KIQ_PIPE);
1636                 if (r)
1637                         goto failure;
1638
1639                 mes_v12_0_set_hw_resources_1(&adev->mes, AMDGPU_MES_KIQ_PIPE);
1640         }
1641
1642         if (adev->mes.enable_legacy_queue_map) {
1643                 r = mes_v12_0_hw_init(ip_block);
1644                 if (r)
1645                         goto failure;
1646         }
1647
1648         return r;
1649
1650 failure:
1651         mes_v12_0_hw_fini(ip_block);
1652         return r;
1653 }
1654
1655 static int mes_v12_0_kiq_hw_fini(struct amdgpu_device *adev)
1656 {
1657         if (adev->mes.ring[0].sched.ready) {
1658                 if (adev->enable_uni_mes)
1659                         amdgpu_mes_unmap_legacy_queue(adev,
1660                                       &adev->mes.ring[AMDGPU_MES_SCHED_PIPE],
1661                                       RESET_QUEUES, 0, 0);
1662                 else
1663                         mes_v12_0_kiq_dequeue_sched(adev);
1664
1665                 adev->mes.ring[0].sched.ready = false;
1666         }
1667
1668         mes_v12_0_enable(adev, false);
1669
1670         return 0;
1671 }
1672
1673 static int mes_v12_0_hw_init(struct amdgpu_ip_block *ip_block)
1674 {
1675         int r;
1676         struct amdgpu_device *adev = ip_block->adev;
1677
1678         if (adev->mes.ring[0].sched.ready)
1679                 goto out;
1680
1681         if (!adev->enable_mes_kiq) {
1682                 if (adev->firmware.load_type == AMDGPU_FW_LOAD_DIRECT) {
1683                         r = mes_v12_0_load_microcode(adev,
1684                                              AMDGPU_MES_SCHED_PIPE, true);
1685                         if (r) {
1686                                 DRM_ERROR("failed to MES fw, r=%d\n", r);
1687                                 return r;
1688                         }
1689
1690                         mes_v12_0_set_ucode_start_addr(adev);
1691
1692                 } else if (adev->firmware.load_type ==
1693                            AMDGPU_FW_LOAD_RLC_BACKDOOR_AUTO) {
1694
1695                         mes_v12_0_set_ucode_start_addr(adev);
1696                 }
1697
1698                 mes_v12_0_enable(adev, true);
1699         }
1700
1701         /* Enable the MES to handle doorbell ring on unmapped queue */
1702         mes_v12_0_enable_unmapped_doorbell_handling(&adev->mes, true);
1703
1704         r = mes_v12_0_queue_init(adev, AMDGPU_MES_SCHED_PIPE);
1705         if (r)
1706                 goto failure;
1707
1708         r = mes_v12_0_set_hw_resources(&adev->mes, AMDGPU_MES_SCHED_PIPE);
1709         if (r)
1710                 goto failure;
1711
1712         if (adev->enable_uni_mes)
1713                 mes_v12_0_set_hw_resources_1(&adev->mes, AMDGPU_MES_SCHED_PIPE);
1714
1715         mes_v12_0_init_aggregated_doorbell(&adev->mes);
1716
1717         r = mes_v12_0_query_sched_status(&adev->mes, AMDGPU_MES_SCHED_PIPE);
1718         if (r) {
1719                 DRM_ERROR("MES is busy\n");
1720                 goto failure;
1721         }
1722
1723 out:
1724         /*
1725          * Disable KIQ ring usage from the driver once MES is enabled.
1726          * MES uses KIQ ring exclusively so driver cannot access KIQ ring
1727          * with MES enabled.
1728          */
1729         adev->gfx.kiq[0].ring.sched.ready = false;
1730         adev->mes.ring[0].sched.ready = true;
1731
1732         return 0;
1733
1734 failure:
1735         mes_v12_0_hw_fini(ip_block);
1736         return r;
1737 }
1738
1739 static int mes_v12_0_hw_fini(struct amdgpu_ip_block *ip_block)
1740 {
1741         return 0;
1742 }
1743
1744 static int mes_v12_0_suspend(struct amdgpu_ip_block *ip_block)
1745 {
1746         int r;
1747
1748         r = amdgpu_mes_suspend(ip_block->adev);
1749         if (r)
1750                 return r;
1751
1752         return mes_v12_0_hw_fini(ip_block);
1753 }
1754
1755 static int mes_v12_0_resume(struct amdgpu_ip_block *ip_block)
1756 {
1757         int r;
1758
1759         r = mes_v12_0_hw_init(ip_block);
1760         if (r)
1761                 return r;
1762
1763         return amdgpu_mes_resume(ip_block->adev);
1764 }
1765
1766 static int mes_v12_0_early_init(struct amdgpu_ip_block *ip_block)
1767 {
1768         struct amdgpu_device *adev = ip_block->adev;
1769         int pipe, r;
1770
1771         for (pipe = 0; pipe < AMDGPU_MAX_MES_PIPES; pipe++) {
1772                 r = amdgpu_mes_init_microcode(adev, pipe);
1773                 if (r)
1774                         return r;
1775         }
1776
1777         return 0;
1778 }
1779
1780 static int mes_v12_0_late_init(struct amdgpu_ip_block *ip_block)
1781 {
1782         struct amdgpu_device *adev = ip_block->adev;
1783
1784         /* it's only intended for use in mes_self_test case, not for s0ix and reset */
1785         if (!amdgpu_in_reset(adev) && !adev->in_s0ix && !adev->in_suspend)
1786                 amdgpu_mes_self_test(adev);
1787
1788         return 0;
1789 }
1790
1791 static const struct amd_ip_funcs mes_v12_0_ip_funcs = {
1792         .name = "mes_v12_0",
1793         .early_init = mes_v12_0_early_init,
1794         .late_init = mes_v12_0_late_init,
1795         .sw_init = mes_v12_0_sw_init,
1796         .sw_fini = mes_v12_0_sw_fini,
1797         .hw_init = mes_v12_0_hw_init,
1798         .hw_fini = mes_v12_0_hw_fini,
1799         .suspend = mes_v12_0_suspend,
1800         .resume = mes_v12_0_resume,
1801 };
1802
1803 const struct amdgpu_ip_block_version mes_v12_0_ip_block = {
1804         .type = AMD_IP_BLOCK_TYPE_MES,
1805         .major = 12,
1806         .minor = 0,
1807         .rev = 0,
1808         .funcs = &mes_v12_0_ip_funcs,
1809 };