drivers/gpu/drm/amd/amdgpu/mes_v12_0.c

   1 /*
   2  * Copyright 2023 Advanced Micro Devices, Inc.
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice shall be included in
  12  * all copies or substantial portions of the Software.
  13  *
  14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  20  * OTHER DEALINGS IN THE SOFTWARE.
  21  *
  22  */
  23
  24 #include <linux/firmware.h>
  25 #include <linux/module.h>
  26 #include "amdgpu.h"
  27 #include "gfx_v12_0.h"
  28 #include "soc15_common.h"
  29 #include "soc21.h"
  30 #include "gc/gc_12_0_0_offset.h"
  31 #include "gc/gc_12_0_0_sh_mask.h"
  32 #include "gc/gc_11_0_0_default.h"
  33 #include "v12_structs.h"
  34 #include "mes_v12_api_def.h"
  35
  36 MODULE_FIRMWARE("amdgpu/gc_12_0_0_mes.bin");
  37 MODULE_FIRMWARE("amdgpu/gc_12_0_0_mes1.bin");
  38 MODULE_FIRMWARE("amdgpu/gc_12_0_0_uni_mes.bin");
  39 MODULE_FIRMWARE("amdgpu/gc_12_0_1_mes.bin");
  40 MODULE_FIRMWARE("amdgpu/gc_12_0_1_mes1.bin");
  41 MODULE_FIRMWARE("amdgpu/gc_12_0_1_uni_mes.bin");
  42
  43 static int mes_v12_0_hw_init(struct amdgpu_ip_block *ip_block);
  44 static int mes_v12_0_hw_fini(struct amdgpu_ip_block *ip_block);
  45 static int mes_v12_0_kiq_hw_init(struct amdgpu_device *adev);
  46 static int mes_v12_0_kiq_hw_fini(struct amdgpu_device *adev);
  47
  48 #define MES_EOP_SIZE   2048
  49
  50 static void mes_v12_0_ring_set_wptr(struct amdgpu_ring *ring)
  51 {
  52         struct amdgpu_device *adev = ring->adev;
  53
  54         if (ring->use_doorbell) {
  55                 atomic64_set((atomic64_t *)ring->wptr_cpu_addr,
  56                              ring->wptr);
  57                 WDOORBELL64(ring->doorbell_index, ring->wptr);
  58         } else {
  59                 BUG();
  60         }
  61 }
  62
  63 static u64 mes_v12_0_ring_get_rptr(struct amdgpu_ring *ring)
  64 {
  65         return *ring->rptr_cpu_addr;
  66 }
  67
  68 static u64 mes_v12_0_ring_get_wptr(struct amdgpu_ring *ring)
  69 {
  70         u64 wptr;
  71
  72         if (ring->use_doorbell)
  73                 wptr = atomic64_read((atomic64_t *)ring->wptr_cpu_addr);
  74         else
  75                 BUG();
  76         return wptr;
  77 }
  78
  79 static const struct amdgpu_ring_funcs mes_v12_0_ring_funcs = {
  80         .type = AMDGPU_RING_TYPE_MES,
  81         .align_mask = 1,
  82         .nop = 0,
  83         .support_64bit_ptrs = true,
  84         .get_rptr = mes_v12_0_ring_get_rptr,
  85         .get_wptr = mes_v12_0_ring_get_wptr,
  86         .set_wptr = mes_v12_0_ring_set_wptr,
  87         .insert_nop = amdgpu_ring_insert_nop,
  88 };
  89
  90 static const char *mes_v12_0_opcodes[] = {
  91         "SET_HW_RSRC",
  92         "SET_SCHEDULING_CONFIG",
  93         "ADD_QUEUE",
  94         "REMOVE_QUEUE",
  95         "PERFORM_YIELD",
  96         "SET_GANG_PRIORITY_LEVEL",
  97         "SUSPEND",
  98         "RESUME",
  99         "RESET",
 100         "SET_LOG_BUFFER",
 101         "CHANGE_GANG_PRORITY",
 102         "QUERY_SCHEDULER_STATUS",
 103         "unused",
 104         "SET_DEBUG_VMID",
 105         "MISC",
 106         "UPDATE_ROOT_PAGE_TABLE",
 107         "AMD_LOG",
 108         "SET_SE_MODE",
 109         "SET_GANG_SUBMIT",
 110         "SET_HW_RSRC_1",
 111 };
 112
 113 static const char *mes_v12_0_misc_opcodes[] = {
 114         "WRITE_REG",
 115         "INV_GART",
 116         "QUERY_STATUS",
 117         "READ_REG",
 118         "WAIT_REG_MEM",
 119         "SET_SHADER_DEBUGGER",
 120         "NOTIFY_WORK_ON_UNMAPPED_QUEUE",
 121         "NOTIFY_TO_UNMAP_PROCESSES",
 122 };
 123
 124 static const char *mes_v12_0_get_op_string(union MESAPI__MISC *x_pkt)
 125 {
 126         const char *op_str = NULL;
 127
 128         if (x_pkt->header.opcode < ARRAY_SIZE(mes_v12_0_opcodes))
 129                 op_str = mes_v12_0_opcodes[x_pkt->header.opcode];
 130
 131         return op_str;
 132 }
 133
 134 static const char *mes_v12_0_get_misc_op_string(union MESAPI__MISC *x_pkt)
 135 {
 136         const char *op_str = NULL;
 137
 138         if ((x_pkt->header.opcode == MES_SCH_API_MISC) &&
 139             (x_pkt->opcode < ARRAY_SIZE(mes_v12_0_misc_opcodes)))
 140                 op_str = mes_v12_0_misc_opcodes[x_pkt->opcode];
 141
 142         return op_str;
 143 }
 144
 145 static int mes_v12_0_submit_pkt_and_poll_completion(struct amdgpu_mes *mes,
 146                                             int pipe, void *pkt, int size,
 147                                             int api_status_off)
 148 {
 149         union MESAPI__QUERY_MES_STATUS mes_status_pkt;
 150         signed long timeout = 2100000; /* 2100 ms */
 151         struct amdgpu_device *adev = mes->adev;
 152         struct amdgpu_ring *ring = &mes->ring[pipe];
 153         spinlock_t *ring_lock = &mes->ring_lock[pipe];
 154         struct MES_API_STATUS *api_status;
 155         union MESAPI__MISC *x_pkt = pkt;
 156         const char *op_str, *misc_op_str;
 157         unsigned long flags;
 158         u64 status_gpu_addr;
 159         u32 seq, status_offset;
 160         u64 *status_ptr;
 161         signed long r;
 162         int ret;
 163
 164         if (x_pkt->header.opcode >= MES_SCH_API_MAX)
 165                 return -EINVAL;
 166
 167         if (amdgpu_emu_mode) {
 168                 timeout *= 100;
 169         } else if (amdgpu_sriov_vf(adev)) {
 170                 /* Worst case in sriov where all other 15 VF timeout, each VF needs about 600ms */
 171                 timeout = 15 * 600 * 1000;
 172         }
 173
 174         ret = amdgpu_device_wb_get(adev, &status_offset);
 175         if (ret)
 176                 return ret;
 177
 178         status_gpu_addr = adev->wb.gpu_addr + (status_offset * 4);
 179         status_ptr = (u64 *)&adev->wb.wb[status_offset];
 180         *status_ptr = 0;
 181
 182         spin_lock_irqsave(ring_lock, flags);
 183         r = amdgpu_ring_alloc(ring, (size + sizeof(mes_status_pkt)) / 4);
 184         if (r)
 185                 goto error_unlock_free;
 186
 187         seq = ++ring->fence_drv.sync_seq;
 188         r = amdgpu_fence_wait_polling(ring,
 189                                       seq - ring->fence_drv.num_fences_mask,
 190                                       timeout);
 191         if (r < 1)
 192                 goto error_undo;
 193
 194         api_status = (struct MES_API_STATUS *)((char *)pkt + api_status_off);
 195         api_status->api_completion_fence_addr = status_gpu_addr;
 196         api_status->api_completion_fence_value = 1;
 197
 198         amdgpu_ring_write_multiple(ring, pkt, size / 4);
 199
 200         memset(&mes_status_pkt, 0, sizeof(mes_status_pkt));
 201         mes_status_pkt.header.type = MES_API_TYPE_SCHEDULER;
 202         mes_status_pkt.header.opcode = MES_SCH_API_QUERY_SCHEDULER_STATUS;
 203         mes_status_pkt.header.dwsize = API_FRAME_SIZE_IN_DWORDS;
 204         mes_status_pkt.api_status.api_completion_fence_addr =
 205                 ring->fence_drv.gpu_addr;
 206         mes_status_pkt.api_status.api_completion_fence_value = seq;
 207
 208         amdgpu_ring_write_multiple(ring, &mes_status_pkt,
 209                                    sizeof(mes_status_pkt) / 4);
 210
 211         amdgpu_ring_commit(ring);
 212         spin_unlock_irqrestore(ring_lock, flags);
 213
 214         op_str = mes_v12_0_get_op_string(x_pkt);
 215         misc_op_str = mes_v12_0_get_misc_op_string(x_pkt);
 216
 217         if (misc_op_str)
 218                 dev_dbg(adev->dev, "MES(%d) msg=%s (%s) was emitted\n",
 219                         pipe, op_str, misc_op_str);
 220         else if (op_str)
 221                 dev_dbg(adev->dev, "MES(%d) msg=%s was emitted\n",
 222                         pipe, op_str);
 223         else
 224                 dev_dbg(adev->dev, "MES(%d) msg=%d was emitted\n",
 225                         pipe, x_pkt->header.opcode);
 226
 227         r = amdgpu_fence_wait_polling(ring, seq, timeout);
 228         if (r < 1 || !*status_ptr) {
 229
 230                 if (misc_op_str)
 231                         dev_err(adev->dev, "MES(%d) failed to respond to msg=%s (%s)\n",
 232                                 pipe, op_str, misc_op_str);
 233                 else if (op_str)
 234                         dev_err(adev->dev, "MES(%d) failed to respond to msg=%s\n",
 235                                 pipe, op_str);
 236                 else
 237                         dev_err(adev->dev, "MES(%d) failed to respond to msg=%d\n",
 238                                 pipe, x_pkt->header.opcode);
 239
 240                 while (halt_if_hws_hang)
 241                         schedule();
 242
 243                 r = -ETIMEDOUT;
 244                 goto error_wb_free;
 245         }
 246
 247         amdgpu_device_wb_free(adev, status_offset);
 248         return 0;
 249
 250 error_undo:
 251         dev_err(adev->dev, "MES ring buffer is full.\n");
 252         amdgpu_ring_undo(ring);
 253
 254 error_unlock_free:
 255         spin_unlock_irqrestore(ring_lock, flags);
 256
 257 error_wb_free:
 258         amdgpu_device_wb_free(adev, status_offset);
 259         return r;
 260 }
 261
 262 static int convert_to_mes_queue_type(int queue_type)
 263 {
 264         if (queue_type == AMDGPU_RING_TYPE_GFX)
 265                 return MES_QUEUE_TYPE_GFX;
 266         else if (queue_type == AMDGPU_RING_TYPE_COMPUTE)
 267                 return MES_QUEUE_TYPE_COMPUTE;
 268         else if (queue_type == AMDGPU_RING_TYPE_SDMA)
 269                 return MES_QUEUE_TYPE_SDMA;
 270         else if (queue_type == AMDGPU_RING_TYPE_MES)
 271                 return MES_QUEUE_TYPE_SCHQ;
 272         else
 273                 BUG();
 274         return -1;
 275 }
 276
 277 static int mes_v12_0_add_hw_queue(struct amdgpu_mes *mes,
 278                                   struct mes_add_queue_input *input)
 279 {
 280         struct amdgpu_device *adev = mes->adev;
 281         union MESAPI__ADD_QUEUE mes_add_queue_pkt;
 282         struct amdgpu_vmhub *hub = &adev->vmhub[AMDGPU_GFXHUB(0)];
 283         uint32_t vm_cntx_cntl = hub->vm_cntx_cntl;
 284
 285         memset(&mes_add_queue_pkt, 0, sizeof(mes_add_queue_pkt));
 286
 287         mes_add_queue_pkt.header.type = MES_API_TYPE_SCHEDULER;
 288         mes_add_queue_pkt.header.opcode = MES_SCH_API_ADD_QUEUE;
 289         mes_add_queue_pkt.header.dwsize = API_FRAME_SIZE_IN_DWORDS;
 290
 291         mes_add_queue_pkt.process_id = input->process_id;
 292         mes_add_queue_pkt.page_table_base_addr = input->page_table_base_addr;
 293         mes_add_queue_pkt.process_va_start = input->process_va_start;
 294         mes_add_queue_pkt.process_va_end = input->process_va_end;
 295         mes_add_queue_pkt.process_quantum = input->process_quantum;
 296         mes_add_queue_pkt.process_context_addr = input->process_context_addr;
 297         mes_add_queue_pkt.gang_quantum = input->gang_quantum;
 298         mes_add_queue_pkt.gang_context_addr = input->gang_context_addr;
 299         mes_add_queue_pkt.inprocess_gang_priority =
 300                 input->inprocess_gang_priority;
 301         mes_add_queue_pkt.gang_global_priority_level =
 302                 input->gang_global_priority_level;
 303         mes_add_queue_pkt.doorbell_offset = input->doorbell_offset;
 304         mes_add_queue_pkt.mqd_addr = input->mqd_addr;
 305
 306         mes_add_queue_pkt.wptr_addr = input->wptr_mc_addr;
 307
 308         mes_add_queue_pkt.queue_type =
 309                 convert_to_mes_queue_type(input->queue_type);
 310         mes_add_queue_pkt.paging = input->paging;
 311         mes_add_queue_pkt.vm_context_cntl = vm_cntx_cntl;
 312         mes_add_queue_pkt.gws_base = input->gws_base;
 313         mes_add_queue_pkt.gws_size = input->gws_size;
 314         mes_add_queue_pkt.trap_handler_addr = input->tba_addr;
 315         mes_add_queue_pkt.tma_addr = input->tma_addr;
 316         mes_add_queue_pkt.trap_en = input->trap_en;
 317         mes_add_queue_pkt.skip_process_ctx_clear = input->skip_process_ctx_clear;
 318         mes_add_queue_pkt.is_kfd_process = input->is_kfd_process;
 319
 320         /* For KFD, gds_size is re-used for queue size (needed in MES for AQL queues) */
 321         mes_add_queue_pkt.is_aql_queue = input->is_aql_queue;
 322         mes_add_queue_pkt.gds_size = input->queue_size;
 323
 324         /* For KFD, gds_size is re-used for queue size (needed in MES for AQL queues) */
 325         mes_add_queue_pkt.is_aql_queue = input->is_aql_queue;
 326         mes_add_queue_pkt.gds_size = input->queue_size;
 327
 328         return mes_v12_0_submit_pkt_and_poll_completion(mes,
 329                         AMDGPU_MES_SCHED_PIPE,
 330                         &mes_add_queue_pkt, sizeof(mes_add_queue_pkt),
 331                         offsetof(union MESAPI__ADD_QUEUE, api_status));
 332 }
 333
 334 static int mes_v12_0_remove_hw_queue(struct amdgpu_mes *mes,
 335                                      struct mes_remove_queue_input *input)
 336 {
 337         union MESAPI__REMOVE_QUEUE mes_remove_queue_pkt;
 338
 339         memset(&mes_remove_queue_pkt, 0, sizeof(mes_remove_queue_pkt));
 340
 341         mes_remove_queue_pkt.header.type = MES_API_TYPE_SCHEDULER;
 342         mes_remove_queue_pkt.header.opcode = MES_SCH_API_REMOVE_QUEUE;
 343         mes_remove_queue_pkt.header.dwsize = API_FRAME_SIZE_IN_DWORDS;
 344
 345         mes_remove_queue_pkt.doorbell_offset = input->doorbell_offset;
 346         mes_remove_queue_pkt.gang_context_addr = input->gang_context_addr;
 347
 348         return mes_v12_0_submit_pkt_and_poll_completion(mes,
 349                         AMDGPU_MES_SCHED_PIPE,
 350                         &mes_remove_queue_pkt, sizeof(mes_remove_queue_pkt),
 351                         offsetof(union MESAPI__REMOVE_QUEUE, api_status));
 352 }
 353
 354 int gfx_v12_0_request_gfx_index_mutex(struct amdgpu_device *adev,
 355                                       bool req)
 356 {
 357         u32 i, tmp, val;
 358
 359         for (i = 0; i < adev->usec_timeout; i++) {
 360                 /* Request with MeId=2, PipeId=0 */
 361                 tmp = REG_SET_FIELD(0, CP_GFX_INDEX_MUTEX, REQUEST, req);
 362                 tmp = REG_SET_FIELD(tmp, CP_GFX_INDEX_MUTEX, CLIENTID, 4);
 363                 WREG32_SOC15(GC, 0, regCP_GFX_INDEX_MUTEX, tmp);
 364
 365                 val = RREG32_SOC15(GC, 0, regCP_GFX_INDEX_MUTEX);
 366                 if (req) {
 367                         if (val == tmp)
 368                                 break;
 369                 } else {
 370                         tmp = REG_SET_FIELD(tmp, CP_GFX_INDEX_MUTEX,
 371                                             REQUEST, 1);
 372
 373                         /* unlocked or locked by firmware */
 374                         if (val != tmp)
 375                                 break;
 376                 }
 377                 udelay(1);
 378         }
 379
 380         if (i >= adev->usec_timeout)
 381                 return -EINVAL;
 382
 383         return 0;
 384 }
 385
 386 static int mes_v12_0_reset_queue_mmio(struct amdgpu_mes *mes, uint32_t queue_type,
 387                                       uint32_t me_id, uint32_t pipe_id,
 388                                       uint32_t queue_id, uint32_t vmid)
 389 {
 390         struct amdgpu_device *adev = mes->adev;
 391         uint32_t value, reg;
 392         int i, r = 0;
 393
 394         amdgpu_gfx_rlc_enter_safe_mode(adev, 0);
 395
 396         if (queue_type == AMDGPU_RING_TYPE_GFX) {
 397                 dev_info(adev->dev, "reset gfx queue (%d:%d:%d: vmid:%d)\n",
 398                          me_id, pipe_id, queue_id, vmid);
 399
 400                 mutex_lock(&adev->gfx.reset_sem_mutex);
 401                 gfx_v12_0_request_gfx_index_mutex(adev, true);
 402                 /* all se allow writes */
 403                 WREG32_SOC15(GC, 0, regGRBM_GFX_INDEX,
 404                              (uint32_t)(0x1 << GRBM_GFX_INDEX__SE_BROADCAST_WRITES__SHIFT));
 405                 value = REG_SET_FIELD(0, CP_VMID_RESET, RESET_REQUEST, 1 << vmid);
 406                 if (pipe_id == 0)
 407                         value = REG_SET_FIELD(value, CP_VMID_RESET, PIPE0_QUEUES, 1 << queue_id);
 408                 else
 409                         value = REG_SET_FIELD(value, CP_VMID_RESET, PIPE1_QUEUES, 1 << queue_id);
 410                 WREG32_SOC15(GC, 0, regCP_VMID_RESET, value);
 411                 gfx_v12_0_request_gfx_index_mutex(adev, false);
 412                 mutex_unlock(&adev->gfx.reset_sem_mutex);
 413
 414                 mutex_lock(&adev->srbm_mutex);
 415                 soc21_grbm_select(adev, me_id, pipe_id, queue_id, 0);
 416                 /* wait till dequeue take effects */
 417                 for (i = 0; i < adev->usec_timeout; i++) {
 418                         if (!(RREG32_SOC15(GC, 0, regCP_GFX_HQD_ACTIVE) & 1))
 419                                 break;
 420                         udelay(1);
 421                 }
 422                 if (i >= adev->usec_timeout) {
 423                         dev_err(adev->dev, "failed to wait on gfx hqd deactivate\n");
 424                         r = -ETIMEDOUT;
 425                 }
 426
 427                 soc21_grbm_select(adev, 0, 0, 0, 0);
 428                 mutex_unlock(&adev->srbm_mutex);
 429         } else if (queue_type == AMDGPU_RING_TYPE_COMPUTE) {
 430                 dev_info(adev->dev, "reset compute queue (%d:%d:%d)\n",
 431                          me_id, pipe_id, queue_id);
 432                 mutex_lock(&adev->srbm_mutex);
 433                 soc21_grbm_select(adev, me_id, pipe_id, queue_id, 0);
 434                 WREG32_SOC15(GC, 0, regCP_HQD_DEQUEUE_REQUEST, 0x2);
 435                 WREG32_SOC15(GC, 0, regSPI_COMPUTE_QUEUE_RESET, 0x1);
 436
 437                 /* wait till dequeue take effects */
 438                 for (i = 0; i < adev->usec_timeout; i++) {
 439                         if (!(RREG32_SOC15(GC, 0, regCP_HQD_ACTIVE) & 1))
 440                                 break;
 441                         udelay(1);
 442                 }
 443                 if (i >= adev->usec_timeout) {
 444                         dev_err(adev->dev, "failed to wait on hqd deactivate\n");
 445                         r = -ETIMEDOUT;
 446                 }
 447                 soc21_grbm_select(adev, 0, 0, 0, 0);
 448                 mutex_unlock(&adev->srbm_mutex);
 449         } else if (queue_type == AMDGPU_RING_TYPE_SDMA) {
 450                 dev_info(adev->dev, "reset sdma queue (%d:%d:%d)\n",
 451                          me_id, pipe_id, queue_id);
 452                 switch (me_id) {
 453                 case 1:
 454                         reg = SOC15_REG_OFFSET(GC, 0, regSDMA1_QUEUE_RESET_REQ);
 455                         break;
 456                 case 0:
 457                 default:
 458                         reg = SOC15_REG_OFFSET(GC, 0, regSDMA0_QUEUE_RESET_REQ);
 459                         break;
 460                 }
 461
 462                 value = 1 << queue_id;
 463                 WREG32(reg, value);
 464                 /* wait for queue reset done */
 465                 for (i = 0; i < adev->usec_timeout; i++) {
 466                         if (!(RREG32(reg) & value))
 467                                 break;
 468                         udelay(1);
 469                 }
 470                 if (i >= adev->usec_timeout) {
 471                         dev_err(adev->dev, "failed to wait on sdma queue reset done\n");
 472                         r = -ETIMEDOUT;
 473                 }
 474         }
 475
 476         amdgpu_gfx_rlc_exit_safe_mode(adev, 0);
 477         return r;
 478 }
 479
 480 static int mes_v12_0_reset_hw_queue(struct amdgpu_mes *mes,
 481                                     struct mes_reset_queue_input *input)
 482 {
 483         union MESAPI__RESET mes_reset_queue_pkt;
 484         int pipe;
 485
 486         memset(&mes_reset_queue_pkt, 0, sizeof(mes_reset_queue_pkt));
 487
 488         mes_reset_queue_pkt.header.type = MES_API_TYPE_SCHEDULER;
 489         mes_reset_queue_pkt.header.opcode = MES_SCH_API_RESET;
 490         mes_reset_queue_pkt.header.dwsize = API_FRAME_SIZE_IN_DWORDS;
 491
 492         mes_reset_queue_pkt.doorbell_offset = input->doorbell_offset;
 493         mes_reset_queue_pkt.gang_context_addr = input->gang_context_addr;
 494         /*mes_reset_queue_pkt.reset_queue_only = 1;*/
 495
 496         if (mes->adev->enable_uni_mes)
 497                 pipe = AMDGPU_MES_KIQ_PIPE;
 498         else
 499                 pipe = AMDGPU_MES_SCHED_PIPE;
 500
 501         return mes_v12_0_submit_pkt_and_poll_completion(mes, pipe,
 502                         &mes_reset_queue_pkt, sizeof(mes_reset_queue_pkt),
 503                         offsetof(union MESAPI__REMOVE_QUEUE, api_status));
 504 }
 505
 506 static int mes_v12_0_map_legacy_queue(struct amdgpu_mes *mes,
 507                                       struct mes_map_legacy_queue_input *input)
 508 {
 509         union MESAPI__ADD_QUEUE mes_add_queue_pkt;
 510         int pipe;
 511
 512         memset(&mes_add_queue_pkt, 0, sizeof(mes_add_queue_pkt));
 513
 514         mes_add_queue_pkt.header.type = MES_API_TYPE_SCHEDULER;
 515         mes_add_queue_pkt.header.opcode = MES_SCH_API_ADD_QUEUE;
 516         mes_add_queue_pkt.header.dwsize = API_FRAME_SIZE_IN_DWORDS;
 517
 518         mes_add_queue_pkt.pipe_id = input->pipe_id;
 519         mes_add_queue_pkt.queue_id = input->queue_id;
 520         mes_add_queue_pkt.doorbell_offset = input->doorbell_offset;
 521         mes_add_queue_pkt.mqd_addr = input->mqd_addr;
 522         mes_add_queue_pkt.wptr_addr = input->wptr_addr;
 523         mes_add_queue_pkt.queue_type =
 524                 convert_to_mes_queue_type(input->queue_type);
 525         mes_add_queue_pkt.map_legacy_kq = 1;
 526
 527         if (mes->adev->enable_uni_mes)
 528                 pipe = AMDGPU_MES_KIQ_PIPE;
 529         else
 530                 pipe = AMDGPU_MES_SCHED_PIPE;
 531
 532         return mes_v12_0_submit_pkt_and_poll_completion(mes, pipe,
 533                         &mes_add_queue_pkt, sizeof(mes_add_queue_pkt),
 534                         offsetof(union MESAPI__ADD_QUEUE, api_status));
 535 }
 536
 537 static int mes_v12_0_unmap_legacy_queue(struct amdgpu_mes *mes,
 538                         struct mes_unmap_legacy_queue_input *input)
 539 {
 540         union MESAPI__REMOVE_QUEUE mes_remove_queue_pkt;
 541         int pipe;
 542
 543         memset(&mes_remove_queue_pkt, 0, sizeof(mes_remove_queue_pkt));
 544
 545         mes_remove_queue_pkt.header.type = MES_API_TYPE_SCHEDULER;
 546         mes_remove_queue_pkt.header.opcode = MES_SCH_API_REMOVE_QUEUE;
 547         mes_remove_queue_pkt.header.dwsize = API_FRAME_SIZE_IN_DWORDS;
 548
 549         mes_remove_queue_pkt.doorbell_offset = input->doorbell_offset;
 550         mes_remove_queue_pkt.gang_context_addr = 0;
 551
 552         mes_remove_queue_pkt.pipe_id = input->pipe_id;
 553         mes_remove_queue_pkt.queue_id = input->queue_id;
 554
 555         if (input->action == PREEMPT_QUEUES_NO_UNMAP) {
 556                 mes_remove_queue_pkt.preempt_legacy_gfx_queue = 1;
 557                 mes_remove_queue_pkt.tf_addr = input->trail_fence_addr;
 558                 mes_remove_queue_pkt.tf_data =
 559                         lower_32_bits(input->trail_fence_data);
 560         } else {
 561                 mes_remove_queue_pkt.unmap_legacy_queue = 1;
 562                 mes_remove_queue_pkt.queue_type =
 563                         convert_to_mes_queue_type(input->queue_type);
 564         }
 565
 566         if (mes->adev->enable_uni_mes)
 567                 pipe = AMDGPU_MES_KIQ_PIPE;
 568         else
 569                 pipe = AMDGPU_MES_SCHED_PIPE;
 570
 571         return mes_v12_0_submit_pkt_and_poll_completion(mes, pipe,
 572                         &mes_remove_queue_pkt, sizeof(mes_remove_queue_pkt),
 573                         offsetof(union MESAPI__REMOVE_QUEUE, api_status));
 574 }
 575
 576 static int mes_v12_0_suspend_gang(struct amdgpu_mes *mes,
 577                                   struct mes_suspend_gang_input *input)
 578 {
 579         return 0;
 580 }
 581
 582 static int mes_v12_0_resume_gang(struct amdgpu_mes *mes,
 583                                  struct mes_resume_gang_input *input)
 584 {
 585         return 0;
 586 }
 587
 588 static int mes_v12_0_query_sched_status(struct amdgpu_mes *mes, int pipe)
 589 {
 590         union MESAPI__QUERY_MES_STATUS mes_status_pkt;
 591
 592         memset(&mes_status_pkt, 0, sizeof(mes_status_pkt));
 593
 594         mes_status_pkt.header.type = MES_API_TYPE_SCHEDULER;
 595         mes_status_pkt.header.opcode = MES_SCH_API_QUERY_SCHEDULER_STATUS;
 596         mes_status_pkt.header.dwsize = API_FRAME_SIZE_IN_DWORDS;
 597
 598         return mes_v12_0_submit_pkt_and_poll_completion(mes, pipe,
 599                         &mes_status_pkt, sizeof(mes_status_pkt),
 600                         offsetof(union MESAPI__QUERY_MES_STATUS, api_status));
 601 }
 602
 603 static int mes_v12_0_misc_op(struct amdgpu_mes *mes,
 604                              struct mes_misc_op_input *input)
 605 {
 606         union MESAPI__MISC misc_pkt;
 607         int pipe;
 608
 609         if (mes->adev->enable_uni_mes)
 610                 pipe = AMDGPU_MES_KIQ_PIPE;
 611         else
 612                 pipe = AMDGPU_MES_SCHED_PIPE;
 613
 614         memset(&misc_pkt, 0, sizeof(misc_pkt));
 615
 616         misc_pkt.header.type = MES_API_TYPE_SCHEDULER;
 617         misc_pkt.header.opcode = MES_SCH_API_MISC;
 618         misc_pkt.header.dwsize = API_FRAME_SIZE_IN_DWORDS;
 619
 620         switch (input->op) {
 621         case MES_MISC_OP_READ_REG:
 622                 misc_pkt.opcode = MESAPI_MISC__READ_REG;
 623                 misc_pkt.read_reg.reg_offset = input->read_reg.reg_offset;
 624                 misc_pkt.read_reg.buffer_addr = input->read_reg.buffer_addr;
 625                 break;
 626         case MES_MISC_OP_WRITE_REG:
 627                 misc_pkt.opcode = MESAPI_MISC__WRITE_REG;
 628                 misc_pkt.write_reg.reg_offset = input->write_reg.reg_offset;
 629                 misc_pkt.write_reg.reg_value = input->write_reg.reg_value;
 630                 break;
 631         case MES_MISC_OP_WRM_REG_WAIT:
 632                 misc_pkt.opcode = MESAPI_MISC__WAIT_REG_MEM;
 633                 misc_pkt.wait_reg_mem.op = WRM_OPERATION__WAIT_REG_MEM;
 634                 misc_pkt.wait_reg_mem.reference = input->wrm_reg.ref;
 635                 misc_pkt.wait_reg_mem.mask = input->wrm_reg.mask;
 636                 misc_pkt.wait_reg_mem.reg_offset1 = input->wrm_reg.reg0;
 637                 misc_pkt.wait_reg_mem.reg_offset2 = 0;
 638                 break;
 639         case MES_MISC_OP_WRM_REG_WR_WAIT:
 640                 misc_pkt.opcode = MESAPI_MISC__WAIT_REG_MEM;
 641                 misc_pkt.wait_reg_mem.op = WRM_OPERATION__WR_WAIT_WR_REG;
 642                 misc_pkt.wait_reg_mem.reference = input->wrm_reg.ref;
 643                 misc_pkt.wait_reg_mem.mask = input->wrm_reg.mask;
 644                 misc_pkt.wait_reg_mem.reg_offset1 = input->wrm_reg.reg0;
 645                 misc_pkt.wait_reg_mem.reg_offset2 = input->wrm_reg.reg1;
 646                 break;
 647         case MES_MISC_OP_SET_SHADER_DEBUGGER:
 648                 pipe = AMDGPU_MES_SCHED_PIPE;
 649                 misc_pkt.opcode = MESAPI_MISC__SET_SHADER_DEBUGGER;
 650                 misc_pkt.set_shader_debugger.process_context_addr =
 651                                 input->set_shader_debugger.process_context_addr;
 652                 misc_pkt.set_shader_debugger.flags.u32all =
 653                                 input->set_shader_debugger.flags.u32all;
 654                 misc_pkt.set_shader_debugger.spi_gdbg_per_vmid_cntl =
 655                                 input->set_shader_debugger.spi_gdbg_per_vmid_cntl;
 656                 memcpy(misc_pkt.set_shader_debugger.tcp_watch_cntl,
 657                                 input->set_shader_debugger.tcp_watch_cntl,
 658                                 sizeof(misc_pkt.set_shader_debugger.tcp_watch_cntl));
 659                 misc_pkt.set_shader_debugger.trap_en = input->set_shader_debugger.trap_en;
 660                 break;
 661         case MES_MISC_OP_CHANGE_CONFIG:
 662                 misc_pkt.opcode = MESAPI_MISC__CHANGE_CONFIG;
 663                 misc_pkt.change_config.opcode =
 664                                 MESAPI_MISC__CHANGE_CONFIG_OPTION_LIMIT_SINGLE_PROCESS;
 665                 misc_pkt.change_config.option.bits.limit_single_process =
 666                                 input->change_config.option.limit_single_process;
 667                 break;
 668
 669         default:
 670                 DRM_ERROR("unsupported misc op (%d) \n", input->op);
 671                 return -EINVAL;
 672         }
 673
 674         return mes_v12_0_submit_pkt_and_poll_completion(mes, pipe,
 675                         &misc_pkt, sizeof(misc_pkt),
 676                         offsetof(union MESAPI__MISC, api_status));
 677 }
 678
 679 static int mes_v12_0_set_hw_resources_1(struct amdgpu_mes *mes, int pipe)
 680 {
 681         union MESAPI_SET_HW_RESOURCES_1 mes_set_hw_res_1_pkt;
 682
 683         memset(&mes_set_hw_res_1_pkt, 0, sizeof(mes_set_hw_res_1_pkt));
 684
 685         mes_set_hw_res_1_pkt.header.type = MES_API_TYPE_SCHEDULER;
 686         mes_set_hw_res_1_pkt.header.opcode = MES_SCH_API_SET_HW_RSRC_1;
 687         mes_set_hw_res_1_pkt.header.dwsize = API_FRAME_SIZE_IN_DWORDS;
 688         mes_set_hw_res_1_pkt.mes_kiq_unmap_timeout = 0xa;
 689
 690         return mes_v12_0_submit_pkt_and_poll_completion(mes, pipe,
 691                         &mes_set_hw_res_1_pkt, sizeof(mes_set_hw_res_1_pkt),
 692                         offsetof(union MESAPI_SET_HW_RESOURCES_1, api_status));
 693 }
 694
 695 static int mes_v12_0_set_hw_resources(struct amdgpu_mes *mes, int pipe)
 696 {
 697         int i;
 698         struct amdgpu_device *adev = mes->adev;
 699         union MESAPI_SET_HW_RESOURCES mes_set_hw_res_pkt;
 700
 701         memset(&mes_set_hw_res_pkt, 0, sizeof(mes_set_hw_res_pkt));
 702
 703         mes_set_hw_res_pkt.header.type = MES_API_TYPE_SCHEDULER;
 704         mes_set_hw_res_pkt.header.opcode = MES_SCH_API_SET_HW_RSRC;
 705         mes_set_hw_res_pkt.header.dwsize = API_FRAME_SIZE_IN_DWORDS;
 706
 707         if (pipe == AMDGPU_MES_SCHED_PIPE) {
 708                 mes_set_hw_res_pkt.vmid_mask_mmhub = mes->vmid_mask_mmhub;
 709                 mes_set_hw_res_pkt.vmid_mask_gfxhub = mes->vmid_mask_gfxhub;
 710                 mes_set_hw_res_pkt.gds_size = adev->gds.gds_size;
 711                 mes_set_hw_res_pkt.paging_vmid = 0;
 712
 713                 for (i = 0; i < MAX_COMPUTE_PIPES; i++)
 714                         mes_set_hw_res_pkt.compute_hqd_mask[i] =
 715                                 mes->compute_hqd_mask[i];
 716
 717                 for (i = 0; i < MAX_GFX_PIPES; i++)
 718                         mes_set_hw_res_pkt.gfx_hqd_mask[i] =
 719                                 mes->gfx_hqd_mask[i];
 720
 721                 for (i = 0; i < MAX_SDMA_PIPES; i++)
 722                         mes_set_hw_res_pkt.sdma_hqd_mask[i] =
 723                                 mes->sdma_hqd_mask[i];
 724
 725                 for (i = 0; i < AMD_PRIORITY_NUM_LEVELS; i++)
 726                         mes_set_hw_res_pkt.aggregated_doorbells[i] =
 727                                 mes->aggregated_doorbells[i];
 728         }
 729
 730         mes_set_hw_res_pkt.g_sch_ctx_gpu_mc_ptr =
 731                 mes->sch_ctx_gpu_addr[pipe];
 732         mes_set_hw_res_pkt.query_status_fence_gpu_mc_ptr =
 733                 mes->query_status_fence_gpu_addr[pipe];
 734
 735         for (i = 0; i < 5; i++) {
 736                 mes_set_hw_res_pkt.gc_base[i] = adev->reg_offset[GC_HWIP][0][i];
 737                 mes_set_hw_res_pkt.mmhub_base[i] =
 738                                 adev->reg_offset[MMHUB_HWIP][0][i];
 739                 mes_set_hw_res_pkt.osssys_base[i] =
 740                 adev->reg_offset[OSSSYS_HWIP][0][i];
 741         }
 742
 743         mes_set_hw_res_pkt.disable_reset = 1;
 744         mes_set_hw_res_pkt.disable_mes_log = 1;
 745         mes_set_hw_res_pkt.use_different_vmid_compute = 1;
 746         mes_set_hw_res_pkt.enable_reg_active_poll = 1;
 747         mes_set_hw_res_pkt.enable_level_process_quantum_check = 1;
 748
 749         /*
 750          * Keep oversubscribe timer for sdma . When we have unmapped doorbell
 751          * handling support, other queue will not use the oversubscribe timer.
 752          * handling  mode - 0: disabled; 1: basic version; 2: basic+ version
 753          */
 754         mes_set_hw_res_pkt.oversubscription_timer = 50;
 755         mes_set_hw_res_pkt.unmapped_doorbell_handling = 1;
 756
 757         if (amdgpu_mes_log_enable) {
 758                 mes_set_hw_res_pkt.enable_mes_event_int_logging = 1;
 759                 mes_set_hw_res_pkt.event_intr_history_gpu_mc_ptr = mes->event_log_gpu_addr +
 760                                 pipe * (AMDGPU_MES_LOG_BUFFER_SIZE + AMDGPU_MES_MSCRATCH_SIZE);
 761         }
 762
 763         if (enforce_isolation)
 764                 mes_set_hw_res_pkt.limit_single_process = 1;
 765
 766         return mes_v12_0_submit_pkt_and_poll_completion(mes, pipe,
 767                         &mes_set_hw_res_pkt, sizeof(mes_set_hw_res_pkt),
 768                         offsetof(union MESAPI_SET_HW_RESOURCES, api_status));
 769 }
 770
 771 static void mes_v12_0_init_aggregated_doorbell(struct amdgpu_mes *mes)
 772 {
 773         struct amdgpu_device *adev = mes->adev;
 774         uint32_t data;
 775
 776         data = RREG32_SOC15(GC, 0, regCP_MES_DOORBELL_CONTROL1);
 777         data &= ~(CP_MES_DOORBELL_CONTROL1__DOORBELL_OFFSET_MASK |
 778                   CP_MES_DOORBELL_CONTROL1__DOORBELL_EN_MASK |
 779                   CP_MES_DOORBELL_CONTROL1__DOORBELL_HIT_MASK);
 780         data |= mes->aggregated_doorbells[AMDGPU_MES_PRIORITY_LEVEL_LOW] <<
 781                 CP_MES_DOORBELL_CONTROL1__DOORBELL_OFFSET__SHIFT;
 782         data |= 1 << CP_MES_DOORBELL_CONTROL1__DOORBELL_EN__SHIFT;
 783         WREG32_SOC15(GC, 0, regCP_MES_DOORBELL_CONTROL1, data);
 784
 785         data = RREG32_SOC15(GC, 0, regCP_MES_DOORBELL_CONTROL2);
 786         data &= ~(CP_MES_DOORBELL_CONTROL2__DOORBELL_OFFSET_MASK |
 787                   CP_MES_DOORBELL_CONTROL2__DOORBELL_EN_MASK |
 788                   CP_MES_DOORBELL_CONTROL2__DOORBELL_HIT_MASK);
 789         data |= mes->aggregated_doorbells[AMDGPU_MES_PRIORITY_LEVEL_NORMAL] <<
 790                 CP_MES_DOORBELL_CONTROL2__DOORBELL_OFFSET__SHIFT;
 791         data |= 1 << CP_MES_DOORBELL_CONTROL2__DOORBELL_EN__SHIFT;
 792         WREG32_SOC15(GC, 0, regCP_MES_DOORBELL_CONTROL2, data);
 793
 794         data = RREG32_SOC15(GC, 0, regCP_MES_DOORBELL_CONTROL3);
 795         data &= ~(CP_MES_DOORBELL_CONTROL3__DOORBELL_OFFSET_MASK |
 796                   CP_MES_DOORBELL_CONTROL3__DOORBELL_EN_MASK |
 797                   CP_MES_DOORBELL_CONTROL3__DOORBELL_HIT_MASK);
 798         data |= mes->aggregated_doorbells[AMDGPU_MES_PRIORITY_LEVEL_MEDIUM] <<
 799                 CP_MES_DOORBELL_CONTROL3__DOORBELL_OFFSET__SHIFT;
 800         data |= 1 << CP_MES_DOORBELL_CONTROL3__DOORBELL_EN__SHIFT;
 801         WREG32_SOC15(GC, 0, regCP_MES_DOORBELL_CONTROL3, data);
 802
 803         data = RREG32_SOC15(GC, 0, regCP_MES_DOORBELL_CONTROL4);
 804         data &= ~(CP_MES_DOORBELL_CONTROL4__DOORBELL_OFFSET_MASK |
 805                   CP_MES_DOORBELL_CONTROL4__DOORBELL_EN_MASK |
 806                   CP_MES_DOORBELL_CONTROL4__DOORBELL_HIT_MASK);
 807         data |= mes->aggregated_doorbells[AMDGPU_MES_PRIORITY_LEVEL_HIGH] <<
 808                 CP_MES_DOORBELL_CONTROL4__DOORBELL_OFFSET__SHIFT;
 809         data |= 1 << CP_MES_DOORBELL_CONTROL4__DOORBELL_EN__SHIFT;
 810         WREG32_SOC15(GC, 0, regCP_MES_DOORBELL_CONTROL4, data);
 811
 812         data = RREG32_SOC15(GC, 0, regCP_MES_DOORBELL_CONTROL5);
 813         data &= ~(CP_MES_DOORBELL_CONTROL5__DOORBELL_OFFSET_MASK |
 814                   CP_MES_DOORBELL_CONTROL5__DOORBELL_EN_MASK |
 815                   CP_MES_DOORBELL_CONTROL5__DOORBELL_HIT_MASK);
 816         data |= mes->aggregated_doorbells[AMDGPU_MES_PRIORITY_LEVEL_REALTIME] <<
 817                 CP_MES_DOORBELL_CONTROL5__DOORBELL_OFFSET__SHIFT;
 818         data |= 1 << CP_MES_DOORBELL_CONTROL5__DOORBELL_EN__SHIFT;
 819         WREG32_SOC15(GC, 0, regCP_MES_DOORBELL_CONTROL5, data);
 820
 821         data = 1 << CP_HQD_GFX_CONTROL__DB_UPDATED_MSG_EN__SHIFT;
 822         WREG32_SOC15(GC, 0, regCP_HQD_GFX_CONTROL, data);
 823 }
 824
 825
 826 static void mes_v12_0_enable_unmapped_doorbell_handling(
 827                 struct amdgpu_mes *mes, bool enable)
 828 {
 829         struct amdgpu_device *adev = mes->adev;
 830         uint32_t data = RREG32_SOC15(GC, 0, regCP_UNMAPPED_DOORBELL);
 831
 832         /*
 833          * The default PROC_LSB settng is 0xc which means doorbell
 834          * addr[16:12] gives the doorbell page number. For kfd, each
 835          * process will use 2 pages of doorbell, we need to change the
 836          * setting to 0xd
 837          */
 838         data &= ~CP_UNMAPPED_DOORBELL__PROC_LSB_MASK;
 839         data |= 0xd <<  CP_UNMAPPED_DOORBELL__PROC_LSB__SHIFT;
 840
 841         data |= (enable ? 1 : 0) << CP_UNMAPPED_DOORBELL__ENABLE__SHIFT;
 842
 843         WREG32_SOC15(GC, 0, regCP_UNMAPPED_DOORBELL, data);
 844 }
 845
 846 static int mes_v12_0_reset_legacy_queue(struct amdgpu_mes *mes,
 847                                         struct mes_reset_legacy_queue_input *input)
 848 {
 849         union MESAPI__RESET mes_reset_queue_pkt;
 850         int pipe;
 851
 852         if (input->use_mmio)
 853                 return mes_v12_0_reset_queue_mmio(mes, input->queue_type,
 854                                                   input->me_id, input->pipe_id,
 855                                                   input->queue_id, input->vmid);
 856
 857         memset(&mes_reset_queue_pkt, 0, sizeof(mes_reset_queue_pkt));
 858
 859         mes_reset_queue_pkt.header.type = MES_API_TYPE_SCHEDULER;
 860         mes_reset_queue_pkt.header.opcode = MES_SCH_API_RESET;
 861         mes_reset_queue_pkt.header.dwsize = API_FRAME_SIZE_IN_DWORDS;
 862
 863         mes_reset_queue_pkt.queue_type =
 864                 convert_to_mes_queue_type(input->queue_type);
 865
 866         if (mes_reset_queue_pkt.queue_type == MES_QUEUE_TYPE_GFX) {
 867                 mes_reset_queue_pkt.reset_legacy_gfx = 1;
 868                 mes_reset_queue_pkt.pipe_id_lp = input->pipe_id;
 869                 mes_reset_queue_pkt.queue_id_lp = input->queue_id;
 870                 mes_reset_queue_pkt.mqd_mc_addr_lp = input->mqd_addr;
 871                 mes_reset_queue_pkt.doorbell_offset_lp = input->doorbell_offset;
 872                 mes_reset_queue_pkt.wptr_addr_lp = input->wptr_addr;
 873                 mes_reset_queue_pkt.vmid_id_lp = input->vmid;
 874         } else {
 875                 mes_reset_queue_pkt.reset_queue_only = 1;
 876                 mes_reset_queue_pkt.doorbell_offset = input->doorbell_offset;
 877         }
 878
 879         if (mes->adev->enable_uni_mes)
 880                 pipe = AMDGPU_MES_KIQ_PIPE;
 881         else
 882                 pipe = AMDGPU_MES_SCHED_PIPE;
 883
 884         return mes_v12_0_submit_pkt_and_poll_completion(mes, pipe,
 885                         &mes_reset_queue_pkt, sizeof(mes_reset_queue_pkt),
 886                         offsetof(union MESAPI__RESET, api_status));
 887 }
 888
 889 static const struct amdgpu_mes_funcs mes_v12_0_funcs = {
 890         .add_hw_queue = mes_v12_0_add_hw_queue,
 891         .remove_hw_queue = mes_v12_0_remove_hw_queue,
 892         .map_legacy_queue = mes_v12_0_map_legacy_queue,
 893         .unmap_legacy_queue = mes_v12_0_unmap_legacy_queue,
 894         .suspend_gang = mes_v12_0_suspend_gang,
 895         .resume_gang = mes_v12_0_resume_gang,
 896         .misc_op = mes_v12_0_misc_op,
 897         .reset_legacy_queue = mes_v12_0_reset_legacy_queue,
 898         .reset_hw_queue = mes_v12_0_reset_hw_queue,
 899 };
 900
 901 static int mes_v12_0_allocate_ucode_buffer(struct amdgpu_device *adev,
 902                                            enum admgpu_mes_pipe pipe)
 903 {
 904         int r;
 905         const struct mes_firmware_header_v1_0 *mes_hdr;
 906         const __le32 *fw_data;
 907         unsigned fw_size;
 908
 909         mes_hdr = (const struct mes_firmware_header_v1_0 *)
 910                 adev->mes.fw[pipe]->data;
 911
 912         fw_data = (const __le32 *)(adev->mes.fw[pipe]->data +
 913                    le32_to_cpu(mes_hdr->mes_ucode_offset_bytes));
 914         fw_size = le32_to_cpu(mes_hdr->mes_ucode_size_bytes);
 915
 916         r = amdgpu_bo_create_reserved(adev, fw_size,
 917                                       PAGE_SIZE,
 918                                       AMDGPU_GEM_DOMAIN_VRAM,
 919                                       &adev->mes.ucode_fw_obj[pipe],
 920                                       &adev->mes.ucode_fw_gpu_addr[pipe],
 921                                       (void **)&adev->mes.ucode_fw_ptr[pipe]);
 922         if (r) {
 923                 dev_err(adev->dev, "(%d) failed to create mes fw bo\n", r);
 924                 return r;
 925         }
 926
 927         memcpy(adev->mes.ucode_fw_ptr[pipe], fw_data, fw_size);
 928
 929         amdgpu_bo_kunmap(adev->mes.ucode_fw_obj[pipe]);
 930         amdgpu_bo_unreserve(adev->mes.ucode_fw_obj[pipe]);
 931
 932         return 0;
 933 }
 934
 935 static int mes_v12_0_allocate_ucode_data_buffer(struct amdgpu_device *adev,
 936                                                 enum admgpu_mes_pipe pipe)
 937 {
 938         int r;
 939         const struct mes_firmware_header_v1_0 *mes_hdr;
 940         const __le32 *fw_data;
 941         unsigned fw_size;
 942
 943         mes_hdr = (const struct mes_firmware_header_v1_0 *)
 944                 adev->mes.fw[pipe]->data;
 945
 946         fw_data = (const __le32 *)(adev->mes.fw[pipe]->data +
 947                    le32_to_cpu(mes_hdr->mes_ucode_data_offset_bytes));
 948         fw_size = le32_to_cpu(mes_hdr->mes_ucode_data_size_bytes);
 949
 950         r = amdgpu_bo_create_reserved(adev, fw_size,
 951                                       64 * 1024,
 952                                       AMDGPU_GEM_DOMAIN_VRAM,
 953                                       &adev->mes.data_fw_obj[pipe],
 954                                       &adev->mes.data_fw_gpu_addr[pipe],
 955                                       (void **)&adev->mes.data_fw_ptr[pipe]);
 956         if (r) {
 957                 dev_err(adev->dev, "(%d) failed to create mes data fw bo\n", r);
 958                 return r;
 959         }
 960
 961         memcpy(adev->mes.data_fw_ptr[pipe], fw_data, fw_size);
 962
 963         amdgpu_bo_kunmap(adev->mes.data_fw_obj[pipe]);
 964         amdgpu_bo_unreserve(adev->mes.data_fw_obj[pipe]);
 965
 966         return 0;
 967 }
 968
 969 static void mes_v12_0_free_ucode_buffers(struct amdgpu_device *adev,
 970                                          enum admgpu_mes_pipe pipe)
 971 {
 972         amdgpu_bo_free_kernel(&adev->mes.data_fw_obj[pipe],
 973                               &adev->mes.data_fw_gpu_addr[pipe],
 974                               (void **)&adev->mes.data_fw_ptr[pipe]);
 975
 976         amdgpu_bo_free_kernel(&adev->mes.ucode_fw_obj[pipe],
 977                               &adev->mes.ucode_fw_gpu_addr[pipe],
 978                               (void **)&adev->mes.ucode_fw_ptr[pipe]);
 979 }
 980
 981 static void mes_v12_0_enable(struct amdgpu_device *adev, bool enable)
 982 {
 983         uint64_t ucode_addr;
 984         uint32_t pipe, data = 0;
 985
 986         if (enable) {
 987                 mutex_lock(&adev->srbm_mutex);
 988                 for (pipe = 0; pipe < AMDGPU_MAX_MES_PIPES; pipe++) {
 989                         soc21_grbm_select(adev, 3, pipe, 0, 0);
 990                         if (amdgpu_mes_log_enable) {
 991                                 u32 log_size = AMDGPU_MES_LOG_BUFFER_SIZE + AMDGPU_MES_MSCRATCH_SIZE;
 992                                 /* In case uni mes is not enabled, only program for pipe 0 */
 993                                 if (adev->mes.event_log_size >= (pipe + 1) * log_size) {
 994                                         WREG32_SOC15(GC, 0, regCP_MES_MSCRATCH_LO,
 995                                                      lower_32_bits(adev->mes.event_log_gpu_addr +
 996                                                      pipe * log_size + AMDGPU_MES_LOG_BUFFER_SIZE));
 997                                         WREG32_SOC15(GC, 0, regCP_MES_MSCRATCH_HI,
 998                                                      upper_32_bits(adev->mes.event_log_gpu_addr +
 999                                                      pipe * log_size + AMDGPU_MES_LOG_BUFFER_SIZE));
1000                                         dev_info(adev->dev, "Setup CP MES MSCRATCH address : 0x%x. 0x%x\n",
1001                                                  RREG32_SOC15(GC, 0, regCP_MES_MSCRATCH_HI),
1002                                                  RREG32_SOC15(GC, 0, regCP_MES_MSCRATCH_LO));
1003                                 }
1004                         }
1005
1006                         data = RREG32_SOC15(GC, 0, regCP_MES_CNTL);
1007                         if (pipe == 0)
1008                                 data = REG_SET_FIELD(data, CP_MES_CNTL, MES_PIPE0_RESET, 1);
1009                         else
1010                                 data = REG_SET_FIELD(data, CP_MES_CNTL, MES_PIPE1_RESET, 1);
1011                         WREG32_SOC15(GC, 0, regCP_MES_CNTL, data);
1012
1013                         ucode_addr = adev->mes.uc_start_addr[pipe] >> 2;
1014                         WREG32_SOC15(GC, 0, regCP_MES_PRGRM_CNTR_START,
1015                                      lower_32_bits(ucode_addr));
1016                         WREG32_SOC15(GC, 0, regCP_MES_PRGRM_CNTR_START_HI,
1017                                      upper_32_bits(ucode_addr));
1018
1019                         /* unhalt MES and activate one pipe each loop */
1020                         data = REG_SET_FIELD(0, CP_MES_CNTL, MES_PIPE0_ACTIVE, 1);
1021                         if (pipe)
1022                                 data = REG_SET_FIELD(data, CP_MES_CNTL, MES_PIPE1_ACTIVE, 1);
1023                         dev_info(adev->dev, "program CP_MES_CNTL : 0x%x\n", data);
1024
1025                         WREG32_SOC15(GC, 0, regCP_MES_CNTL, data);
1026
1027                 }
1028                 soc21_grbm_select(adev, 0, 0, 0, 0);
1029                 mutex_unlock(&adev->srbm_mutex);
1030
1031                 if (amdgpu_emu_mode)
1032                         msleep(100);
1033                 else if (adev->enable_uni_mes)
1034                         udelay(500);
1035                 else
1036                         udelay(50);
1037         } else {
1038                 data = RREG32_SOC15(GC, 0, regCP_MES_CNTL);
1039                 data = REG_SET_FIELD(data, CP_MES_CNTL, MES_PIPE0_ACTIVE, 0);
1040                 data = REG_SET_FIELD(data, CP_MES_CNTL, MES_PIPE1_ACTIVE, 0);
1041                 data = REG_SET_FIELD(data, CP_MES_CNTL,
1042                                      MES_INVALIDATE_ICACHE, 1);
1043                 data = REG_SET_FIELD(data, CP_MES_CNTL, MES_PIPE0_RESET, 1);
1044                 data = REG_SET_FIELD(data, CP_MES_CNTL, MES_PIPE1_RESET, 1);
1045                 data = REG_SET_FIELD(data, CP_MES_CNTL, MES_HALT, 1);
1046                 WREG32_SOC15(GC, 0, regCP_MES_CNTL, data);
1047         }
1048 }
1049
1050 static void mes_v12_0_set_ucode_start_addr(struct amdgpu_device *adev)
1051 {
1052         uint64_t ucode_addr;
1053         int pipe;
1054
1055         mes_v12_0_enable(adev, false);
1056
1057         mutex_lock(&adev->srbm_mutex);
1058         for (pipe = 0; pipe < AMDGPU_MAX_MES_PIPES; pipe++) {
1059                 /* me=3, queue=0 */
1060                 soc21_grbm_select(adev, 3, pipe, 0, 0);
1061
1062                 /* set ucode start address */
1063                 ucode_addr = adev->mes.uc_start_addr[pipe] >> 2;
1064                 WREG32_SOC15(GC, 0, regCP_MES_PRGRM_CNTR_START,
1065                                 lower_32_bits(ucode_addr));
1066                 WREG32_SOC15(GC, 0, regCP_MES_PRGRM_CNTR_START_HI,
1067                                 upper_32_bits(ucode_addr));
1068
1069                 soc21_grbm_select(adev, 0, 0, 0, 0);
1070         }
1071         mutex_unlock(&adev->srbm_mutex);
1072 }
1073
1074 /* This function is for backdoor MES firmware */
1075 static int mes_v12_0_load_microcode(struct amdgpu_device *adev,
1076                                     enum admgpu_mes_pipe pipe, bool prime_icache)
1077 {
1078         int r;
1079         uint32_t data;
1080
1081         mes_v12_0_enable(adev, false);
1082
1083         if (!adev->mes.fw[pipe])
1084                 return -EINVAL;
1085
1086         r = mes_v12_0_allocate_ucode_buffer(adev, pipe);
1087         if (r)
1088                 return r;
1089
1090         r = mes_v12_0_allocate_ucode_data_buffer(adev, pipe);
1091         if (r) {
1092                 mes_v12_0_free_ucode_buffers(adev, pipe);
1093                 return r;
1094         }
1095
1096         mutex_lock(&adev->srbm_mutex);
1097         /* me=3, pipe=0, queue=0 */
1098         soc21_grbm_select(adev, 3, pipe, 0, 0);
1099
1100         WREG32_SOC15(GC, 0, regCP_MES_IC_BASE_CNTL, 0);
1101
1102         /* set ucode fimrware address */
1103         WREG32_SOC15(GC, 0, regCP_MES_IC_BASE_LO,
1104                      lower_32_bits(adev->mes.ucode_fw_gpu_addr[pipe]));
1105         WREG32_SOC15(GC, 0, regCP_MES_IC_BASE_HI,
1106                      upper_32_bits(adev->mes.ucode_fw_gpu_addr[pipe]));
1107
1108         /* set ucode instruction cache boundary to 2M-1 */
1109         WREG32_SOC15(GC, 0, regCP_MES_MIBOUND_LO, 0x1FFFFF);
1110
1111         /* set ucode data firmware address */
1112         WREG32_SOC15(GC, 0, regCP_MES_MDBASE_LO,
1113                      lower_32_bits(adev->mes.data_fw_gpu_addr[pipe]));
1114         WREG32_SOC15(GC, 0, regCP_MES_MDBASE_HI,
1115                      upper_32_bits(adev->mes.data_fw_gpu_addr[pipe]));
1116
1117         /* Set data cache boundary CP_MES_MDBOUND_LO */
1118         WREG32_SOC15(GC, 0, regCP_MES_MDBOUND_LO, 0x7FFFF);
1119
1120         if (prime_icache) {
1121                 /* invalidate ICACHE */
1122                 data = RREG32_SOC15(GC, 0, regCP_MES_IC_OP_CNTL);
1123                 data = REG_SET_FIELD(data, CP_MES_IC_OP_CNTL, PRIME_ICACHE, 0);
1124                 data = REG_SET_FIELD(data, CP_MES_IC_OP_CNTL, INVALIDATE_CACHE, 1);
1125                 WREG32_SOC15(GC, 0, regCP_MES_IC_OP_CNTL, data);
1126
1127                 /* prime the ICACHE. */
1128                 data = RREG32_SOC15(GC, 0, regCP_MES_IC_OP_CNTL);
1129                 data = REG_SET_FIELD(data, CP_MES_IC_OP_CNTL, PRIME_ICACHE, 1);
1130                 WREG32_SOC15(GC, 0, regCP_MES_IC_OP_CNTL, data);
1131         }
1132
1133         soc21_grbm_select(adev, 0, 0, 0, 0);
1134         mutex_unlock(&adev->srbm_mutex);
1135
1136         return 0;
1137 }
1138
1139 static int mes_v12_0_allocate_eop_buf(struct amdgpu_device *adev,
1140                                       enum admgpu_mes_pipe pipe)
1141 {
1142         int r;
1143         u32 *eop;
1144
1145         r = amdgpu_bo_create_reserved(adev, MES_EOP_SIZE, PAGE_SIZE,
1146                               AMDGPU_GEM_DOMAIN_GTT,
1147                               &adev->mes.eop_gpu_obj[pipe],
1148                               &adev->mes.eop_gpu_addr[pipe],
1149                               (void **)&eop);
1150         if (r) {
1151                 dev_warn(adev->dev, "(%d) create EOP bo failed\n", r);
1152                 return r;
1153         }
1154
1155         memset(eop, 0,
1156                adev->mes.eop_gpu_obj[pipe]->tbo.base.size);
1157
1158         amdgpu_bo_kunmap(adev->mes.eop_gpu_obj[pipe]);
1159         amdgpu_bo_unreserve(adev->mes.eop_gpu_obj[pipe]);
1160
1161         return 0;
1162 }
1163
1164 static int mes_v12_0_mqd_init(struct amdgpu_ring *ring)
1165 {
1166         struct v12_compute_mqd *mqd = ring->mqd_ptr;
1167         uint64_t hqd_gpu_addr, wb_gpu_addr, eop_base_addr;
1168         uint32_t tmp;
1169
1170         mqd->header = 0xC0310800;
1171         mqd->compute_pipelinestat_enable = 0x00000001;
1172         mqd->compute_static_thread_mgmt_se0 = 0xffffffff;
1173         mqd->compute_static_thread_mgmt_se1 = 0xffffffff;
1174         mqd->compute_static_thread_mgmt_se2 = 0xffffffff;
1175         mqd->compute_static_thread_mgmt_se3 = 0xffffffff;
1176         mqd->compute_misc_reserved = 0x00000007;
1177
1178         eop_base_addr = ring->eop_gpu_addr >> 8;
1179
1180         /* set the EOP size, register value is 2^(EOP_SIZE+1) dwords */
1181         tmp = regCP_HQD_EOP_CONTROL_DEFAULT;
1182         tmp = REG_SET_FIELD(tmp, CP_HQD_EOP_CONTROL, EOP_SIZE,
1183                         (order_base_2(MES_EOP_SIZE / 4) - 1));
1184
1185         mqd->cp_hqd_eop_base_addr_lo = lower_32_bits(eop_base_addr);
1186         mqd->cp_hqd_eop_base_addr_hi = upper_32_bits(eop_base_addr);
1187         mqd->cp_hqd_eop_control = tmp;
1188
1189         /* disable the queue if it's active */
1190         ring->wptr = 0;
1191         mqd->cp_hqd_pq_rptr = 0;
1192         mqd->cp_hqd_pq_wptr_lo = 0;
1193         mqd->cp_hqd_pq_wptr_hi = 0;
1194
1195         /* set the pointer to the MQD */
1196         mqd->cp_mqd_base_addr_lo = ring->mqd_gpu_addr & 0xfffffffc;
1197         mqd->cp_mqd_base_addr_hi = upper_32_bits(ring->mqd_gpu_addr);
1198
1199         /* set MQD vmid to 0 */
1200         tmp = regCP_MQD_CONTROL_DEFAULT;
1201         tmp = REG_SET_FIELD(tmp, CP_MQD_CONTROL, VMID, 0);
1202         mqd->cp_mqd_control = tmp;
1203
1204         /* set the pointer to the HQD, this is similar CP_RB0_BASE/_HI */
1205         hqd_gpu_addr = ring->gpu_addr >> 8;
1206         mqd->cp_hqd_pq_base_lo = lower_32_bits(hqd_gpu_addr);
1207         mqd->cp_hqd_pq_base_hi = upper_32_bits(hqd_gpu_addr);
1208
1209         /* set the wb address whether it's enabled or not */
1210         wb_gpu_addr = ring->rptr_gpu_addr;
1211         mqd->cp_hqd_pq_rptr_report_addr_lo = wb_gpu_addr & 0xfffffffc;
1212         mqd->cp_hqd_pq_rptr_report_addr_hi =
1213                 upper_32_bits(wb_gpu_addr) & 0xffff;
1214
1215         /* only used if CP_PQ_WPTR_POLL_CNTL.CP_PQ_WPTR_POLL_CNTL__EN_MASK=1 */
1216         wb_gpu_addr = ring->wptr_gpu_addr;
1217         mqd->cp_hqd_pq_wptr_poll_addr_lo = wb_gpu_addr & 0xfffffff8;
1218         mqd->cp_hqd_pq_wptr_poll_addr_hi = upper_32_bits(wb_gpu_addr) & 0xffff;
1219
1220         /* set up the HQD, this is similar to CP_RB0_CNTL */
1221         tmp = regCP_HQD_PQ_CONTROL_DEFAULT;
1222         tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, QUEUE_SIZE,
1223                             (order_base_2(ring->ring_size / 4) - 1));
1224         tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, RPTR_BLOCK_SIZE,
1225                             ((order_base_2(AMDGPU_GPU_PAGE_SIZE / 4) - 1) << 8));
1226         tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, UNORD_DISPATCH, 1);
1227         tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, TUNNEL_DISPATCH, 0);
1228         tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, PRIV_STATE, 1);
1229         tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, KMD_QUEUE, 1);
1230         tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, NO_UPDATE_RPTR, 1);
1231         mqd->cp_hqd_pq_control = tmp;
1232
1233         /* enable doorbell */
1234         tmp = 0;
1235         if (ring->use_doorbell) {
1236                 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
1237                                     DOORBELL_OFFSET, ring->doorbell_index);
1238                 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
1239                                     DOORBELL_EN, 1);
1240                 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
1241                                     DOORBELL_SOURCE, 0);
1242                 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
1243                                     DOORBELL_HIT, 0);
1244         } else {
1245                 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
1246                                     DOORBELL_EN, 0);
1247         }
1248         mqd->cp_hqd_pq_doorbell_control = tmp;
1249
1250         mqd->cp_hqd_vmid = 0;
1251         /* activate the queue */
1252         mqd->cp_hqd_active = 1;
1253
1254         tmp = regCP_HQD_PERSISTENT_STATE_DEFAULT;
1255         tmp = REG_SET_FIELD(tmp, CP_HQD_PERSISTENT_STATE,
1256                             PRELOAD_SIZE, 0x55);
1257         mqd->cp_hqd_persistent_state = tmp;
1258
1259         mqd->cp_hqd_ib_control = regCP_HQD_IB_CONTROL_DEFAULT;
1260         mqd->cp_hqd_iq_timer = regCP_HQD_IQ_TIMER_DEFAULT;
1261         mqd->cp_hqd_quantum = regCP_HQD_QUANTUM_DEFAULT;
1262
1263         /*
1264          * Set CP_HQD_GFX_CONTROL.DB_UPDATED_MSG_EN[15] to enable unmapped
1265          * doorbell handling. This is a reserved CP internal register can
1266          * not be accesss by others
1267          */
1268         mqd->reserved_184 = BIT(15);
1269
1270         return 0;
1271 }
1272
1273 static void mes_v12_0_queue_init_register(struct amdgpu_ring *ring)
1274 {
1275         struct v12_compute_mqd *mqd = ring->mqd_ptr;
1276         struct amdgpu_device *adev = ring->adev;
1277         uint32_t data = 0;
1278
1279         mutex_lock(&adev->srbm_mutex);
1280         soc21_grbm_select(adev, 3, ring->pipe, 0, 0);
1281
1282         /* set CP_HQD_VMID.VMID = 0. */
1283         data = RREG32_SOC15(GC, 0, regCP_HQD_VMID);
1284         data = REG_SET_FIELD(data, CP_HQD_VMID, VMID, 0);
1285         WREG32_SOC15(GC, 0, regCP_HQD_VMID, data);
1286
1287         /* set CP_HQD_PQ_DOORBELL_CONTROL.DOORBELL_EN=0 */
1288         data = RREG32_SOC15(GC, 0, regCP_HQD_PQ_DOORBELL_CONTROL);
1289         data = REG_SET_FIELD(data, CP_HQD_PQ_DOORBELL_CONTROL,
1290                              DOORBELL_EN, 0);
1291         WREG32_SOC15(GC, 0, regCP_HQD_PQ_DOORBELL_CONTROL, data);
1292
1293         /* set CP_MQD_BASE_ADDR/HI with the MQD base address */
1294         WREG32_SOC15(GC, 0, regCP_MQD_BASE_ADDR, mqd->cp_mqd_base_addr_lo);
1295         WREG32_SOC15(GC, 0, regCP_MQD_BASE_ADDR_HI, mqd->cp_mqd_base_addr_hi);
1296
1297         /* set CP_MQD_CONTROL.VMID=0 */
1298         data = RREG32_SOC15(GC, 0, regCP_MQD_CONTROL);
1299         data = REG_SET_FIELD(data, CP_MQD_CONTROL, VMID, 0);
1300         WREG32_SOC15(GC, 0, regCP_MQD_CONTROL, 0);
1301
1302         /* set CP_HQD_PQ_BASE/HI with the ring buffer base address */
1303         WREG32_SOC15(GC, 0, regCP_HQD_PQ_BASE, mqd->cp_hqd_pq_base_lo);
1304         WREG32_SOC15(GC, 0, regCP_HQD_PQ_BASE_HI, mqd->cp_hqd_pq_base_hi);
1305
1306         /* set CP_HQD_PQ_RPTR_REPORT_ADDR/HI */
1307         WREG32_SOC15(GC, 0, regCP_HQD_PQ_RPTR_REPORT_ADDR,
1308                      mqd->cp_hqd_pq_rptr_report_addr_lo);
1309         WREG32_SOC15(GC, 0, regCP_HQD_PQ_RPTR_REPORT_ADDR_HI,
1310                      mqd->cp_hqd_pq_rptr_report_addr_hi);
1311
1312         /* set CP_HQD_PQ_CONTROL */
1313         WREG32_SOC15(GC, 0, regCP_HQD_PQ_CONTROL, mqd->cp_hqd_pq_control);
1314
1315         /* set CP_HQD_PQ_WPTR_POLL_ADDR/HI */
1316         WREG32_SOC15(GC, 0, regCP_HQD_PQ_WPTR_POLL_ADDR,
1317                      mqd->cp_hqd_pq_wptr_poll_addr_lo);
1318         WREG32_SOC15(GC, 0, regCP_HQD_PQ_WPTR_POLL_ADDR_HI,
1319                      mqd->cp_hqd_pq_wptr_poll_addr_hi);
1320
1321         /* set CP_HQD_PQ_DOORBELL_CONTROL */
1322         WREG32_SOC15(GC, 0, regCP_HQD_PQ_DOORBELL_CONTROL,
1323                      mqd->cp_hqd_pq_doorbell_control);
1324
1325         /* set CP_HQD_PERSISTENT_STATE.PRELOAD_SIZE=0x53 */
1326         WREG32_SOC15(GC, 0, regCP_HQD_PERSISTENT_STATE, mqd->cp_hqd_persistent_state);
1327
1328         /* set CP_HQD_ACTIVE.ACTIVE=1 */
1329         WREG32_SOC15(GC, 0, regCP_HQD_ACTIVE, mqd->cp_hqd_active);
1330
1331         soc21_grbm_select(adev, 0, 0, 0, 0);
1332         mutex_unlock(&adev->srbm_mutex);
1333 }
1334
1335 static int mes_v12_0_kiq_enable_queue(struct amdgpu_device *adev)
1336 {
1337         struct amdgpu_kiq *kiq = &adev->gfx.kiq[0];
1338         struct amdgpu_ring *kiq_ring = &adev->gfx.kiq[0].ring;
1339         int r;
1340
1341         if (!kiq->pmf || !kiq->pmf->kiq_map_queues)
1342                 return -EINVAL;
1343
1344         r = amdgpu_ring_alloc(kiq_ring, kiq->pmf->map_queues_size);
1345         if (r) {
1346                 DRM_ERROR("Failed to lock KIQ (%d).\n", r);
1347                 return r;
1348         }
1349
1350         kiq->pmf->kiq_map_queues(kiq_ring, &adev->mes.ring[0]);
1351
1352         r = amdgpu_ring_test_ring(kiq_ring);
1353         if (r) {
1354                 DRM_ERROR("kfq enable failed\n");
1355                 kiq_ring->sched.ready = false;
1356         }
1357         return r;
1358 }
1359
1360 static int mes_v12_0_queue_init(struct amdgpu_device *adev,
1361                                 enum admgpu_mes_pipe pipe)
1362 {
1363         struct amdgpu_ring *ring;
1364         int r;
1365
1366         if (!adev->enable_uni_mes && pipe == AMDGPU_MES_KIQ_PIPE)
1367                 ring = &adev->gfx.kiq[0].ring;
1368         else
1369                 ring = &adev->mes.ring[pipe];
1370
1371         if ((adev->enable_uni_mes || pipe == AMDGPU_MES_SCHED_PIPE) &&
1372             (amdgpu_in_reset(adev) || adev->in_suspend)) {
1373                 *(ring->wptr_cpu_addr) = 0;
1374                 *(ring->rptr_cpu_addr) = 0;
1375                 amdgpu_ring_clear_ring(ring);
1376         }
1377
1378         r = mes_v12_0_mqd_init(ring);
1379         if (r)
1380                 return r;
1381
1382         if (pipe == AMDGPU_MES_SCHED_PIPE) {
1383                 if (adev->enable_uni_mes)
1384                         r = amdgpu_mes_map_legacy_queue(adev, ring);
1385                 else
1386                         r = mes_v12_0_kiq_enable_queue(adev);
1387                 if (r)
1388                         return r;
1389         } else {
1390                 mes_v12_0_queue_init_register(ring);
1391         }
1392
1393         /* get MES scheduler/KIQ versions */
1394         mutex_lock(&adev->srbm_mutex);
1395         soc21_grbm_select(adev, 3, pipe, 0, 0);
1396
1397         if (pipe == AMDGPU_MES_SCHED_PIPE)
1398                 adev->mes.sched_version = RREG32_SOC15(GC, 0, regCP_MES_GP3_LO);
1399         else if (pipe == AMDGPU_MES_KIQ_PIPE && adev->enable_mes_kiq)
1400                 adev->mes.kiq_version = RREG32_SOC15(GC, 0, regCP_MES_GP3_LO);
1401
1402         soc21_grbm_select(adev, 0, 0, 0, 0);
1403         mutex_unlock(&adev->srbm_mutex);
1404
1405         return 0;
1406 }
1407
1408 static int mes_v12_0_ring_init(struct amdgpu_device *adev, int pipe)
1409 {
1410         struct amdgpu_ring *ring;
1411
1412         ring = &adev->mes.ring[pipe];
1413
1414         ring->funcs = &mes_v12_0_ring_funcs;
1415
1416         ring->me = 3;
1417         ring->pipe = pipe;
1418         ring->queue = 0;
1419
1420         ring->ring_obj = NULL;
1421         ring->use_doorbell = true;
1422         ring->eop_gpu_addr = adev->mes.eop_gpu_addr[pipe];
1423         ring->no_scheduler = true;
1424         sprintf(ring->name, "mes_%d.%d.%d", ring->me, ring->pipe, ring->queue);
1425
1426         if (pipe == AMDGPU_MES_SCHED_PIPE)
1427                 ring->doorbell_index = adev->doorbell_index.mes_ring0 << 1;
1428         else
1429                 ring->doorbell_index = adev->doorbell_index.mes_ring1 << 1;
1430
1431         return amdgpu_ring_init(adev, ring, 1024, NULL, 0,
1432                                 AMDGPU_RING_PRIO_DEFAULT, NULL);
1433 }
1434
1435 static int mes_v12_0_kiq_ring_init(struct amdgpu_device *adev)
1436 {
1437         struct amdgpu_ring *ring;
1438
1439         spin_lock_init(&adev->gfx.kiq[0].ring_lock);
1440
1441         ring = &adev->gfx.kiq[0].ring;
1442
1443         ring->me = 3;
1444         ring->pipe = 1;
1445         ring->queue = 0;
1446
1447         ring->adev = NULL;
1448         ring->ring_obj = NULL;
1449         ring->use_doorbell = true;
1450         ring->doorbell_index = adev->doorbell_index.mes_ring1 << 1;
1451         ring->eop_gpu_addr = adev->mes.eop_gpu_addr[AMDGPU_MES_KIQ_PIPE];
1452         ring->no_scheduler = true;
1453         sprintf(ring->name, "mes_kiq_%d.%d.%d",
1454                 ring->me, ring->pipe, ring->queue);
1455
1456         return amdgpu_ring_init(adev, ring, 1024, NULL, 0,
1457                                 AMDGPU_RING_PRIO_DEFAULT, NULL);
1458 }
1459
1460 static int mes_v12_0_mqd_sw_init(struct amdgpu_device *adev,
1461                                  enum admgpu_mes_pipe pipe)
1462 {
1463         int r, mqd_size = sizeof(struct v12_compute_mqd);
1464         struct amdgpu_ring *ring;
1465
1466         if (!adev->enable_uni_mes && pipe == AMDGPU_MES_KIQ_PIPE)
1467                 ring = &adev->gfx.kiq[0].ring;
1468         else
1469                 ring = &adev->mes.ring[pipe];
1470
1471         if (ring->mqd_obj)
1472                 return 0;
1473
1474         r = amdgpu_bo_create_kernel(adev, mqd_size, PAGE_SIZE,
1475                                     AMDGPU_GEM_DOMAIN_GTT, &ring->mqd_obj,
1476                                     &ring->mqd_gpu_addr, &ring->mqd_ptr);
1477         if (r) {
1478                 dev_warn(adev->dev, "failed to create ring mqd bo (%d)", r);
1479                 return r;
1480         }
1481
1482         memset(ring->mqd_ptr, 0, mqd_size);
1483
1484         /* prepare MQD backup */
1485         adev->mes.mqd_backup[pipe] = kmalloc(mqd_size, GFP_KERNEL);
1486         if (!adev->mes.mqd_backup[pipe])
1487                 dev_warn(adev->dev,
1488                          "no memory to create MQD backup for ring %s\n",
1489                          ring->name);
1490
1491         return 0;
1492 }
1493
1494 static int mes_v12_0_sw_init(struct amdgpu_ip_block *ip_block)
1495 {
1496         struct amdgpu_device *adev = ip_block->adev;
1497         int pipe, r;
1498
1499         adev->mes.funcs = &mes_v12_0_funcs;
1500         adev->mes.kiq_hw_init = &mes_v12_0_kiq_hw_init;
1501         adev->mes.kiq_hw_fini = &mes_v12_0_kiq_hw_fini;
1502         adev->mes.enable_legacy_queue_map = true;
1503
1504         adev->mes.event_log_size = adev->enable_uni_mes ?
1505                 (AMDGPU_MAX_MES_PIPES * (AMDGPU_MES_LOG_BUFFER_SIZE + AMDGPU_MES_MSCRATCH_SIZE)) :
1506                 (AMDGPU_MES_LOG_BUFFER_SIZE + AMDGPU_MES_MSCRATCH_SIZE);
1507         r = amdgpu_mes_init(adev);
1508         if (r)
1509                 return r;
1510
1511         for (pipe = 0; pipe < AMDGPU_MAX_MES_PIPES; pipe++) {
1512                 r = mes_v12_0_allocate_eop_buf(adev, pipe);
1513                 if (r)
1514                         return r;
1515
1516                 r = mes_v12_0_mqd_sw_init(adev, pipe);
1517                 if (r)
1518                         return r;
1519
1520                 if (!adev->enable_uni_mes && pipe == AMDGPU_MES_KIQ_PIPE)
1521                         r = mes_v12_0_kiq_ring_init(adev);
1522                 else
1523                         r = mes_v12_0_ring_init(adev, pipe);
1524                 if (r)
1525                         return r;
1526         }
1527
1528         return 0;
1529 }
1530
1531 static int mes_v12_0_sw_fini(struct amdgpu_ip_block *ip_block)
1532 {
1533         struct amdgpu_device *adev = ip_block->adev;
1534         int pipe;
1535
1536         for (pipe = 0; pipe < AMDGPU_MAX_MES_PIPES; pipe++) {
1537                 kfree(adev->mes.mqd_backup[pipe]);
1538
1539                 amdgpu_bo_free_kernel(&adev->mes.eop_gpu_obj[pipe],
1540                                       &adev->mes.eop_gpu_addr[pipe],
1541                                       NULL);
1542                 amdgpu_ucode_release(&adev->mes.fw[pipe]);
1543
1544                 if (adev->enable_uni_mes || pipe == AMDGPU_MES_SCHED_PIPE) {
1545                         amdgpu_bo_free_kernel(&adev->mes.ring[pipe].mqd_obj,
1546                                               &adev->mes.ring[pipe].mqd_gpu_addr,
1547                                               &adev->mes.ring[pipe].mqd_ptr);
1548                         amdgpu_ring_fini(&adev->mes.ring[pipe]);
1549                 }
1550         }
1551
1552         if (!adev->enable_uni_mes) {
1553                 amdgpu_bo_free_kernel(&adev->gfx.kiq[0].ring.mqd_obj,
1554                                       &adev->gfx.kiq[0].ring.mqd_gpu_addr,
1555                                       &adev->gfx.kiq[0].ring.mqd_ptr);
1556                 amdgpu_ring_fini(&adev->gfx.kiq[0].ring);
1557         }
1558
1559         if (adev->firmware.load_type == AMDGPU_FW_LOAD_DIRECT) {
1560                 mes_v12_0_free_ucode_buffers(adev, AMDGPU_MES_KIQ_PIPE);
1561                 mes_v12_0_free_ucode_buffers(adev, AMDGPU_MES_SCHED_PIPE);
1562         }
1563
1564         amdgpu_mes_fini(adev);
1565         return 0;
1566 }
1567
1568 static void mes_v12_0_kiq_dequeue_sched(struct amdgpu_device *adev)
1569 {
1570         uint32_t data;
1571         int i;
1572
1573         mutex_lock(&adev->srbm_mutex);
1574         soc21_grbm_select(adev, 3, AMDGPU_MES_SCHED_PIPE, 0, 0);
1575
1576         /* disable the queue if it's active */
1577         if (RREG32_SOC15(GC, 0, regCP_HQD_ACTIVE) & 1) {
1578                 WREG32_SOC15(GC, 0, regCP_HQD_DEQUEUE_REQUEST, 1);
1579                 for (i = 0; i < adev->usec_timeout; i++) {
1580                         if (!(RREG32_SOC15(GC, 0, regCP_HQD_ACTIVE) & 1))
1581                                 break;
1582                         udelay(1);
1583                 }
1584         }
1585         data = RREG32_SOC15(GC, 0, regCP_HQD_PQ_DOORBELL_CONTROL);
1586         data = REG_SET_FIELD(data, CP_HQD_PQ_DOORBELL_CONTROL,
1587                                 DOORBELL_EN, 0);
1588         data = REG_SET_FIELD(data, CP_HQD_PQ_DOORBELL_CONTROL,
1589                                 DOORBELL_HIT, 1);
1590         WREG32_SOC15(GC, 0, regCP_HQD_PQ_DOORBELL_CONTROL, data);
1591
1592         WREG32_SOC15(GC, 0, regCP_HQD_PQ_DOORBELL_CONTROL, 0);
1593
1594         WREG32_SOC15(GC, 0, regCP_HQD_PQ_WPTR_LO, 0);
1595         WREG32_SOC15(GC, 0, regCP_HQD_PQ_WPTR_HI, 0);
1596         WREG32_SOC15(GC, 0, regCP_HQD_PQ_RPTR, 0);
1597
1598         soc21_grbm_select(adev, 0, 0, 0, 0);
1599         mutex_unlock(&adev->srbm_mutex);
1600
1601         adev->mes.ring[0].sched.ready = false;
1602 }
1603
1604 static void mes_v12_0_kiq_setting(struct amdgpu_ring *ring)
1605 {
1606         uint32_t tmp;
1607         struct amdgpu_device *adev = ring->adev;
1608
1609         /* tell RLC which is KIQ queue */
1610         tmp = RREG32_SOC15(GC, 0, regRLC_CP_SCHEDULERS);
1611         tmp &= 0xffffff00;
1612         tmp |= (ring->me << 5) | (ring->pipe << 3) | (ring->queue);
1613         WREG32_SOC15(GC, 0, regRLC_CP_SCHEDULERS, tmp | 0x80);
1614 }
1615
1616 static int mes_v12_0_kiq_hw_init(struct amdgpu_device *adev)
1617 {
1618         int r = 0;
1619         struct amdgpu_ip_block *ip_block;
1620
1621         if (adev->enable_uni_mes)
1622                 mes_v12_0_kiq_setting(&adev->mes.ring[AMDGPU_MES_KIQ_PIPE]);
1623         else
1624                 mes_v12_0_kiq_setting(&adev->gfx.kiq[0].ring);
1625
1626         if (adev->firmware.load_type == AMDGPU_FW_LOAD_DIRECT) {
1627
1628                 r = mes_v12_0_load_microcode(adev, AMDGPU_MES_SCHED_PIPE, false);
1629                 if (r) {
1630                         DRM_ERROR("failed to load MES fw, r=%d\n", r);
1631                         return r;
1632                 }
1633
1634                 r = mes_v12_0_load_microcode(adev, AMDGPU_MES_KIQ_PIPE, true);
1635                 if (r) {
1636                         DRM_ERROR("failed to load MES kiq fw, r=%d\n", r);
1637                         return r;
1638                 }
1639
1640                 mes_v12_0_set_ucode_start_addr(adev);
1641
1642         } else if (adev->firmware.load_type == AMDGPU_FW_LOAD_RLC_BACKDOOR_AUTO)
1643                 mes_v12_0_set_ucode_start_addr(adev);
1644
1645         mes_v12_0_enable(adev, true);
1646
1647         ip_block = amdgpu_device_ip_get_ip_block(adev, AMD_IP_BLOCK_TYPE_MES);
1648         if (unlikely(!ip_block)) {
1649                 dev_err(adev->dev, "Failed to get MES handle\n");
1650                 return -EINVAL;
1651         }
1652
1653         r = mes_v12_0_queue_init(adev, AMDGPU_MES_KIQ_PIPE);
1654         if (r)
1655                 goto failure;
1656
1657         if (adev->enable_uni_mes) {
1658                 r = mes_v12_0_set_hw_resources(&adev->mes, AMDGPU_MES_KIQ_PIPE);
1659                 if (r)
1660                         goto failure;
1661
1662                 mes_v12_0_set_hw_resources_1(&adev->mes, AMDGPU_MES_KIQ_PIPE);
1663         }
1664
1665         if (adev->mes.enable_legacy_queue_map) {
1666                 r = mes_v12_0_hw_init(ip_block);
1667                 if (r)
1668                         goto failure;
1669         }
1670
1671         return r;
1672
1673 failure:
1674         mes_v12_0_hw_fini(ip_block);
1675         return r;
1676 }
1677
1678 static int mes_v12_0_kiq_hw_fini(struct amdgpu_device *adev)
1679 {
1680         if (adev->mes.ring[0].sched.ready) {
1681                 if (adev->enable_uni_mes)
1682                         amdgpu_mes_unmap_legacy_queue(adev,
1683                                       &adev->mes.ring[AMDGPU_MES_SCHED_PIPE],
1684                                       RESET_QUEUES, 0, 0);
1685                 else
1686                         mes_v12_0_kiq_dequeue_sched(adev);
1687
1688                 adev->mes.ring[0].sched.ready = false;
1689         }
1690
1691         mes_v12_0_enable(adev, false);
1692
1693         return 0;
1694 }
1695
1696 static int mes_v12_0_hw_init(struct amdgpu_ip_block *ip_block)
1697 {
1698         int r;
1699         struct amdgpu_device *adev = ip_block->adev;
1700
1701         if (adev->mes.ring[0].sched.ready)
1702                 goto out;
1703
1704         if (!adev->enable_mes_kiq) {
1705                 if (adev->firmware.load_type == AMDGPU_FW_LOAD_DIRECT) {
1706                         r = mes_v12_0_load_microcode(adev,
1707                                              AMDGPU_MES_SCHED_PIPE, true);
1708                         if (r) {
1709                                 DRM_ERROR("failed to MES fw, r=%d\n", r);
1710                                 return r;
1711                         }
1712
1713                         mes_v12_0_set_ucode_start_addr(adev);
1714
1715                 } else if (adev->firmware.load_type ==
1716                            AMDGPU_FW_LOAD_RLC_BACKDOOR_AUTO) {
1717
1718                         mes_v12_0_set_ucode_start_addr(adev);
1719                 }
1720
1721                 mes_v12_0_enable(adev, true);
1722         }
1723
1724         /* Enable the MES to handle doorbell ring on unmapped queue */
1725         mes_v12_0_enable_unmapped_doorbell_handling(&adev->mes, true);
1726
1727         r = mes_v12_0_queue_init(adev, AMDGPU_MES_SCHED_PIPE);
1728         if (r)
1729                 goto failure;
1730
1731         r = mes_v12_0_set_hw_resources(&adev->mes, AMDGPU_MES_SCHED_PIPE);
1732         if (r)
1733                 goto failure;
1734
1735         if (adev->enable_uni_mes)
1736                 mes_v12_0_set_hw_resources_1(&adev->mes, AMDGPU_MES_SCHED_PIPE);
1737
1738         mes_v12_0_init_aggregated_doorbell(&adev->mes);
1739
1740         r = mes_v12_0_query_sched_status(&adev->mes, AMDGPU_MES_SCHED_PIPE);
1741         if (r) {
1742                 DRM_ERROR("MES is busy\n");
1743                 goto failure;
1744         }
1745
1746 out:
1747         /*
1748          * Disable KIQ ring usage from the driver once MES is enabled.
1749          * MES uses KIQ ring exclusively so driver cannot access KIQ ring
1750          * with MES enabled.
1751          */
1752         adev->gfx.kiq[0].ring.sched.ready = false;
1753         adev->mes.ring[0].sched.ready = true;
1754
1755         return 0;
1756
1757 failure:
1758         mes_v12_0_hw_fini(ip_block);
1759         return r;
1760 }
1761
1762 static int mes_v12_0_hw_fini(struct amdgpu_ip_block *ip_block)
1763 {
1764         return 0;
1765 }
1766
1767 static int mes_v12_0_suspend(struct amdgpu_ip_block *ip_block)
1768 {
1769         int r;
1770
1771         r = amdgpu_mes_suspend(ip_block->adev);
1772         if (r)
1773                 return r;
1774
1775         return mes_v12_0_hw_fini(ip_block);
1776 }
1777
1778 static int mes_v12_0_resume(struct amdgpu_ip_block *ip_block)
1779 {
1780         int r;
1781
1782         r = mes_v12_0_hw_init(ip_block);
1783         if (r)
1784                 return r;
1785
1786         return amdgpu_mes_resume(ip_block->adev);
1787 }
1788
1789 static int mes_v12_0_early_init(struct amdgpu_ip_block *ip_block)
1790 {
1791         struct amdgpu_device *adev = ip_block->adev;
1792         int pipe, r;
1793
1794         for (pipe = 0; pipe < AMDGPU_MAX_MES_PIPES; pipe++) {
1795                 r = amdgpu_mes_init_microcode(adev, pipe);
1796                 if (r)
1797                         return r;
1798         }
1799
1800         return 0;
1801 }
1802
1803 static int mes_v12_0_late_init(struct amdgpu_ip_block *ip_block)
1804 {
1805         struct amdgpu_device *adev = ip_block->adev;
1806
1807         /* it's only intended for use in mes_self_test case, not for s0ix and reset */
1808         if (!amdgpu_in_reset(adev) && !adev->in_s0ix && !adev->in_suspend)
1809                 amdgpu_mes_self_test(adev);
1810
1811         return 0;
1812 }
1813
1814 static const struct amd_ip_funcs mes_v12_0_ip_funcs = {
1815         .name = "mes_v12_0",
1816         .early_init = mes_v12_0_early_init,
1817         .late_init = mes_v12_0_late_init,
1818         .sw_init = mes_v12_0_sw_init,
1819         .sw_fini = mes_v12_0_sw_fini,
1820         .hw_init = mes_v12_0_hw_init,
1821         .hw_fini = mes_v12_0_hw_fini,
1822         .suspend = mes_v12_0_suspend,
1823         .resume = mes_v12_0_resume,
1824 };
1825
1826 const struct amdgpu_ip_block_version mes_v12_0_ip_block = {
1827         .type = AMD_IP_BLOCK_TYPE_MES,
1828         .major = 12,
1829         .minor = 0,
1830         .rev = 0,
1831         .funcs = &mes_v12_0_ip_funcs,
1832 };