drivers/gpu/drm/amd/amdgpu/mes_v11_0.c

   1 /*
   2  * Copyright 2019 Advanced Micro Devices, Inc.
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice shall be included in
  12  * all copies or substantial portions of the Software.
  13  *
  14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  20  * OTHER DEALINGS IN THE SOFTWARE.
  21  *
  22  */
  23
  24 #include <linux/firmware.h>
  25 #include <linux/module.h>
  26 #include "amdgpu.h"
  27 #include "soc15_common.h"
  28 #include "soc21.h"
  29 #include "gfx_v11_0.h"
  30 #include "gc/gc_11_0_0_offset.h"
  31 #include "gc/gc_11_0_0_sh_mask.h"
  32 #include "gc/gc_11_0_0_default.h"
  33 #include "v11_structs.h"
  34 #include "mes_v11_api_def.h"
  35
  36 MODULE_FIRMWARE("amdgpu/gc_11_0_0_mes.bin");
  37 MODULE_FIRMWARE("amdgpu/gc_11_0_0_mes_2.bin");
  38 MODULE_FIRMWARE("amdgpu/gc_11_0_0_mes1.bin");
  39 MODULE_FIRMWARE("amdgpu/gc_11_0_1_mes.bin");
  40 MODULE_FIRMWARE("amdgpu/gc_11_0_1_mes_2.bin");
  41 MODULE_FIRMWARE("amdgpu/gc_11_0_1_mes1.bin");
  42 MODULE_FIRMWARE("amdgpu/gc_11_0_2_mes.bin");
  43 MODULE_FIRMWARE("amdgpu/gc_11_0_2_mes_2.bin");
  44 MODULE_FIRMWARE("amdgpu/gc_11_0_2_mes1.bin");
  45 MODULE_FIRMWARE("amdgpu/gc_11_0_3_mes.bin");
  46 MODULE_FIRMWARE("amdgpu/gc_11_0_3_mes_2.bin");
  47 MODULE_FIRMWARE("amdgpu/gc_11_0_3_mes1.bin");
  48 MODULE_FIRMWARE("amdgpu/gc_11_0_4_mes.bin");
  49 MODULE_FIRMWARE("amdgpu/gc_11_0_4_mes_2.bin");
  50 MODULE_FIRMWARE("amdgpu/gc_11_0_4_mes1.bin");
  51 MODULE_FIRMWARE("amdgpu/gc_11_5_0_mes_2.bin");
  52 MODULE_FIRMWARE("amdgpu/gc_11_5_0_mes1.bin");
  53 MODULE_FIRMWARE("amdgpu/gc_11_5_1_mes_2.bin");
  54 MODULE_FIRMWARE("amdgpu/gc_11_5_1_mes1.bin");
  55 MODULE_FIRMWARE("amdgpu/gc_11_5_2_mes_2.bin");
  56 MODULE_FIRMWARE("amdgpu/gc_11_5_2_mes1.bin");
  57
  58 static int mes_v11_0_hw_init(void *handle);
  59 static int mes_v11_0_hw_fini(void *handle);
  60 static int mes_v11_0_kiq_hw_init(struct amdgpu_device *adev);
  61 static int mes_v11_0_kiq_hw_fini(struct amdgpu_device *adev);
  62
  63 #define MES_EOP_SIZE   2048
  64 #define GFX_MES_DRAM_SIZE       0x80000
  65
  66 static void mes_v11_0_ring_set_wptr(struct amdgpu_ring *ring)
  67 {
  68         struct amdgpu_device *adev = ring->adev;
  69
  70         if (ring->use_doorbell) {
  71                 atomic64_set((atomic64_t *)ring->wptr_cpu_addr,
  72                              ring->wptr);
  73                 WDOORBELL64(ring->doorbell_index, ring->wptr);
  74         } else {
  75                 BUG();
  76         }
  77 }
  78
  79 static u64 mes_v11_0_ring_get_rptr(struct amdgpu_ring *ring)
  80 {
  81         return *ring->rptr_cpu_addr;
  82 }
  83
  84 static u64 mes_v11_0_ring_get_wptr(struct amdgpu_ring *ring)
  85 {
  86         u64 wptr;
  87
  88         if (ring->use_doorbell)
  89                 wptr = atomic64_read((atomic64_t *)ring->wptr_cpu_addr);
  90         else
  91                 BUG();
  92         return wptr;
  93 }
  94
  95 static const struct amdgpu_ring_funcs mes_v11_0_ring_funcs = {
  96         .type = AMDGPU_RING_TYPE_MES,
  97         .align_mask = 1,
  98         .nop = 0,
  99         .support_64bit_ptrs = true,
 100         .get_rptr = mes_v11_0_ring_get_rptr,
 101         .get_wptr = mes_v11_0_ring_get_wptr,
 102         .set_wptr = mes_v11_0_ring_set_wptr,
 103         .insert_nop = amdgpu_ring_insert_nop,
 104 };
 105
 106 static const char *mes_v11_0_opcodes[] = {
 107         "SET_HW_RSRC",
 108         "SET_SCHEDULING_CONFIG",
 109         "ADD_QUEUE",
 110         "REMOVE_QUEUE",
 111         "PERFORM_YIELD",
 112         "SET_GANG_PRIORITY_LEVEL",
 113         "SUSPEND",
 114         "RESUME",
 115         "RESET",
 116         "SET_LOG_BUFFER",
 117         "CHANGE_GANG_PRORITY",
 118         "QUERY_SCHEDULER_STATUS",
 119         "PROGRAM_GDS",
 120         "SET_DEBUG_VMID",
 121         "MISC",
 122         "UPDATE_ROOT_PAGE_TABLE",
 123         "AMD_LOG",
 124         "unused",
 125         "unused",
 126         "SET_HW_RSRC_1",
 127 };
 128
 129 static const char *mes_v11_0_misc_opcodes[] = {
 130         "WRITE_REG",
 131         "INV_GART",
 132         "QUERY_STATUS",
 133         "READ_REG",
 134         "WAIT_REG_MEM",
 135         "SET_SHADER_DEBUGGER",
 136 };
 137
 138 static const char *mes_v11_0_get_op_string(union MESAPI__MISC *x_pkt)
 139 {
 140         const char *op_str = NULL;
 141
 142         if (x_pkt->header.opcode < ARRAY_SIZE(mes_v11_0_opcodes))
 143                 op_str = mes_v11_0_opcodes[x_pkt->header.opcode];
 144
 145         return op_str;
 146 }
 147
 148 static const char *mes_v11_0_get_misc_op_string(union MESAPI__MISC *x_pkt)
 149 {
 150         const char *op_str = NULL;
 151
 152         if ((x_pkt->header.opcode == MES_SCH_API_MISC) &&
 153             (x_pkt->opcode < ARRAY_SIZE(mes_v11_0_misc_opcodes)))
 154                 op_str = mes_v11_0_misc_opcodes[x_pkt->opcode];
 155
 156         return op_str;
 157 }
 158
 159 static int mes_v11_0_submit_pkt_and_poll_completion(struct amdgpu_mes *mes,
 160                                                     void *pkt, int size,
 161                                                     int api_status_off)
 162 {
 163         union MESAPI__QUERY_MES_STATUS mes_status_pkt;
 164         signed long timeout = 2100000; /* 2100 ms */
 165         struct amdgpu_device *adev = mes->adev;
 166         struct amdgpu_ring *ring = &mes->ring[0];
 167         struct MES_API_STATUS *api_status;
 168         union MESAPI__MISC *x_pkt = pkt;
 169         const char *op_str, *misc_op_str;
 170         unsigned long flags;
 171         u64 status_gpu_addr;
 172         u32 seq, status_offset;
 173         u64 *status_ptr;
 174         signed long r;
 175         int ret;
 176
 177         if (x_pkt->header.opcode >= MES_SCH_API_MAX)
 178                 return -EINVAL;
 179
 180         if (amdgpu_emu_mode) {
 181                 timeout *= 100;
 182         } else if (amdgpu_sriov_vf(adev)) {
 183                 /* Worst case in sriov where all other 15 VF timeout, each VF needs about 600ms */
 184                 timeout = 15 * 600 * 1000;
 185         }
 186
 187         ret = amdgpu_device_wb_get(adev, &status_offset);
 188         if (ret)
 189                 return ret;
 190
 191         status_gpu_addr = adev->wb.gpu_addr + (status_offset * 4);
 192         status_ptr = (u64 *)&adev->wb.wb[status_offset];
 193         *status_ptr = 0;
 194
 195         spin_lock_irqsave(&mes->ring_lock[0], flags);
 196         r = amdgpu_ring_alloc(ring, (size + sizeof(mes_status_pkt)) / 4);
 197         if (r)
 198                 goto error_unlock_free;
 199
 200         seq = ++ring->fence_drv.sync_seq;
 201         r = amdgpu_fence_wait_polling(ring,
 202                                       seq - ring->fence_drv.num_fences_mask,
 203                                       timeout);
 204         if (r < 1)
 205                 goto error_undo;
 206
 207         api_status = (struct MES_API_STATUS *)((char *)pkt + api_status_off);
 208         api_status->api_completion_fence_addr = status_gpu_addr;
 209         api_status->api_completion_fence_value = 1;
 210
 211         amdgpu_ring_write_multiple(ring, pkt, size / 4);
 212
 213         memset(&mes_status_pkt, 0, sizeof(mes_status_pkt));
 214         mes_status_pkt.header.type = MES_API_TYPE_SCHEDULER;
 215         mes_status_pkt.header.opcode = MES_SCH_API_QUERY_SCHEDULER_STATUS;
 216         mes_status_pkt.header.dwsize = API_FRAME_SIZE_IN_DWORDS;
 217         mes_status_pkt.api_status.api_completion_fence_addr =
 218                 ring->fence_drv.gpu_addr;
 219         mes_status_pkt.api_status.api_completion_fence_value = seq;
 220
 221         amdgpu_ring_write_multiple(ring, &mes_status_pkt,
 222                                    sizeof(mes_status_pkt) / 4);
 223
 224         amdgpu_ring_commit(ring);
 225         spin_unlock_irqrestore(&mes->ring_lock[0], flags);
 226
 227         op_str = mes_v11_0_get_op_string(x_pkt);
 228         misc_op_str = mes_v11_0_get_misc_op_string(x_pkt);
 229
 230         if (misc_op_str)
 231                 dev_dbg(adev->dev, "MES msg=%s (%s) was emitted\n", op_str,
 232                         misc_op_str);
 233         else if (op_str)
 234                 dev_dbg(adev->dev, "MES msg=%s was emitted\n", op_str);
 235         else
 236                 dev_dbg(adev->dev, "MES msg=%d was emitted\n",
 237                         x_pkt->header.opcode);
 238
 239         r = amdgpu_fence_wait_polling(ring, seq, timeout);
 240         if (r < 1 || !*status_ptr) {
 241
 242                 if (misc_op_str)
 243                         dev_err(adev->dev, "MES failed to respond to msg=%s (%s)\n",
 244                                 op_str, misc_op_str);
 245                 else if (op_str)
 246                         dev_err(adev->dev, "MES failed to respond to msg=%s\n",
 247                                 op_str);
 248                 else
 249                         dev_err(adev->dev, "MES failed to respond to msg=%d\n",
 250                                 x_pkt->header.opcode);
 251
 252                 while (halt_if_hws_hang)
 253                         schedule();
 254
 255                 r = -ETIMEDOUT;
 256                 goto error_wb_free;
 257         }
 258
 259         amdgpu_device_wb_free(adev, status_offset);
 260         return 0;
 261
 262 error_undo:
 263         dev_err(adev->dev, "MES ring buffer is full.\n");
 264         amdgpu_ring_undo(ring);
 265
 266 error_unlock_free:
 267         spin_unlock_irqrestore(&mes->ring_lock[0], flags);
 268
 269 error_wb_free:
 270         amdgpu_device_wb_free(adev, status_offset);
 271         return r;
 272 }
 273
 274 static int convert_to_mes_queue_type(int queue_type)
 275 {
 276         if (queue_type == AMDGPU_RING_TYPE_GFX)
 277                 return MES_QUEUE_TYPE_GFX;
 278         else if (queue_type == AMDGPU_RING_TYPE_COMPUTE)
 279                 return MES_QUEUE_TYPE_COMPUTE;
 280         else if (queue_type == AMDGPU_RING_TYPE_SDMA)
 281                 return MES_QUEUE_TYPE_SDMA;
 282         else
 283                 BUG();
 284         return -1;
 285 }
 286
 287 static int mes_v11_0_add_hw_queue(struct amdgpu_mes *mes,
 288                                   struct mes_add_queue_input *input)
 289 {
 290         struct amdgpu_device *adev = mes->adev;
 291         union MESAPI__ADD_QUEUE mes_add_queue_pkt;
 292         struct amdgpu_vmhub *hub = &adev->vmhub[AMDGPU_GFXHUB(0)];
 293         uint32_t vm_cntx_cntl = hub->vm_cntx_cntl;
 294
 295         memset(&mes_add_queue_pkt, 0, sizeof(mes_add_queue_pkt));
 296
 297         mes_add_queue_pkt.header.type = MES_API_TYPE_SCHEDULER;
 298         mes_add_queue_pkt.header.opcode = MES_SCH_API_ADD_QUEUE;
 299         mes_add_queue_pkt.header.dwsize = API_FRAME_SIZE_IN_DWORDS;
 300
 301         mes_add_queue_pkt.process_id = input->process_id;
 302         mes_add_queue_pkt.page_table_base_addr = input->page_table_base_addr;
 303         mes_add_queue_pkt.process_va_start = input->process_va_start;
 304         mes_add_queue_pkt.process_va_end = input->process_va_end;
 305         mes_add_queue_pkt.process_quantum = input->process_quantum;
 306         mes_add_queue_pkt.process_context_addr = input->process_context_addr;
 307         mes_add_queue_pkt.gang_quantum = input->gang_quantum;
 308         mes_add_queue_pkt.gang_context_addr = input->gang_context_addr;
 309         mes_add_queue_pkt.inprocess_gang_priority =
 310                 input->inprocess_gang_priority;
 311         mes_add_queue_pkt.gang_global_priority_level =
 312                 input->gang_global_priority_level;
 313         mes_add_queue_pkt.doorbell_offset = input->doorbell_offset;
 314         mes_add_queue_pkt.mqd_addr = input->mqd_addr;
 315
 316         if (((adev->mes.sched_version & AMDGPU_MES_API_VERSION_MASK) >>
 317                         AMDGPU_MES_API_VERSION_SHIFT) >= 2)
 318                 mes_add_queue_pkt.wptr_addr = input->wptr_mc_addr;
 319         else
 320                 mes_add_queue_pkt.wptr_addr = input->wptr_addr;
 321
 322         mes_add_queue_pkt.queue_type =
 323                 convert_to_mes_queue_type(input->queue_type);
 324         mes_add_queue_pkt.paging = input->paging;
 325         mes_add_queue_pkt.vm_context_cntl = vm_cntx_cntl;
 326         mes_add_queue_pkt.gws_base = input->gws_base;
 327         mes_add_queue_pkt.gws_size = input->gws_size;
 328         mes_add_queue_pkt.trap_handler_addr = input->tba_addr;
 329         mes_add_queue_pkt.tma_addr = input->tma_addr;
 330         mes_add_queue_pkt.trap_en = input->trap_en;
 331         mes_add_queue_pkt.skip_process_ctx_clear = input->skip_process_ctx_clear;
 332         mes_add_queue_pkt.is_kfd_process = input->is_kfd_process;
 333
 334         /* For KFD, gds_size is re-used for queue size (needed in MES for AQL queues) */
 335         mes_add_queue_pkt.is_aql_queue = input->is_aql_queue;
 336         mes_add_queue_pkt.gds_size = input->queue_size;
 337
 338         mes_add_queue_pkt.exclusively_scheduled = input->exclusively_scheduled;
 339
 340         return mes_v11_0_submit_pkt_and_poll_completion(mes,
 341                         &mes_add_queue_pkt, sizeof(mes_add_queue_pkt),
 342                         offsetof(union MESAPI__ADD_QUEUE, api_status));
 343 }
 344
 345 static int mes_v11_0_remove_hw_queue(struct amdgpu_mes *mes,
 346                                      struct mes_remove_queue_input *input)
 347 {
 348         union MESAPI__REMOVE_QUEUE mes_remove_queue_pkt;
 349
 350         memset(&mes_remove_queue_pkt, 0, sizeof(mes_remove_queue_pkt));
 351
 352         mes_remove_queue_pkt.header.type = MES_API_TYPE_SCHEDULER;
 353         mes_remove_queue_pkt.header.opcode = MES_SCH_API_REMOVE_QUEUE;
 354         mes_remove_queue_pkt.header.dwsize = API_FRAME_SIZE_IN_DWORDS;
 355
 356         mes_remove_queue_pkt.doorbell_offset = input->doorbell_offset;
 357         mes_remove_queue_pkt.gang_context_addr = input->gang_context_addr;
 358
 359         return mes_v11_0_submit_pkt_and_poll_completion(mes,
 360                         &mes_remove_queue_pkt, sizeof(mes_remove_queue_pkt),
 361                         offsetof(union MESAPI__REMOVE_QUEUE, api_status));
 362 }
 363
 364 static int mes_v11_0_reset_queue_mmio(struct amdgpu_mes *mes, uint32_t queue_type,
 365                                       uint32_t me_id, uint32_t pipe_id,
 366                                       uint32_t queue_id, uint32_t vmid)
 367 {
 368         struct amdgpu_device *adev = mes->adev;
 369         uint32_t value;
 370         int i, r = 0;
 371
 372         amdgpu_gfx_rlc_enter_safe_mode(adev, 0);
 373
 374         if (queue_type == AMDGPU_RING_TYPE_GFX) {
 375                 dev_info(adev->dev, "reset gfx queue (%d:%d:%d: vmid:%d)\n",
 376                          me_id, pipe_id, queue_id, vmid);
 377
 378                 mutex_lock(&adev->gfx.reset_sem_mutex);
 379                 gfx_v11_0_request_gfx_index_mutex(adev, true);
 380                 /* all se allow writes */
 381                 WREG32_SOC15(GC, 0, regGRBM_GFX_INDEX,
 382                              (uint32_t)(0x1 << GRBM_GFX_INDEX__SE_BROADCAST_WRITES__SHIFT));
 383                 value = REG_SET_FIELD(0, CP_VMID_RESET, RESET_REQUEST, 1 << vmid);
 384                 if (pipe_id == 0)
 385                         value = REG_SET_FIELD(value, CP_VMID_RESET, PIPE0_QUEUES, 1 << queue_id);
 386                 else
 387                         value = REG_SET_FIELD(value, CP_VMID_RESET, PIPE1_QUEUES, 1 << queue_id);
 388                 WREG32_SOC15(GC, 0, regCP_VMID_RESET, value);
 389                 gfx_v11_0_request_gfx_index_mutex(adev, false);
 390                 mutex_unlock(&adev->gfx.reset_sem_mutex);
 391
 392                 mutex_lock(&adev->srbm_mutex);
 393                 soc21_grbm_select(adev, me_id, pipe_id, queue_id, 0);
 394                 /* wait till dequeue take effects */
 395                 for (i = 0; i < adev->usec_timeout; i++) {
 396                         if (!(RREG32_SOC15(GC, 0, regCP_GFX_HQD_ACTIVE) & 1))
 397                                 break;
 398                         udelay(1);
 399                 }
 400                 if (i >= adev->usec_timeout) {
 401                         dev_err(adev->dev, "failed to wait on gfx hqd deactivate\n");
 402                         r = -ETIMEDOUT;
 403                 }
 404
 405                 soc21_grbm_select(adev, 0, 0, 0, 0);
 406                 mutex_unlock(&adev->srbm_mutex);
 407         } else if (queue_type == AMDGPU_RING_TYPE_COMPUTE) {
 408                 dev_info(adev->dev, "reset compute queue (%d:%d:%d)\n",
 409                          me_id, pipe_id, queue_id);
 410                 mutex_lock(&adev->srbm_mutex);
 411                 soc21_grbm_select(adev, me_id, pipe_id, queue_id, 0);
 412                 WREG32_SOC15(GC, 0, regCP_HQD_DEQUEUE_REQUEST, 0x2);
 413                 WREG32_SOC15(GC, 0, regSPI_COMPUTE_QUEUE_RESET, 0x1);
 414
 415                 /* wait till dequeue take effects */
 416                 for (i = 0; i < adev->usec_timeout; i++) {
 417                         if (!(RREG32_SOC15(GC, 0, regCP_HQD_ACTIVE) & 1))
 418                                 break;
 419                         udelay(1);
 420                 }
 421                 if (i >= adev->usec_timeout) {
 422                         dev_err(adev->dev, "failed to wait on hqd deactivate\n");
 423                         r = -ETIMEDOUT;
 424                 }
 425                 soc21_grbm_select(adev, 0, 0, 0, 0);
 426                 mutex_unlock(&adev->srbm_mutex);
 427         }
 428
 429         amdgpu_gfx_rlc_exit_safe_mode(adev, 0);
 430         return r;
 431 }
 432
 433 static int mes_v11_0_reset_hw_queue(struct amdgpu_mes *mes,
 434                                     struct mes_reset_queue_input *input)
 435 {
 436         if (input->use_mmio)
 437                 return mes_v11_0_reset_queue_mmio(mes, input->queue_type,
 438                                                   input->me_id, input->pipe_id,
 439                                                   input->queue_id, input->vmid);
 440
 441         union MESAPI__RESET mes_reset_queue_pkt;
 442
 443         memset(&mes_reset_queue_pkt, 0, sizeof(mes_reset_queue_pkt));
 444
 445         mes_reset_queue_pkt.header.type = MES_API_TYPE_SCHEDULER;
 446         mes_reset_queue_pkt.header.opcode = MES_SCH_API_RESET;
 447         mes_reset_queue_pkt.header.dwsize = API_FRAME_SIZE_IN_DWORDS;
 448
 449         mes_reset_queue_pkt.doorbell_offset = input->doorbell_offset;
 450         mes_reset_queue_pkt.gang_context_addr = input->gang_context_addr;
 451         /*mes_reset_queue_pkt.reset_queue_only = 1;*/
 452
 453         return mes_v11_0_submit_pkt_and_poll_completion(mes,
 454                         &mes_reset_queue_pkt, sizeof(mes_reset_queue_pkt),
 455                         offsetof(union MESAPI__REMOVE_QUEUE, api_status));
 456 }
 457
 458 static int mes_v11_0_map_legacy_queue(struct amdgpu_mes *mes,
 459                                       struct mes_map_legacy_queue_input *input)
 460 {
 461         union MESAPI__ADD_QUEUE mes_add_queue_pkt;
 462
 463         memset(&mes_add_queue_pkt, 0, sizeof(mes_add_queue_pkt));
 464
 465         mes_add_queue_pkt.header.type = MES_API_TYPE_SCHEDULER;
 466         mes_add_queue_pkt.header.opcode = MES_SCH_API_ADD_QUEUE;
 467         mes_add_queue_pkt.header.dwsize = API_FRAME_SIZE_IN_DWORDS;
 468
 469         mes_add_queue_pkt.pipe_id = input->pipe_id;
 470         mes_add_queue_pkt.queue_id = input->queue_id;
 471         mes_add_queue_pkt.doorbell_offset = input->doorbell_offset;
 472         mes_add_queue_pkt.mqd_addr = input->mqd_addr;
 473         mes_add_queue_pkt.wptr_addr = input->wptr_addr;
 474         mes_add_queue_pkt.queue_type =
 475                 convert_to_mes_queue_type(input->queue_type);
 476         mes_add_queue_pkt.map_legacy_kq = 1;
 477
 478         return mes_v11_0_submit_pkt_and_poll_completion(mes,
 479                         &mes_add_queue_pkt, sizeof(mes_add_queue_pkt),
 480                         offsetof(union MESAPI__ADD_QUEUE, api_status));
 481 }
 482
 483 static int mes_v11_0_unmap_legacy_queue(struct amdgpu_mes *mes,
 484                         struct mes_unmap_legacy_queue_input *input)
 485 {
 486         union MESAPI__REMOVE_QUEUE mes_remove_queue_pkt;
 487
 488         memset(&mes_remove_queue_pkt, 0, sizeof(mes_remove_queue_pkt));
 489
 490         mes_remove_queue_pkt.header.type = MES_API_TYPE_SCHEDULER;
 491         mes_remove_queue_pkt.header.opcode = MES_SCH_API_REMOVE_QUEUE;
 492         mes_remove_queue_pkt.header.dwsize = API_FRAME_SIZE_IN_DWORDS;
 493
 494         mes_remove_queue_pkt.doorbell_offset = input->doorbell_offset;
 495         mes_remove_queue_pkt.gang_context_addr = 0;
 496
 497         mes_remove_queue_pkt.pipe_id = input->pipe_id;
 498         mes_remove_queue_pkt.queue_id = input->queue_id;
 499
 500         if (input->action == PREEMPT_QUEUES_NO_UNMAP) {
 501                 mes_remove_queue_pkt.preempt_legacy_gfx_queue = 1;
 502                 mes_remove_queue_pkt.tf_addr = input->trail_fence_addr;
 503                 mes_remove_queue_pkt.tf_data =
 504                         lower_32_bits(input->trail_fence_data);
 505         } else {
 506                 mes_remove_queue_pkt.unmap_legacy_queue = 1;
 507                 mes_remove_queue_pkt.queue_type =
 508                         convert_to_mes_queue_type(input->queue_type);
 509         }
 510
 511         return mes_v11_0_submit_pkt_and_poll_completion(mes,
 512                         &mes_remove_queue_pkt, sizeof(mes_remove_queue_pkt),
 513                         offsetof(union MESAPI__REMOVE_QUEUE, api_status));
 514 }
 515
 516 static int mes_v11_0_suspend_gang(struct amdgpu_mes *mes,
 517                                   struct mes_suspend_gang_input *input)
 518 {
 519         union MESAPI__SUSPEND mes_suspend_gang_pkt;
 520
 521         memset(&mes_suspend_gang_pkt, 0, sizeof(mes_suspend_gang_pkt));
 522
 523         mes_suspend_gang_pkt.header.type = MES_API_TYPE_SCHEDULER;
 524         mes_suspend_gang_pkt.header.opcode = MES_SCH_API_SUSPEND;
 525         mes_suspend_gang_pkt.header.dwsize = API_FRAME_SIZE_IN_DWORDS;
 526
 527         mes_suspend_gang_pkt.suspend_all_gangs = input->suspend_all_gangs;
 528         mes_suspend_gang_pkt.gang_context_addr = input->gang_context_addr;
 529         mes_suspend_gang_pkt.suspend_fence_addr = input->suspend_fence_addr;
 530         mes_suspend_gang_pkt.suspend_fence_value = input->suspend_fence_value;
 531
 532         return mes_v11_0_submit_pkt_and_poll_completion(mes,
 533                         &mes_suspend_gang_pkt, sizeof(mes_suspend_gang_pkt),
 534                         offsetof(union MESAPI__SUSPEND, api_status));
 535 }
 536
 537 static int mes_v11_0_resume_gang(struct amdgpu_mes *mes,
 538                                  struct mes_resume_gang_input *input)
 539 {
 540         union MESAPI__RESUME mes_resume_gang_pkt;
 541
 542         memset(&mes_resume_gang_pkt, 0, sizeof(mes_resume_gang_pkt));
 543
 544         mes_resume_gang_pkt.header.type = MES_API_TYPE_SCHEDULER;
 545         mes_resume_gang_pkt.header.opcode = MES_SCH_API_RESUME;
 546         mes_resume_gang_pkt.header.dwsize = API_FRAME_SIZE_IN_DWORDS;
 547
 548         mes_resume_gang_pkt.resume_all_gangs = input->resume_all_gangs;
 549         mes_resume_gang_pkt.gang_context_addr = input->gang_context_addr;
 550
 551         return mes_v11_0_submit_pkt_and_poll_completion(mes,
 552                         &mes_resume_gang_pkt, sizeof(mes_resume_gang_pkt),
 553                         offsetof(union MESAPI__RESUME, api_status));
 554 }
 555
 556 static int mes_v11_0_query_sched_status(struct amdgpu_mes *mes)
 557 {
 558         union MESAPI__QUERY_MES_STATUS mes_status_pkt;
 559
 560         memset(&mes_status_pkt, 0, sizeof(mes_status_pkt));
 561
 562         mes_status_pkt.header.type = MES_API_TYPE_SCHEDULER;
 563         mes_status_pkt.header.opcode = MES_SCH_API_QUERY_SCHEDULER_STATUS;
 564         mes_status_pkt.header.dwsize = API_FRAME_SIZE_IN_DWORDS;
 565
 566         return mes_v11_0_submit_pkt_and_poll_completion(mes,
 567                         &mes_status_pkt, sizeof(mes_status_pkt),
 568                         offsetof(union MESAPI__QUERY_MES_STATUS, api_status));
 569 }
 570
 571 static int mes_v11_0_misc_op(struct amdgpu_mes *mes,
 572                              struct mes_misc_op_input *input)
 573 {
 574         union MESAPI__MISC misc_pkt;
 575
 576         memset(&misc_pkt, 0, sizeof(misc_pkt));
 577
 578         misc_pkt.header.type = MES_API_TYPE_SCHEDULER;
 579         misc_pkt.header.opcode = MES_SCH_API_MISC;
 580         misc_pkt.header.dwsize = API_FRAME_SIZE_IN_DWORDS;
 581
 582         switch (input->op) {
 583         case MES_MISC_OP_READ_REG:
 584                 misc_pkt.opcode = MESAPI_MISC__READ_REG;
 585                 misc_pkt.read_reg.reg_offset = input->read_reg.reg_offset;
 586                 misc_pkt.read_reg.buffer_addr = input->read_reg.buffer_addr;
 587                 break;
 588         case MES_MISC_OP_WRITE_REG:
 589                 misc_pkt.opcode = MESAPI_MISC__WRITE_REG;
 590                 misc_pkt.write_reg.reg_offset = input->write_reg.reg_offset;
 591                 misc_pkt.write_reg.reg_value = input->write_reg.reg_value;
 592                 break;
 593         case MES_MISC_OP_WRM_REG_WAIT:
 594                 misc_pkt.opcode = MESAPI_MISC__WAIT_REG_MEM;
 595                 misc_pkt.wait_reg_mem.op = WRM_OPERATION__WAIT_REG_MEM;
 596                 misc_pkt.wait_reg_mem.reference = input->wrm_reg.ref;
 597                 misc_pkt.wait_reg_mem.mask = input->wrm_reg.mask;
 598                 misc_pkt.wait_reg_mem.reg_offset1 = input->wrm_reg.reg0;
 599                 misc_pkt.wait_reg_mem.reg_offset2 = 0;
 600                 break;
 601         case MES_MISC_OP_WRM_REG_WR_WAIT:
 602                 misc_pkt.opcode = MESAPI_MISC__WAIT_REG_MEM;
 603                 misc_pkt.wait_reg_mem.op = WRM_OPERATION__WR_WAIT_WR_REG;
 604                 misc_pkt.wait_reg_mem.reference = input->wrm_reg.ref;
 605                 misc_pkt.wait_reg_mem.mask = input->wrm_reg.mask;
 606                 misc_pkt.wait_reg_mem.reg_offset1 = input->wrm_reg.reg0;
 607                 misc_pkt.wait_reg_mem.reg_offset2 = input->wrm_reg.reg1;
 608                 break;
 609         case MES_MISC_OP_SET_SHADER_DEBUGGER:
 610                 misc_pkt.opcode = MESAPI_MISC__SET_SHADER_DEBUGGER;
 611                 misc_pkt.set_shader_debugger.process_context_addr =
 612                                 input->set_shader_debugger.process_context_addr;
 613                 misc_pkt.set_shader_debugger.flags.u32all =
 614                                 input->set_shader_debugger.flags.u32all;
 615                 misc_pkt.set_shader_debugger.spi_gdbg_per_vmid_cntl =
 616                                 input->set_shader_debugger.spi_gdbg_per_vmid_cntl;
 617                 memcpy(misc_pkt.set_shader_debugger.tcp_watch_cntl,
 618                                 input->set_shader_debugger.tcp_watch_cntl,
 619                                 sizeof(misc_pkt.set_shader_debugger.tcp_watch_cntl));
 620                 misc_pkt.set_shader_debugger.trap_en = input->set_shader_debugger.trap_en;
 621                 break;
 622         default:
 623                 DRM_ERROR("unsupported misc op (%d) \n", input->op);
 624                 return -EINVAL;
 625         }
 626
 627         return mes_v11_0_submit_pkt_and_poll_completion(mes,
 628                         &misc_pkt, sizeof(misc_pkt),
 629                         offsetof(union MESAPI__MISC, api_status));
 630 }
 631
 632 static int mes_v11_0_set_hw_resources(struct amdgpu_mes *mes)
 633 {
 634         int i;
 635         struct amdgpu_device *adev = mes->adev;
 636         union MESAPI_SET_HW_RESOURCES mes_set_hw_res_pkt;
 637
 638         memset(&mes_set_hw_res_pkt, 0, sizeof(mes_set_hw_res_pkt));
 639
 640         mes_set_hw_res_pkt.header.type = MES_API_TYPE_SCHEDULER;
 641         mes_set_hw_res_pkt.header.opcode = MES_SCH_API_SET_HW_RSRC;
 642         mes_set_hw_res_pkt.header.dwsize = API_FRAME_SIZE_IN_DWORDS;
 643
 644         mes_set_hw_res_pkt.vmid_mask_mmhub = mes->vmid_mask_mmhub;
 645         mes_set_hw_res_pkt.vmid_mask_gfxhub = mes->vmid_mask_gfxhub;
 646         mes_set_hw_res_pkt.gds_size = adev->gds.gds_size;
 647         mes_set_hw_res_pkt.paging_vmid = 0;
 648         mes_set_hw_res_pkt.g_sch_ctx_gpu_mc_ptr = mes->sch_ctx_gpu_addr[0];
 649         mes_set_hw_res_pkt.query_status_fence_gpu_mc_ptr =
 650                 mes->query_status_fence_gpu_addr[0];
 651
 652         for (i = 0; i < MAX_COMPUTE_PIPES; i++)
 653                 mes_set_hw_res_pkt.compute_hqd_mask[i] =
 654                         mes->compute_hqd_mask[i];
 655
 656         for (i = 0; i < MAX_GFX_PIPES; i++)
 657                 mes_set_hw_res_pkt.gfx_hqd_mask[i] = mes->gfx_hqd_mask[i];
 658
 659         for (i = 0; i < MAX_SDMA_PIPES; i++)
 660                 mes_set_hw_res_pkt.sdma_hqd_mask[i] = mes->sdma_hqd_mask[i];
 661
 662         for (i = 0; i < AMD_PRIORITY_NUM_LEVELS; i++)
 663                 mes_set_hw_res_pkt.aggregated_doorbells[i] =
 664                         mes->aggregated_doorbells[i];
 665
 666         for (i = 0; i < 5; i++) {
 667                 mes_set_hw_res_pkt.gc_base[i] = adev->reg_offset[GC_HWIP][0][i];
 668                 mes_set_hw_res_pkt.mmhub_base[i] =
 669                                 adev->reg_offset[MMHUB_HWIP][0][i];
 670                 mes_set_hw_res_pkt.osssys_base[i] =
 671                 adev->reg_offset[OSSSYS_HWIP][0][i];
 672         }
 673
 674         mes_set_hw_res_pkt.disable_reset = 1;
 675         mes_set_hw_res_pkt.disable_mes_log = 1;
 676         mes_set_hw_res_pkt.use_different_vmid_compute = 1;
 677         mes_set_hw_res_pkt.enable_reg_active_poll = 1;
 678         mes_set_hw_res_pkt.enable_level_process_quantum_check = 1;
 679         mes_set_hw_res_pkt.oversubscription_timer = 50;
 680         if (amdgpu_mes_log_enable) {
 681                 mes_set_hw_res_pkt.enable_mes_event_int_logging = 1;
 682                 mes_set_hw_res_pkt.event_intr_history_gpu_mc_ptr =
 683                                         mes->event_log_gpu_addr;
 684         }
 685
 686         return mes_v11_0_submit_pkt_and_poll_completion(mes,
 687                         &mes_set_hw_res_pkt, sizeof(mes_set_hw_res_pkt),
 688                         offsetof(union MESAPI_SET_HW_RESOURCES, api_status));
 689 }
 690
 691 static int mes_v11_0_set_hw_resources_1(struct amdgpu_mes *mes)
 692 {
 693         int size = 128 * PAGE_SIZE;
 694         int ret = 0;
 695         struct amdgpu_device *adev = mes->adev;
 696         union MESAPI_SET_HW_RESOURCES_1 mes_set_hw_res_pkt;
 697         memset(&mes_set_hw_res_pkt, 0, sizeof(mes_set_hw_res_pkt));
 698
 699         mes_set_hw_res_pkt.header.type = MES_API_TYPE_SCHEDULER;
 700         mes_set_hw_res_pkt.header.opcode = MES_SCH_API_SET_HW_RSRC_1;
 701         mes_set_hw_res_pkt.header.dwsize = API_FRAME_SIZE_IN_DWORDS;
 702         mes_set_hw_res_pkt.enable_mes_info_ctx = 1;
 703
 704         ret = amdgpu_bo_create_kernel(adev, size, PAGE_SIZE,
 705                                 AMDGPU_GEM_DOMAIN_VRAM,
 706                                 &mes->resource_1,
 707                                 &mes->resource_1_gpu_addr,
 708                                 &mes->resource_1_addr);
 709         if (ret) {
 710                 dev_err(adev->dev, "(%d) failed to create mes resource_1 bo\n", ret);
 711                 return ret;
 712         }
 713
 714         mes_set_hw_res_pkt.mes_info_ctx_mc_addr = mes->resource_1_gpu_addr;
 715         mes_set_hw_res_pkt.mes_info_ctx_size = mes->resource_1->tbo.base.size;
 716         return mes_v11_0_submit_pkt_and_poll_completion(mes,
 717                         &mes_set_hw_res_pkt, sizeof(mes_set_hw_res_pkt),
 718                         offsetof(union MESAPI_SET_HW_RESOURCES_1, api_status));
 719 }
 720
 721 static int mes_v11_0_reset_legacy_queue(struct amdgpu_mes *mes,
 722                                         struct mes_reset_legacy_queue_input *input)
 723 {
 724         union MESAPI__RESET mes_reset_queue_pkt;
 725
 726         if (input->use_mmio)
 727                 return mes_v11_0_reset_queue_mmio(mes, input->queue_type,
 728                                                   input->me_id, input->pipe_id,
 729                                                   input->queue_id, input->vmid);
 730
 731         memset(&mes_reset_queue_pkt, 0, sizeof(mes_reset_queue_pkt));
 732
 733         mes_reset_queue_pkt.header.type = MES_API_TYPE_SCHEDULER;
 734         mes_reset_queue_pkt.header.opcode = MES_SCH_API_RESET;
 735         mes_reset_queue_pkt.header.dwsize = API_FRAME_SIZE_IN_DWORDS;
 736
 737         mes_reset_queue_pkt.queue_type =
 738                 convert_to_mes_queue_type(input->queue_type);
 739
 740         if (mes_reset_queue_pkt.queue_type == MES_QUEUE_TYPE_GFX) {
 741                 mes_reset_queue_pkt.reset_legacy_gfx = 1;
 742                 mes_reset_queue_pkt.pipe_id_lp = input->pipe_id;
 743                 mes_reset_queue_pkt.queue_id_lp = input->queue_id;
 744                 mes_reset_queue_pkt.mqd_mc_addr_lp = input->mqd_addr;
 745                 mes_reset_queue_pkt.doorbell_offset_lp = input->doorbell_offset;
 746                 mes_reset_queue_pkt.wptr_addr_lp = input->wptr_addr;
 747                 mes_reset_queue_pkt.vmid_id_lp = input->vmid;
 748         } else {
 749                 mes_reset_queue_pkt.reset_queue_only = 1;
 750                 mes_reset_queue_pkt.doorbell_offset = input->doorbell_offset;
 751         }
 752
 753         return mes_v11_0_submit_pkt_and_poll_completion(mes,
 754                         &mes_reset_queue_pkt, sizeof(mes_reset_queue_pkt),
 755                         offsetof(union MESAPI__RESET, api_status));
 756 }
 757
 758 static const struct amdgpu_mes_funcs mes_v11_0_funcs = {
 759         .add_hw_queue = mes_v11_0_add_hw_queue,
 760         .remove_hw_queue = mes_v11_0_remove_hw_queue,
 761         .map_legacy_queue = mes_v11_0_map_legacy_queue,
 762         .unmap_legacy_queue = mes_v11_0_unmap_legacy_queue,
 763         .suspend_gang = mes_v11_0_suspend_gang,
 764         .resume_gang = mes_v11_0_resume_gang,
 765         .misc_op = mes_v11_0_misc_op,
 766         .reset_legacy_queue = mes_v11_0_reset_legacy_queue,
 767         .reset_hw_queue = mes_v11_0_reset_hw_queue,
 768 };
 769
 770 static int mes_v11_0_allocate_ucode_buffer(struct amdgpu_device *adev,
 771                                            enum admgpu_mes_pipe pipe)
 772 {
 773         int r;
 774         const struct mes_firmware_header_v1_0 *mes_hdr;
 775         const __le32 *fw_data;
 776         unsigned fw_size;
 777
 778         mes_hdr = (const struct mes_firmware_header_v1_0 *)
 779                 adev->mes.fw[pipe]->data;
 780
 781         fw_data = (const __le32 *)(adev->mes.fw[pipe]->data +
 782                    le32_to_cpu(mes_hdr->mes_ucode_offset_bytes));
 783         fw_size = le32_to_cpu(mes_hdr->mes_ucode_size_bytes);
 784
 785         r = amdgpu_bo_create_reserved(adev, fw_size,
 786                                       PAGE_SIZE,
 787                                       AMDGPU_GEM_DOMAIN_VRAM |
 788                                       AMDGPU_GEM_DOMAIN_GTT,
 789                                       &adev->mes.ucode_fw_obj[pipe],
 790                                       &adev->mes.ucode_fw_gpu_addr[pipe],
 791                                       (void **)&adev->mes.ucode_fw_ptr[pipe]);
 792         if (r) {
 793                 dev_err(adev->dev, "(%d) failed to create mes fw bo\n", r);
 794                 return r;
 795         }
 796
 797         memcpy(adev->mes.ucode_fw_ptr[pipe], fw_data, fw_size);
 798
 799         amdgpu_bo_kunmap(adev->mes.ucode_fw_obj[pipe]);
 800         amdgpu_bo_unreserve(adev->mes.ucode_fw_obj[pipe]);
 801
 802         return 0;
 803 }
 804
 805 static int mes_v11_0_allocate_ucode_data_buffer(struct amdgpu_device *adev,
 806                                                 enum admgpu_mes_pipe pipe)
 807 {
 808         int r;
 809         const struct mes_firmware_header_v1_0 *mes_hdr;
 810         const __le32 *fw_data;
 811         unsigned fw_size;
 812
 813         mes_hdr = (const struct mes_firmware_header_v1_0 *)
 814                 adev->mes.fw[pipe]->data;
 815
 816         fw_data = (const __le32 *)(adev->mes.fw[pipe]->data +
 817                    le32_to_cpu(mes_hdr->mes_ucode_data_offset_bytes));
 818         fw_size = le32_to_cpu(mes_hdr->mes_ucode_data_size_bytes);
 819
 820         if (fw_size > GFX_MES_DRAM_SIZE) {
 821                 dev_err(adev->dev, "PIPE%d ucode data fw size (%d) is greater than dram size (%d)\n",
 822                         pipe, fw_size, GFX_MES_DRAM_SIZE);
 823                 return -EINVAL;
 824         }
 825
 826         r = amdgpu_bo_create_reserved(adev, GFX_MES_DRAM_SIZE,
 827                                       64 * 1024,
 828                                       AMDGPU_GEM_DOMAIN_VRAM |
 829                                       AMDGPU_GEM_DOMAIN_GTT,
 830                                       &adev->mes.data_fw_obj[pipe],
 831                                       &adev->mes.data_fw_gpu_addr[pipe],
 832                                       (void **)&adev->mes.data_fw_ptr[pipe]);
 833         if (r) {
 834                 dev_err(adev->dev, "(%d) failed to create mes data fw bo\n", r);
 835                 return r;
 836         }
 837
 838         memcpy(adev->mes.data_fw_ptr[pipe], fw_data, fw_size);
 839
 840         amdgpu_bo_kunmap(adev->mes.data_fw_obj[pipe]);
 841         amdgpu_bo_unreserve(adev->mes.data_fw_obj[pipe]);
 842
 843         return 0;
 844 }
 845
 846 static void mes_v11_0_free_ucode_buffers(struct amdgpu_device *adev,
 847                                          enum admgpu_mes_pipe pipe)
 848 {
 849         amdgpu_bo_free_kernel(&adev->mes.data_fw_obj[pipe],
 850                               &adev->mes.data_fw_gpu_addr[pipe],
 851                               (void **)&adev->mes.data_fw_ptr[pipe]);
 852
 853         amdgpu_bo_free_kernel(&adev->mes.ucode_fw_obj[pipe],
 854                               &adev->mes.ucode_fw_gpu_addr[pipe],
 855                               (void **)&adev->mes.ucode_fw_ptr[pipe]);
 856 }
 857
 858 static void mes_v11_0_get_fw_version(struct amdgpu_device *adev)
 859 {
 860         int pipe;
 861
 862         /* get MES scheduler/KIQ versions */
 863         mutex_lock(&adev->srbm_mutex);
 864
 865         for (pipe = 0; pipe < AMDGPU_MAX_MES_PIPES; pipe++) {
 866                 soc21_grbm_select(adev, 3, pipe, 0, 0);
 867
 868                 if (pipe == AMDGPU_MES_SCHED_PIPE)
 869                         adev->mes.sched_version =
 870                                 RREG32_SOC15(GC, 0, regCP_MES_GP3_LO);
 871                 else if (pipe == AMDGPU_MES_KIQ_PIPE && adev->enable_mes_kiq)
 872                         adev->mes.kiq_version =
 873                                 RREG32_SOC15(GC, 0, regCP_MES_GP3_LO);
 874         }
 875
 876         soc21_grbm_select(adev, 0, 0, 0, 0);
 877         mutex_unlock(&adev->srbm_mutex);
 878 }
 879
 880 static void mes_v11_0_enable(struct amdgpu_device *adev, bool enable)
 881 {
 882         uint64_t ucode_addr;
 883         uint32_t pipe, data = 0;
 884
 885         if (enable) {
 886                 data = RREG32_SOC15(GC, 0, regCP_MES_CNTL);
 887                 data = REG_SET_FIELD(data, CP_MES_CNTL, MES_PIPE0_RESET, 1);
 888                 data = REG_SET_FIELD(data, CP_MES_CNTL,
 889                              MES_PIPE1_RESET, adev->enable_mes_kiq ? 1 : 0);
 890                 WREG32_SOC15(GC, 0, regCP_MES_CNTL, data);
 891
 892                 mutex_lock(&adev->srbm_mutex);
 893                 for (pipe = 0; pipe < AMDGPU_MAX_MES_PIPES; pipe++) {
 894                         if (!adev->enable_mes_kiq &&
 895                             pipe == AMDGPU_MES_KIQ_PIPE)
 896                                 continue;
 897
 898                         soc21_grbm_select(adev, 3, pipe, 0, 0);
 899
 900                         ucode_addr = adev->mes.uc_start_addr[pipe] >> 2;
 901                         WREG32_SOC15(GC, 0, regCP_MES_PRGRM_CNTR_START,
 902                                      lower_32_bits(ucode_addr));
 903                         WREG32_SOC15(GC, 0, regCP_MES_PRGRM_CNTR_START_HI,
 904                                      upper_32_bits(ucode_addr));
 905                 }
 906                 soc21_grbm_select(adev, 0, 0, 0, 0);
 907                 mutex_unlock(&adev->srbm_mutex);
 908
 909                 /* unhalt MES and activate pipe0 */
 910                 data = REG_SET_FIELD(0, CP_MES_CNTL, MES_PIPE0_ACTIVE, 1);
 911                 data = REG_SET_FIELD(data, CP_MES_CNTL, MES_PIPE1_ACTIVE,
 912                                      adev->enable_mes_kiq ? 1 : 0);
 913                 WREG32_SOC15(GC, 0, regCP_MES_CNTL, data);
 914
 915                 if (amdgpu_emu_mode)
 916                         msleep(100);
 917                 else
 918                         udelay(500);
 919         } else {
 920                 data = RREG32_SOC15(GC, 0, regCP_MES_CNTL);
 921                 data = REG_SET_FIELD(data, CP_MES_CNTL, MES_PIPE0_ACTIVE, 0);
 922                 data = REG_SET_FIELD(data, CP_MES_CNTL, MES_PIPE1_ACTIVE, 0);
 923                 data = REG_SET_FIELD(data, CP_MES_CNTL,
 924                                      MES_INVALIDATE_ICACHE, 1);
 925                 data = REG_SET_FIELD(data, CP_MES_CNTL, MES_PIPE0_RESET, 1);
 926                 data = REG_SET_FIELD(data, CP_MES_CNTL, MES_PIPE1_RESET,
 927                                      adev->enable_mes_kiq ? 1 : 0);
 928                 data = REG_SET_FIELD(data, CP_MES_CNTL, MES_HALT, 1);
 929                 WREG32_SOC15(GC, 0, regCP_MES_CNTL, data);
 930         }
 931 }
 932
 933 /* This function is for backdoor MES firmware */
 934 static int mes_v11_0_load_microcode(struct amdgpu_device *adev,
 935                                     enum admgpu_mes_pipe pipe, bool prime_icache)
 936 {
 937         int r;
 938         uint32_t data;
 939         uint64_t ucode_addr;
 940
 941         mes_v11_0_enable(adev, false);
 942
 943         if (!adev->mes.fw[pipe])
 944                 return -EINVAL;
 945
 946         r = mes_v11_0_allocate_ucode_buffer(adev, pipe);
 947         if (r)
 948                 return r;
 949
 950         r = mes_v11_0_allocate_ucode_data_buffer(adev, pipe);
 951         if (r) {
 952                 mes_v11_0_free_ucode_buffers(adev, pipe);
 953                 return r;
 954         }
 955
 956         mutex_lock(&adev->srbm_mutex);
 957         /* me=3, pipe=0, queue=0 */
 958         soc21_grbm_select(adev, 3, pipe, 0, 0);
 959
 960         WREG32_SOC15(GC, 0, regCP_MES_IC_BASE_CNTL, 0);
 961
 962         /* set ucode start address */
 963         ucode_addr = adev->mes.uc_start_addr[pipe] >> 2;
 964         WREG32_SOC15(GC, 0, regCP_MES_PRGRM_CNTR_START,
 965                      lower_32_bits(ucode_addr));
 966         WREG32_SOC15(GC, 0, regCP_MES_PRGRM_CNTR_START_HI,
 967                      upper_32_bits(ucode_addr));
 968
 969         /* set ucode fimrware address */
 970         WREG32_SOC15(GC, 0, regCP_MES_IC_BASE_LO,
 971                      lower_32_bits(adev->mes.ucode_fw_gpu_addr[pipe]));
 972         WREG32_SOC15(GC, 0, regCP_MES_IC_BASE_HI,
 973                      upper_32_bits(adev->mes.ucode_fw_gpu_addr[pipe]));
 974
 975         /* set ucode instruction cache boundary to 2M-1 */
 976         WREG32_SOC15(GC, 0, regCP_MES_MIBOUND_LO, 0x1FFFFF);
 977
 978         /* set ucode data firmware address */
 979         WREG32_SOC15(GC, 0, regCP_MES_MDBASE_LO,
 980                      lower_32_bits(adev->mes.data_fw_gpu_addr[pipe]));
 981         WREG32_SOC15(GC, 0, regCP_MES_MDBASE_HI,
 982                      upper_32_bits(adev->mes.data_fw_gpu_addr[pipe]));
 983
 984         /* Set 0x7FFFF (512K-1) to CP_MES_MDBOUND_LO */
 985         WREG32_SOC15(GC, 0, regCP_MES_MDBOUND_LO, 0x7FFFF);
 986
 987         if (prime_icache) {
 988                 /* invalidate ICACHE */
 989                 data = RREG32_SOC15(GC, 0, regCP_MES_IC_OP_CNTL);
 990                 data = REG_SET_FIELD(data, CP_MES_IC_OP_CNTL, PRIME_ICACHE, 0);
 991                 data = REG_SET_FIELD(data, CP_MES_IC_OP_CNTL, INVALIDATE_CACHE, 1);
 992                 WREG32_SOC15(GC, 0, regCP_MES_IC_OP_CNTL, data);
 993
 994                 /* prime the ICACHE. */
 995                 data = RREG32_SOC15(GC, 0, regCP_MES_IC_OP_CNTL);
 996                 data = REG_SET_FIELD(data, CP_MES_IC_OP_CNTL, PRIME_ICACHE, 1);
 997                 WREG32_SOC15(GC, 0, regCP_MES_IC_OP_CNTL, data);
 998         }
 999
1000         soc21_grbm_select(adev, 0, 0, 0, 0);
1001         mutex_unlock(&adev->srbm_mutex);
1002
1003         return 0;
1004 }
1005
1006 static int mes_v11_0_allocate_eop_buf(struct amdgpu_device *adev,
1007                                       enum admgpu_mes_pipe pipe)
1008 {
1009         int r;
1010         u32 *eop;
1011
1012         r = amdgpu_bo_create_reserved(adev, MES_EOP_SIZE, PAGE_SIZE,
1013                               AMDGPU_GEM_DOMAIN_GTT,
1014                               &adev->mes.eop_gpu_obj[pipe],
1015                               &adev->mes.eop_gpu_addr[pipe],
1016                               (void **)&eop);
1017         if (r) {
1018                 dev_warn(adev->dev, "(%d) create EOP bo failed\n", r);
1019                 return r;
1020         }
1021
1022         memset(eop, 0,
1023                adev->mes.eop_gpu_obj[pipe]->tbo.base.size);
1024
1025         amdgpu_bo_kunmap(adev->mes.eop_gpu_obj[pipe]);
1026         amdgpu_bo_unreserve(adev->mes.eop_gpu_obj[pipe]);
1027
1028         return 0;
1029 }
1030
1031 static int mes_v11_0_mqd_init(struct amdgpu_ring *ring)
1032 {
1033         struct v11_compute_mqd *mqd = ring->mqd_ptr;
1034         uint64_t hqd_gpu_addr, wb_gpu_addr, eop_base_addr;
1035         uint32_t tmp;
1036
1037         memset(mqd, 0, sizeof(*mqd));
1038
1039         mqd->header = 0xC0310800;
1040         mqd->compute_pipelinestat_enable = 0x00000001;
1041         mqd->compute_static_thread_mgmt_se0 = 0xffffffff;
1042         mqd->compute_static_thread_mgmt_se1 = 0xffffffff;
1043         mqd->compute_static_thread_mgmt_se2 = 0xffffffff;
1044         mqd->compute_static_thread_mgmt_se3 = 0xffffffff;
1045         mqd->compute_misc_reserved = 0x00000007;
1046
1047         eop_base_addr = ring->eop_gpu_addr >> 8;
1048
1049         /* set the EOP size, register value is 2^(EOP_SIZE+1) dwords */
1050         tmp = regCP_HQD_EOP_CONTROL_DEFAULT;
1051         tmp = REG_SET_FIELD(tmp, CP_HQD_EOP_CONTROL, EOP_SIZE,
1052                         (order_base_2(MES_EOP_SIZE / 4) - 1));
1053
1054         mqd->cp_hqd_eop_base_addr_lo = lower_32_bits(eop_base_addr);
1055         mqd->cp_hqd_eop_base_addr_hi = upper_32_bits(eop_base_addr);
1056         mqd->cp_hqd_eop_control = tmp;
1057
1058         /* disable the queue if it's active */
1059         ring->wptr = 0;
1060         mqd->cp_hqd_pq_rptr = 0;
1061         mqd->cp_hqd_pq_wptr_lo = 0;
1062         mqd->cp_hqd_pq_wptr_hi = 0;
1063
1064         /* set the pointer to the MQD */
1065         mqd->cp_mqd_base_addr_lo = ring->mqd_gpu_addr & 0xfffffffc;
1066         mqd->cp_mqd_base_addr_hi = upper_32_bits(ring->mqd_gpu_addr);
1067
1068         /* set MQD vmid to 0 */
1069         tmp = regCP_MQD_CONTROL_DEFAULT;
1070         tmp = REG_SET_FIELD(tmp, CP_MQD_CONTROL, VMID, 0);
1071         mqd->cp_mqd_control = tmp;
1072
1073         /* set the pointer to the HQD, this is similar CP_RB0_BASE/_HI */
1074         hqd_gpu_addr = ring->gpu_addr >> 8;
1075         mqd->cp_hqd_pq_base_lo = lower_32_bits(hqd_gpu_addr);
1076         mqd->cp_hqd_pq_base_hi = upper_32_bits(hqd_gpu_addr);
1077
1078         /* set the wb address whether it's enabled or not */
1079         wb_gpu_addr = ring->rptr_gpu_addr;
1080         mqd->cp_hqd_pq_rptr_report_addr_lo = wb_gpu_addr & 0xfffffffc;
1081         mqd->cp_hqd_pq_rptr_report_addr_hi =
1082                 upper_32_bits(wb_gpu_addr) & 0xffff;
1083
1084         /* only used if CP_PQ_WPTR_POLL_CNTL.CP_PQ_WPTR_POLL_CNTL__EN_MASK=1 */
1085         wb_gpu_addr = ring->wptr_gpu_addr;
1086         mqd->cp_hqd_pq_wptr_poll_addr_lo = wb_gpu_addr & 0xfffffff8;
1087         mqd->cp_hqd_pq_wptr_poll_addr_hi = upper_32_bits(wb_gpu_addr) & 0xffff;
1088
1089         /* set up the HQD, this is similar to CP_RB0_CNTL */
1090         tmp = regCP_HQD_PQ_CONTROL_DEFAULT;
1091         tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, QUEUE_SIZE,
1092                             (order_base_2(ring->ring_size / 4) - 1));
1093         tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, RPTR_BLOCK_SIZE,
1094                             ((order_base_2(AMDGPU_GPU_PAGE_SIZE / 4) - 1) << 8));
1095         tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, UNORD_DISPATCH, 1);
1096         tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, TUNNEL_DISPATCH, 0);
1097         tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, PRIV_STATE, 1);
1098         tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, KMD_QUEUE, 1);
1099         tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, NO_UPDATE_RPTR, 1);
1100         mqd->cp_hqd_pq_control = tmp;
1101
1102         /* enable doorbell */
1103         tmp = 0;
1104         if (ring->use_doorbell) {
1105                 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
1106                                     DOORBELL_OFFSET, ring->doorbell_index);
1107                 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
1108                                     DOORBELL_EN, 1);
1109                 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
1110                                     DOORBELL_SOURCE, 0);
1111                 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
1112                                     DOORBELL_HIT, 0);
1113         } else
1114                 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
1115                                     DOORBELL_EN, 0);
1116         mqd->cp_hqd_pq_doorbell_control = tmp;
1117
1118         mqd->cp_hqd_vmid = 0;
1119         /* activate the queue */
1120         mqd->cp_hqd_active = 1;
1121
1122         tmp = regCP_HQD_PERSISTENT_STATE_DEFAULT;
1123         tmp = REG_SET_FIELD(tmp, CP_HQD_PERSISTENT_STATE,
1124                             PRELOAD_SIZE, 0x55);
1125         mqd->cp_hqd_persistent_state = tmp;
1126
1127         mqd->cp_hqd_ib_control = regCP_HQD_IB_CONTROL_DEFAULT;
1128         mqd->cp_hqd_iq_timer = regCP_HQD_IQ_TIMER_DEFAULT;
1129         mqd->cp_hqd_quantum = regCP_HQD_QUANTUM_DEFAULT;
1130
1131         amdgpu_device_flush_hdp(ring->adev, NULL);
1132         return 0;
1133 }
1134
1135 static void mes_v11_0_queue_init_register(struct amdgpu_ring *ring)
1136 {
1137         struct v11_compute_mqd *mqd = ring->mqd_ptr;
1138         struct amdgpu_device *adev = ring->adev;
1139         uint32_t data = 0;
1140
1141         mutex_lock(&adev->srbm_mutex);
1142         soc21_grbm_select(adev, 3, ring->pipe, 0, 0);
1143
1144         /* set CP_HQD_VMID.VMID = 0. */
1145         data = RREG32_SOC15(GC, 0, regCP_HQD_VMID);
1146         data = REG_SET_FIELD(data, CP_HQD_VMID, VMID, 0);
1147         WREG32_SOC15(GC, 0, regCP_HQD_VMID, data);
1148
1149         /* set CP_HQD_PQ_DOORBELL_CONTROL.DOORBELL_EN=0 */
1150         data = RREG32_SOC15(GC, 0, regCP_HQD_PQ_DOORBELL_CONTROL);
1151         data = REG_SET_FIELD(data, CP_HQD_PQ_DOORBELL_CONTROL,
1152                              DOORBELL_EN, 0);
1153         WREG32_SOC15(GC, 0, regCP_HQD_PQ_DOORBELL_CONTROL, data);
1154
1155         /* set CP_MQD_BASE_ADDR/HI with the MQD base address */
1156         WREG32_SOC15(GC, 0, regCP_MQD_BASE_ADDR, mqd->cp_mqd_base_addr_lo);
1157         WREG32_SOC15(GC, 0, regCP_MQD_BASE_ADDR_HI, mqd->cp_mqd_base_addr_hi);
1158
1159         /* set CP_MQD_CONTROL.VMID=0 */
1160         data = RREG32_SOC15(GC, 0, regCP_MQD_CONTROL);
1161         data = REG_SET_FIELD(data, CP_MQD_CONTROL, VMID, 0);
1162         WREG32_SOC15(GC, 0, regCP_MQD_CONTROL, 0);
1163
1164         /* set CP_HQD_PQ_BASE/HI with the ring buffer base address */
1165         WREG32_SOC15(GC, 0, regCP_HQD_PQ_BASE, mqd->cp_hqd_pq_base_lo);
1166         WREG32_SOC15(GC, 0, regCP_HQD_PQ_BASE_HI, mqd->cp_hqd_pq_base_hi);
1167
1168         /* set CP_HQD_PQ_RPTR_REPORT_ADDR/HI */
1169         WREG32_SOC15(GC, 0, regCP_HQD_PQ_RPTR_REPORT_ADDR,
1170                      mqd->cp_hqd_pq_rptr_report_addr_lo);
1171         WREG32_SOC15(GC, 0, regCP_HQD_PQ_RPTR_REPORT_ADDR_HI,
1172                      mqd->cp_hqd_pq_rptr_report_addr_hi);
1173
1174         /* set CP_HQD_PQ_CONTROL */
1175         WREG32_SOC15(GC, 0, regCP_HQD_PQ_CONTROL, mqd->cp_hqd_pq_control);
1176
1177         /* set CP_HQD_PQ_WPTR_POLL_ADDR/HI */
1178         WREG32_SOC15(GC, 0, regCP_HQD_PQ_WPTR_POLL_ADDR,
1179                      mqd->cp_hqd_pq_wptr_poll_addr_lo);
1180         WREG32_SOC15(GC, 0, regCP_HQD_PQ_WPTR_POLL_ADDR_HI,
1181                      mqd->cp_hqd_pq_wptr_poll_addr_hi);
1182
1183         /* set CP_HQD_PQ_DOORBELL_CONTROL */
1184         WREG32_SOC15(GC, 0, regCP_HQD_PQ_DOORBELL_CONTROL,
1185                      mqd->cp_hqd_pq_doorbell_control);
1186
1187         /* set CP_HQD_PERSISTENT_STATE.PRELOAD_SIZE=0x53 */
1188         WREG32_SOC15(GC, 0, regCP_HQD_PERSISTENT_STATE, mqd->cp_hqd_persistent_state);
1189
1190         /* set CP_HQD_ACTIVE.ACTIVE=1 */
1191         WREG32_SOC15(GC, 0, regCP_HQD_ACTIVE, mqd->cp_hqd_active);
1192
1193         soc21_grbm_select(adev, 0, 0, 0, 0);
1194         mutex_unlock(&adev->srbm_mutex);
1195 }
1196
1197 static int mes_v11_0_kiq_enable_queue(struct amdgpu_device *adev)
1198 {
1199         struct amdgpu_kiq *kiq = &adev->gfx.kiq[0];
1200         struct amdgpu_ring *kiq_ring = &adev->gfx.kiq[0].ring;
1201         int r;
1202
1203         if (!kiq->pmf || !kiq->pmf->kiq_map_queues)
1204                 return -EINVAL;
1205
1206         r = amdgpu_ring_alloc(kiq_ring, kiq->pmf->map_queues_size);
1207         if (r) {
1208                 DRM_ERROR("Failed to lock KIQ (%d).\n", r);
1209                 return r;
1210         }
1211
1212         kiq->pmf->kiq_map_queues(kiq_ring, &adev->mes.ring[0]);
1213
1214         return amdgpu_ring_test_helper(kiq_ring);
1215 }
1216
1217 static int mes_v11_0_queue_init(struct amdgpu_device *adev,
1218                                 enum admgpu_mes_pipe pipe)
1219 {
1220         struct amdgpu_ring *ring;
1221         int r;
1222
1223         if (pipe == AMDGPU_MES_KIQ_PIPE)
1224                 ring = &adev->gfx.kiq[0].ring;
1225         else if (pipe == AMDGPU_MES_SCHED_PIPE)
1226                 ring = &adev->mes.ring[0];
1227         else
1228                 BUG();
1229
1230         if ((pipe == AMDGPU_MES_SCHED_PIPE) &&
1231             (amdgpu_in_reset(adev) || adev->in_suspend)) {
1232                 *(ring->wptr_cpu_addr) = 0;
1233                 *(ring->rptr_cpu_addr) = 0;
1234                 amdgpu_ring_clear_ring(ring);
1235         }
1236
1237         r = mes_v11_0_mqd_init(ring);
1238         if (r)
1239                 return r;
1240
1241         if (pipe == AMDGPU_MES_SCHED_PIPE) {
1242                 r = mes_v11_0_kiq_enable_queue(adev);
1243                 if (r)
1244                         return r;
1245         } else {
1246                 mes_v11_0_queue_init_register(ring);
1247         }
1248
1249         return 0;
1250 }
1251
1252 static int mes_v11_0_ring_init(struct amdgpu_device *adev)
1253 {
1254         struct amdgpu_ring *ring;
1255
1256         ring = &adev->mes.ring[0];
1257
1258         ring->funcs = &mes_v11_0_ring_funcs;
1259
1260         ring->me = 3;
1261         ring->pipe = 0;
1262         ring->queue = 0;
1263
1264         ring->ring_obj = NULL;
1265         ring->use_doorbell = true;
1266         ring->doorbell_index = adev->doorbell_index.mes_ring0 << 1;
1267         ring->eop_gpu_addr = adev->mes.eop_gpu_addr[AMDGPU_MES_SCHED_PIPE];
1268         ring->no_scheduler = true;
1269         sprintf(ring->name, "mes_%d.%d.%d", ring->me, ring->pipe, ring->queue);
1270
1271         return amdgpu_ring_init(adev, ring, 1024, NULL, 0,
1272                                 AMDGPU_RING_PRIO_DEFAULT, NULL);
1273 }
1274
1275 static int mes_v11_0_kiq_ring_init(struct amdgpu_device *adev)
1276 {
1277         struct amdgpu_ring *ring;
1278
1279         spin_lock_init(&adev->gfx.kiq[0].ring_lock);
1280
1281         ring = &adev->gfx.kiq[0].ring;
1282
1283         ring->me = 3;
1284         ring->pipe = 1;
1285         ring->queue = 0;
1286
1287         ring->adev = NULL;
1288         ring->ring_obj = NULL;
1289         ring->use_doorbell = true;
1290         ring->doorbell_index = adev->doorbell_index.mes_ring1 << 1;
1291         ring->eop_gpu_addr = adev->mes.eop_gpu_addr[AMDGPU_MES_KIQ_PIPE];
1292         ring->no_scheduler = true;
1293         sprintf(ring->name, "mes_kiq_%d.%d.%d",
1294                 ring->me, ring->pipe, ring->queue);
1295
1296         return amdgpu_ring_init(adev, ring, 1024, NULL, 0,
1297                                 AMDGPU_RING_PRIO_DEFAULT, NULL);
1298 }
1299
1300 static int mes_v11_0_mqd_sw_init(struct amdgpu_device *adev,
1301                                  enum admgpu_mes_pipe pipe)
1302 {
1303         int r, mqd_size = sizeof(struct v11_compute_mqd);
1304         struct amdgpu_ring *ring;
1305
1306         if (pipe == AMDGPU_MES_KIQ_PIPE)
1307                 ring = &adev->gfx.kiq[0].ring;
1308         else if (pipe == AMDGPU_MES_SCHED_PIPE)
1309                 ring = &adev->mes.ring[0];
1310         else
1311                 BUG();
1312
1313         if (ring->mqd_obj)
1314                 return 0;
1315
1316         r = amdgpu_bo_create_kernel(adev, mqd_size, PAGE_SIZE,
1317                                     AMDGPU_GEM_DOMAIN_VRAM |
1318                                     AMDGPU_GEM_DOMAIN_GTT, &ring->mqd_obj,
1319                                     &ring->mqd_gpu_addr, &ring->mqd_ptr);
1320         if (r) {
1321                 dev_warn(adev->dev, "failed to create ring mqd bo (%d)", r);
1322                 return r;
1323         }
1324
1325         memset(ring->mqd_ptr, 0, mqd_size);
1326
1327         /* prepare MQD backup */
1328         adev->mes.mqd_backup[pipe] = kmalloc(mqd_size, GFP_KERNEL);
1329         if (!adev->mes.mqd_backup[pipe]) {
1330                 dev_warn(adev->dev,
1331                          "no memory to create MQD backup for ring %s\n",
1332                          ring->name);
1333                 return -ENOMEM;
1334         }
1335
1336         return 0;
1337 }
1338
1339 static int mes_v11_0_sw_init(void *handle)
1340 {
1341         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1342         int pipe, r;
1343
1344         adev->mes.funcs = &mes_v11_0_funcs;
1345         adev->mes.kiq_hw_init = &mes_v11_0_kiq_hw_init;
1346         adev->mes.kiq_hw_fini = &mes_v11_0_kiq_hw_fini;
1347
1348         adev->mes.event_log_size = AMDGPU_MES_LOG_BUFFER_SIZE;
1349
1350         r = amdgpu_mes_init(adev);
1351         if (r)
1352                 return r;
1353
1354         for (pipe = 0; pipe < AMDGPU_MAX_MES_PIPES; pipe++) {
1355                 if (!adev->enable_mes_kiq && pipe == AMDGPU_MES_KIQ_PIPE)
1356                         continue;
1357
1358                 r = mes_v11_0_allocate_eop_buf(adev, pipe);
1359                 if (r)
1360                         return r;
1361
1362                 r = mes_v11_0_mqd_sw_init(adev, pipe);
1363                 if (r)
1364                         return r;
1365         }
1366
1367         if (adev->enable_mes_kiq) {
1368                 r = mes_v11_0_kiq_ring_init(adev);
1369                 if (r)
1370                         return r;
1371         }
1372
1373         r = mes_v11_0_ring_init(adev);
1374         if (r)
1375                 return r;
1376
1377         return 0;
1378 }
1379
1380 static int mes_v11_0_sw_fini(void *handle)
1381 {
1382         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1383         int pipe;
1384
1385         for (pipe = 0; pipe < AMDGPU_MAX_MES_PIPES; pipe++) {
1386                 kfree(adev->mes.mqd_backup[pipe]);
1387
1388                 amdgpu_bo_free_kernel(&adev->mes.eop_gpu_obj[pipe],
1389                                       &adev->mes.eop_gpu_addr[pipe],
1390                                       NULL);
1391                 amdgpu_ucode_release(&adev->mes.fw[pipe]);
1392         }
1393
1394         amdgpu_bo_free_kernel(&adev->gfx.kiq[0].ring.mqd_obj,
1395                               &adev->gfx.kiq[0].ring.mqd_gpu_addr,
1396                               &adev->gfx.kiq[0].ring.mqd_ptr);
1397
1398         amdgpu_bo_free_kernel(&adev->mes.ring[0].mqd_obj,
1399                               &adev->mes.ring[0].mqd_gpu_addr,
1400                               &adev->mes.ring[0].mqd_ptr);
1401
1402         amdgpu_ring_fini(&adev->gfx.kiq[0].ring);
1403         amdgpu_ring_fini(&adev->mes.ring[0]);
1404
1405         if (adev->firmware.load_type == AMDGPU_FW_LOAD_DIRECT) {
1406                 mes_v11_0_free_ucode_buffers(adev, AMDGPU_MES_KIQ_PIPE);
1407                 mes_v11_0_free_ucode_buffers(adev, AMDGPU_MES_SCHED_PIPE);
1408         }
1409
1410         amdgpu_mes_fini(adev);
1411         return 0;
1412 }
1413
1414 static void mes_v11_0_kiq_dequeue(struct amdgpu_ring *ring)
1415 {
1416         uint32_t data;
1417         int i;
1418         struct amdgpu_device *adev = ring->adev;
1419
1420         mutex_lock(&adev->srbm_mutex);
1421         soc21_grbm_select(adev, 3, ring->pipe, 0, 0);
1422
1423         /* disable the queue if it's active */
1424         if (RREG32_SOC15(GC, 0, regCP_HQD_ACTIVE) & 1) {
1425                 WREG32_SOC15(GC, 0, regCP_HQD_DEQUEUE_REQUEST, 1);
1426                 for (i = 0; i < adev->usec_timeout; i++) {
1427                         if (!(RREG32_SOC15(GC, 0, regCP_HQD_ACTIVE) & 1))
1428                                 break;
1429                         udelay(1);
1430                 }
1431         }
1432         data = RREG32_SOC15(GC, 0, regCP_HQD_PQ_DOORBELL_CONTROL);
1433         data = REG_SET_FIELD(data, CP_HQD_PQ_DOORBELL_CONTROL,
1434                                 DOORBELL_EN, 0);
1435         data = REG_SET_FIELD(data, CP_HQD_PQ_DOORBELL_CONTROL,
1436                                 DOORBELL_HIT, 1);
1437         WREG32_SOC15(GC, 0, regCP_HQD_PQ_DOORBELL_CONTROL, data);
1438
1439         WREG32_SOC15(GC, 0, regCP_HQD_PQ_DOORBELL_CONTROL, 0);
1440
1441         WREG32_SOC15(GC, 0, regCP_HQD_PQ_WPTR_LO, 0);
1442         WREG32_SOC15(GC, 0, regCP_HQD_PQ_WPTR_HI, 0);
1443         WREG32_SOC15(GC, 0, regCP_HQD_PQ_RPTR, 0);
1444
1445         soc21_grbm_select(adev, 0, 0, 0, 0);
1446         mutex_unlock(&adev->srbm_mutex);
1447 }
1448
1449 static void mes_v11_0_kiq_setting(struct amdgpu_ring *ring)
1450 {
1451         uint32_t tmp;
1452         struct amdgpu_device *adev = ring->adev;
1453
1454         /* tell RLC which is KIQ queue */
1455         tmp = RREG32_SOC15(GC, 0, regRLC_CP_SCHEDULERS);
1456         tmp &= 0xffffff00;
1457         tmp |= (ring->me << 5) | (ring->pipe << 3) | (ring->queue);
1458         WREG32_SOC15(GC, 0, regRLC_CP_SCHEDULERS, tmp);
1459         tmp |= 0x80;
1460         WREG32_SOC15(GC, 0, regRLC_CP_SCHEDULERS, tmp);
1461 }
1462
1463 static void mes_v11_0_kiq_clear(struct amdgpu_device *adev)
1464 {
1465         uint32_t tmp;
1466
1467         /* tell RLC which is KIQ dequeue */
1468         tmp = RREG32_SOC15(GC, 0, regRLC_CP_SCHEDULERS);
1469         tmp &= ~RLC_CP_SCHEDULERS__scheduler0_MASK;
1470         WREG32_SOC15(GC, 0, regRLC_CP_SCHEDULERS, tmp);
1471 }
1472
1473 static int mes_v11_0_kiq_hw_init(struct amdgpu_device *adev)
1474 {
1475         int r = 0;
1476
1477         if (adev->firmware.load_type == AMDGPU_FW_LOAD_DIRECT) {
1478
1479                 r = mes_v11_0_load_microcode(adev, AMDGPU_MES_SCHED_PIPE, false);
1480                 if (r) {
1481                         DRM_ERROR("failed to load MES fw, r=%d\n", r);
1482                         return r;
1483                 }
1484
1485                 r = mes_v11_0_load_microcode(adev, AMDGPU_MES_KIQ_PIPE, true);
1486                 if (r) {
1487                         DRM_ERROR("failed to load MES kiq fw, r=%d\n", r);
1488                         return r;
1489                 }
1490
1491         }
1492
1493         mes_v11_0_enable(adev, true);
1494
1495         mes_v11_0_get_fw_version(adev);
1496
1497         mes_v11_0_kiq_setting(&adev->gfx.kiq[0].ring);
1498
1499         r = mes_v11_0_queue_init(adev, AMDGPU_MES_KIQ_PIPE);
1500         if (r)
1501                 goto failure;
1502
1503         if ((adev->mes.sched_version & AMDGPU_MES_VERSION_MASK) >= 0x47)
1504                 adev->mes.enable_legacy_queue_map = true;
1505         else
1506                 adev->mes.enable_legacy_queue_map = false;
1507
1508         if (adev->mes.enable_legacy_queue_map) {
1509                 r = mes_v11_0_hw_init(adev);
1510                 if (r)
1511                         goto failure;
1512         }
1513
1514         return r;
1515
1516 failure:
1517         mes_v11_0_hw_fini(adev);
1518         return r;
1519 }
1520
1521 static int mes_v11_0_kiq_hw_fini(struct amdgpu_device *adev)
1522 {
1523         if (adev->mes.ring[0].sched.ready) {
1524                 mes_v11_0_kiq_dequeue(&adev->mes.ring[0]);
1525                 adev->mes.ring[0].sched.ready = false;
1526         }
1527
1528         if (amdgpu_sriov_vf(adev)) {
1529                 mes_v11_0_kiq_dequeue(&adev->gfx.kiq[0].ring);
1530                 mes_v11_0_kiq_clear(adev);
1531         }
1532
1533         mes_v11_0_enable(adev, false);
1534
1535         return 0;
1536 }
1537
1538 static int mes_v11_0_hw_init(void *handle)
1539 {
1540         int r;
1541         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1542
1543         if (adev->mes.ring[0].sched.ready)
1544                 goto out;
1545
1546         if (!adev->enable_mes_kiq) {
1547                 if (adev->firmware.load_type == AMDGPU_FW_LOAD_DIRECT) {
1548                         r = mes_v11_0_load_microcode(adev,
1549                                              AMDGPU_MES_SCHED_PIPE, true);
1550                         if (r) {
1551                                 DRM_ERROR("failed to MES fw, r=%d\n", r);
1552                                 return r;
1553                         }
1554                 }
1555
1556                 mes_v11_0_enable(adev, true);
1557         }
1558
1559         r = mes_v11_0_queue_init(adev, AMDGPU_MES_SCHED_PIPE);
1560         if (r)
1561                 goto failure;
1562
1563         r = mes_v11_0_set_hw_resources(&adev->mes);
1564         if (r)
1565                 goto failure;
1566
1567         if (amdgpu_sriov_is_mes_info_enable(adev)) {
1568                 r = mes_v11_0_set_hw_resources_1(&adev->mes);
1569                 if (r) {
1570                         DRM_ERROR("failed mes_v11_0_set_hw_resources_1, r=%d\n", r);
1571                         goto failure;
1572                 }
1573         }
1574
1575         r = mes_v11_0_query_sched_status(&adev->mes);
1576         if (r) {
1577                 DRM_ERROR("MES is busy\n");
1578                 goto failure;
1579         }
1580
1581 out:
1582         /*
1583          * Disable KIQ ring usage from the driver once MES is enabled.
1584          * MES uses KIQ ring exclusively so driver cannot access KIQ ring
1585          * with MES enabled.
1586          */
1587         adev->gfx.kiq[0].ring.sched.ready = false;
1588         adev->mes.ring[0].sched.ready = true;
1589
1590         return 0;
1591
1592 failure:
1593         mes_v11_0_hw_fini(adev);
1594         return r;
1595 }
1596
1597 static int mes_v11_0_hw_fini(void *handle)
1598 {
1599         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1600         if (amdgpu_sriov_is_mes_info_enable(adev)) {
1601                 amdgpu_bo_free_kernel(&adev->mes.resource_1, &adev->mes.resource_1_gpu_addr,
1602                                         &adev->mes.resource_1_addr);
1603         }
1604         return 0;
1605 }
1606
1607 static int mes_v11_0_suspend(void *handle)
1608 {
1609         int r;
1610         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1611
1612         r = amdgpu_mes_suspend(adev);
1613         if (r)
1614                 return r;
1615
1616         return mes_v11_0_hw_fini(adev);
1617 }
1618
1619 static int mes_v11_0_resume(void *handle)
1620 {
1621         int r;
1622         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1623
1624         r = mes_v11_0_hw_init(adev);
1625         if (r)
1626                 return r;
1627
1628         return amdgpu_mes_resume(adev);
1629 }
1630
1631 static int mes_v11_0_early_init(void *handle)
1632 {
1633         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1634         int pipe, r;
1635
1636         for (pipe = 0; pipe < AMDGPU_MAX_MES_PIPES; pipe++) {
1637                 if (!adev->enable_mes_kiq && pipe == AMDGPU_MES_KIQ_PIPE)
1638                         continue;
1639                 r = amdgpu_mes_init_microcode(adev, pipe);
1640                 if (r)
1641                         return r;
1642         }
1643
1644         return 0;
1645 }
1646
1647 static int mes_v11_0_late_init(void *handle)
1648 {
1649         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1650
1651         /* it's only intended for use in mes_self_test case, not for s0ix and reset */
1652         if (!amdgpu_in_reset(adev) && !adev->in_s0ix && !adev->in_suspend &&
1653             (amdgpu_ip_version(adev, GC_HWIP, 0) != IP_VERSION(11, 0, 3)))
1654                 amdgpu_mes_self_test(adev);
1655
1656         return 0;
1657 }
1658
1659 static const struct amd_ip_funcs mes_v11_0_ip_funcs = {
1660         .name = "mes_v11_0",
1661         .early_init = mes_v11_0_early_init,
1662         .late_init = mes_v11_0_late_init,
1663         .sw_init = mes_v11_0_sw_init,
1664         .sw_fini = mes_v11_0_sw_fini,
1665         .hw_init = mes_v11_0_hw_init,
1666         .hw_fini = mes_v11_0_hw_fini,
1667         .suspend = mes_v11_0_suspend,
1668         .resume = mes_v11_0_resume,
1669         .dump_ip_state = NULL,
1670         .print_ip_state = NULL,
1671 };
1672
1673 const struct amdgpu_ip_block_version mes_v11_0_ip_block = {
1674         .type = AMD_IP_BLOCK_TYPE_MES,
1675         .major = 11,
1676         .minor = 0,
1677         .rev = 0,
1678         .funcs = &mes_v11_0_ip_funcs,
1679 };