drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v9.c

   1 // SPDX-License-Identifier: GPL-2.0 OR MIT
   2 /*
   3  * Copyright 2016-2022 Advanced Micro Devices, Inc.
   4  *
   5  * Permission is hereby granted, free of charge, to any person obtaining a
   6  * copy of this software and associated documentation files (the "Software"),
   7  * to deal in the Software without restriction, including without limitation
   8  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   9  * and/or sell copies of the Software, and to permit persons to whom the
  10  * Software is furnished to do so, subject to the following conditions:
  11  *
  12  * The above copyright notice and this permission notice shall be included in
  13  * all copies or substantial portions of the Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  19  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  20  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  21  * OTHER DEALINGS IN THE SOFTWARE.
  22  *
  23  */
  24
  25 #include <linux/printk.h>
  26 #include <linux/slab.h>
  27 #include <linux/uaccess.h>
  28 #include "kfd_priv.h"
  29 #include "kfd_mqd_manager.h"
  30 #include "v9_structs.h"
  31 #include "gc/gc_9_0_offset.h"
  32 #include "gc/gc_9_0_sh_mask.h"
  33 #include "sdma0/sdma0_4_0_sh_mask.h"
  34 #include "amdgpu_amdkfd.h"
  35 #include "kfd_device_queue_manager.h"
  36
  37 static void update_mqd(struct mqd_manager *mm, void *mqd,
  38                        struct queue_properties *q,
  39                        struct mqd_update_info *minfo);
  40
  41 static uint64_t mqd_stride_v9(struct mqd_manager *mm,
  42                                 struct queue_properties *q)
  43 {
  44         if (mm->dev->kfd->cwsr_enabled &&
  45             q->type == KFD_QUEUE_TYPE_COMPUTE)
  46                 return ALIGN(q->ctl_stack_size, PAGE_SIZE) +
  47                         ALIGN(sizeof(struct v9_mqd), PAGE_SIZE);
  48
  49         return mm->mqd_size;
  50 }
  51
  52 static inline struct v9_mqd *get_mqd(void *mqd)
  53 {
  54         return (struct v9_mqd *)mqd;
  55 }
  56
  57 static inline struct v9_sdma_mqd *get_sdma_mqd(void *mqd)
  58 {
  59         return (struct v9_sdma_mqd *)mqd;
  60 }
  61
  62 static void update_cu_mask(struct mqd_manager *mm, void *mqd,
  63                         struct mqd_update_info *minfo, uint32_t inst)
  64 {
  65         struct v9_mqd *m;
  66         uint32_t se_mask[KFD_MAX_NUM_SE] = {0};
  67
  68         if (!minfo || !minfo->cu_mask.ptr)
  69                 return;
  70
  71         mqd_symmetrically_map_cu_mask(mm,
  72                 minfo->cu_mask.ptr, minfo->cu_mask.count, se_mask, inst);
  73
  74         m = get_mqd(mqd);
  75
  76         m->compute_static_thread_mgmt_se0 = se_mask[0];
  77         m->compute_static_thread_mgmt_se1 = se_mask[1];
  78         m->compute_static_thread_mgmt_se2 = se_mask[2];
  79         m->compute_static_thread_mgmt_se3 = se_mask[3];
  80         if (KFD_GC_VERSION(mm->dev) != IP_VERSION(9, 4, 3) &&
  81             KFD_GC_VERSION(mm->dev) != IP_VERSION(9, 4, 4) &&
  82             KFD_GC_VERSION(mm->dev) != IP_VERSION(9, 5, 0)) {
  83                 m->compute_static_thread_mgmt_se4 = se_mask[4];
  84                 m->compute_static_thread_mgmt_se5 = se_mask[5];
  85                 m->compute_static_thread_mgmt_se6 = se_mask[6];
  86                 m->compute_static_thread_mgmt_se7 = se_mask[7];
  87
  88                 pr_debug("update cu mask to %#x %#x %#x %#x %#x %#x %#x %#x\n",
  89                         m->compute_static_thread_mgmt_se0,
  90                         m->compute_static_thread_mgmt_se1,
  91                         m->compute_static_thread_mgmt_se2,
  92                         m->compute_static_thread_mgmt_se3,
  93                         m->compute_static_thread_mgmt_se4,
  94                         m->compute_static_thread_mgmt_se5,
  95                         m->compute_static_thread_mgmt_se6,
  96                         m->compute_static_thread_mgmt_se7);
  97         } else {
  98                 pr_debug("inst: %u, update cu mask to %#x %#x %#x %#x\n",
  99                         inst, m->compute_static_thread_mgmt_se0,
 100                         m->compute_static_thread_mgmt_se1,
 101                         m->compute_static_thread_mgmt_se2,
 102                         m->compute_static_thread_mgmt_se3);
 103         }
 104 }
 105
 106 static void set_priority(struct v9_mqd *m, struct queue_properties *q)
 107 {
 108         m->cp_hqd_pipe_priority = pipe_priority_map[q->priority];
 109         m->cp_hqd_queue_priority = q->priority;
 110 }
 111
 112 static struct kfd_mem_obj *allocate_mqd(struct kfd_node *node,
 113                 struct queue_properties *q)
 114 {
 115         int retval;
 116         struct kfd_mem_obj *mqd_mem_obj = NULL;
 117
 118         /* For V9 only, due to a HW bug, the control stack of a user mode
 119          * compute queue needs to be allocated just behind the page boundary
 120          * of its regular MQD buffer. So we allocate an enlarged MQD buffer:
 121          * the first page of the buffer serves as the regular MQD buffer
 122          * purpose and the remaining is for control stack. Although the two
 123          * parts are in the same buffer object, they need different memory
 124          * types: MQD part needs UC (uncached) as usual, while control stack
 125          * needs NC (non coherent), which is different from the UC type which
 126          * is used when control stack is allocated in user space.
 127          *
 128          * Because of all those, we use the gtt allocation function instead
 129          * of sub-allocation function for this enlarged MQD buffer. Moreover,
 130          * in order to achieve two memory types in a single buffer object, we
 131          * pass a special bo flag AMDGPU_GEM_CREATE_CP_MQD_GFX9 to instruct
 132          * amdgpu memory functions to do so.
 133          */
 134         if (node->kfd->cwsr_enabled && (q->type == KFD_QUEUE_TYPE_COMPUTE)) {
 135                 mqd_mem_obj = kzalloc(sizeof(struct kfd_mem_obj), GFP_KERNEL);
 136                 if (!mqd_mem_obj)
 137                         return NULL;
 138                 retval = amdgpu_amdkfd_alloc_gtt_mem(node->adev,
 139                         (ALIGN(q->ctl_stack_size, PAGE_SIZE) +
 140                         ALIGN(sizeof(struct v9_mqd), PAGE_SIZE)) *
 141                         NUM_XCC(node->xcc_mask),
 142                         &(mqd_mem_obj->gtt_mem),
 143                         &(mqd_mem_obj->gpu_addr),
 144                         (void *)&(mqd_mem_obj->cpu_ptr), true);
 145
 146                 if (retval) {
 147                         kfree(mqd_mem_obj);
 148                         return NULL;
 149                 }
 150         } else {
 151                 retval = kfd_gtt_sa_allocate(node, sizeof(struct v9_mqd),
 152                                 &mqd_mem_obj);
 153                 if (retval)
 154                         return NULL;
 155         }
 156
 157         return mqd_mem_obj;
 158 }
 159
 160 static void init_mqd(struct mqd_manager *mm, void **mqd,
 161                         struct kfd_mem_obj *mqd_mem_obj, uint64_t *gart_addr,
 162                         struct queue_properties *q)
 163 {
 164         uint64_t addr;
 165         struct v9_mqd *m;
 166
 167         m = (struct v9_mqd *) mqd_mem_obj->cpu_ptr;
 168         addr = mqd_mem_obj->gpu_addr;
 169
 170         memset(m, 0, sizeof(struct v9_mqd));
 171
 172         m->header = 0xC0310800;
 173         m->compute_pipelinestat_enable = 1;
 174         m->compute_static_thread_mgmt_se0 = 0xFFFFFFFF;
 175         m->compute_static_thread_mgmt_se1 = 0xFFFFFFFF;
 176         m->compute_static_thread_mgmt_se2 = 0xFFFFFFFF;
 177         m->compute_static_thread_mgmt_se3 = 0xFFFFFFFF;
 178         m->compute_static_thread_mgmt_se4 = 0xFFFFFFFF;
 179         m->compute_static_thread_mgmt_se5 = 0xFFFFFFFF;
 180         m->compute_static_thread_mgmt_se6 = 0xFFFFFFFF;
 181         m->compute_static_thread_mgmt_se7 = 0xFFFFFFFF;
 182
 183         m->cp_hqd_persistent_state = CP_HQD_PERSISTENT_STATE__PRELOAD_REQ_MASK |
 184                         0x53 << CP_HQD_PERSISTENT_STATE__PRELOAD_SIZE__SHIFT;
 185
 186         m->cp_mqd_control = 1 << CP_MQD_CONTROL__PRIV_STATE__SHIFT;
 187
 188         m->cp_mqd_base_addr_lo        = lower_32_bits(addr);
 189         m->cp_mqd_base_addr_hi        = upper_32_bits(addr);
 190
 191         m->cp_hqd_quantum = 1 << CP_HQD_QUANTUM__QUANTUM_EN__SHIFT |
 192                         1 << CP_HQD_QUANTUM__QUANTUM_SCALE__SHIFT |
 193                         1 << CP_HQD_QUANTUM__QUANTUM_DURATION__SHIFT;
 194
 195         /* Set cp_hqd_hq_scheduler0 bit 14 to 1 to have the CP set up the
 196          * DISPATCH_PTR.  This is required for the kfd debugger
 197          */
 198         m->cp_hqd_hq_status0 = 1 << 14;
 199
 200         if (q->format == KFD_QUEUE_FORMAT_AQL)
 201                 m->cp_hqd_aql_control =
 202                         1 << CP_HQD_AQL_CONTROL__CONTROL0__SHIFT;
 203
 204         if (q->tba_addr) {
 205                 m->compute_pgm_rsrc2 |=
 206                         (1 << COMPUTE_PGM_RSRC2__TRAP_PRESENT__SHIFT);
 207         }
 208
 209         if (mm->dev->kfd->cwsr_enabled && q->ctx_save_restore_area_address) {
 210                 m->cp_hqd_persistent_state |=
 211                         (1 << CP_HQD_PERSISTENT_STATE__QSWITCH_MODE__SHIFT);
 212                 m->cp_hqd_ctx_save_base_addr_lo =
 213                         lower_32_bits(q->ctx_save_restore_area_address);
 214                 m->cp_hqd_ctx_save_base_addr_hi =
 215                         upper_32_bits(q->ctx_save_restore_area_address);
 216                 m->cp_hqd_ctx_save_size = q->ctx_save_restore_area_size;
 217                 m->cp_hqd_cntl_stack_size = q->ctl_stack_size;
 218                 m->cp_hqd_cntl_stack_offset = q->ctl_stack_size;
 219                 m->cp_hqd_wg_state_offset = q->ctl_stack_size;
 220         }
 221
 222         *mqd = m;
 223         if (gart_addr)
 224                 *gart_addr = addr;
 225         update_mqd(mm, m, q, NULL);
 226 }
 227
 228 static int load_mqd(struct mqd_manager *mm, void *mqd,
 229                         uint32_t pipe_id, uint32_t queue_id,
 230                         struct queue_properties *p, struct mm_struct *mms)
 231 {
 232         /* AQL write pointer counts in 64B packets, PM4/CP counts in dwords. */
 233         uint32_t wptr_shift = (p->format == KFD_QUEUE_FORMAT_AQL ? 4 : 0);
 234
 235         return mm->dev->kfd2kgd->hqd_load(mm->dev->adev, mqd, pipe_id, queue_id,
 236                                           (uint32_t __user *)p->write_ptr,
 237                                           wptr_shift, 0, mms, 0);
 238 }
 239
 240 static void update_mqd(struct mqd_manager *mm, void *mqd,
 241                         struct queue_properties *q,
 242                         struct mqd_update_info *minfo)
 243 {
 244         struct v9_mqd *m;
 245
 246         m = get_mqd(mqd);
 247
 248         m->cp_hqd_pq_control = 5 << CP_HQD_PQ_CONTROL__RPTR_BLOCK_SIZE__SHIFT;
 249         m->cp_hqd_pq_control |= order_base_2(q->queue_size / 4) - 1;
 250         pr_debug("cp_hqd_pq_control 0x%x\n", m->cp_hqd_pq_control);
 251
 252         m->cp_hqd_pq_base_lo = lower_32_bits((uint64_t)q->queue_address >> 8);
 253         m->cp_hqd_pq_base_hi = upper_32_bits((uint64_t)q->queue_address >> 8);
 254
 255         m->cp_hqd_pq_rptr_report_addr_lo = lower_32_bits((uint64_t)q->read_ptr);
 256         m->cp_hqd_pq_rptr_report_addr_hi = upper_32_bits((uint64_t)q->read_ptr);
 257         m->cp_hqd_pq_wptr_poll_addr_lo = lower_32_bits((uint64_t)q->write_ptr);
 258         m->cp_hqd_pq_wptr_poll_addr_hi = upper_32_bits((uint64_t)q->write_ptr);
 259
 260         m->cp_hqd_pq_doorbell_control =
 261                 q->doorbell_off <<
 262                         CP_HQD_PQ_DOORBELL_CONTROL__DOORBELL_OFFSET__SHIFT;
 263         pr_debug("cp_hqd_pq_doorbell_control 0x%x\n",
 264                         m->cp_hqd_pq_doorbell_control);
 265
 266         m->cp_hqd_ib_control =
 267                 3 << CP_HQD_IB_CONTROL__MIN_IB_AVAIL_SIZE__SHIFT |
 268                 1 << CP_HQD_IB_CONTROL__IB_EXE_DISABLE__SHIFT;
 269
 270         /*
 271          * HW does not clamp this field correctly. Maximum EOP queue size
 272          * is constrained by per-SE EOP done signal count, which is 8-bit.
 273          * Limit is 0xFF EOP entries (= 0x7F8 dwords). CP will not submit
 274          * more than (EOP entry count - 1) so a queue size of 0x800 dwords
 275          * is safe, giving a maximum field value of 0xA.
 276          *
 277          * Also, do calculation only if EOP is used (size > 0), otherwise
 278          * the order_base_2 calculation provides incorrect result.
 279          *
 280          */
 281         m->cp_hqd_eop_control = q->eop_ring_buffer_size ?
 282                 min(0xA, order_base_2(q->eop_ring_buffer_size / 4) - 1) : 0;
 283
 284         m->cp_hqd_eop_base_addr_lo =
 285                         lower_32_bits(q->eop_ring_buffer_address >> 8);
 286         m->cp_hqd_eop_base_addr_hi =
 287                         upper_32_bits(q->eop_ring_buffer_address >> 8);
 288
 289         m->cp_hqd_iq_timer = 0;
 290
 291         m->cp_hqd_vmid = q->vmid;
 292
 293         if (q->format == KFD_QUEUE_FORMAT_AQL) {
 294                 m->cp_hqd_pq_control |= CP_HQD_PQ_CONTROL__NO_UPDATE_RPTR_MASK |
 295                                 2 << CP_HQD_PQ_CONTROL__SLOT_BASED_WPTR__SHIFT |
 296                                 1 << CP_HQD_PQ_CONTROL__QUEUE_FULL_EN__SHIFT |
 297                                 1 << CP_HQD_PQ_CONTROL__WPP_CLAMP_EN__SHIFT;
 298                 m->cp_hqd_pq_doorbell_control |= 1 <<
 299                         CP_HQD_PQ_DOORBELL_CONTROL__DOORBELL_BIF_DROP__SHIFT;
 300         }
 301         if (mm->dev->kfd->cwsr_enabled && q->ctx_save_restore_area_address)
 302                 m->cp_hqd_ctx_save_control = 0;
 303
 304         if (KFD_GC_VERSION(mm->dev) != IP_VERSION(9, 4, 3) &&
 305             KFD_GC_VERSION(mm->dev) != IP_VERSION(9, 4, 4) &&
 306             KFD_GC_VERSION(mm->dev) != IP_VERSION(9, 5, 0))
 307                 update_cu_mask(mm, mqd, minfo, 0);
 308         set_priority(m, q);
 309
 310         if (minfo && KFD_GC_VERSION(mm->dev) >= IP_VERSION(9, 4, 2)) {
 311                 if (minfo->update_flag & UPDATE_FLAG_IS_GWS)
 312                         m->compute_resource_limits |=
 313                                 COMPUTE_RESOURCE_LIMITS__FORCE_SIMD_DIST_MASK;
 314                 else
 315                         m->compute_resource_limits &=
 316                                 ~COMPUTE_RESOURCE_LIMITS__FORCE_SIMD_DIST_MASK;
 317         }
 318
 319         q->is_active = QUEUE_IS_ACTIVE(*q);
 320 }
 321
 322
 323 static bool check_preemption_failed(struct mqd_manager *mm, void *mqd)
 324 {
 325         struct v9_mqd *m = (struct v9_mqd *)mqd;
 326         uint32_t doorbell_id = m->queue_doorbell_id0;
 327
 328         m->queue_doorbell_id0 = 0;
 329
 330         return kfd_check_hiq_mqd_doorbell_id(mm->dev, doorbell_id, 0);
 331 }
 332
 333 static int get_wave_state(struct mqd_manager *mm, void *mqd,
 334                           struct queue_properties *q,
 335                           void __user *ctl_stack,
 336                           u32 *ctl_stack_used_size,
 337                           u32 *save_area_used_size)
 338 {
 339         struct v9_mqd *m;
 340         struct kfd_context_save_area_header header;
 341
 342         /* Control stack is located one page after MQD. */
 343         void *mqd_ctl_stack = (void *)((uintptr_t)mqd + PAGE_SIZE);
 344
 345         m = get_mqd(mqd);
 346
 347         *ctl_stack_used_size = m->cp_hqd_cntl_stack_size -
 348                 m->cp_hqd_cntl_stack_offset;
 349         *save_area_used_size = m->cp_hqd_wg_state_offset -
 350                 m->cp_hqd_cntl_stack_size;
 351
 352         header.wave_state.control_stack_size = *ctl_stack_used_size;
 353         header.wave_state.wave_state_size = *save_area_used_size;
 354
 355         header.wave_state.wave_state_offset = m->cp_hqd_wg_state_offset;
 356         header.wave_state.control_stack_offset = m->cp_hqd_cntl_stack_offset;
 357
 358         if (copy_to_user(ctl_stack, &header, sizeof(header.wave_state)))
 359                 return -EFAULT;
 360
 361         if (copy_to_user(ctl_stack + m->cp_hqd_cntl_stack_offset,
 362                                 mqd_ctl_stack + m->cp_hqd_cntl_stack_offset,
 363                                 *ctl_stack_used_size))
 364                 return -EFAULT;
 365
 366         return 0;
 367 }
 368
 369 static void get_checkpoint_info(struct mqd_manager *mm, void *mqd, u32 *ctl_stack_size)
 370 {
 371         struct v9_mqd *m = get_mqd(mqd);
 372
 373         *ctl_stack_size = m->cp_hqd_cntl_stack_size;
 374 }
 375
 376 static void checkpoint_mqd(struct mqd_manager *mm, void *mqd, void *mqd_dst, void *ctl_stack_dst)
 377 {
 378         struct v9_mqd *m;
 379         /* Control stack is located one page after MQD. */
 380         void *ctl_stack = (void *)((uintptr_t)mqd + PAGE_SIZE);
 381
 382         m = get_mqd(mqd);
 383
 384         memcpy(mqd_dst, m, sizeof(struct v9_mqd));
 385         memcpy(ctl_stack_dst, ctl_stack, m->cp_hqd_cntl_stack_size);
 386 }
 387
 388 static void restore_mqd(struct mqd_manager *mm, void **mqd,
 389                         struct kfd_mem_obj *mqd_mem_obj, uint64_t *gart_addr,
 390                         struct queue_properties *qp,
 391                         const void *mqd_src,
 392                         const void *ctl_stack_src, u32 ctl_stack_size)
 393 {
 394         uint64_t addr;
 395         struct v9_mqd *m;
 396         void *ctl_stack;
 397
 398         m = (struct v9_mqd *) mqd_mem_obj->cpu_ptr;
 399         addr = mqd_mem_obj->gpu_addr;
 400
 401         memcpy(m, mqd_src, sizeof(*m));
 402
 403         *mqd = m;
 404         if (gart_addr)
 405                 *gart_addr = addr;
 406
 407         /* Control stack is located one page after MQD. */
 408         ctl_stack = (void *)((uintptr_t)*mqd + PAGE_SIZE);
 409         memcpy(ctl_stack, ctl_stack_src, ctl_stack_size);
 410
 411         m->cp_hqd_pq_doorbell_control =
 412                 qp->doorbell_off <<
 413                         CP_HQD_PQ_DOORBELL_CONTROL__DOORBELL_OFFSET__SHIFT;
 414         pr_debug("cp_hqd_pq_doorbell_control 0x%x\n",
 415                                 m->cp_hqd_pq_doorbell_control);
 416
 417         qp->is_active = 0;
 418 }
 419
 420 static void init_mqd_hiq(struct mqd_manager *mm, void **mqd,
 421                         struct kfd_mem_obj *mqd_mem_obj, uint64_t *gart_addr,
 422                         struct queue_properties *q)
 423 {
 424         struct v9_mqd *m;
 425
 426         init_mqd(mm, mqd, mqd_mem_obj, gart_addr, q);
 427
 428         m = get_mqd(*mqd);
 429
 430         m->cp_hqd_pq_control |= 1 << CP_HQD_PQ_CONTROL__PRIV_STATE__SHIFT |
 431                         1 << CP_HQD_PQ_CONTROL__KMD_QUEUE__SHIFT;
 432 }
 433
 434 static int destroy_hiq_mqd(struct mqd_manager *mm, void *mqd,
 435                         enum kfd_preempt_type type, unsigned int timeout,
 436                         uint32_t pipe_id, uint32_t queue_id)
 437 {
 438         int err;
 439         struct v9_mqd *m;
 440         u32 doorbell_off;
 441
 442         m = get_mqd(mqd);
 443
 444         doorbell_off = m->cp_hqd_pq_doorbell_control >>
 445                         CP_HQD_PQ_DOORBELL_CONTROL__DOORBELL_OFFSET__SHIFT;
 446         err = amdgpu_amdkfd_unmap_hiq(mm->dev->adev, doorbell_off, 0);
 447         if (err)
 448                 pr_debug("Destroy HIQ MQD failed: %d\n", err);
 449
 450         return err;
 451 }
 452
 453 static void init_mqd_sdma(struct mqd_manager *mm, void **mqd,
 454                 struct kfd_mem_obj *mqd_mem_obj, uint64_t *gart_addr,
 455                 struct queue_properties *q)
 456 {
 457         struct v9_sdma_mqd *m;
 458
 459         m = (struct v9_sdma_mqd *) mqd_mem_obj->cpu_ptr;
 460
 461         memset(m, 0, sizeof(struct v9_sdma_mqd));
 462
 463         *mqd = m;
 464         if (gart_addr)
 465                 *gart_addr = mqd_mem_obj->gpu_addr;
 466
 467         mm->update_mqd(mm, m, q, NULL);
 468 }
 469
 470 #define SDMA_RLC_DUMMY_DEFAULT 0xf
 471
 472 static void update_mqd_sdma(struct mqd_manager *mm, void *mqd,
 473                         struct queue_properties *q,
 474                         struct mqd_update_info *minfo)
 475 {
 476         struct v9_sdma_mqd *m;
 477
 478         m = get_sdma_mqd(mqd);
 479         m->sdmax_rlcx_rb_cntl = order_base_2(q->queue_size / 4)
 480                 << SDMA0_RLC0_RB_CNTL__RB_SIZE__SHIFT |
 481                 q->vmid << SDMA0_RLC0_RB_CNTL__RB_VMID__SHIFT |
 482                 1 << SDMA0_RLC0_RB_CNTL__RPTR_WRITEBACK_ENABLE__SHIFT |
 483                 6 << SDMA0_RLC0_RB_CNTL__RPTR_WRITEBACK_TIMER__SHIFT;
 484
 485         m->sdmax_rlcx_rb_base = lower_32_bits(q->queue_address >> 8);
 486         m->sdmax_rlcx_rb_base_hi = upper_32_bits(q->queue_address >> 8);
 487         m->sdmax_rlcx_rb_rptr_addr_lo = lower_32_bits((uint64_t)q->read_ptr);
 488         m->sdmax_rlcx_rb_rptr_addr_hi = upper_32_bits((uint64_t)q->read_ptr);
 489         m->sdmax_rlcx_doorbell_offset =
 490                 q->doorbell_off << SDMA0_RLC0_DOORBELL_OFFSET__OFFSET__SHIFT;
 491
 492         m->sdma_engine_id = q->sdma_engine_id;
 493         m->sdma_queue_id = q->sdma_queue_id;
 494         m->sdmax_rlcx_dummy_reg = SDMA_RLC_DUMMY_DEFAULT;
 495
 496         q->is_active = QUEUE_IS_ACTIVE(*q);
 497 }
 498
 499 static void checkpoint_mqd_sdma(struct mqd_manager *mm,
 500                                 void *mqd,
 501                                 void *mqd_dst,
 502                                 void *ctl_stack_dst)
 503 {
 504         struct v9_sdma_mqd *m;
 505
 506         m = get_sdma_mqd(mqd);
 507
 508         memcpy(mqd_dst, m, sizeof(struct v9_sdma_mqd));
 509 }
 510
 511 static void restore_mqd_sdma(struct mqd_manager *mm, void **mqd,
 512                              struct kfd_mem_obj *mqd_mem_obj, uint64_t *gart_addr,
 513                              struct queue_properties *qp,
 514                              const void *mqd_src,
 515                              const void *ctl_stack_src, const u32 ctl_stack_size)
 516 {
 517         uint64_t addr;
 518         struct v9_sdma_mqd *m;
 519
 520         m = (struct v9_sdma_mqd *) mqd_mem_obj->cpu_ptr;
 521         addr = mqd_mem_obj->gpu_addr;
 522
 523         memcpy(m, mqd_src, sizeof(*m));
 524
 525         m->sdmax_rlcx_doorbell_offset =
 526                 qp->doorbell_off << SDMA0_RLC0_DOORBELL_OFFSET__OFFSET__SHIFT;
 527
 528         *mqd = m;
 529         if (gart_addr)
 530                 *gart_addr = addr;
 531
 532         qp->is_active = 0;
 533 }
 534
 535 static void init_mqd_hiq_v9_4_3(struct mqd_manager *mm, void **mqd,
 536                         struct kfd_mem_obj *mqd_mem_obj, uint64_t *gart_addr,
 537                         struct queue_properties *q)
 538 {
 539         struct v9_mqd *m;
 540         int xcc = 0;
 541         struct kfd_mem_obj xcc_mqd_mem_obj;
 542         uint64_t xcc_gart_addr = 0;
 543
 544         memset(&xcc_mqd_mem_obj, 0x0, sizeof(struct kfd_mem_obj));
 545
 546         for (xcc = 0; xcc < NUM_XCC(mm->dev->xcc_mask); xcc++) {
 547                 kfd_get_hiq_xcc_mqd(mm->dev, &xcc_mqd_mem_obj, xcc);
 548
 549                 init_mqd(mm, (void **)&m, &xcc_mqd_mem_obj, &xcc_gart_addr, q);
 550
 551                 m->cp_hqd_pq_control |= CP_HQD_PQ_CONTROL__NO_UPDATE_RPTR_MASK |
 552                                         1 << CP_HQD_PQ_CONTROL__PRIV_STATE__SHIFT |
 553                                         1 << CP_HQD_PQ_CONTROL__KMD_QUEUE__SHIFT;
 554                 if (amdgpu_sriov_vf(mm->dev->adev))
 555                         m->cp_hqd_pq_doorbell_control |= 1 <<
 556                                 CP_HQD_PQ_DOORBELL_CONTROL__DOORBELL_MODE__SHIFT;
 557                 m->cp_mqd_stride_size = kfd_hiq_mqd_stride(mm->dev);
 558                 if (xcc == 0) {
 559                         /* Set no_update_rptr = 0 in Master XCC */
 560                         m->cp_hqd_pq_control &= ~CP_HQD_PQ_CONTROL__NO_UPDATE_RPTR_MASK;
 561
 562                         /* Set the MQD pointer and gart address to XCC0 MQD */
 563                         *mqd = m;
 564                         *gart_addr = xcc_gart_addr;
 565                 }
 566         }
 567 }
 568
 569 static int hiq_load_mqd_kiq_v9_4_3(struct mqd_manager *mm, void *mqd,
 570                         uint32_t pipe_id, uint32_t queue_id,
 571                         struct queue_properties *p, struct mm_struct *mms)
 572 {
 573         uint32_t xcc_mask = mm->dev->xcc_mask;
 574         int xcc_id, err, inst = 0;
 575         void *xcc_mqd;
 576         uint64_t hiq_mqd_size = kfd_hiq_mqd_stride(mm->dev);
 577
 578         for_each_inst(xcc_id, xcc_mask) {
 579                 xcc_mqd = mqd + hiq_mqd_size * inst;
 580                 err = mm->dev->kfd2kgd->hiq_mqd_load(mm->dev->adev, xcc_mqd,
 581                                                      pipe_id, queue_id,
 582                                                      p->doorbell_off, xcc_id);
 583                 if (err) {
 584                         pr_debug("Failed to load HIQ MQD for XCC: %d\n", inst);
 585                         break;
 586                 }
 587                 ++inst;
 588         }
 589
 590         return err;
 591 }
 592
 593 static int destroy_hiq_mqd_v9_4_3(struct mqd_manager *mm, void *mqd,
 594                         enum kfd_preempt_type type, unsigned int timeout,
 595                         uint32_t pipe_id, uint32_t queue_id)
 596 {
 597         uint32_t xcc_mask = mm->dev->xcc_mask;
 598         int xcc_id, err, inst = 0;
 599         uint64_t hiq_mqd_size = kfd_hiq_mqd_stride(mm->dev);
 600         struct v9_mqd *m;
 601         u32 doorbell_off;
 602
 603         for_each_inst(xcc_id, xcc_mask) {
 604                 m = get_mqd(mqd + hiq_mqd_size * inst);
 605
 606                 doorbell_off = m->cp_hqd_pq_doorbell_control >>
 607                                 CP_HQD_PQ_DOORBELL_CONTROL__DOORBELL_OFFSET__SHIFT;
 608
 609                 err = amdgpu_amdkfd_unmap_hiq(mm->dev->adev, doorbell_off, xcc_id);
 610                 if (err) {
 611                         pr_debug("Destroy HIQ MQD failed for xcc: %d\n", inst);
 612                         break;
 613                 }
 614                 ++inst;
 615         }
 616
 617         return err;
 618 }
 619
 620 static bool check_preemption_failed_v9_4_3(struct mqd_manager *mm, void *mqd)
 621 {
 622         uint64_t hiq_mqd_size = kfd_hiq_mqd_stride(mm->dev);
 623         uint32_t xcc_mask = mm->dev->xcc_mask;
 624         int inst = 0, xcc_id;
 625         struct v9_mqd *m;
 626         bool ret = false;
 627
 628         for_each_inst(xcc_id, xcc_mask) {
 629                 m = get_mqd(mqd + hiq_mqd_size * inst);
 630                 ret |= kfd_check_hiq_mqd_doorbell_id(mm->dev,
 631                                         m->queue_doorbell_id0, inst);
 632                 m->queue_doorbell_id0 = 0;
 633                 ++inst;
 634         }
 635
 636         return ret;
 637 }
 638
 639 static void get_xcc_mqd(struct kfd_mem_obj *mqd_mem_obj,
 640                                struct kfd_mem_obj *xcc_mqd_mem_obj,
 641                                uint64_t offset)
 642 {
 643         xcc_mqd_mem_obj->gtt_mem = (offset == 0) ?
 644                                         mqd_mem_obj->gtt_mem : NULL;
 645         xcc_mqd_mem_obj->gpu_addr = mqd_mem_obj->gpu_addr + offset;
 646         xcc_mqd_mem_obj->cpu_ptr = (uint32_t *)((uintptr_t)mqd_mem_obj->cpu_ptr
 647                                                 + offset);
 648 }
 649
 650 static void init_mqd_v9_4_3(struct mqd_manager *mm, void **mqd,
 651                         struct kfd_mem_obj *mqd_mem_obj, uint64_t *gart_addr,
 652                         struct queue_properties *q)
 653 {
 654         struct v9_mqd *m;
 655         int xcc = 0;
 656         struct kfd_mem_obj xcc_mqd_mem_obj;
 657         uint64_t xcc_gart_addr = 0;
 658         uint64_t xcc_ctx_save_restore_area_address;
 659         uint64_t offset = mm->mqd_stride(mm, q);
 660         uint32_t local_xcc_start = mm->dev->dqm->current_logical_xcc_start++;
 661
 662         memset(&xcc_mqd_mem_obj, 0x0, sizeof(struct kfd_mem_obj));
 663         for (xcc = 0; xcc < NUM_XCC(mm->dev->xcc_mask); xcc++) {
 664                 get_xcc_mqd(mqd_mem_obj, &xcc_mqd_mem_obj, offset*xcc);
 665
 666                 init_mqd(mm, (void **)&m, &xcc_mqd_mem_obj, &xcc_gart_addr, q);
 667
 668                 m->cp_mqd_stride_size = offset;
 669
 670                 /*
 671                  * Update the CWSR address for each XCC if CWSR is enabled
 672                  * and CWSR area is allocated in thunk
 673                  */
 674                 if (mm->dev->kfd->cwsr_enabled &&
 675                     q->ctx_save_restore_area_address) {
 676                         xcc_ctx_save_restore_area_address =
 677                                 q->ctx_save_restore_area_address +
 678                                 (xcc * q->ctx_save_restore_area_size);
 679
 680                         m->cp_hqd_ctx_save_base_addr_lo =
 681                                 lower_32_bits(xcc_ctx_save_restore_area_address);
 682                         m->cp_hqd_ctx_save_base_addr_hi =
 683                                 upper_32_bits(xcc_ctx_save_restore_area_address);
 684                 }
 685
 686                 if (q->format == KFD_QUEUE_FORMAT_AQL) {
 687                         m->compute_tg_chunk_size = 1;
 688                         m->compute_current_logic_xcc_id =
 689                                         (local_xcc_start + xcc) %
 690                                         NUM_XCC(mm->dev->xcc_mask);
 691
 692                         switch (xcc) {
 693                         case 0:
 694                                 /* Master XCC */
 695                                 m->cp_hqd_pq_control &=
 696                                         ~CP_HQD_PQ_CONTROL__NO_UPDATE_RPTR_MASK;
 697                                 break;
 698                         default:
 699                                 break;
 700                         }
 701                 } else {
 702                         /* PM4 Queue */
 703                         m->compute_current_logic_xcc_id = 0;
 704                         m->compute_tg_chunk_size = 0;
 705                         m->pm4_target_xcc_in_xcp = q->pm4_target_xcc;
 706                 }
 707
 708                 if (xcc == 0) {
 709                         /* Set the MQD pointer and gart address to XCC0 MQD */
 710                         *mqd = m;
 711                         *gart_addr = xcc_gart_addr;
 712                 }
 713         }
 714 }
 715
 716 static void update_mqd_v9_4_3(struct mqd_manager *mm, void *mqd,
 717                       struct queue_properties *q, struct mqd_update_info *minfo)
 718 {
 719         struct v9_mqd *m;
 720         int xcc = 0;
 721         uint64_t size = mm->mqd_stride(mm, q);
 722
 723         for (xcc = 0; xcc < NUM_XCC(mm->dev->xcc_mask); xcc++) {
 724                 m = get_mqd(mqd + size * xcc);
 725                 update_mqd(mm, m, q, minfo);
 726
 727                 update_cu_mask(mm, m, minfo, xcc);
 728
 729                 if (q->format == KFD_QUEUE_FORMAT_AQL) {
 730                         switch (xcc) {
 731                         case 0:
 732                                 /* Master XCC */
 733                                 m->cp_hqd_pq_control &=
 734                                         ~CP_HQD_PQ_CONTROL__NO_UPDATE_RPTR_MASK;
 735                                 break;
 736                         default:
 737                                 break;
 738                         }
 739                         m->compute_tg_chunk_size = 1;
 740                 } else {
 741                         /* PM4 Queue */
 742                         m->compute_current_logic_xcc_id = 0;
 743                         m->compute_tg_chunk_size = 0;
 744                         m->pm4_target_xcc_in_xcp = q->pm4_target_xcc;
 745                 }
 746         }
 747 }
 748
 749 static int destroy_mqd_v9_4_3(struct mqd_manager *mm, void *mqd,
 750                    enum kfd_preempt_type type, unsigned int timeout,
 751                    uint32_t pipe_id, uint32_t queue_id)
 752 {
 753         uint32_t xcc_mask = mm->dev->xcc_mask;
 754         int xcc_id, err, inst = 0;
 755         void *xcc_mqd;
 756         struct v9_mqd *m;
 757         uint64_t mqd_offset;
 758
 759         m = get_mqd(mqd);
 760         mqd_offset = m->cp_mqd_stride_size;
 761
 762         for_each_inst(xcc_id, xcc_mask) {
 763                 xcc_mqd = mqd + mqd_offset * inst;
 764                 err = mm->dev->kfd2kgd->hqd_destroy(mm->dev->adev, xcc_mqd,
 765                                                     type, timeout, pipe_id,
 766                                                     queue_id, xcc_id);
 767                 if (err) {
 768                         pr_debug("Destroy MQD failed for xcc: %d\n", inst);
 769                         break;
 770                 }
 771                 ++inst;
 772         }
 773
 774         return err;
 775 }
 776
 777 static int load_mqd_v9_4_3(struct mqd_manager *mm, void *mqd,
 778                         uint32_t pipe_id, uint32_t queue_id,
 779                         struct queue_properties *p, struct mm_struct *mms)
 780 {
 781         /* AQL write pointer counts in 64B packets, PM4/CP counts in dwords. */
 782         uint32_t wptr_shift = (p->format == KFD_QUEUE_FORMAT_AQL ? 4 : 0);
 783         uint32_t xcc_mask = mm->dev->xcc_mask;
 784         int xcc_id, err, inst = 0;
 785         void *xcc_mqd;
 786         uint64_t mqd_stride_size = mm->mqd_stride(mm, p);
 787
 788         for_each_inst(xcc_id, xcc_mask) {
 789                 xcc_mqd = mqd + mqd_stride_size * inst;
 790                 err = mm->dev->kfd2kgd->hqd_load(
 791                         mm->dev->adev, xcc_mqd, pipe_id, queue_id,
 792                         (uint32_t __user *)p->write_ptr, wptr_shift, 0, mms,
 793                         xcc_id);
 794                 if (err) {
 795                         pr_debug("Load MQD failed for xcc: %d\n", inst);
 796                         break;
 797                 }
 798                 ++inst;
 799         }
 800
 801         return err;
 802 }
 803
 804 static int get_wave_state_v9_4_3(struct mqd_manager *mm, void *mqd,
 805                                  struct queue_properties *q,
 806                                  void __user *ctl_stack,
 807                                  u32 *ctl_stack_used_size,
 808                                  u32 *save_area_used_size)
 809 {
 810         int xcc, err = 0;
 811         void *xcc_mqd;
 812         void __user *xcc_ctl_stack;
 813         uint64_t mqd_stride_size = mm->mqd_stride(mm, q);
 814         u32 tmp_ctl_stack_used_size = 0, tmp_save_area_used_size = 0;
 815
 816         for (xcc = 0; xcc < NUM_XCC(mm->dev->xcc_mask); xcc++) {
 817                 xcc_mqd = mqd + mqd_stride_size * xcc;
 818                 xcc_ctl_stack = (void __user *)((uintptr_t)ctl_stack +
 819                                         q->ctx_save_restore_area_size * xcc);
 820
 821                 err = get_wave_state(mm, xcc_mqd, q, xcc_ctl_stack,
 822                                      &tmp_ctl_stack_used_size,
 823                                      &tmp_save_area_used_size);
 824                 if (err)
 825                         break;
 826
 827                 /*
 828                  * Set the ctl_stack_used_size and save_area_used_size to
 829                  * ctl_stack_used_size and save_area_used_size of XCC 0 when
 830                  * passing the info the user-space.
 831                  * For multi XCC, user-space would have to look at the header
 832                  * info of each Control stack area to determine the control
 833                  * stack size and save area used.
 834                  */
 835                 if (xcc == 0) {
 836                         *ctl_stack_used_size = tmp_ctl_stack_used_size;
 837                         *save_area_used_size = tmp_save_area_used_size;
 838                 }
 839         }
 840
 841         return err;
 842 }
 843
 844 #if defined(CONFIG_DEBUG_FS)
 845
 846 static int debugfs_show_mqd(struct seq_file *m, void *data)
 847 {
 848         seq_hex_dump(m, "    ", DUMP_PREFIX_OFFSET, 32, 4,
 849                      data, sizeof(struct v9_mqd), false);
 850         return 0;
 851 }
 852
 853 static int debugfs_show_mqd_sdma(struct seq_file *m, void *data)
 854 {
 855         seq_hex_dump(m, "    ", DUMP_PREFIX_OFFSET, 32, 4,
 856                      data, sizeof(struct v9_sdma_mqd), false);
 857         return 0;
 858 }
 859
 860 #endif
 861
 862 struct mqd_manager *mqd_manager_init_v9(enum KFD_MQD_TYPE type,
 863                 struct kfd_node *dev)
 864 {
 865         struct mqd_manager *mqd;
 866
 867         if (WARN_ON(type >= KFD_MQD_TYPE_MAX))
 868                 return NULL;
 869
 870         mqd = kzalloc(sizeof(*mqd), GFP_KERNEL);
 871         if (!mqd)
 872                 return NULL;
 873
 874         mqd->dev = dev;
 875
 876         switch (type) {
 877         case KFD_MQD_TYPE_CP:
 878                 mqd->allocate_mqd = allocate_mqd;
 879                 mqd->free_mqd = kfd_free_mqd_cp;
 880                 mqd->is_occupied = kfd_is_occupied_cp;
 881                 mqd->get_checkpoint_info = get_checkpoint_info;
 882                 mqd->checkpoint_mqd = checkpoint_mqd;
 883                 mqd->restore_mqd = restore_mqd;
 884                 mqd->mqd_size = sizeof(struct v9_mqd);
 885                 mqd->mqd_stride = mqd_stride_v9;
 886 #if defined(CONFIG_DEBUG_FS)
 887                 mqd->debugfs_show_mqd = debugfs_show_mqd;
 888 #endif
 889                 if (KFD_GC_VERSION(dev) == IP_VERSION(9, 4, 3) ||
 890                     KFD_GC_VERSION(dev) == IP_VERSION(9, 4, 4) ||
 891                     KFD_GC_VERSION(dev) == IP_VERSION(9, 5, 0)) {
 892                         mqd->init_mqd = init_mqd_v9_4_3;
 893                         mqd->load_mqd = load_mqd_v9_4_3;
 894                         mqd->update_mqd = update_mqd_v9_4_3;
 895                         mqd->destroy_mqd = destroy_mqd_v9_4_3;
 896                         mqd->get_wave_state = get_wave_state_v9_4_3;
 897                 } else {
 898                         mqd->init_mqd = init_mqd;
 899                         mqd->load_mqd = load_mqd;
 900                         mqd->update_mqd = update_mqd;
 901                         mqd->destroy_mqd = kfd_destroy_mqd_cp;
 902                         mqd->get_wave_state = get_wave_state;
 903                 }
 904                 break;
 905         case KFD_MQD_TYPE_HIQ:
 906                 mqd->allocate_mqd = allocate_hiq_mqd;
 907                 mqd->free_mqd = free_mqd_hiq_sdma;
 908                 mqd->update_mqd = update_mqd;
 909                 mqd->is_occupied = kfd_is_occupied_cp;
 910                 mqd->mqd_size = sizeof(struct v9_mqd);
 911                 mqd->mqd_stride = kfd_mqd_stride;
 912 #if defined(CONFIG_DEBUG_FS)
 913                 mqd->debugfs_show_mqd = debugfs_show_mqd;
 914 #endif
 915                 mqd->check_preemption_failed = check_preemption_failed;
 916                 if (KFD_GC_VERSION(dev) == IP_VERSION(9, 4, 3) ||
 917                     KFD_GC_VERSION(dev) == IP_VERSION(9, 4, 4) ||
 918                     KFD_GC_VERSION(dev) == IP_VERSION(9, 5, 0)) {
 919                         mqd->init_mqd = init_mqd_hiq_v9_4_3;
 920                         mqd->load_mqd = hiq_load_mqd_kiq_v9_4_3;
 921                         mqd->destroy_mqd = destroy_hiq_mqd_v9_4_3;
 922                         mqd->check_preemption_failed = check_preemption_failed_v9_4_3;
 923                 } else {
 924                         mqd->init_mqd = init_mqd_hiq;
 925                         mqd->load_mqd = kfd_hiq_load_mqd_kiq;
 926                         mqd->destroy_mqd = destroy_hiq_mqd;
 927                         mqd->check_preemption_failed = check_preemption_failed;
 928                 }
 929                 break;
 930         case KFD_MQD_TYPE_DIQ:
 931                 mqd->allocate_mqd = allocate_mqd;
 932                 mqd->init_mqd = init_mqd_hiq;
 933                 mqd->free_mqd = kfd_free_mqd_cp;
 934                 mqd->load_mqd = load_mqd;
 935                 mqd->update_mqd = update_mqd;
 936                 mqd->destroy_mqd = kfd_destroy_mqd_cp;
 937                 mqd->is_occupied = kfd_is_occupied_cp;
 938                 mqd->mqd_size = sizeof(struct v9_mqd);
 939 #if defined(CONFIG_DEBUG_FS)
 940                 mqd->debugfs_show_mqd = debugfs_show_mqd;
 941 #endif
 942                 break;
 943         case KFD_MQD_TYPE_SDMA:
 944                 mqd->allocate_mqd = allocate_sdma_mqd;
 945                 mqd->init_mqd = init_mqd_sdma;
 946                 mqd->free_mqd = free_mqd_hiq_sdma;
 947                 mqd->load_mqd = kfd_load_mqd_sdma;
 948                 mqd->update_mqd = update_mqd_sdma;
 949                 mqd->destroy_mqd = kfd_destroy_mqd_sdma;
 950                 mqd->is_occupied = kfd_is_occupied_sdma;
 951                 mqd->checkpoint_mqd = checkpoint_mqd_sdma;
 952                 mqd->restore_mqd = restore_mqd_sdma;
 953                 mqd->mqd_size = sizeof(struct v9_sdma_mqd);
 954                 mqd->mqd_stride = kfd_mqd_stride;
 955 #if defined(CONFIG_DEBUG_FS)
 956                 mqd->debugfs_show_mqd = debugfs_show_mqd_sdma;
 957 #endif
 958                 break;
 959         default:
 960                 kfree(mqd);
 961                 return NULL;
 962         }
 963
 964         return mqd;
 965 }